botholomew 0.21.2 → 0.22.0

package/README.md

@@ -232,6 +232,7 @@ semantic search, append-only versioning, and URL refresh all live there.
 | Command | Purpose |
 |---|---|
 | `botholomew init` | Initialize the current directory as a project (refuses on iCloud/Dropbox/NFS without `--force`) |
+| `botholomew status` | One-command dashboard: workers, task counts/claims, schedules (with a live "due?" check), quarantined files, store info. `--json` for scripting, `--no-evaluate` to skip the LLM schedule check |
 | `botholomew worker run\|start` | Run a worker (foreground or background); `--persist` for long-running, `--task-id <id>` to target one task |
 | `botholomew worker list\|status\|stop\|kill\|reap` | Inspect and manage running workers |
 | `botholomew chat` | Interactive Ink/React TUI |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "botholomew",
-  "version": "0.21.2",
+  "version": "0.22.0",
   "description": "An autonomous AI agent for knowledge work — works your task queue while you sleep.",
   "type": "module",
   "bin": {

package/src/chat/agent.ts

@@ -28,6 +28,7 @@ import { maybeStoreResult } from "../worker/large-results.ts";
 import {
   buildMetaHeader,
   extractKeywords,
+  LARGE_JSON_SECTION,
   loadPersistentContext,
   MEMBOT_PROMPT_SECTION,
   STYLE_RULES,
@@ -64,6 +65,7 @@ const CHAT_TOOL_NAMES = new Set([
   "membot_count_lines",
   "membot_copy",
   "membot_pipe",
+  "membot_query",
   "list_threads",
   "view_thread",
   "search_threads",
@@ -120,6 +122,7 @@ Format your responses using Markdown. Use headings, bold, italic, lists, and cod
 `;
 
   prompt += `\n${MEMBOT_PROMPT_SECTION}`;
+  prompt += `\n${LARGE_JSON_SECTION}`;
 
   if (options?.hasMcpTools) {
     prompt += `

package/src/cli.ts

@@ -13,6 +13,7 @@ import { registerPrepareCommand } from "./commands/prepare.ts";
 import { registerPromptsCommand } from "./commands/prompts.ts";
 import { registerScheduleCommand } from "./commands/schedule.ts";
 import { registerSkillCommand } from "./commands/skill.ts";
+import { registerStatusCommand } from "./commands/status.ts";
 import { registerTaskCommand } from "./commands/task.ts";
 import { registerThreadCommand } from "./commands/thread.ts";
 import { registerUpgradeCommand } from "./commands/upgrade.ts";
@@ -57,6 +58,7 @@ program
   });
 
 registerInitCommand(program);
+registerStatusCommand(program);
 registerWorkerCommand(program);
 registerTaskCommand(program);
 registerThreadCommand(program);

package/src/commands/status.ts

@@ -0,0 +1,194 @@
+import ansis from "ansis";
+import type { Command } from "commander";
+import { loadConfig } from "../config/loader.ts";
+import {
+  collectStatus,
+  type ScheduleEntry,
+  type StatusReport,
+  type WorkerSummary,
+} from "../status/collect.ts";
+import type { TaskStatus } from "../tasks/schema.ts";
+import type { WorkerStatus } from "../workers/store.ts";
+
+function formatAge(fromIso: string | null, to = new Date()): string {
+  if (!fromIso) return "never";
+  const from = new Date(fromIso);
+  const secs = Math.max(0, Math.floor((to.getTime() - from.getTime()) / 1000));
+  if (secs < 60) return `${secs}s ago`;
+  const mins = Math.floor(secs / 60);
+  if (mins < 60) return `${mins}m ago`;
+  const hours = Math.floor(mins / 60);
+  if (hours < 24) return `${hours}h ago`;
+  return `${Math.floor(hours / 24)}d ago`;
+}
+
+function padColored(colored: string, raw: string, width: number): string {
+  return colored + " ".repeat(Math.max(0, width - raw.length));
+}
+
+function workerStatusColor(status: WorkerStatus): string {
+  switch (status) {
+    case "running":
+      return ansis.green(status);
+    case "stopped":
+      return ansis.dim(status);
+    case "dead":
+      return ansis.red(status);
+  }
+}
+
+function taskStatusColor(status: TaskStatus, label: string): string {
+  switch (status) {
+    case "pending":
+      return ansis.yellow(label);
+    case "in_progress":
+      return ansis.blue(label);
+    case "complete":
+      return ansis.green(label);
+    case "failed":
+      return ansis.red(label);
+    case "waiting":
+      return ansis.magenta(label);
+  }
+}
+
+function section(title: string): void {
+  console.log(`\n${ansis.bold.underline(title)}`);
+}
+
+function renderWorkers(w: WorkerSummary): void {
+  section("Workers");
+  const parts = [
+    `${ansis.green(`${w.running} running`)}`,
+    `${ansis.dim(`${w.stopped} stopped`)}`,
+    `${w.dead > 0 ? ansis.red(`${w.dead} dead`) : ansis.dim("0 dead")}`,
+  ];
+  console.log(
+    `  ${w.total} total — ${parts.join(", ")}   latest heartbeat: ${formatAge(w.latest_heartbeat_at)}`,
+  );
+  if (w.list.length === 0) {
+    console.log(ansis.dim("  (none)"));
+    return;
+  }
+  for (const wk of w.list) {
+    const short = ansis.bold(wk.id.slice(0, 8));
+    const status = workerStatusColor(wk.status);
+    const task = wk.task_id ? `  task=${wk.task_id.slice(0, 8)}` : "";
+    console.log(
+      `  ${short}  ${status}  mode=${wk.mode}  pid=${wk.pid}  heartbeat ${formatAge(wk.last_heartbeat_at)}${task}`,
+    );
+  }
+}
+
+function renderTasks(t: StatusReport["tasks"]): void {
+  section("Tasks");
+  const order: TaskStatus[] = [
+    "pending",
+    "in_progress",
+    "waiting",
+    "complete",
+    "failed",
+  ];
+  const counts = order
+    .map((s) => taskStatusColor(s, `${t.by_status[s]} ${s}`))
+    .join(", ");
+  console.log(`  ${t.total} total — ${counts}`);
+
+  if (t.claimed.length > 0) {
+    console.log(ansis.bold("\n  Claimed:"));
+    for (const c of t.claimed) {
+      console.log(
+        `  ${ansis.dim(c.id.slice(0, 8))}  ${c.name}  ${ansis.dim(`by ${c.claimed_by.slice(0, 8)} (${formatAge(c.claimed_at)})`)}`,
+      );
+    }
+  }
+}
+
+function dueLabel(entry: ScheduleEntry): string {
+  if (!entry.evaluation) return ansis.dim("—");
+  return entry.evaluation.is_due
+    ? ansis.green.bold("due")
+    : ansis.dim("not due");
+}
+
+function renderSchedules(s: StatusReport["schedules"]): void {
+  section("Schedules");
+  console.log(
+    `  ${s.total} total — ${ansis.green(`${s.enabled} enabled`)}, ${ansis.dim(`${s.disabled} disabled`)}`,
+  );
+  if (s.list.length === 0) {
+    console.log(ansis.dim("  (none)"));
+    return;
+  }
+  for (const e of s.list) {
+    const name = e.enabled ? e.name : ansis.dim(e.name);
+    const freq = ansis.cyan(e.frequency);
+    const last = ansis.dim(`last run ${formatAge(e.last_run_at)}`);
+    let line = `  ${name}  (${freq})  ${last}  ${dueLabel(e)}`;
+    if (e.evaluation?.is_due && e.evaluation.tasks_to_create > 0) {
+      line += ansis.dim(` → ${e.evaluation.tasks_to_create} task(s)`);
+    }
+    console.log(line);
+    if (e.evaluation?.is_due) {
+      console.log(ansis.dim(`      ${e.evaluation.reasoning}`));
+    }
+  }
+}
+
+function renderQuarantined(report: StatusReport): void {
+  const items = [
+    ...report.tasks.quarantined.map((q) => ({ kind: "task", ...q })),
+    ...report.schedules.quarantined.map((q) => ({ kind: "schedule", ...q })),
+  ];
+  if (items.length === 0) return;
+  section("Quarantined (invalid frontmatter)");
+  for (const q of items) {
+    console.log(
+      `  ${ansis.red(q.kind)} ${ansis.dim(q.id)} — ${ansis.yellow(q.reason)}`,
+    );
+  }
+}
+
+function renderStore(store: StatusReport["store"]): void {
+  section("Store");
+  console.log(
+    `  membot: ${ansis.cyan(store.membot_scope)}  ${ansis.dim(store.membot_dir)}`,
+  );
+  console.log(
+    `  mcpx:   ${ansis.cyan(store.mcpx_scope)}  ${ansis.dim(store.mcpx_dir)}`,
+  );
+}
+
+function renderText(report: StatusReport): void {
+  console.log(ansis.bold(`Project: ${report.project_dir}`));
+  renderWorkers(report.workers);
+  renderTasks(report.tasks);
+  renderSchedules(report.schedules);
+  renderQuarantined(report);
+  renderStore(report.store);
+}
+
+export function registerStatusCommand(program: Command): void {
+  program
+    .command("status")
+    .description(
+      "One-command project dashboard: workers, tasks, schedules, quarantined files, and store info",
+    )
+    .option("--json", "output a serialized StatusReport as JSON", false)
+    .option(
+      "--no-evaluate",
+      "skip the LLM evaluation of enabled schedules (faster / offline)",
+    )
+    .action(async (opts: { json?: boolean; evaluate?: boolean }) => {
+      const dir = program.opts().dir;
+      const config = await loadConfig(dir);
+      const report = await collectStatus(dir, config, {
+        evaluateSchedules: opts.evaluate !== false,
+      });
+      if (opts.json) {
+        console.log(JSON.stringify(report, null, 2));
+        return;
+      }
+      renderText(report);
+    });
+}

package/src/prompts/capabilities.ts

@@ -339,7 +339,7 @@ function renderFallback(inv: RawInventory, now: Date): string {
     schedule:
       "create and list recurring schedules that automatically generate tasks",
     membot:
-      "add, read, write, edit, move, copy, delete, and search content in the agent's knowledge store; track every version and refresh from URL sources",
+      "add, read, write, edit, move, copy, delete, and search content in the agent's knowledge store; track every version and refresh from URL sources; capture large tool outputs into the store and reduce/reshape stored JSON without loading it into context",
     prompt:
       "list, read, create, edit, and delete the project's prompt files (goals, beliefs, capabilities, plus any agent-authored ones)",
     skill: "list, read, write, edit, delete, and search slash-command skills",

package/src/status/collect.ts

@@ -0,0 +1,241 @@
+import type { BotholomewConfig, Scope } from "../config/schemas.ts";
+import { readWithMtime } from "../fs/atomic.ts";
+import { resolveMcpxDir } from "../mcpx/client.ts";
+import { resolveMembotDir } from "../mem/client.ts";
+import type { Schedule } from "../schedules/schema.ts";
+import {
+  listScheduleFiles,
+  parseScheduleFile,
+  scheduleFilePath,
+} from "../schedules/store.ts";
+import { TASK_STATUSES, type TaskStatus } from "../tasks/schema.ts";
+import { listTaskFiles, parseTaskFile, taskFilePath } from "../tasks/store.ts";
+import { evaluateSchedule } from "../worker/schedules.ts";
+import { listWorkers, type Worker } from "../workers/store.ts";
+
+export interface QuarantinedFile {
+  id: string;
+  reason: string;
+}
+
+export interface WorkerSummary {
+  total: number;
+  running: number;
+  stopped: number;
+  dead: number;
+  /** Most recent heartbeat across all workers, or null if none. */
+  latest_heartbeat_at: string | null;
+  list: Array<
+    Pick<
+      Worker,
+      "id" | "status" | "mode" | "pid" | "last_heartbeat_at" | "task_id"
+    >
+  >;
+}
+
+export interface TaskSummary {
+  /** Count of valid (parseable) tasks. */
+  total: number;
+  by_status: Record<TaskStatus, number>;
+  claimed: Array<{
+    id: string;
+    name: string;
+    claimed_by: string;
+    claimed_at: string | null;
+  }>;
+  quarantined: QuarantinedFile[];
+}
+
+export interface ScheduleEntry {
+  id: string;
+  name: string;
+  frequency: string;
+  enabled: boolean;
+  last_run_at: string | null;
+  /** Present only when schedule evaluation ran (enabled schedules, --evaluate). */
+  evaluation?: {
+    is_due: boolean;
+    reasoning: string;
+    tasks_to_create: number;
+  };
+}
+
+export interface ScheduleSummary {
+  total: number;
+  enabled: number;
+  disabled: number;
+  list: ScheduleEntry[];
+  quarantined: QuarantinedFile[];
+}
+
+export interface StoreSummary {
+  membot_scope: Scope;
+  membot_dir: string;
+  mcpx_scope: Scope;
+  mcpx_dir: string;
+}
+
+export interface StatusReport {
+  project_dir: string;
+  workers: WorkerSummary;
+  tasks: TaskSummary;
+  schedules: ScheduleSummary;
+  store: StoreSummary;
+}
+
+export interface CollectStatusOptions {
+  /** Run the LLM schedule evaluator for each enabled schedule. */
+  evaluateSchedules: boolean;
+}
+
+async function collectWorkers(projectDir: string): Promise<WorkerSummary> {
+  const workers = await listWorkers(projectDir);
+  let running = 0;
+  let stopped = 0;
+  let dead = 0;
+  let latest: string | null = null;
+  for (const w of workers) {
+    if (w.status === "running") running++;
+    else if (w.status === "stopped") stopped++;
+    else dead++;
+    if (!latest || w.last_heartbeat_at > latest) latest = w.last_heartbeat_at;
+  }
+  return {
+    total: workers.length,
+    running,
+    stopped,
+    dead,
+    latest_heartbeat_at: latest,
+    list: workers.map((w) => ({
+      id: w.id,
+      status: w.status,
+      mode: w.mode,
+      pid: w.pid,
+      last_heartbeat_at: w.last_heartbeat_at,
+      task_id: w.task_id,
+    })),
+  };
+}
+
+/**
+ * Single-pass task scan. Parses each `tasks/<id>.md` directly via
+ * `parseTaskFile` (rather than `getTask`/`listTasks`) so malformed files land
+ * in `quarantined` without spraying `logger.warn` over the dashboard output.
+ */
+async function collectTasks(projectDir: string): Promise<TaskSummary> {
+  const by_status = Object.fromEntries(
+    TASK_STATUSES.map((s) => [s, 0]),
+  ) as Record<TaskStatus, number>;
+  const claimed: TaskSummary["claimed"] = [];
+  const quarantined: QuarantinedFile[] = [];
+  let total = 0;
+
+  for (const id of await listTaskFiles(projectDir)) {
+    const file = await readWithMtime(taskFilePath(projectDir, id));
+    if (!file) continue;
+    const parsed = parseTaskFile(file.content, file.mtimeMs);
+    if (!parsed.ok) {
+      quarantined.push({ id, reason: parsed.reason });
+      continue;
+    }
+    const t = parsed.task;
+    total++;
+    by_status[t.status]++;
+    if (t.status === "in_progress" && t.claimed_by) {
+      claimed.push({
+        id: t.id,
+        name: t.name,
+        claimed_by: t.claimed_by,
+        claimed_at: t.claimed_at,
+      });
+    }
+  }
+
+  return { total, by_status, claimed, quarantined };
+}
+
+async function collectSchedules(
+  projectDir: string,
+  config: BotholomewConfig,
+  evaluateSchedules: boolean,
+): Promise<ScheduleSummary> {
+  const quarantined: QuarantinedFile[] = [];
+  // Keep each entry paired with its full parsed Schedule so the evaluator
+  // receives the real description, last_run_at, etc.
+  const parsed: Array<{ entry: ScheduleEntry; schedule: Schedule }> = [];
+  let enabled = 0;
+  let disabled = 0;
+
+  for (const id of await listScheduleFiles(projectDir)) {
+    const file = await readWithMtime(scheduleFilePath(projectDir, id));
+    if (!file) continue;
+    const result = parseScheduleFile(file.content, file.mtimeMs);
+    if (!result.ok) {
+      quarantined.push({ id, reason: result.reason });
+      continue;
+    }
+    const s = result.schedule;
+    if (s.enabled) enabled++;
+    else disabled++;
+    parsed.push({
+      entry: {
+        id: s.id,
+        name: s.name,
+        frequency: s.frequency,
+        enabled: s.enabled,
+        last_run_at: s.last_run_at,
+      },
+      schedule: s,
+    });
+  }
+
+  // Evaluate enabled schedules in parallel. evaluateSchedule has no disk
+  // side-effects and already degrades gracefully on LLM error.
+  if (evaluateSchedules) {
+    await Promise.all(
+      parsed
+        .filter((p) => p.schedule.enabled)
+        .map(async ({ entry, schedule }) => {
+          const evaluation = await evaluateSchedule(config, schedule);
+          entry.evaluation = {
+            is_due: evaluation.isDue,
+            reasoning: evaluation.reasoning,
+            tasks_to_create: evaluation.tasksToCreate.length,
+          };
+        }),
+    );
+  }
+
+  const list = parsed.map((p) => p.entry);
+  return { total: list.length, enabled, disabled, list, quarantined };
+}
+
+/**
+ * Gather a read-only snapshot of project state for `botholomew status`. Pure
+ * data-gathering: no disk mutation (schedule evaluation is read-only). The
+ * returned object is fully serializable for `--json`.
+ */
+export async function collectStatus(
+  projectDir: string,
+  config: BotholomewConfig,
+  opts: CollectStatusOptions,
+): Promise<StatusReport> {
+  const [workers, tasks, schedules] = await Promise.all([
+    collectWorkers(projectDir),
+    collectTasks(projectDir),
+    collectSchedules(projectDir, config, opts.evaluateSchedules),
+  ]);
+
+  return {
+    project_dir: projectDir,
+    workers,
+    tasks,
+    schedules,
+    store: {
+      membot_scope: config.membot_scope,
+      membot_dir: resolveMembotDir(projectDir, config),
+      mcpx_scope: config.mcpx_scope,
+      mcpx_dir: resolveMcpxDir(projectDir, config),
+    },
+  };
+}

package/src/worker/prompt.ts

@@ -18,6 +18,20 @@ export const MEMBOT_PROMPT_SECTION = `## Knowledge store (membot)
 ${MEMBOT_INSTRUCTIONS}
 `;
 
+/**
+ * Teaches the `membot_pipe` → `membot_query` pattern for large JSON. Shared
+ * verbatim by the worker and chat prompts (chat imports it). These are core
+ * knowledge-store tools — not MCP-only — so the section renders unconditionally;
+ * without it the model only learns about `membot_query` from its tool schema,
+ * exactly when it's most context-pressured (holding a big blob).
+ */
+export const LARGE_JSON_SECTION = `## Large JSON results
+When a tool would return a large JSON payload (mcp_exec dumps, search results, web fetches) that you don't need to read verbatim, don't pull it into context:
+1. \`membot_pipe\` the call into a \`logical_path\` — the bytes land in the store, you get back only an ack.
+2. \`membot_query\` that \`logical_path\` with a JSONata expression to filter / pluck / group / dedup / sort / aggregate down to the small slice you actually need (pass \`expression="?"\` for the syntax reference). Chain with \`output_logical_path\` to refine in steps.
+This keeps big blobs out of the conversation. Reach for it whenever you expect a result bigger than what you need.
+`;
+
 export const STYLE_RULES = `## Style
 - Open with the result, action, or next step. Skip preambles like "Great question", "You're absolutely right", "Let me…", "I'll go ahead and…".
 - Don't flatter the user or their ideas. If a request is wrong, ambiguous, or risky, say so plainly with the reason.
@@ -136,6 +150,7 @@ When calling complete_task, write a summary that captures your key findings, dec
 `;
 
   prompt += `\n${MEMBOT_PROMPT_SECTION}`;
+  prompt += `\n${LARGE_JSON_SECTION}`;
 
   if (options?.hasMcpTools) {
     prompt += `