botholomew 0.21.2 → 0.22.1

package/README.md

@@ -232,6 +232,7 @@ semantic search, append-only versioning, and URL refresh all live there.
 | Command | Purpose |
 |---|---|
 | `botholomew init` | Initialize the current directory as a project (refuses on iCloud/Dropbox/NFS without `--force`) |
+| `botholomew status` | One-command dashboard: workers, task counts/claims, schedules (with a live "due?" check), quarantined files, store info. `--json` for scripting, `--no-evaluate` to skip the LLM schedule check |
 | `botholomew worker run\|start` | Run a worker (foreground or background); `--persist` for long-running, `--task-id <id>` to target one task |
 | `botholomew worker list\|status\|stop\|kill\|reap` | Inspect and manage running workers |
 | `botholomew chat` | Interactive Ink/React TUI |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "botholomew",
-  "version": "0.21.2",
+  "version": "0.22.1",
   "description": "An autonomous AI agent for knowledge work — works your task queue while you sleep.",
   "type": "module",
   "bin": {

package/src/chat/agent.ts

@@ -28,6 +28,7 @@ import { maybeStoreResult } from "../worker/large-results.ts";
 import {
   buildMetaHeader,
   extractKeywords,
+  LARGE_JSON_SECTION,
   loadPersistentContext,
   MEMBOT_PROMPT_SECTION,
   STYLE_RULES,
@@ -64,6 +65,7 @@ const CHAT_TOOL_NAMES = new Set([
   "membot_count_lines",
   "membot_copy",
   "membot_pipe",
+  "membot_query",
   "list_threads",
   "view_thread",
   "search_threads",
@@ -120,6 +122,7 @@ Format your responses using Markdown. Use headings, bold, italic, lists, and cod
 `;
 
   prompt += `\n${MEMBOT_PROMPT_SECTION}`;
+  prompt += `\n${LARGE_JSON_SECTION}`;
 
   if (options?.hasMcpTools) {
     prompt += `

package/src/cli.ts

@@ -13,6 +13,7 @@ import { registerPrepareCommand } from "./commands/prepare.ts";
 import { registerPromptsCommand } from "./commands/prompts.ts";
 import { registerScheduleCommand } from "./commands/schedule.ts";
 import { registerSkillCommand } from "./commands/skill.ts";
+import { registerStatusCommand } from "./commands/status.ts";
 import { registerTaskCommand } from "./commands/task.ts";
 import { registerThreadCommand } from "./commands/thread.ts";
 import { registerUpgradeCommand } from "./commands/upgrade.ts";
@@ -57,6 +58,7 @@ program
   });
 
 registerInitCommand(program);
+registerStatusCommand(program);
 registerWorkerCommand(program);
 registerTaskCommand(program);
 registerThreadCommand(program);

package/src/commands/status.ts

@@ -0,0 +1,194 @@
+import ansis from "ansis";
+import type { Command } from "commander";
+import { loadConfig } from "../config/loader.ts";
+import {
+  collectStatus,
+  type ScheduleEntry,
+  type StatusReport,
+  type WorkerSummary,
+} from "../status/collect.ts";
+import type { TaskStatus } from "../tasks/schema.ts";
+import type { WorkerStatus } from "../workers/store.ts";
+
+function formatAge(fromIso: string | null, to = new Date()): string {
+  if (!fromIso) return "never";
+  const from = new Date(fromIso);
+  const secs = Math.max(0, Math.floor((to.getTime() - from.getTime()) / 1000));
+  if (secs < 60) return `${secs}s ago`;
+  const mins = Math.floor(secs / 60);
+  if (mins < 60) return `${mins}m ago`;
+  const hours = Math.floor(mins / 60);
+  if (hours < 24) return `${hours}h ago`;
+  return `${Math.floor(hours / 24)}d ago`;
+}
+
+function padColored(colored: string, raw: string, width: number): string {
+  return colored + " ".repeat(Math.max(0, width - raw.length));
+}
+
+function workerStatusColor(status: WorkerStatus): string {
+  switch (status) {
+    case "running":
+      return ansis.green(status);
+    case "stopped":
+      return ansis.dim(status);
+    case "dead":
+      return ansis.red(status);
+  }
+}
+
+function taskStatusColor(status: TaskStatus, label: string): string {
+  switch (status) {
+    case "pending":
+      return ansis.yellow(label);
+    case "in_progress":
+      return ansis.blue(label);
+    case "complete":
+      return ansis.green(label);
+    case "failed":
+      return ansis.red(label);
+    case "waiting":
+      return ansis.magenta(label);
+  }
+}
+
+function section(title: string): void {
+  console.log(`\n${ansis.bold.underline(title)}`);
+}
+
+function renderWorkers(w: WorkerSummary): void {
+  section("Workers");
+  const parts = [
+    `${ansis.green(`${w.running} running`)}`,
+    `${ansis.dim(`${w.stopped} stopped`)}`,
+    `${w.dead > 0 ? ansis.red(`${w.dead} dead`) : ansis.dim("0 dead")}`,
+  ];
+  console.log(
+    `  ${w.total} total — ${parts.join(", ")}   latest heartbeat: ${formatAge(w.latest_heartbeat_at)}`,
+  );
+  if (w.list.length === 0) {
+    console.log(ansis.dim("  (none)"));
+    return;
+  }
+  for (const wk of w.list) {
+    const short = ansis.bold(wk.id.slice(0, 8));
+    const status = workerStatusColor(wk.status);
+    const task = wk.task_id ? `  task=${wk.task_id.slice(0, 8)}` : "";
+    console.log(
+      `  ${short}  ${status}  mode=${wk.mode}  pid=${wk.pid}  heartbeat ${formatAge(wk.last_heartbeat_at)}${task}`,
+    );
+  }
+}
+
+function renderTasks(t: StatusReport["tasks"]): void {
+  section("Tasks");
+  const order: TaskStatus[] = [
+    "pending",
+    "in_progress",
+    "waiting",
+    "complete",
+    "failed",
+  ];
+  const counts = order
+    .map((s) => taskStatusColor(s, `${t.by_status[s]} ${s}`))
+    .join(", ");
+  console.log(`  ${t.total} total — ${counts}`);
+
+  if (t.claimed.length > 0) {
+    console.log(ansis.bold("\n  Claimed:"));
+    for (const c of t.claimed) {
+      console.log(
+        `  ${ansis.dim(c.id.slice(0, 8))}  ${c.name}  ${ansis.dim(`by ${c.claimed_by.slice(0, 8)} (${formatAge(c.claimed_at)})`)}`,
+      );
+    }
+  }
+}
+
+function dueLabel(entry: ScheduleEntry): string {
+  if (!entry.evaluation) return ansis.dim("—");
+  return entry.evaluation.is_due
+    ? ansis.green.bold("due")
+    : ansis.dim("not due");
+}
+
+function renderSchedules(s: StatusReport["schedules"]): void {
+  section("Schedules");
+  console.log(
+    `  ${s.total} total — ${ansis.green(`${s.enabled} enabled`)}, ${ansis.dim(`${s.disabled} disabled`)}`,
+  );
+  if (s.list.length === 0) {
+    console.log(ansis.dim("  (none)"));
+    return;
+  }
+  for (const e of s.list) {
+    const name = e.enabled ? e.name : ansis.dim(e.name);
+    const freq = ansis.cyan(e.frequency);
+    const last = ansis.dim(`last run ${formatAge(e.last_run_at)}`);
+    let line = `  ${name}  (${freq})  ${last}  ${dueLabel(e)}`;
+    if (e.evaluation?.is_due && e.evaluation.tasks_to_create > 0) {
+      line += ansis.dim(` → ${e.evaluation.tasks_to_create} task(s)`);
+    }
+    console.log(line);
+    if (e.evaluation?.is_due) {
+      console.log(ansis.dim(`      ${e.evaluation.reasoning}`));
+    }
+  }
+}
+
+function renderQuarantined(report: StatusReport): void {
+  const items = [
+    ...report.tasks.quarantined.map((q) => ({ kind: "task", ...q })),
+    ...report.schedules.quarantined.map((q) => ({ kind: "schedule", ...q })),
+  ];
+  if (items.length === 0) return;
+  section("Quarantined (invalid frontmatter)");
+  for (const q of items) {
+    console.log(
+      `  ${ansis.red(q.kind)} ${ansis.dim(q.id)} — ${ansis.yellow(q.reason)}`,
+    );
+  }
+}
+
+function renderStore(store: StatusReport["store"]): void {
+  section("Store");
+  console.log(
+    `  membot: ${ansis.cyan(store.membot_scope)}  ${ansis.dim(store.membot_dir)}`,
+  );
+  console.log(
+    `  mcpx:   ${ansis.cyan(store.mcpx_scope)}  ${ansis.dim(store.mcpx_dir)}`,
+  );
+}
+
+function renderText(report: StatusReport): void {
+  console.log(ansis.bold(`Project: ${report.project_dir}`));
+  renderWorkers(report.workers);
+  renderTasks(report.tasks);
+  renderSchedules(report.schedules);
+  renderQuarantined(report);
+  renderStore(report.store);
+}
+
+export function registerStatusCommand(program: Command): void {
+  program
+    .command("status")
+    .description(
+      "One-command project dashboard: workers, tasks, schedules, quarantined files, and store info",
+    )
+    .option("--json", "output a serialized StatusReport as JSON", false)
+    .option(
+      "--no-evaluate",
+      "skip the LLM evaluation of enabled schedules (faster / offline)",
+    )
+    .action(async (opts: { json?: boolean; evaluate?: boolean }) => {
+      const dir = program.opts().dir;
+      const config = await loadConfig(dir);
+      const report = await collectStatus(dir, config, {
+        evaluateSchedules: opts.evaluate !== false,
+      });
+      if (opts.json) {
+        console.log(JSON.stringify(report, null, 2));
+        return;
+      }
+      renderText(report);
+    });
+}

package/src/prompts/capabilities.ts

@@ -339,7 +339,7 @@ function renderFallback(inv: RawInventory, now: Date): string {
     schedule:
       "create and list recurring schedules that automatically generate tasks",
     membot:
-      "add, read, write, edit, move, copy, delete, and search content in the agent's knowledge store; track every version and refresh from URL sources",
+      "add, read, write, edit, move, copy, delete, and search content in the agent's knowledge store; track every version and refresh from URL sources; capture large tool outputs into the store and reduce/reshape stored JSON without loading it into context",
     prompt:
       "list, read, create, edit, and delete the project's prompt files (goals, beliefs, capabilities, plus any agent-authored ones)",
     skill: "list, read, write, edit, delete, and search slash-command skills",

package/src/status/collect.ts

@@ -0,0 +1,241 @@
+import type { BotholomewConfig, Scope } from "../config/schemas.ts";
+import { readWithMtime } from "../fs/atomic.ts";
+import { resolveMcpxDir } from "../mcpx/client.ts";
+import { resolveMembotDir } from "../mem/client.ts";
+import type { Schedule } from "../schedules/schema.ts";
+import {
+  listScheduleFiles,
+  parseScheduleFile,
+  scheduleFilePath,
+} from "../schedules/store.ts";
+import { TASK_STATUSES, type TaskStatus } from "../tasks/schema.ts";
+import { listTaskFiles, parseTaskFile, taskFilePath } from "../tasks/store.ts";
+import { evaluateSchedule } from "../worker/schedules.ts";
+import { listWorkers, type Worker } from "../workers/store.ts";
+
+export interface QuarantinedFile {
+  id: string;
+  reason: string;
+}
+
+export interface WorkerSummary {
+  total: number;
+  running: number;
+  stopped: number;
+  dead: number;
+  /** Most recent heartbeat across all workers, or null if none. */
+  latest_heartbeat_at: string | null;
+  list: Array<
+    Pick<
+      Worker,
+      "id" | "status" | "mode" | "pid" | "last_heartbeat_at" | "task_id"
+    >
+  >;
+}
+
+export interface TaskSummary {
+  /** Count of valid (parseable) tasks. */
+  total: number;
+  by_status: Record<TaskStatus, number>;
+  claimed: Array<{
+    id: string;
+    name: string;
+    claimed_by: string;
+    claimed_at: string | null;
+  }>;
+  quarantined: QuarantinedFile[];
+}
+
+export interface ScheduleEntry {
+  id: string;
+  name: string;
+  frequency: string;
+  enabled: boolean;
+  last_run_at: string | null;
+  /** Present only when schedule evaluation ran (enabled schedules, --evaluate). */
+  evaluation?: {
+    is_due: boolean;
+    reasoning: string;
+    tasks_to_create: number;
+  };
+}
+
+export interface ScheduleSummary {
+  total: number;
+  enabled: number;
+  disabled: number;
+  list: ScheduleEntry[];
+  quarantined: QuarantinedFile[];
+}
+
+export interface StoreSummary {
+  membot_scope: Scope;
+  membot_dir: string;
+  mcpx_scope: Scope;
+  mcpx_dir: string;
+}
+
+export interface StatusReport {
+  project_dir: string;
+  workers: WorkerSummary;
+  tasks: TaskSummary;
+  schedules: ScheduleSummary;
+  store: StoreSummary;
+}
+
+export interface CollectStatusOptions {
+  /** Run the LLM schedule evaluator for each enabled schedule. */
+  evaluateSchedules: boolean;
+}
+
+async function collectWorkers(projectDir: string): Promise<WorkerSummary> {
+  const workers = await listWorkers(projectDir);
+  let running = 0;
+  let stopped = 0;
+  let dead = 0;
+  let latest: string | null = null;
+  for (const w of workers) {
+    if (w.status === "running") running++;
+    else if (w.status === "stopped") stopped++;
+    else dead++;
+    if (!latest || w.last_heartbeat_at > latest) latest = w.last_heartbeat_at;
+  }
+  return {
+    total: workers.length,
+    running,
+    stopped,
+    dead,
+    latest_heartbeat_at: latest,
+    list: workers.map((w) => ({
+      id: w.id,
+      status: w.status,
+      mode: w.mode,
+      pid: w.pid,
+      last_heartbeat_at: w.last_heartbeat_at,
+      task_id: w.task_id,
+    })),
+  };
+}
+
+/**
+ * Single-pass task scan. Parses each `tasks/<id>.md` directly via
+ * `parseTaskFile` (rather than `getTask`/`listTasks`) so malformed files land
+ * in `quarantined` without spraying `logger.warn` over the dashboard output.
+ */
+async function collectTasks(projectDir: string): Promise<TaskSummary> {
+  const by_status = Object.fromEntries(
+    TASK_STATUSES.map((s) => [s, 0]),
+  ) as Record<TaskStatus, number>;
+  const claimed: TaskSummary["claimed"] = [];
+  const quarantined: QuarantinedFile[] = [];
+  let total = 0;
+
+  for (const id of await listTaskFiles(projectDir)) {
+    const file = await readWithMtime(taskFilePath(projectDir, id));
+    if (!file) continue;
+    const parsed = parseTaskFile(file.content, file.mtimeMs);
+    if (!parsed.ok) {
+      quarantined.push({ id, reason: parsed.reason });
+      continue;
+    }
+    const t = parsed.task;
+    total++;
+    by_status[t.status]++;
+    if (t.status === "in_progress" && t.claimed_by) {
+      claimed.push({
+        id: t.id,
+        name: t.name,
+        claimed_by: t.claimed_by,
+        claimed_at: t.claimed_at,
+      });
+    }
+  }
+
+  return { total, by_status, claimed, quarantined };
+}
+
+async function collectSchedules(
+  projectDir: string,
+  config: BotholomewConfig,
+  evaluateSchedules: boolean,
+): Promise<ScheduleSummary> {
+  const quarantined: QuarantinedFile[] = [];
+  // Keep each entry paired with its full parsed Schedule so the evaluator
+  // receives the real description, last_run_at, etc.
+  const parsed: Array<{ entry: ScheduleEntry; schedule: Schedule }> = [];
+  let enabled = 0;
+  let disabled = 0;
+
+  for (const id of await listScheduleFiles(projectDir)) {
+    const file = await readWithMtime(scheduleFilePath(projectDir, id));
+    if (!file) continue;
+    const result = parseScheduleFile(file.content, file.mtimeMs);
+    if (!result.ok) {
+      quarantined.push({ id, reason: result.reason });
+      continue;
+    }
+    const s = result.schedule;
+    if (s.enabled) enabled++;
+    else disabled++;
+    parsed.push({
+      entry: {
+        id: s.id,
+        name: s.name,
+        frequency: s.frequency,
+        enabled: s.enabled,
+        last_run_at: s.last_run_at,
+      },
+      schedule: s,
+    });
+  }
+
+  // Evaluate enabled schedules in parallel. evaluateSchedule has no disk
+  // side-effects and already degrades gracefully on LLM error.
+  if (evaluateSchedules) {
+    await Promise.all(
+      parsed
+        .filter((p) => p.schedule.enabled)
+        .map(async ({ entry, schedule }) => {
+          const evaluation = await evaluateSchedule(config, schedule);
+          entry.evaluation = {
+            is_due: evaluation.isDue,
+            reasoning: evaluation.reasoning,
+            tasks_to_create: evaluation.tasksToCreate.length,
+          };
+        }),
+    );
+  }
+
+  const list = parsed.map((p) => p.entry);
+  return { total: list.length, enabled, disabled, list, quarantined };
+}
+
+/**
+ * Gather a read-only snapshot of project state for `botholomew status`. Pure
+ * data-gathering: no disk mutation (schedule evaluation is read-only). The
+ * returned object is fully serializable for `--json`.
+ */
+export async function collectStatus(
+  projectDir: string,
+  config: BotholomewConfig,
+  opts: CollectStatusOptions,
+): Promise<StatusReport> {
+  const [workers, tasks, schedules] = await Promise.all([
+    collectWorkers(projectDir),
+    collectTasks(projectDir),
+    collectSchedules(projectDir, config, opts.evaluateSchedules),
+  ]);
+
+  return {
+    project_dir: projectDir,
+    workers,
+    tasks,
+    schedules,
+    store: {
+      membot_scope: config.membot_scope,
+      membot_dir: resolveMembotDir(projectDir, config),
+      mcpx_scope: config.mcpx_scope,
+      mcpx_dir: resolveMcpxDir(projectDir, config),
+    },
+  };
+}

package/src/worker/llm.ts

@@ -114,6 +114,7 @@ export async function runAgentLoop(input: {
   const maxInputTokens = await getMaxInputTokens(config.llm);
 
   const maxTurns = config.max_turns;
+  let nudgeCount = 0;
   for (let turn = 0; !maxTurns || turn < maxTurns; turn++) {
     const startTime = Date.now();
     fitToContextWindow(messages, systemPrompt, maxInputTokens);
@@ -188,9 +189,29 @@ export async function runAgentLoop(input: {
     }
 
     if (collectedToolCalls.length === 0) {
+      // An implicit tick-end (the model stopped emitting tool calls) is
+      // ambiguous evidence — it usually means the agent hit a dead end,
+      // exhausted its output budget mid-thought, or forgot to declare a
+      // terminal status. Don't treat it as success. Nudge once to give the
+      // agent a chance to recover (e.g. emit the final tool call it was about
+      // to make, or fail_task on a capability gap); fail if it still doesn't.
+      if (nudgeCount === 0) {
+        nudgeCount++;
+        const nudge =
+          "You ended your turn without calling a terminal status tool. Every tick must end with exactly one of: complete_task (only if the required deliverable truly exists — verify it), fail_task (if you are blocked or a required tool/capability is unavailable — state the gap), or wait_task (if you must wait on something external). Call the appropriate one now.";
+        messages.push({ role: "user", content: nudge });
+        await logInteraction(projectDir, threadId, {
+          role: "system",
+          kind: "status_change",
+          content:
+            "Agent ended its turn without a terminal status tool; nudging to call complete_task/fail_task/wait_task.",
+        });
+        continue;
+      }
       return {
-        status: "complete",
-        reason: "Agent completed without explicit status tool call",
+        status: "failed",
+        reason:
+          "Agent ended its tick without calling a terminal status tool (complete_task/fail_task/wait_task)",
       };
     }
 

package/src/worker/prompt.ts

@@ -18,6 +18,20 @@ export const MEMBOT_PROMPT_SECTION = `## Knowledge store (membot)
 ${MEMBOT_INSTRUCTIONS}
 `;
 
+/**
+ * Teaches the `membot_pipe` → `membot_query` pattern for large JSON. Shared
+ * verbatim by the worker and chat prompts (chat imports it). These are core
+ * knowledge-store tools — not MCP-only — so the section renders unconditionally;
+ * without it the model only learns about `membot_query` from its tool schema,
+ * exactly when it's most context-pressured (holding a big blob).
+ */
+export const LARGE_JSON_SECTION = `## Large JSON results
+When a tool would return a large JSON payload (mcp_exec dumps, search results, web fetches) that you don't need to read verbatim, don't pull it into context:
+1. \`membot_pipe\` the call into a \`logical_path\` — the bytes land in the store, you get back only an ack.
+2. \`membot_query\` that \`logical_path\` with a JSONata expression to filter / pluck / group / dedup / sort / aggregate down to the small slice you actually need (pass \`expression="?"\` for the syntax reference). Chain with \`output_logical_path\` to refine in steps.
+This keeps big blobs out of the conversation. Reach for it whenever you expect a result bigger than what you need.
+`;
+
 export const STYLE_RULES = `## Style
 - Open with the result, action, or next step. Skip preambles like "Great question", "You're absolutely right", "Let me…", "I'll go ahead and…".
 - Don't flatter the user or their ideas. If a request is wrong, ambiguous, or risky, say so plainly with the reason.
@@ -132,10 +146,13 @@ export async function buildSystemPrompt(
   prompt += `## Instructions
 You are Botholomew, a wise-owl worker that works through tasks. Use available tools to complete your assigned task, then call complete_task, fail_task, or wait_task. Use create_task for subtasks and update_task to refine pending tasks. Batch independent tool calls in a single response for parallel execution.
 
+Always end your tick by calling exactly one terminal status tool — never just stop. Call complete_task ONLY if the required deliverable actually exists (verify it). If you are blocked or a required tool/capability is unavailable (e.g. no way to produce the requested output), call fail_task and state the gap — do not pretend success. If you must wait on something external, call wait_task.
+
 When calling complete_task, write a summary that captures your key findings, decisions, and outputs. This summary becomes the task's output and is provided to any downstream tasks that depend on this one. Include specific results (data, names, paths, conclusions) rather than vague descriptions of what you did — downstream tasks will rely on this information to do their work.
 `;
 
   prompt += `\n${MEMBOT_PROMPT_SECTION}`;
+  prompt += `\n${LARGE_JSON_SECTION}`;
 
   if (options?.hasMcpTools) {
     prompt += `