@mercury-fw/core 0.25.0

package/src/wiki/self-review-runner.ts

@@ -0,0 +1,133 @@
+/**
+ * The nightly self-review job's three LLM passes. Each is a fresh,
+ * stateless `generateText` call — never a persisted conversation history
+ * like `agent-turn.ts`'s `runTurn` — so nothing carries over between
+ * passes or between nights except what actually landed on disk. Running
+ * all four checks as one giant multi-step call would let tool-call
+ * context accumulate across unrelated jobs at once; splitting into three
+ * independent calls keeps each one's context bounded to its own job.
+ *
+ * `generateText` comes from `ai-sdk-ollama`, matching `agent-turn.ts` —
+ * plain `ai`'s version is documented there to return empty text after a
+ * tool call on this codebase's Ollama setup, which applies here too
+ * since every pass uses tools.
+ */
+import { isStepCount, type LanguageModel, type Tool } from "ai";
+import { generateText } from "ai-sdk-ollama";
+import type { Message } from "../session/history.ts";
+import { createSelfReviewTools } from "./self-review-tools.ts";
+
+/** Empirical, tuned later — generous headroom for reading several docs and
+ * writing/deleting within one pass, well above a conversational turn's. */
+export const SELF_REVIEW_STEP_COUNT = 15;
+
+type GenerateTextFn = (params: {
+  model: LanguageModel;
+  messages: Message[];
+  tools: Record<string, Tool>;
+  instructions: string;
+}) => Promise<{ text: string }>;
+
+const defaultGenerateTextFn: GenerateTextFn = (params) =>
+  generateText({ ...params, stopWhen: isStepCount(SELF_REVIEW_STEP_COUNT) });
+
+async function runPass(params: {
+  vaultPath: string;
+  model: LanguageModel;
+  instructions: string;
+  userMessage: string;
+  generateTextFn?: GenerateTextFn;
+}): Promise<void> {
+  const tools = createSelfReviewTools({ vaultPath: params.vaultPath });
+  const generate = params.generateTextFn ?? defaultGenerateTextFn;
+  await generate({
+    model: params.model,
+    messages: [{ role: "user", content: params.userMessage }],
+    tools,
+    instructions: params.instructions,
+  });
+}
+
+const SHARED_BOUNDARIES =
+  "You can read/list/grep curated/ and raw/, write curated docs, add/update or remove a curated " +
+  "doc's index.md entry (update_index_entry/remove_index_entry — never hand-write index.md's " +
+  "content yourself, those tools own its exact format), and delete resolved raw/ entries. You have " +
+  "no access to inferred/ — it is written exclusively by a separate deterministic process, never by " +
+  "judgment calls like this one.";
+
+const RAW_TRIAGE_SYSTEM =
+  "You are performing Mercury's periodic wiki self-review — the raw/ triage pass. For each entry " +
+  "in raw/, read it and decide: merge its content into an existing curated doc, promote it into a " +
+  "new curated doc, or discard it (already superseded, duplicate, or simply not wiki-worthy — one " +
+  "bucket, not three). Then delete it from raw/ once resolved. If you create or meaningfully change " +
+  "a curated doc, call update_index_entry for it too. " +
+  SHARED_BOUNDARIES;
+
+const INDEX_AND_ORPHAN_SYSTEM =
+  "You are performing Mercury's periodic wiki self-review — the index.md and orphan-page pass. For " +
+  "each orphaned curated doc listed below, decide whether it needs an index.md entry " +
+  "(update_index_entry), a cross-link from a related doc, or both. " +
+  SHARED_BOUNDARIES;
+
+const CONTRADICTION_CHECK_SYSTEM =
+  "You are performing Mercury's periodic wiki self-review — the contradiction and cross-link check. " +
+  "Read through curated/ and look for direct contradictions between documents, or clearly-related " +
+  "documents missing a cross-link between them. Fix what you're confident about; leave the rest — " +
+  "this is best-effort, not exhaustive. If you delete a doc as a resolved duplicate, call " +
+  "remove_index_entry for it in the same pass. " +
+  SHARED_BOUNDARIES;
+
+export type RawTriagePassDeps = {
+  vaultPath: string;
+  model: LanguageModel;
+  rawEntries: string[];
+  generateTextFn?: GenerateTextFn;
+};
+
+export async function runRawTriagePass(deps: RawTriagePassDeps): Promise<void> {
+  const userMessage = ["raw/ entries to triage:", ...deps.rawEntries.map((e) => `- ${e}`)].join("\n");
+  await runPass({
+    vaultPath: deps.vaultPath,
+    model: deps.model,
+    instructions: RAW_TRIAGE_SYSTEM,
+    userMessage,
+    generateTextFn: deps.generateTextFn,
+  });
+}
+
+export type IndexAndOrphanPassDeps = {
+  vaultPath: string;
+  model: LanguageModel;
+  orphans: string[];
+  generateTextFn?: GenerateTextFn;
+};
+
+export async function runIndexAndOrphanPass(deps: IndexAndOrphanPassDeps): Promise<void> {
+  const userMessage = [
+    "Orphaned curated docs (not referenced by index.md nor any [[wikilink]]):",
+    ...deps.orphans.map((o) => `- ${o}`),
+  ].join("\n");
+  await runPass({
+    vaultPath: deps.vaultPath,
+    model: deps.model,
+    instructions: INDEX_AND_ORPHAN_SYSTEM,
+    userMessage,
+    generateTextFn: deps.generateTextFn,
+  });
+}
+
+export type ContradictionCheckPassDeps = {
+  vaultPath: string;
+  model: LanguageModel;
+  generateTextFn?: GenerateTextFn;
+};
+
+export async function runContradictionCheckPass(deps: ContradictionCheckPassDeps): Promise<void> {
+  await runPass({
+    vaultPath: deps.vaultPath,
+    model: deps.model,
+    instructions: CONTRADICTION_CHECK_SYSTEM,
+    userMessage: "Review curated/ for contradictions and missing cross-links.",
+    generateTextFn: deps.generateTextFn,
+  });
+}

package/src/wiki/self-review-tools.ts

@@ -0,0 +1,162 @@
+/**
+ * The nightly self-review job's own tool set — distinct from
+ * `wiki-tools.ts` (the model-invocable tools a normal conversation gets),
+ * same reasoning already applied to `writeInferredNote` never being
+ * wired into those: this is a separate trust context (a stateless admin
+ * batch job, not a live conversation), scoped via `selfReviewRoots`
+ * (curated/ + raw/, never inferred/ — reserved for deterministic,
+ * mechanically-written notes, not an LLM's own judgment call) and with
+ * capabilities no conversational tool
+ * has (deleting an entry, updating/removing an index.md entry — the
+ * model supplies a doc path and a description, never the file's raw text,
+ * so an update can't get the `[[wikilink]]` format wrong).
+ *
+ * All three nightly sub-passes (`self-review-runner.ts`) share this
+ * exact tool set — they differ only in system prompt and pre-computed
+ * input data, not in which tools they can call.
+ */
+import type { ExecutableTool } from "@mercury-fw/plugin-types";
+import { tool } from "ai";
+import { z } from "zod";
+import { listWikiFilesInRoots, readWikiFileInRoots, grepWikiInRoots, selfReviewRoots, readIndexFile } from "./wiki-read.ts";
+import { writeCuratedNote, writeIndexFile, deleteRawEntry, deleteCuratedEntry } from "./wiki-note.ts";
+import { normalizeIndexKey, upsertIndexEntry, removeIndexEntry } from "./index-entry.ts";
+
+export type SelfReviewToolsDeps = { vaultPath: string };
+
+export function createSelfReviewTools(
+  deps: SelfReviewToolsDeps,
+): Record<
+  | "list_files"
+  | "read_file"
+  | "grep"
+  | "write_curated"
+  | "update_index_entry"
+  | "remove_index_entry"
+  | "delete_raw"
+  | "delete_curated",
+  ExecutableTool
+> {
+  const { vaultPath } = deps;
+  const roots = selfReviewRoots(vaultPath);
+
+  const list_files = tool({
+    description: "List every file under curated/ and raw/ (never inferred/). Returns paths relative to the vault root.",
+    inputSchema: z.object({}),
+    execute: async () => {
+      const files = await listWikiFilesInRoots(vaultPath, roots);
+      return { ok: true as const, files };
+    },
+  });
+
+  const read_file = tool({
+    description: 'Read a file by path, e.g. "curated/standards/x.md" or "raw/notes/y.md". Only curated/ and raw/ are readable.',
+    inputSchema: z.object({ path: z.string().min(1) }),
+    execute: async ({ path }) => {
+      try {
+        const content = await readWikiFileInRoots(vaultPath, roots, path);
+        return { ok: true as const, content };
+      } catch (err) {
+        return { ok: false as const, error: String(err) };
+      }
+    },
+  });
+
+  const grep = tool({
+    description: "Search curated/ and raw/ files for a regular expression pattern. Returns matching lines with their file path and line number.",
+    inputSchema: z.object({ pattern: z.string().min(1) }),
+    execute: async ({ pattern }) => {
+      try {
+        const matches = await grepWikiInRoots(vaultPath, roots, pattern);
+        return { ok: true as const, matches };
+      } catch (err) {
+        return { ok: false as const, error: String(err) };
+      }
+    },
+  });
+
+  const write_curated = tool({
+    description: 'Create or overwrite a curated doc. "path" is relative to curated/, e.g. "standards/jira-fields.md".',
+    inputSchema: z.object({ path: z.string().min(1), content: z.string() }),
+    execute: async ({ path, content }) => {
+      try {
+        await writeCuratedNote(vaultPath, path, {}, content);
+        return { ok: true as const };
+      } catch (err) {
+        return { ok: false as const, error: String(err) };
+      }
+    },
+  });
+
+  const update_index_entry = tool({
+    description:
+      'Add or update index.md\'s line for one curated doc — pass its path in any form ("curated/projects/x.md", "projects/x.md", or "projects/x", all normalize the same way) and a short description. Writes it as a [[wikilink]] so the orphan check recognizes it; never hand-write index.md yourself, it has to match this exact format to count.',
+    inputSchema: z.object({ path: z.string().min(1), description: z.string().min(1) }),
+    execute: async ({ path, description }) => {
+      const key = normalizeIndexKey(path);
+      const curatedPath = `curated/${key}.md`;
+      try {
+        await readWikiFileInRoots(vaultPath, roots, curatedPath);
+      } catch {
+        return { ok: false as const, error: `${curatedPath} does not exist — create it first with write_curated` };
+      }
+      try {
+        const current = await readIndexFile(vaultPath);
+        await writeIndexFile(vaultPath, upsertIndexEntry(current, key, description));
+        return { ok: true as const };
+      } catch (err) {
+        return { ok: false as const, error: String(err) };
+      }
+    },
+  });
+
+  const remove_index_entry = tool({
+    description:
+      'Remove index.md\'s line for one curated doc (e.g. after delete_curated) — same path forms as update_index_entry. A no-op if it has no line.',
+    inputSchema: z.object({ path: z.string().min(1) }),
+    execute: async ({ path }) => {
+      const key = normalizeIndexKey(path);
+      try {
+        const current = await readIndexFile(vaultPath);
+        await writeIndexFile(vaultPath, removeIndexEntry(current, key));
+        return { ok: true as const };
+      } catch (err) {
+        return { ok: false as const, error: String(err) };
+      }
+    },
+  });
+
+  const delete_raw = tool({
+    description: 'Delete a raw/ entry once it has been triaged (merged, promoted, or discarded). "path" must start with "raw/".',
+    inputSchema: z.object({ path: z.string().min(1) }),
+    execute: async ({ path }) => {
+      if (!path.startsWith("raw/")) {
+        return { ok: false as const, error: `path must start with "raw/" (got "${path}")` };
+      }
+      try {
+        await deleteRawEntry(vaultPath, path.slice("raw/".length));
+        return { ok: true as const };
+      } catch (err) {
+        return { ok: false as const, error: String(err) };
+      }
+    },
+  });
+
+  const delete_curated = tool({
+    description: 'Delete a curated doc that is redundant or superseded. "path" must start with "curated/". Remove its index.md line too, if it has one.',
+    inputSchema: z.object({ path: z.string().min(1) }),
+    execute: async ({ path }) => {
+      if (!path.startsWith("curated/")) {
+        return { ok: false as const, error: `path must start with "curated/" (got "${path}")` };
+      }
+      try {
+        await deleteCuratedEntry(vaultPath, path.slice("curated/".length));
+        return { ok: true as const };
+      } catch (err) {
+        return { ok: false as const, error: String(err) };
+      }
+    },
+  });
+
+  return { list_files, read_file, grep, write_curated, update_index_entry, remove_index_entry, delete_raw, delete_curated };
+}

package/src/wiki/vault-cli.ts

@@ -0,0 +1,143 @@
+#!/usr/bin/env bun
+/**
+ * Maintenance CLI for the wiki vault — runs INSIDE the Mercury container
+ * (the vault is a Docker named volume, not a host path, see `scripts/vault.sh`
+ * and CLAUDE.md § "Manutenzione della vault wiki"). Thin argv wrapper around
+ * functions that already exist and are already tested (`wiki-note.ts`,
+ * `vault-init.ts`) — no new write/read logic here, only routing.
+ *
+ * Deliberately does NOT expose `writeInferredNote` — that writer is
+ * reserved exclusively for a deterministic, mechanical consolidation
+ * process (see its own docstring in `wiki-note.ts`); a manual CLI writing
+ * "agent-sourced" notes by hand would defeat that guarantee.
+ *
+ * `write-raw` has no `--author`/frontmatter options, unlike `write-curated`
+ * — raw/ content is verbatim, un-triaged material (a pasted README, notes),
+ * and its provenance is recovered from git history rather than a
+ * schema field. The nightly self-review job (`self-review-tools.ts`) is
+ * the only thing that reads raw/ back to triage it into curated/.
+ *
+ * `list`/`read`/`grep` intentionally bypass `wiki-read.ts`'s per-user
+ * scoping (`allowedRoots`) — that scoping exists to isolate what the
+ * MODEL can see per caller; a maintainer running this CLI is already a
+ * trusted admin context with no such boundary, so these just walk the
+ * whole vault directly.
+ */
+import { initVault } from "./vault-init.ts";
+import { writeCuratedNote, writeRawEntry } from "./wiki-note.ts";
+
+function usage(): never {
+  console.error(
+    [
+      "Usage: vault-cli <command> [args]",
+      "",
+      "Commands:",
+      "  write-curated <curated/...path.md> [--author NAME]   body read from stdin",
+      "  write-raw <raw/...path.md>                           body read from stdin",
+      "  read <path>",
+      "  list",
+      "  grep <pattern>",
+      "",
+      "Every path is relative to the vault root (as printed by `list`), including",
+      "the leading `curated/`/`raw/` — write-curated/write-raw require it too, so",
+      "paths are consistent across every command instead of meaning different things.",
+    ].join("\n"),
+  );
+  process.exit(1);
+}
+
+async function readStdin(): Promise<string> {
+  return await Bun.stdin.text();
+}
+
+async function main(): Promise<void> {
+  const vaultPath = process.env.WIKI_VAULT_PATH;
+  if (!vaultPath) {
+    console.error("WIKI_VAULT_PATH is not set");
+    process.exit(1);
+  }
+  await initVault(vaultPath);
+
+  const [command, ...args] = process.argv.slice(2);
+
+  switch (command) {
+    case "write-curated": {
+      const vaultRelativePath = args[0];
+      if (!vaultRelativePath) usage();
+      if (!vaultRelativePath.startsWith("curated/")) {
+        console.error(`path must start with "curated/" (got "${vaultRelativePath}") — see \`list\` for real examples`);
+        process.exit(1);
+      }
+      const curatedRelativePath = vaultRelativePath.slice("curated/".length);
+      const authorFlagIndex = args.indexOf("--author");
+      const author = authorFlagIndex !== -1 ? args[authorFlagIndex + 1] : undefined;
+      const body = await readStdin();
+      if (!body.trim()) {
+        console.error("empty body on stdin — nothing to write");
+        process.exit(1);
+      }
+      await writeCuratedNote(
+        vaultPath,
+        curatedRelativePath,
+        { author, last_updated: new Date().toISOString().slice(0, 10) },
+        body.trimEnd(),
+      );
+      console.log(`wrote ${vaultRelativePath}`);
+      break;
+    }
+
+    case "write-raw": {
+      const vaultRelativePath = args[0];
+      if (!vaultRelativePath) usage();
+      if (!vaultRelativePath.startsWith("raw/")) {
+        console.error(`path must start with "raw/" (got "${vaultRelativePath}")`);
+        process.exit(1);
+      }
+      const rawRelativePath = vaultRelativePath.slice("raw/".length);
+      const body = await readStdin();
+      if (!body.trim()) {
+        console.error("empty body on stdin — nothing to write");
+        process.exit(1);
+      }
+      await writeRawEntry(vaultPath, rawRelativePath, body.trimEnd());
+      console.log(`wrote ${vaultRelativePath}`);
+      break;
+    }
+
+    case "read": {
+      const relativePath = args[0];
+      if (!relativePath) usage();
+      console.log(await Bun.file(`${vaultPath}/${relativePath}`).text());
+      break;
+    }
+
+    case "list": {
+      const glob = new Bun.Glob("**/*.md");
+      for await (const file of glob.scan({ cwd: vaultPath })) {
+        console.log(file);
+      }
+      break;
+    }
+
+    case "grep": {
+      const pattern = args[0];
+      if (!pattern) usage();
+      const regex = new RegExp(pattern);
+      const glob = new Bun.Glob("**/*.md");
+      for await (const file of glob.scan({ cwd: vaultPath })) {
+        const content = await Bun.file(`${vaultPath}/${file}`).text();
+        content.split("\n").forEach((line, i) => {
+          if (regex.test(line)) {
+            console.log(`${file}:${i + 1}:${line}`);
+          }
+        });
+      }
+      break;
+    }
+
+    default:
+      usage();
+  }
+}
+
+await main();

package/src/wiki/vault-init.ts

@@ -0,0 +1,43 @@
+/**
+ * Idempotent scaffolding for the wiki vault (Layer 2) — the vault itself
+ * is a separate git repository, mounted as an external volume
+ * (`wiki-vault:/app/wiki-vault` in docker-compose.yml), not part of
+ * Mercury's own repo. This runs at Mercury startup against
+ * `WIKI_VAULT_PATH` to make sure the expected curated/inferred structure
+ * and the vault's own git repo exist, without disturbing whatever
+ * content is already there.
+ */
+import { mkdir, stat } from "node:fs/promises";
+import { join } from "node:path";
+
+const SUBDIRS = ["curated/design", "curated/standards", "curated/decisions", "inferred/users", "raw"];
+
+async function pathExists(path: string): Promise<boolean> {
+  try {
+    await stat(path);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Creates the vault's curated/inferred subdirectories and git-inits the
+ * vault if it isn't already a git repo. Safe to call on every startup:
+ * pre-existing directories/content are left untouched, and re-running
+ * `git init` on an already-initialized repo is a no-op.
+ */
+export async function initVault(vaultPath: string): Promise<void> {
+  for (const sub of SUBDIRS) {
+    await mkdir(join(vaultPath, sub), { recursive: true });
+  }
+
+  if (!(await pathExists(join(vaultPath, ".git")))) {
+    const proc = Bun.spawn(["git", "init"], { cwd: vaultPath, stdout: "pipe", stderr: "pipe" });
+    const exitCode = await proc.exited;
+    if (exitCode !== 0) {
+      const stderr = await new Response(proc.stderr).text();
+      throw new Error(`git init failed in ${vaultPath}: ${stderr}`);
+    }
+  }
+}