@mercury-fw/core 0.25.0 → 0.26.0

package/CHANGELOG.md

@@ -1,5 +1,38 @@
 # @mercury-fw/core
 
+## 0.26.0
+
+### Minor Changes
+
+- 8b92abb: - `mfw` operates an app from inside its folder: `start`, `stop` and `restart` (with `--no-cache`), `logs`, `repl`, `shell`, all as `docker compose` calls.
+  - `mfw vault` maintains the wiki vault (`list`, `read`, `grep`, `write-curated`, `write-raw`) in a one-off container.
+  - `mfw memory list` and `mfw memory read <collection>` read the memory on Qdrant, newest first where the collection has a timestamp index.
+  - `mfw reset memory` and `mfw reset wiki` delete a memory volume after you type the app's name, then bring the service back up empty; after a memory reset a running app is restarted, so it sets up its collections again.
+  - The command line is declared with commander: `--help` at every level, a suggestion for a mistyped command, and every argument checked before anything runs.
+  - A scaffolded app lists `@mercury-fw/cli` among its devDependencies, at the framework's version, and its README runs it through `bunx mfw`.
+  - A scaffolded app's `@types/bun` and `typescript` use caret ranges instead of exact versions.
+  - The core ships a read-only memory CLI (`src/memory/memory-cli.ts`) next to the vault one, which is what `mfw memory` runs.
+  - The vault CLI says so when asked to read a note that doesn't exist, instead of printing a stack trace.
+
+### Patch Changes
+
+- @mercury-fw/plugin-types@0.26.0
+- @mercury-fw/channel-types@0.26.0
+- @mercury-fw/cli-engine@0.26.0
+- @mercury-fw/confirm-engine@0.26.0
+
+## 0.25.1
+
+### Patch Changes
+
+- 5613e5e: - Mercury starts even when Qdrant isn't answering yet: the memory collections are set up in the background and retried until Qdrant is reachable, instead of crashing the process at startup.
+  - A new session's context primer goes on without the last-session recap when Qdrant doesn't answer, instead of failing the turn.
+  - The scaffolded app's `docker-compose.yml` restarts the app service unless it was stopped.
+  - @mercury-fw/plugin-types@0.25.1
+  - @mercury-fw/channel-types@0.25.1
+  - @mercury-fw/cli-engine@0.25.1
+  - @mercury-fw/confirm-engine@0.25.1
+
 ## 0.25.0
 
 ### Minor Changes

package/dist/src/memory/collection-setup.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Layer 3's startup setup (creating the Qdrant collections and their indexes),
+ * run in the background so an unreachable Qdrant never stops Mercury from
+ * starting: the episodic store is enrichment and fails soft (principle 3).
+ */
+/**
+ * Runs `setup` now and, while it fails, again every `retryMs`, on a timer that
+ * doesn't keep the process alive. Logs once when Qdrant goes missing and once
+ * when it's back, never per retry. `done` resolves after the first success.
+ */
+export declare function setUpWhenReachable(setup: () => Promise<void>, { log, retryMs }: {
+    log: (msg: string) => void;
+    retryMs?: number;
+}): {
+    done: Promise<void>;
+};

package/dist/src/memory/memory-cli.d.ts

@@ -0,0 +1,38 @@
+#!/usr/bin/env bun
+type ScrollOffset = string | number | Record<string, unknown> | null;
+/** The slice of the Qdrant client this CLI uses. */
+export type MemoryCliClient = {
+    getCollections(): Promise<{
+        collections: Array<{
+            name: string;
+        }>;
+    }>;
+    count(collection: string, params: {
+        exact: boolean;
+    }): Promise<{
+        count: number;
+    }>;
+    scroll(collection: string, params: {
+        limit: number;
+        offset?: ScrollOffset;
+        with_payload: boolean;
+        order_by?: {
+            key: string;
+            direction: "asc" | "desc";
+        };
+    }): Promise<{
+        points: Array<{
+            id: string | number;
+            payload?: Record<string, unknown> | null;
+        }>;
+        next_page_offset?: ScrollOffset;
+    }>;
+};
+type Io = {
+    client: MemoryCliClient;
+    out: (line: string) => void;
+    err: (line: string) => void;
+};
+/** Runs the CLI on `argv` against `io.client`; returns the exit code. */
+export declare function runMemoryCli(argv: string[], { client, out, err }: Io): Promise<number>;
+export {};

package/dist/src/session/context-primer.d.ts

@@ -7,6 +7,8 @@ export type ContextPrimerDeps = {
     listWikiFilesInRootsFn: typeof listWikiFilesInRoots;
     readWikiFileInRootsFn: typeof readWikiFileInRoots;
     readIndexFileFn: typeof readIndexFile;
+    /** Where a skipped section says why (the recap, when Qdrant doesn't answer). */
+    log: (msg: string) => void;
 };
 /**
  * Text of the primer for `userId`, built from injected deps only — never

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mercury-fw/core",
-  "version": "0.25.0",
+  "version": "0.26.0",
   "type": "module",
   "license": "MIT",
   "repository": {
@@ -31,10 +31,10 @@
     "test": "bun test"
   },
   "dependencies": {
-    "@mercury-fw/channel-types": "0.25.0",
-    "@mercury-fw/cli-engine": "0.25.0",
-    "@mercury-fw/confirm-engine": "0.25.0",
-    "@mercury-fw/plugin-types": "0.25.0",
+    "@mercury-fw/channel-types": "0.26.0",
+    "@mercury-fw/cli-engine": "0.26.0",
+    "@mercury-fw/confirm-engine": "0.26.0",
+    "@mercury-fw/plugin-types": "0.26.0",
     "@qdrant/js-client-rest": "^1.19.0",
     "ai": "^7.0.77",
     "ai-sdk-ollama": "^4.2.0",

package/src/compose.ts

@@ -56,6 +56,7 @@ import { ensureVerbatimCollection, listVerbatimBySession, listVerbatimSessions }
 import { createVerbatimArchiveProvider } from "./memory/memory-provider.ts";
 import { ensureSemanticFactsCollection, storeSemanticFact, searchSemanticFactsByTopic } from "./memory/semantic-facts-store.ts";
 import { ensureToolCorrectionsCollection, storeToolCorrection, searchToolCorrectionsByTopic } from "./memory/tool-corrections-store.ts";
+import { setUpWhenReachable } from "./memory/collection-setup.ts";
 import { consolidateSemanticFact, consolidateToolCorrection, type ToolCorrectionConsolidationDeps } from "./cron/semantic-consolidation.ts";
 import { createToolCorrectionExtractor } from "./session/tool-correction-extractor.ts";
 import { createEmbedder } from "./memory/embedder.ts";
@@ -227,14 +228,12 @@ export async function composeMercury(config: MercuryConfig): Promise<ComposedApp
   const qdrant = new QdrantClient({ url: process.env.QDRANT_URL ?? "http://qdrant:6333" });
   const episodicCollection = process.env.QDRANT_EPISODIC_COLLECTION ?? "episodic_memory";
   const episodicVectorSize = Number(process.env.QDRANT_EPISODIC_VECTOR_SIZE ?? "768");
-  await ensureEpisodicCollection(qdrant, episodicCollection, episodicVectorSize);
 
   // Verbatim conversation archive (#4): a distinct collection holding the raw
   // user<->model exchange, lossless and durable — separate from the lossy
   // Layer-1 window and the derived episodic summaries above.
   const verbatimCollection = process.env.QDRANT_VERBATIM_COLLECTION ?? "verbatim_archive";
   const verbatimVectorSize = Number(process.env.QDRANT_VERBATIM_VECTOR_SIZE ?? "768");
-  await ensureVerbatimCollection(qdrant, verbatimCollection, verbatimVectorSize);
   const verbatimProvider = createVerbatimArchiveProvider({ client: qdrant, collectionName: verbatimCollection, embed });
 
   // Semantic consolidation (D-22/D-34): a separate Qdrant collection from
@@ -242,7 +241,6 @@ export async function composeMercury(config: MercuryConfig): Promise<ComposedApp
   // vector embedded on the topic alone (see semantic-facts-store.ts for why).
   const semanticFactsCollection = process.env.QDRANT_SEMANTIC_FACTS_COLLECTION ?? "semantic_facts";
   const semanticFactsVectorSize = Number(process.env.QDRANT_SEMANTIC_FACTS_VECTOR_SIZE ?? "768");
-  await ensureSemanticFactsCollection(qdrant, semanticFactsCollection, semanticFactsVectorSize);
   const extractFacts = createSemanticFactExtractor(model);
 
   // Idempotent self-heal: the vault lives on a named Docker volume, empty on
@@ -322,7 +320,17 @@ export async function composeMercury(config: MercuryConfig): Promise<ComposedApp
   // the tool-call trace only exists in memory for the duration of its turn.
   const toolCorrectionsCollection = process.env.QDRANT_TOOL_CORRECTIONS_COLLECTION ?? "tool_corrections";
   const toolCorrectionsVectorSize = Number(process.env.QDRANT_TOOL_CORRECTIONS_VECTOR_SIZE ?? "768");
-  await ensureToolCorrectionsCollection(qdrant, toolCorrectionsCollection, toolCorrectionsVectorSize);
+  // Every Layer-3 collection, set up in the background: Mercury starts even
+  // while Qdrant isn't answering yet, and memory switches on once it does.
+  setUpWhenReachable(
+    async () => {
+      await ensureEpisodicCollection(qdrant, episodicCollection, episodicVectorSize);
+      await ensureVerbatimCollection(qdrant, verbatimCollection, verbatimVectorSize);
+      await ensureSemanticFactsCollection(qdrant, semanticFactsCollection, semanticFactsVectorSize);
+      await ensureToolCorrectionsCollection(qdrant, toolCorrectionsCollection, toolCorrectionsVectorSize);
+    },
+    { log: (msg) => console.error(`[memory] ${msg}`) },
+  );
   const extractToolCorrections = createToolCorrectionExtractor(model, undefined, {
     log: (msg) => console.error(`[cron] ${msg}`),
   });
@@ -464,6 +472,7 @@ export async function composeMercury(config: MercuryConfig): Promise<ComposedApp
           listWikiFilesInRootsFn: listWikiFilesInRoots,
           readWikiFileInRootsFn: readWikiFileInRoots,
           readIndexFileFn: readIndexFile,
+          log: (msg) => console.error(`[memory] ${msg}`),
         });
         return getOrCreateHistory(key, trackForCapture, primer);
       }

package/src/memory/collection-setup.ts

@@ -0,0 +1,34 @@
+/**
+ * Layer 3's startup setup (creating the Qdrant collections and their indexes),
+ * run in the background so an unreachable Qdrant never stops Mercury from
+ * starting: the episodic store is enrichment and fails soft (principle 3).
+ */
+
+/**
+ * Runs `setup` now and, while it fails, again every `retryMs`, on a timer that
+ * doesn't keep the process alive. Logs once when Qdrant goes missing and once
+ * when it's back, never per retry. `done` resolves after the first success.
+ */
+export function setUpWhenReachable(
+  setup: () => Promise<void>,
+  { log, retryMs = 5000 }: { log: (msg: string) => void; retryMs?: number },
+): { done: Promise<void> } {
+  let failing = false;
+  const done = new Promise<void>((resolve) => {
+    const attempt = async () => {
+      try {
+        await setup();
+        if (failing) log("Qdrant reachable, memory collections ready");
+        resolve();
+      } catch (err) {
+        if (!failing) {
+          log(`Qdrant unreachable, Layer-3 memory is off until it answers (retrying every ${retryMs / 1000}s): ${String(err)}`);
+        }
+        failing = true;
+        setTimeout(attempt, retryMs).unref();
+      }
+    };
+    void attempt();
+  });
+  return { done };
+}

package/src/memory/memory-cli.ts

@@ -0,0 +1,140 @@
+#!/usr/bin/env bun
+/**
+ * Read-only CLI for Layer-3 memory on Qdrant — runs INSIDE the Mercury
+ * container (where `QDRANT_URL` reaches Qdrant): `mfw memory` runs it in a
+ * one-off container, by this path.
+ * `list` shows the collections and their sizes, `read` a collection's points:
+ * newest first where the collection has a `timestamp` payload index (episodic
+ * memory, the verbatim archive), in Qdrant's own order otherwise, and it says
+ * which. Nothing here writes.
+ */
+import { parseArgs } from "node:util";
+import { QdrantClient } from "@qdrant/js-client-rest";
+import { scrollCollection } from "../admin/qdrant-scroll.ts";
+
+type ScrollOffset = string | number | Record<string, unknown> | null;
+
+/** The slice of the Qdrant client this CLI uses. */
+export type MemoryCliClient = {
+  getCollections(): Promise<{ collections: Array<{ name: string }> }>;
+  count(collection: string, params: { exact: boolean }): Promise<{ count: number }>;
+  scroll(
+    collection: string,
+    params: {
+      limit: number;
+      offset?: ScrollOffset;
+      with_payload: boolean;
+      order_by?: { key: string; direction: "asc" | "desc" };
+    },
+  ): Promise<{
+    points: Array<{ id: string | number; payload?: Record<string, unknown> | null }>;
+    next_page_offset?: ScrollOffset;
+  }>;
+};
+
+type Io = { client: MemoryCliClient; out: (line: string) => void; err: (line: string) => void };
+
+const USAGE = [
+  "Usage: mfw memory <command> [args]",
+  "",
+  "Commands:",
+  "  list                              the collections and their points",
+  "  read <collection> [--limit N]     a collection's points, newest first where it can (default 20)",
+];
+
+/** A point as the lines `read` prints: its id, then one indented line per payload field. */
+function pointLines(point: { id: string | number; payload?: Record<string, unknown> | null }): string[] {
+  const fields = Object.entries(point.payload ?? {}).map(
+    ([key, value]) => `  ${key}: ${typeof value === "string" ? value : JSON.stringify(value)}`,
+  );
+  return [String(point.id), ...fields, ""];
+}
+
+/** Whether `err` is Qdrant refusing to order by a field it has no index on
+ * (HTTP 400, "No range index for `order_by` key"). */
+function isMissingOrderIndex(err: unknown): boolean {
+  const e = err as { status?: number; message?: string; data?: unknown };
+  const text = `${e?.message ?? ""} ${JSON.stringify(e?.data ?? "")}`;
+  return (e?.status === 400 || /\b400\b/.test(text)) && /index/i.test(text);
+}
+
+/** Runs the CLI on `argv` against `io.client`; returns the exit code. */
+export async function runMemoryCli(argv: string[], { client, out, err }: Io): Promise<number> {
+  const usage = () => {
+    USAGE.forEach((line) => err(line));
+    return 1;
+  };
+  let parsed;
+  try {
+    parsed = parseArgs({ args: argv, options: { limit: { type: "string" } }, allowPositionals: true });
+  } catch {
+    return usage();
+  }
+  const [command, collection, ...extra] = parsed.positionals;
+  if (extra.length > 0 || (command === "list" && (collection !== undefined || parsed.values.limit !== undefined))) {
+    return usage();
+  }
+  if (command !== "list" && !(command === "read" && collection !== undefined)) return usage();
+
+  const limitText = parsed.values.limit ?? "20";
+  const limit = Number(limitText);
+  if (!/^\d+$/.test(limitText) || limit < 1) {
+    err(`--limit takes a positive whole number (got "${limitText}").`);
+    return 1;
+  }
+
+  let names: string[];
+  try {
+    names = (await client.getCollections()).collections.map((c) => c.name).sort();
+  } catch (e) {
+    err(`Can't reach Qdrant: ${String(e)}`);
+    return 1;
+  }
+
+  try {
+    if (command === "list") {
+      if (names.length === 0) {
+        out("No collections yet.");
+        return 0;
+      }
+      for (const name of names) {
+        out(`${name}  ${(await client.count(name, { exact: true })).count} points`);
+      }
+      return 0;
+    }
+
+    if (!names.includes(collection as string)) {
+      err(`No collection "${collection}". There are: ${names.join(", ") || "none"}.`);
+      return 1;
+    }
+    let points: Array<{ id: string | number; payload?: Record<string, unknown> | null }>;
+    try {
+      // Qdrant orders by a payload field only when it has an index on it.
+      ({ points } = await client.scroll(collection as string, {
+        limit,
+        with_payload: true,
+        order_by: { key: "timestamp", direction: "desc" },
+      }));
+    } catch (e) {
+      if (!isMissingOrderIndex(e)) throw e;
+      err(`${collection} has no timestamp index: points in Qdrant's own order.`);
+      ({ points } = await scrollCollection(client, collection as string, { limit }));
+    }
+    points.flatMap(pointLines).forEach((line) => out(line));
+    return 0;
+  } catch (e) {
+    err(`Qdrant error: ${String(e)}`);
+    return 1;
+  }
+}
+
+if (import.meta.main) {
+  const client = new QdrantClient({ url: process.env.QDRANT_URL ?? "http://qdrant:6333" });
+  process.exit(
+    await runMemoryCli(process.argv.slice(2), {
+      client,
+      out: (line) => console.log(line),
+      err: (line) => console.error(line),
+    }),
+  );
+}

package/src/session/context-primer.ts

@@ -21,6 +21,8 @@ export type ContextPrimerDeps = {
   listWikiFilesInRootsFn: typeof listWikiFilesInRoots;
   readWikiFileInRootsFn: typeof readWikiFileInRoots;
   readIndexFileFn: typeof readIndexFile;
+  /** Where a skipped section says why (the recap, when Qdrant doesn't answer). */
+  log: (msg: string) => void;
 };
 
 const FRONTMATTER_RE = /^---\n([\s\S]*?)\n---\n/;
@@ -78,7 +80,12 @@ async function pendingConfirmationTokens(userId: string, deps: ContextPrimerDeps
  * `index.ts` supplies the real Qdrant-backed episodic query and wiki reads.
  */
 export async function buildContextPrimer(userId: string, deps: ContextPrimerDeps): Promise<string> {
-  const entries = await deps.getLastSessionEntries(userId);
+  // Episodic memory is enrichment: without Qdrant the primer goes on without
+  // the recap instead of failing the turn.
+  const entries = await deps.getLastSessionEntries(userId).catch((err: unknown) => {
+    deps.log(`last session for ${userId} unavailable, primer built without it: ${String(err)}`);
+    return [];
+  });
   // Checked regardless of `entries` — a pending confirmation isn't tied to
   // "was there a prior episodic session", it's simply still open right now.
   const pendingTokens = await pendingConfirmationTokens(userId, deps);

package/src/wiki/vault-cli.ts

@@ -1,8 +1,8 @@
 #!/usr/bin/env bun
 /**
  * Maintenance CLI for the wiki vault — runs INSIDE the Mercury container
- * (the vault is a Docker named volume, not a host path, see `scripts/vault.sh`
- * and CLAUDE.md § "Manutenzione della vault wiki"). Thin argv wrapper around
+ * (the vault is a Docker named volume, not a host path): `mfw vault` runs it
+ * in a one-off container, by this path. Thin argv wrapper around
  * functions that already exist and are already tested (`wiki-note.ts`,
  * `vault-init.ts`) — no new write/read logic here, only routing.
  *
@@ -29,7 +29,7 @@ import { writeCuratedNote, writeRawEntry } from "./wiki-note.ts";
 function usage(): never {
   console.error(
     [
-      "Usage: vault-cli <command> [args]",
+      "Usage: mfw vault <command> [args]",
       "",
       "Commands:",
       "  write-curated <curated/...path.md> [--author NAME]   body read from stdin",
@@ -107,7 +107,12 @@ async function main(): Promise<void> {
     case "read": {
       const relativePath = args[0];
       if (!relativePath) usage();
-      console.log(await Bun.file(`${vaultPath}/${relativePath}`).text());
+      const note = Bun.file(`${vaultPath}/${relativePath}`);
+      if (!(await note.exists())) {
+        console.error(`no note at ${relativePath} (mfw vault list shows them)`);
+        process.exit(1);
+      }
+      console.log(await note.text());
       break;
     }