@mercury-fw/core 0.25.0

package/src/cron/idle-session-cron.ts

@@ -0,0 +1,144 @@
+/**
+ * The session-persistence cron loop: periodically sweeps for sessions idle past the
+ * configured timeout, summarizes each one (episodic-summarizer.ts),
+ * writes the result to Qdrant (episodic-store.ts), and discards the raw
+ * transcript (`deps.closeSession`). Every dependency is injected — this
+ * file owns only the sweep/interval mechanics, not session storage, the
+ * LLM call, or Qdrant itself.
+ */
+import type { IdleSessionScanner } from "./idle-session-scanner.ts";
+import type { Message } from "../session/history.ts";
+import type { EpisodicSummary } from "../memory/episodic-store.ts";
+import type { SemanticFact } from "../session/semantic-fact-extractor.ts";
+import type { SemanticFactEntry } from "../memory/semantic-facts-store.ts";
+
+export type IdleSession = { key: string; userId: string; messages: Message[] };
+
+/**
+ * Dependencies for `captureSessionToMemory` — a subset of
+ * `IdleSessionSweepDeps`, without `getSession`/`closeSession`: this
+ * function never decides *whether* a session should be captured or
+ * whether it should be closed afterwards, only *how* to capture a given
+ * slice of messages. Reused by the idle sweep below (final capture +
+ * close) and, without touching this file's own tests, by the two
+ * mid-conversation triggers wired in `index.ts` (message-count threshold,
+ * Layer 1 compression) — neither of which closes the session.
+ */
+export type CaptureDeps = {
+  summarize: (messages: Message[]) => Promise<string>;
+  store: (entry: EpisodicSummary) => Promise<void>;
+  /**
+   * Semantic fact extraction/consolidation (D-22/D-34) — an enrichment on
+   * top of the episodic summary above, not a required part of it: omit
+   * all three and capture behaves exactly as before. When present, a
+   * failure here is logged and never propagates — the episodic write
+   * already succeeded and is the source of truth being preserved, same
+   * "system must work when this enrichment is absent" boundary as every
+   * other Layer 2/3 store in Mercury.
+   */
+  extractFacts?: (messages: Message[]) => Promise<SemanticFact[]>;
+  storeFact?: (entry: SemanticFactEntry) => Promise<void>;
+  consolidateFact?: (userId: string, topic: string) => Promise<void>;
+  log?: (msg: string) => void;
+};
+
+/**
+ * Summarizes+stores an episodic entry for `messages`, then (when the
+ * semantic deps are provided) extracts and consolidates semantic facts —
+ * the one place this logic lives, shared by every capture trigger. A
+ * failure summarizing/storing propagates to the caller (it decides what
+ * "capture failed" means for its own trigger — e.g. the idle sweep below
+ * leaves the session tracked for retry and skips `closeSession`); a
+ * failure in the semantic enrichment layer is caught and logged here,
+ * never propagated, since it must never undo an episodic write that
+ * already succeeded.
+ */
+export async function captureSessionToMemory(
+  userId: string,
+  sessionKey: string,
+  messages: Message[],
+  now: number,
+  deps: CaptureDeps,
+): Promise<void> {
+  const log = deps.log ?? ((msg: string) => console.error(msg));
+
+  const summary = await deps.summarize(messages);
+  const timestamp = new Date(now).toISOString();
+  await deps.store({ userId, sessionKey, summary, timestamp });
+
+  if (deps.extractFacts && deps.storeFact && deps.consolidateFact) {
+    try {
+      const facts = await deps.extractFacts(messages);
+      for (const fact of facts) {
+        try {
+          await deps.storeFact({ userId, topic: fact.topic, value: fact.value, timestamp });
+          await deps.consolidateFact(userId, fact.topic);
+        } catch (err) {
+          log(`semantic fact consolidation failed for ${sessionKey}/${fact.topic}: ${String(err)}`);
+        }
+      }
+    } catch (err) {
+      log(`semantic fact extraction failed for ${sessionKey}: ${String(err)}`);
+    }
+  }
+}
+
+export type IdleSessionSweepDeps = CaptureDeps & {
+  /** Looks up a session's current content by key; `undefined` if it's already gone (e.g. closed by something else in the meantime). */
+  getSession: (key: string) => IdleSession | undefined;
+  /** Discards the session's raw transcript — called only after a successful summarize+store. */
+  closeSession: (key: string) => void;
+};
+
+/**
+ * Runs one sweep at `now`: every session `scanner` reports idle (past
+ * `idleTimeoutMs`) gets captured (`captureSessionToMemory`) and closed,
+ * then cleared from `scanner`. A failure capturing is logged and leaves
+ * that session's tracking untouched (retried on the next sweep) — it must
+ * never stop the sweep from processing the others (hard-won convention:
+ * one bad tick can't take down the rest of Mercury).
+ */
+export async function runIdleSessionSweep(
+  scanner: IdleSessionScanner,
+  now: number,
+  idleTimeoutMs: number,
+  deps: IdleSessionSweepDeps,
+): Promise<void> {
+  const log = deps.log ?? ((msg: string) => console.error(msg));
+
+  for (const key of scanner.scanIdle(now, idleTimeoutMs)) {
+    try {
+      const session = deps.getSession(key);
+      if (!session) {
+        scanner.clear(key);
+        continue;
+      }
+
+      await captureSessionToMemory(session.userId, key, session.messages, now, deps);
+
+      deps.closeSession(key);
+      scanner.clear(key);
+    } catch (err) {
+      log(`idle session sweep failed for ${key}: ${String(err)}`);
+    }
+  }
+}
+
+export type IdleSessionCron = { stop: () => void };
+
+/** Starts the periodic sweep on `opts.checkIntervalMs`, gated on `opts.idleTimeoutMs`. `stop()` halts it. */
+export function startIdleSessionCron(
+  scanner: IdleSessionScanner,
+  deps: IdleSessionSweepDeps,
+  opts: { idleTimeoutMs: number; checkIntervalMs: number },
+): IdleSessionCron {
+  const interval = setInterval(() => {
+    runIdleSessionSweep(scanner, Date.now(), opts.idleTimeoutMs, deps).catch((err) => {
+      (deps.log ?? ((msg: string) => console.error(msg)))(`idle session cron tick failed: ${String(err)}`);
+    });
+  }, opts.checkIntervalMs);
+
+  return {
+    stop: () => clearInterval(interval),
+  };
+}

package/src/cron/idle-session-scanner.ts

@@ -0,0 +1,37 @@
+/**
+ * Pure idle-tracking for session persistence: remembers the last
+ * activity time per session key and reports which are idle at a given
+ * moment. Time is always passed in, never read internally (`Date.now()`
+ * lives in the caller, e.g. `src/index.ts`/`idle-session-cron.ts`) — this
+ * is what makes `scanIdle`'s threshold behavior exactly testable.
+ */
+export type IdleSessionScanner = {
+  /** Records activity for `key` at `now`, resetting its idle clock. */
+  touch(key: string, now: number): void;
+  /** Returns every tracked key whose last activity is at least `idleTimeoutMs` before `now`. */
+  scanIdle(now: number, idleTimeoutMs: number): string[];
+  /** Stops tracking `key` — call after a session has been consolidated and its raw transcript discarded. */
+  clear(key: string): void;
+};
+
+export function createIdleSessionScanner(): IdleSessionScanner {
+  const lastActivity = new Map<string, number>();
+
+  return {
+    touch(key, now) {
+      lastActivity.set(key, now);
+    },
+    scanIdle(now, idleTimeoutMs) {
+      const idle: string[] = [];
+      for (const [key, lastSeen] of lastActivity) {
+        if (now - lastSeen >= idleTimeoutMs) {
+          idle.push(key);
+        }
+      }
+      return idle;
+    },
+    clear(key) {
+      lastActivity.delete(key);
+    },
+  };
+}

package/src/cron/self-review-cron.ts

@@ -0,0 +1,103 @@
+/**
+ * The wiki self-review cron: runs once nightly (a fixed local hour,
+ * hardcoded, no env override — deliberately different from every other
+ * interval in this codebase, per an explicit request to keep this one
+ * out of runtime configurability), running raw/ triage, index.md/orphan
+ * maintenance, and a contradiction check as three independent sub-passes
+ * (see `self-review-runner.ts` for why they're independent, not one
+ * multi-step call). Raw-triage and index/orphan each skip when their own
+ * cheap pre-check finds nothing to do; the contradiction check has no
+ * such pre-check and always runs on a triggered tick — running the whole
+ * thing at night, when nothing else contends for the shared local model,
+ * is what makes that affordable.
+ *
+ * This file owns only the scheduling/orchestration mechanics — vault
+ * access and the actual LLM calls are injected, same separation
+ * `idle-session-cron.ts` uses for session storage and the LLM summarizer.
+ */
+export type SelfReviewTickDeps = {
+  listRawEntries: () => Promise<string[]>;
+  findOrphans: () => Promise<string[]>;
+  runRawTriage: (rawEntries: string[]) => Promise<void>;
+  runIndexAndOrphan: (orphans: string[]) => Promise<void>;
+  runContradictionCheck: () => Promise<void>;
+  log?: (msg: string) => void;
+};
+
+/**
+ * Runs one nightly pass: raw-triage and index/orphan each run only if
+ * their own cheap signal found something; the contradiction check always
+ * runs, since nothing cheap can tell it whether there's anything to find.
+ * Each sub-pass is wrapped in its own try/catch — one failing must not
+ * stop the other two from running in the same pass (hard-won convention:
+ * one bad tick can't take down the rest of Mercury).
+ */
+export async function runSelfReviewTick(deps: SelfReviewTickDeps): Promise<void> {
+  const log = deps.log ?? ((msg: string) => console.error(msg));
+
+  const [rawEntries, orphans] = await Promise.all([deps.listRawEntries(), deps.findOrphans()]);
+
+  if (rawEntries.length > 0) {
+    try {
+      await deps.runRawTriage(rawEntries);
+    } catch (err) {
+      log(`self-review raw-triage pass failed: ${String(err)}`);
+    }
+  }
+
+  if (orphans.length > 0) {
+    try {
+      await deps.runIndexAndOrphan(orphans);
+    } catch (err) {
+      log(`self-review index/orphan pass failed: ${String(err)}`);
+    }
+  }
+
+  try {
+    await deps.runContradictionCheck();
+  } catch (err) {
+    log(`self-review contradiction-check pass failed: ${String(err)}`);
+  }
+}
+
+/** 3 AM local time — hardcoded on purpose, see file header. */
+export const SELF_REVIEW_HOUR = 3;
+/** How often to check whether it's time to run — mirrors idle-session-cron's
+ * check-vs-timeout split, hardcoded for the same reason as the hour above. */
+export const SELF_REVIEW_CHECK_INTERVAL_MS = 15 * 60_000;
+
+function localDateKey(d: Date): string {
+  return `${d.getFullYear()}-${d.getMonth()}-${d.getDate()}`;
+}
+
+export type SelfReviewCron = { stop: () => void };
+
+/**
+ * Checks every `opts.checkIntervalMs` whether the current local hour
+ * matches `opts.hour` and today hasn't run yet; if so, runs one tick.
+ * `lastRunDate` is in-memory only — if the process restarts mid-window it
+ * just waits for tomorrow's, which is fine, nothing here needs to survive
+ * a restart. `opts.now`/`opts.hour`/`opts.checkIntervalMs` exist purely as
+ * test seams; production (`index.ts`) never overrides them.
+ */
+export function startSelfReviewCron(deps: SelfReviewTickDeps, opts: { hour?: number; checkIntervalMs?: number; now?: () => Date } = {}): SelfReviewCron {
+  const hour = opts.hour ?? SELF_REVIEW_HOUR;
+  const checkIntervalMs = opts.checkIntervalMs ?? SELF_REVIEW_CHECK_INTERVAL_MS;
+  const now = opts.now ?? (() => new Date());
+  const log = deps.log ?? ((msg: string) => console.error(msg));
+  let lastRunDate: string | null = null;
+
+  const interval = setInterval(() => {
+    const current = now();
+    if (current.getHours() !== hour) return;
+    const today = localDateKey(current);
+    if (lastRunDate === today) return;
+    lastRunDate = today;
+
+    runSelfReviewTick(deps).catch((err) => {
+      log(`self-review cron tick failed: ${String(err)}`);
+    });
+  }, checkIntervalMs);
+
+  return { stop: () => clearInterval(interval) };
+}

package/src/cron/semantic-consolidation.ts

@@ -0,0 +1,228 @@
+/**
+ * Deterministic promotion of clustered semantic facts to a standing wiki
+ * note — the consolidation half of D-22/D-34, paired with
+ * `semantic-fact-extractor.ts` (the LLM half, which only ever proposes
+ * candidate facts). Zero model judgment here: given the last `k`
+ * occurrences of a topic for a user, count the most common value and
+ * compare it against whatever's already written at
+ * `inferred/users/<userId>/<topic>.md` — write only if the challenger's
+ * count strictly exceeds the incumbent's (never on a tie, and never when
+ * no single value is unambiguously dominant in the current window).
+ */
+import { parse as parseYaml } from "yaml";
+import type { readWikiFile } from "../wiki/wiki-read.ts";
+import type { writeInferredNote, writeToolCorrectionNote } from "../wiki/wiki-note.ts";
+import type { SemanticFactEntry } from "../memory/semantic-facts-store.ts";
+import type { ToolCorrectionEntry } from "../memory/tool-corrections-store.ts";
+
+type ClusterFn = (userId: string, topic: string, limit: number) => Promise<SemanticFactEntry[]>;
+type Confidence = "low" | "medium" | "high";
+
+export type ConsolidationDeps = {
+  vaultPath: string;
+  clusterFn: ClusterFn;
+  readWikiFileFn: typeof readWikiFile;
+  writeInferredNoteFn: typeof writeInferredNote;
+  k?: number;
+  confidenceForCount?: (dominantCount: number, k: number) => Confidence;
+  now?: () => string;
+};
+
+type ToolCorrectionClusterFn = (tool: string, topic: string, limit: number) => Promise<ToolCorrectionEntry[]>;
+
+/**
+ * Same shape as `ConsolidationDeps`, keyed by `tool` instead of `userId` —
+ * `readNoteFn`/`writeNoteFn` deliberately don't take a userId at all
+ * (unlike `readWikiFileFn`/`writeInferredNoteFn` above): a procedural
+ * correction lives under `curated/standards/`, visible to every session
+ * regardless of who asks, never scoped to one user's own
+ * `inferred/users/<userId>/`.
+ */
+export type ToolCorrectionConsolidationDeps = {
+  vaultPath: string;
+  clusterFn: ToolCorrectionClusterFn;
+  readNoteFn: (vaultPath: string, relativePath: string) => Promise<string>;
+  writeNoteFn: typeof writeToolCorrectionNote;
+  k?: number;
+  confidenceForCount?: (dominantCount: number, k: number) => Confidence;
+  now?: () => string;
+};
+
+/** Window size for consolidation — how many recent occurrences of a topic to consider. Uncalibrated: chosen without real usage data, to revisit once there's actual traffic to tune against. */
+export const DEFAULT_CONSOLIDATION_K = 3;
+
+/**
+ * Uncalibrated confidence bands, count relative to `k`: a single
+ * occurrence is unconfirmed (low); repeated but not unanimous within the
+ * tracked window is medium; the dominant value filling the whole window
+ * is high. Same "revisit with real usage" caveat as `DEFAULT_CONSOLIDATION_K`.
+ */
+export function defaultConfidenceForCount(dominantCount: number, k: number): Confidence {
+  if (dominantCount >= k) {
+    return "high";
+  }
+  if (dominantCount > 1) {
+    return "medium";
+  }
+  return "low";
+}
+
+// Generic over anything shaped like {value, timestamp} — both
+// SemanticFactEntry and ToolCorrectionEntry satisfy this structurally,
+// reused by consolidateSemanticFact and consolidateToolCorrection alike.
+function dominantValue(
+  entries: Array<{ value: string; timestamp: string }>,
+): { value: string; supportingTimestamps: string[] } | null {
+  const byValue = new Map<string, string[]>();
+  for (const e of entries) {
+    const timestamps = byValue.get(e.value) ?? [];
+    timestamps.push(e.timestamp);
+    byValue.set(e.value, timestamps);
+  }
+
+  let best: { value: string; timestamps: string[] } | null = null;
+  let tie = false;
+  for (const [value, timestamps] of byValue) {
+    if (!best || timestamps.length > best.timestamps.length) {
+      best = { value, timestamps };
+      tie = false;
+    } else if (timestamps.length === best.timestamps.length) {
+      tie = true;
+    }
+  }
+
+  if (!best || tie) {
+    return null;
+  }
+  return { value: best.value, supportingTimestamps: best.timestamps };
+}
+
+async function readIncumbentCount(deps: ConsolidationDeps, userId: string, topic: string): Promise<number> {
+  let text: string;
+  try {
+    text = await deps.readWikiFileFn(deps.vaultPath, userId, `inferred/users/${userId}/${topic}.md`);
+  } catch {
+    return 0;
+  }
+
+  const match = /^---\n([\s\S]*?)\n---\n/.exec(text);
+  if (!match) {
+    return 0;
+  }
+  const frontmatter = parseYaml(match[1] as string) as { derived_from?: unknown };
+  return Array.isArray(frontmatter.derived_from) ? frontmatter.derived_from.length : 0;
+}
+
+/**
+ * Re-clusters `topic` for `userId`, and promotes the dominant value to a
+ * wiki note if it beats the current incumbent's count. No-op if the
+ * cluster is empty or has no single dominant value.
+ *
+ * `userId` here is Qdrant's own storage form (e.g. `"users/42"`, a raw
+ * Google Chat resource name) — `clusterFn` above searches with it as-is,
+ * matching how `storeSemanticFact` wrote it. The wiki's
+ * `inferred/users/<userId>/` convention expects a different,
+ * `encodeURIComponent`-encoded form instead (the same one the model's
+ * own wiki tools already use) — a raw userId containing "/" would
+ * otherwise be rejected outright by
+ * `writeInferredNote`'s own path-separator guard, and even without that
+ * guard would land in a directory the model's wiki tools never look at.
+ * Two different representations of the same identity, for two different
+ * purposes — `wikiUserId` below is used only for the wiki-facing calls,
+ * never for `clusterFn`.
+ */
+export async function consolidateSemanticFact(userId: string, topic: string, deps: ConsolidationDeps): Promise<void> {
+  const k = deps.k ?? DEFAULT_CONSOLIDATION_K;
+  const confidenceForCount = deps.confidenceForCount ?? defaultConfidenceForCount;
+  const cluster = (await deps.clusterFn(userId, topic, k)).filter((e) => e.topic === topic);
+
+  const dominant = dominantValue(cluster);
+  if (!dominant) {
+    return;
+  }
+
+  const wikiUserId = encodeURIComponent(userId);
+  const incumbentCount = await readIncumbentCount(deps, wikiUserId, topic);
+  if (dominant.supportingTimestamps.length <= incumbentCount) {
+    return;
+  }
+
+  const now = deps.now ?? (() => new Date().toISOString());
+  await deps.writeInferredNoteFn(
+    deps.vaultPath,
+    wikiUserId,
+    topic,
+    {
+      confidence: confidenceForCount(dominant.supportingTimestamps.length, k),
+      derived_from: dominant.supportingTimestamps,
+      last_reviewed: now(),
+    },
+    dominant.value,
+  );
+}
+
+async function readToolCorrectionIncumbentCount(
+  deps: ToolCorrectionConsolidationDeps,
+  tool: string,
+  topic: string,
+): Promise<number> {
+  let text: string;
+  try {
+    // Full vault-root-relative path, matching how readWikiFileInRoots (the
+    // real implementation) resolves it — always against the vault root,
+    // never against whichever specific root in the allowed list happens to
+    // match (same convention readIncumbentCount above uses for
+    // inferred/users/<userId>/<topic>.md).
+    text = await deps.readNoteFn(deps.vaultPath, `curated/standards/${tool}-${topic}.md`);
+  } catch {
+    return 0;
+  }
+
+  const match = /^---\n([\s\S]*?)\n---\n/.exec(text);
+  if (!match) {
+    return 0;
+  }
+  const frontmatter = parseYaml(match[1] as string) as { derived_from?: unknown };
+  return Array.isArray(frontmatter.derived_from) ? frontmatter.derived_from.length : 0;
+}
+
+/**
+ * Same promotion logic as `consolidateSemanticFact` — re-clusters `topic`
+ * for `tool`, promotes the dominant value if it beats the incumbent's
+ * count — but keyed by `tool` (a CLI, not a person) and writing to
+ * `curated/standards/<tool>-<topic>.md` (global) instead of
+ * `inferred/users/<userId>/<topic>.md` (per-user). No userId encoding
+ * needed here: a tool name (e.g. "jira") never contains a path separator.
+ */
+export async function consolidateToolCorrection(
+  tool: string,
+  topic: string,
+  deps: ToolCorrectionConsolidationDeps,
+): Promise<void> {
+  const k = deps.k ?? DEFAULT_CONSOLIDATION_K;
+  const confidenceForCount = deps.confidenceForCount ?? defaultConfidenceForCount;
+  const cluster = (await deps.clusterFn(tool, topic, k)).filter((e) => e.topic === topic);
+
+  const dominant = dominantValue(cluster);
+  if (!dominant) {
+    return;
+  }
+
+  const incumbentCount = await readToolCorrectionIncumbentCount(deps, tool, topic);
+  if (dominant.supportingTimestamps.length <= incumbentCount) {
+    return;
+  }
+
+  const now = deps.now ?? (() => new Date().toISOString());
+  await deps.writeNoteFn(
+    deps.vaultPath,
+    tool,
+    topic,
+    {
+      confidence: confidenceForCount(dominant.supportingTimestamps.length, k),
+      derived_from: dominant.supportingTimestamps,
+      last_reviewed: now(),
+    },
+    dominant.value,
+  );
+}

package/src/memory/embedder.ts

@@ -0,0 +1,15 @@
+/**
+ * Thin glue turning an embedding model into the `(text) => Promise<number[]>`
+ * shape `episodic-store.ts`'s `storeEpisodicSummary` expects. Same "not
+ * worth mocking deeply" reasoning as `session/summarizer.ts` — one line
+ * of glue around the AI SDK's `embed`, no dedicated test file.
+ */
+import { embed, type EmbeddingModel } from "ai";
+
+/** Returns a function that embeds a string using `model`. */
+export function createEmbedder(model: EmbeddingModel): (text: string) => Promise<number[]> {
+  return async (text) => {
+    const { embedding } = await embed({ model, value: text });
+    return embedding;
+  };
+}