@0xmaxma/claude-gateway 1.6.8 → 1.7.0

package/mcp/tools/memory/archive-reader.ts

@@ -0,0 +1,253 @@
+/**
+ * Read-only reader for the per-agent knowledge archive (planning-64 K1).
+ *
+ * The archive `kb.sqlite` is BUILT by src/ under Node via `node:sqlite`
+ * (src/agent/knowledge). These MCP tools run under **Bun**, which has no
+ * `node:sqlite`, so the read path here uses **`bun:sqlite`** instead. Same
+ * on-disk SQLite/FTS5 file — verified that node:sqlite writes and bun:sqlite
+ * reads it back via `bm25()`. This is a deliberate, read-only mirror of the
+ * query half of `src/agent/knowledge/archive-db.ts`; keep the two in sync.
+ */
+
+import { Database } from 'bun:sqlite';
+import * as fs from 'fs';
+import * as path from 'path';
+import { recordRetrievalHits } from './retrieval-recorder';
+
+export interface SearchHit {
+  path: string;
+  startLine: number;
+  endLine: number;
+  score: number; // bm25 (lower = more relevant); surfaced for transparency
+  snippet: string;
+  originClass: string | null;
+  importance: number | null;
+  /** Stable archive-entry identity (planning-66); null outside any entry block. */
+  entryHash: string | null;
+}
+
+/** Archive DB path for an agent: sibling of history.db at agents/<id>/kb.sqlite. */
+export function archiveDbPath(workspaceDir: string): string {
+  return path.join(path.dirname(workspaceDir), 'kb.sqlite');
+}
+
+/**
+ * Shared, cross-agent vault DB path from the gateway-provided env
+ * (`GATEWAY_SHARED_KB_DIR` = the resolved <root>/<project> dir), or null when
+ * shared KB is disabled / not configured (planning-64 K3).
+ */
+export function sharedDbPathFromEnv(): string | null {
+  const dir = process.env.GATEWAY_SHARED_KB_DIR;
+  if (!dir || !dir.trim()) return null;
+  return path.join(dir, 'kb.sqlite');
+}
+
+/**
+ * Merge hit lists (per-agent + shared), keeping the best `limit`. bm25 scores are
+ * corpus-relative — the value depends on each index's term/document statistics —
+ * so scores from two independent FTS indexes are NOT on a comparable scale and a
+ * global sort can spuriously rank one corpus over the other. Each list is already
+ * sorted best-first within its own corpus, so interleave them round-robin for a
+ * fair blend instead of a cross-corpus numeric sort.
+ */
+export function mergeHits<T extends SearchHit>(a: T[], b: T[], limit: number): T[] {
+  const out: T[] = [];
+  const max = Math.max(a.length, b.length);
+  for (let i = 0; i < max && out.length < limit; i++) {
+    if (i < a.length) out.push(a[i]);
+    if (out.length < limit && i < b.length) out.push(b[i]);
+  }
+  return out;
+}
+
+/** Build an FTS5 MATCH string: quote each token so punctuation can't inject syntax. */
+function toFtsMatch(query: string): string {
+  return query
+    .split(/[^\p{L}\p{N}_]+/u)
+    .filter(Boolean)
+    .map((t) => `"${t.replace(/"/g, '""')}"`)
+    .join(' ');
+}
+
+/**
+ * Keyword search over the archive (FTS5 BM25). Returns [] when the index does
+ * not exist yet or the query has no usable tokens. Read-only; never creates a DB.
+ *
+ * When `recordRetrievals` is set (planning-66), each returned entry's stable
+ * `entry_hash` is appended to `kb_retrieval_log` through a SEPARATE, dedicated
+ * writable handle AFTER the results are built — append-only and fire-and-forget,
+ * so the read path never read-modify-writes chunk rows and never blocks/locks the
+ * search response. The Node side folds this log into per-entry recency at GC time.
+ */
+/** True when kb_chunks carries the planning-66 `entry_hash` column. */
+function hasEntryHashColumn(db: Database): boolean {
+  const cols = db.query(`PRAGMA table_info(kb_chunks)`).all() as Array<{ name: string }>;
+  return cols.some((c) => c.name === 'entry_hash');
+}
+
+/**
+ * Idempotent, best-effort heal for a pre-#346 archive DB. The `entry_hash`
+ * column is only added by the writer (`ArchiveDb` constructor migration); a
+ * `kb.sqlite` that has not been reindexed since #346 still lacks it, so the
+ * read-only search query below crashes with `no such column: entry_hash`. Add
+ * the column in place via a short writable handle (mirroring the recall-logging
+ * handle further down). A read-only filesystem simply skips — the column-aware
+ * query then falls back to a NULL `entryHash` so search still returns hits.
+ */
+function ensureEntryHashColumn(dbPath: string): void {
+  try {
+    const w = new Database(dbPath);
+    try {
+      w.exec('PRAGMA busy_timeout=2000');
+      if (!hasEntryHashColumn(w)) {
+        w.exec(`ALTER TABLE kb_chunks ADD COLUMN entry_hash TEXT`);
+        w.exec(`CREATE INDEX IF NOT EXISTS idx_kb_chunks_entry_hash ON kb_chunks(entry_hash)`);
+      }
+    } finally {
+      w.close();
+    }
+  } catch {
+    /* read-only / locked DB — the column-aware query below degrades gracefully */
+  }
+}
+
+export function searchArchive(
+  dbPath: string,
+  query: string,
+  maxResults: number,
+  opts?: { recordRetrievals?: boolean; now?: number },
+): SearchHit[] {
+  if (!fs.existsSync(dbPath)) return [];
+  const match = toFtsMatch(query);
+  if (!match) return [];
+
+  // Heal pre-#346 DBs in place so the query never crashes on a missing column.
+  ensureEntryHashColumn(dbPath);
+
+  const db = new Database(dbPath, { readonly: true });
+  // If the heal above could not run (read-only FS), select a NULL entry_hash
+  // rather than referencing a column that does not exist.
+  const entryHashSelect = hasEntryHashColumn(db) ? 'c.entry_hash' : 'NULL';
+  let hits: SearchHit[];
+  try {
+    const rows = db
+      .query(
+        `SELECT c.id AS id, c.path AS path, c.start_line AS startLine, c.end_line AS endLine,
+                c.text AS text, ${entryHashSelect} AS entryHash, bm25(kb_chunks_fts) AS score,
+                p.origin_class AS originClass, r.importance AS importance
+         FROM kb_chunks_fts f
+         JOIN kb_chunks c ON c.rowid_id = f.rowid
+         LEFT JOIN kb_chunk_provenance p ON p.chunk_id = c.id
+         LEFT JOIN kb_chunk_recall r ON r.chunk_id = c.id
+         WHERE kb_chunks_fts MATCH ?
+         ORDER BY bm25(kb_chunks_fts)
+         LIMIT ?`,
+      )
+      .all(match, maxResults) as Array<{
+      path: string;
+      startLine: number;
+      endLine: number;
+      text: string;
+      entryHash: string | null;
+      score: number;
+      originClass: string | null;
+      importance: number | null;
+    }>;
+    hits = rows.map((r) => ({
+      path: r.path,
+      startLine: r.startLine,
+      endLine: r.endLine,
+      score: r.score,
+      snippet: r.text.length > 500 ? `${r.text.slice(0, 500)}…` : r.text,
+      originClass: r.originClass,
+      importance: r.importance,
+      entryHash: r.entryHash,
+    }));
+  } finally {
+    db.close();
+  }
+
+  // Recall counter (planning-66): fire-and-forget, append-only, best-effort. A
+  // dedicated writable handle (WAL + busy_timeout so it never fails a concurrent
+  // reindex) logs the retrieved entries AFTER the read handle is closed. Any error
+  // is swallowed inside recordRetrievalHits — telemetry never fails a search.
+  if (opts?.recordRetrievals && hits.length > 0) {
+    try {
+      const w = new Database(dbPath);
+      try {
+        w.exec('PRAGMA busy_timeout=2000');
+        recordRetrievalHits(
+          { run: (sql, params = []) => void w.query(sql).run(...(params as never[])) },
+          hits.map((h) => h.entryHash),
+          opts.now ?? Date.now(),
+        );
+      } finally {
+        w.close();
+      }
+    } catch {
+      /* best-effort telemetry — never affect the search response */
+    }
+  }
+  return hits;
+}
+
+/** A memory-scoped path: MEMORY.md, USER.md, or memory/**.md — no traversal. */
+export function isMemoryScopedPath(relPath: string): boolean {
+  const p = relPath.replace(/\\/g, '/');
+  if (p.includes('..') || path.isAbsolute(p)) return false;
+  if (p === 'MEMORY.md' || p === 'USER.md') return true;
+  return p.startsWith('memory/') && p.endsWith('.md');
+}
+
+export interface Excerpt {
+  path: string;
+  from: number;
+  to: number;
+  text: string;
+  totalLines: number;
+  truncated: boolean;
+}
+
+/**
+ * Bounded exact excerpt of a memory file by line range (1-indexed). Reads the
+ * file directly (it is the source of truth). Returns null for an out-of-scope
+ * path or a missing file.
+ */
+export function getExcerpt(
+  workspaceDir: string,
+  relPath: string,
+  from = 1,
+  lines = 200,
+): Excerpt | null {
+  // Validate and read the SAME normalized path so the check can't diverge from
+  // what is actually opened.
+  const rel = relPath.replace(/\\/g, '/');
+  if (!isMemoryScopedPath(rel)) return null;
+  const abs = path.join(workspaceDir, rel);
+  if (!fs.existsSync(abs)) return null;
+  // Containment against symlink escape: a memory/x.md symlink pointing outside
+  // the workspace passes the string check but must not be readable through the
+  // tool. Resolve real paths and require the target stays under the workspace.
+  try {
+    const realBase = fs.realpathSync(workspaceDir);
+    const realTarget = fs.realpathSync(abs);
+    if (realTarget !== realBase && !realTarget.startsWith(realBase + path.sep)) return null;
+  } catch {
+    return null; // realpath failed (broken symlink, race) → refuse
+  }
+
+  const all = fs.readFileSync(abs, 'utf8').split('\n');
+  const total = all.length;
+  const start = Math.max(1, Math.floor(from));
+  const count = Math.max(1, Math.floor(lines));
+  const end = Math.min(total, start + count - 1);
+  const slice = start > total ? [] : all.slice(start - 1, end);
+  return {
+    path: relPath,
+    from: start,
+    to: start > total ? start : end,
+    text: slice.join('\n'),
+    totalLines: total,
+    truncated: end < total,
+  };
+}

package/mcp/tools/memory/module.ts

@@ -0,0 +1,152 @@
+/**
+ * Memory tool module (planning-64 K1) — implements ToolModule.
+ *
+ * Read-only, on-demand retrieval over the per-agent knowledge archive
+ * (`kb.sqlite`, built by src/ under Node). Two lanes of memory: the small core
+ * (MEMORY.md/USER.md) is injected every turn; everything else lives in the
+ * searchable archive and is reached here via `memory_search` / `memory_get` —
+ * so the bulk never has to sit in the prompt. Runs under Bun; reads via
+ * `bun:sqlite` (see archive-reader.ts).
+ */
+
+import type { ToolModule, McpToolDefinition, McpToolResult, ToolVisibility } from '../../types';
+import { searchArchive, getExcerpt, archiveDbPath, sharedDbPathFromEnv, mergeHits } from './archive-reader';
+
+/** Corpora the tool can serve. */
+const SUPPORTED_CORPORA = new Set(['memory', 'shared', 'all']);
+
+export class MemoryModule implements ToolModule {
+  id = 'memory';
+  toolVisibility: ToolVisibility = 'all-configured';
+
+  isEnabled(): boolean {
+    // Available whenever the agent's workspace dir is known (same gate as skills).
+    return Boolean(process.env.GATEWAY_WORKSPACE_DIR);
+  }
+
+  getTools(): McpToolDefinition[] {
+    return [
+      {
+        name: 'memory_search',
+        description:
+          'Search your long-term memory archive (past notes under memory/, plus MEMORY.md/USER.md) by keyword and get the most relevant snippets with their file + line range. Use this to recall details that are NOT in the memory currently loaded into your context — the archive holds far more than fits in the prompt. Read-only.',
+        inputSchema: {
+          type: 'object',
+          properties: {
+            query: { type: 'string', description: 'Keywords to search for.' },
+            maxResults: {
+              type: 'number',
+              description: 'Max snippets to return (default 6, max 20).',
+            },
+            corpus: {
+              type: 'string',
+              enum: ['memory', 'shared', 'all'],
+              description:
+                'Which corpus to search: "memory" (this agent, default), "shared" (the cross-agent shared KB), or "all" (both).',
+            },
+          },
+          required: ['query'],
+          additionalProperties: false,
+        },
+      },
+      {
+        name: 'memory_get',
+        description:
+          'Read an exact line range from one of your memory files (a path returned by memory_search, e.g. "memory/foo.md", or "MEMORY.md"/"USER.md"). Use after memory_search to pull the full context around a snippet. Read-only.',
+        inputSchema: {
+          type: 'object',
+          properties: {
+            path: { type: 'string', description: 'Memory-scoped path (memory/*.md, MEMORY.md, USER.md).' },
+            from: { type: 'number', description: '1-indexed start line (default 1).' },
+            lines: { type: 'number', description: 'Number of lines to return (default 200).' },
+          },
+          required: ['path'],
+          additionalProperties: false,
+        },
+      },
+    ];
+  }
+
+  async handleTool(name: string, args: Record<string, unknown>): Promise<McpToolResult> {
+    const workspaceDir = process.env.GATEWAY_WORKSPACE_DIR;
+    if (!workspaceDir) {
+      return this.err('Memory archive unavailable: no workspace directory configured.');
+    }
+
+    try {
+      switch (name) {
+        case 'memory_search': {
+          const query = typeof args.query === 'string' ? args.query.trim() : '';
+          if (!query) return this.err('memory_search requires a non-empty "query".');
+
+          // Fail closed on an unsupported corpus rather than silently widening scope.
+          const corpus = typeof args.corpus === 'string' ? args.corpus : 'memory';
+          if (!SUPPORTED_CORPORA.has(corpus)) {
+            return this.json({
+              results: [],
+              unavailable: true,
+              warning: `corpus "${corpus}" is not available yet (shared/all arrive in a later phase). Use "memory".`,
+            });
+          }
+
+          const rawMax = Number.isFinite(args.maxResults as number) ? (args.maxResults as number) : 6;
+          const maxResults = Math.max(1, Math.min(20, Math.floor(rawMax)));
+
+          const personalDb = archiveDbPath(workspaceDir);
+          const sharedDb = sharedDbPathFromEnv();
+          // Recall counter (planning-66): record retrievals ONLY against the
+          // per-agent archive (the GC tier). Gated by the gateway via env, mirroring
+          // GATEWAY_SHARED_KB_DIR — off unless dreaming.staleness.recordRetrievals.
+          const rec = { recordRetrievals: process.env.GATEWAY_RECORD_RETRIEVALS === '1' };
+          let results;
+          if (corpus === 'memory') {
+            results = searchArchive(personalDb, query, maxResults, rec).map((h) => ({ ...h, corpus: 'memory' }));
+          } else if (corpus === 'shared') {
+            if (!sharedDb) return this.json({ results: [], unavailable: true, warning: 'shared KB is not enabled.' });
+            results = searchArchive(sharedDb, query, maxResults).map((h) => ({ ...h, corpus: 'shared' }));
+          } else {
+            // "all": merge personal + shared, keep the best by bm25.
+            type Tagged = ReturnType<typeof searchArchive>[number] & { corpus: string };
+            const mine: Tagged[] = searchArchive(personalDb, query, maxResults, rec).map((h) => ({ ...h, corpus: 'memory' }));
+            const shared: Tagged[] = sharedDb
+              ? searchArchive(sharedDb, query, maxResults).map((h) => ({ ...h, corpus: 'shared' }))
+              : [];
+            results = mergeHits(mine, shared, maxResults);
+          }
+
+          if (results.length === 0) {
+            return this.json({
+              results: [],
+              note: 'No matches (or the archive has not been indexed yet).',
+            });
+          }
+          return this.json({ results, corpus });
+        }
+
+        case 'memory_get': {
+          const p = typeof args.path === 'string' ? args.path : '';
+          const from = Number.isFinite(args.from as number) ? (args.from as number) : 1;
+          const lines = Number.isFinite(args.lines as number) ? (args.lines as number) : 200;
+          const excerpt = getExcerpt(workspaceDir, p, from, lines);
+          if (!excerpt) {
+            return this.err(`memory_get: "${p}" is not a readable memory-scoped file (memory/*.md, MEMORY.md, USER.md).`);
+          }
+          return this.json(excerpt);
+        }
+
+        default:
+          return this.err(`Unknown memory tool: ${name}`);
+      }
+    } catch (e) {
+      return this.err(`memory tool error: ${e instanceof Error ? e.message : String(e)}`);
+    }
+  }
+
+  private json(obj: unknown): McpToolResult {
+    return { content: [{ type: 'text', text: JSON.stringify(obj, null, 2) }] };
+  }
+
+  private err(text: string): McpToolResult {
+    return { content: [{ type: 'text', text }], isError: true };
+  }
+}

package/mcp/tools/memory/retrieval-recorder.ts

@@ -0,0 +1,66 @@
+/**
+ * Runtime-agnostic append-only retrieval recorder (planning-66).
+ *
+ * The read path (`archive-reader.ts`) runs under Bun (`bun:sqlite`); the tests and
+ * the src/ aggregator run under Node (`node:sqlite`). Both need to write the SAME
+ * append-only `kb_retrieval_log` table, so the actual SQL lives here behind a tiny
+ * `SqlRunner` seam that each runtime adapts to its own driver. This file imports
+ * NO sqlite driver, so it is safe to import from a Node/jest test (importing
+ * `archive-reader.ts` directly is not — it top-level-imports `bun:sqlite`).
+ *
+ * The recall counter is deliberately APPEND-ONLY and fire-and-forget: the Bun read
+ * path must never read-modify-write chunk rows (that would break the "mcp reads
+ * only / src writes" invariant and risk locking the hot search path). It only ever
+ * INSERTs into a dedicated log table; the Node side aggregates that log into
+ * `kb_entry_lifecycle.last_retrieved`/`retrieval_count` at GC time.
+ */
+
+/** Minimal write surface both `bun:sqlite` and `node:sqlite` can satisfy. */
+export interface SqlRunner {
+  run(sql: string, params?: unknown[]): void;
+}
+
+/** DDL for the append-only retrieval log. Idempotent; safe to call every write. */
+export const RETRIEVAL_LOG_DDL =
+  'CREATE TABLE IF NOT EXISTS kb_retrieval_log (' +
+  'rowid_id INTEGER PRIMARY KEY AUTOINCREMENT, entry_hash TEXT NOT NULL, retrieved_at INTEGER NOT NULL)';
+
+/** Ensure the log table exists (the Bun side may run before any Node index pass). */
+export function ensureRetrievalLog(db: SqlRunner): void {
+  db.run(RETRIEVAL_LOG_DDL);
+}
+
+/**
+ * Append one retrieval event, keyed by the STABLE `entry_hash` (never the line-
+ * based chunk id). Best-effort: a null/empty hash is ignored (a chunk outside any
+ * entry block carries no lifecycle identity). Callers wrap this in try/catch and
+ * never block their response on it.
+ */
+export function appendRetrieval(db: SqlRunner, entryHash: string | null | undefined, retrievedAt: number): void {
+  if (!entryHash) return;
+  ensureRetrievalLog(db);
+  db.run('INSERT INTO kb_retrieval_log (entry_hash, retrieved_at) VALUES (?, ?)', [entryHash, retrievedAt]);
+}
+
+/**
+ * Record a batch of retrieval hits (the entry_hashes of one search's results),
+ * de-duplicated so a single search that returns two chunks of the same entry
+ * counts once. Fire-and-forget; never throws out (errors are swallowed so a
+ * telemetry write can never fail a search response).
+ */
+export function recordRetrievalHits(
+  db: SqlRunner,
+  entryHashes: Array<string | null | undefined>,
+  retrievedAt: number,
+): void {
+  try {
+    const seen = new Set<string>();
+    for (const h of entryHashes) {
+      if (!h || seen.has(h)) continue;
+      seen.add(h);
+      appendRetrieval(db, h, retrievedAt);
+    }
+  } catch {
+    /* best-effort telemetry — never affect the search response */
+  }
+}

package/mcp/tools/shared/share-client.ts

@@ -12,7 +12,26 @@ const REQUEST_TIMEOUT_MS = 15_000;
 
 export type ShareRef = { artifact_id?: string; path?: string };
 
-export type ShareItem = { share_id: string; token: string; url?: string; expires_at: string };
+export type ShareItem = {
+  share_id: string;
+  token: string;
+  url?: string;
+  expires_at: string;
+  /** Present only when this share was minted from an artifact_id ref whose
+   *  generation recorded a provider task id — the hook a resume-capable
+   *  provider needs to continue that generation's session. Duplicated from
+   *  ArtifactRow
+   *  (src/share/share-store.ts) rather than imported — this file must stay
+   *  self-contained (see the module comment above). */
+  task_id?: string;
+  provider?: string;
+  /** The prompt that produced this artifact, when recorded
+   *  (handoff-on-model-switch) — deterministic reuse source for continuing
+   *  an edit across a model switch where session resume isn't possible.
+   *  Independent of task_id: present even when resume isn't. Duplicated from
+   *  ArtifactRow rather than imported — see the module comment above. */
+  prior_prompt?: string;
+};
 
 export type ArtifactItem = { artifact_id: string; artifact_ref: string; index: number; path: string };
 

package/mcp/tools/skills/module.ts

@@ -108,6 +108,15 @@ export class SkillsModule implements ToolModule {
           additionalProperties: false,
         },
       },
+      {
+        name: 'skill_metrics',
+        description: 'Read the skill self-improvement effectiveness rollup for this agent: adoption funnel (auto-skill stickiness), cost-to-complete deltas, recovery-rate trend, enabled/disabled cohort, and the net-token ledger (tokens saved by reuse minus tokens spent reviewing). Read-only.',
+        inputSchema: {
+          type: 'object',
+          properties: {},
+          additionalProperties: false,
+        },
+      },
     ];
   }
 
@@ -152,6 +161,25 @@ export class SkillsModule implements ToolModule {
           const result = await installSkill(params);
           return { content: [{ type: 'text', text: result }] };
         }
+        case 'skill_metrics': {
+          const apiUrl = (process.env.GATEWAY_API_URL || '').replace(/\/+$/, '');
+          const apiKey = process.env.GATEWAY_API_KEY;
+          const agentId = process.env.GATEWAY_AGENT_ID;
+          if (!apiUrl || !apiKey || !agentId) {
+            return {
+              content: [{ type: 'text', text: 'skill_metrics unavailable: gateway API env not configured' }],
+              isError: true,
+            };
+          }
+          const res = await fetch(`${apiUrl}/api/v1/agents/${encodeURIComponent(agentId)}/skill-metrics`, {
+            headers: { Authorization: `Bearer ${apiKey}` },
+          });
+          const body = await res.text();
+          if (!res.ok) {
+            return { content: [{ type: 'text', text: `skill_metrics failed (${res.status}): ${body}` }], isError: true };
+          }
+          return { content: [{ type: 'text', text: body }] };
+        }
         default:
           return { content: [{ type: 'text', text: `unknown tool: ${name}` }], isError: true };
       }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@0xmaxma/claude-gateway",
-  "version": "1.6.8",
+  "version": "1.7.0",
   "description": "Multi-agent gateway for Claude",
   "repository": {
     "type": "git",