context-vault 3.17.0 → 3.19.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "context-vault",
-  "version": "3.17.0",
+  "version": "3.19.0",
   "type": "module",
   "description": "Persistent memory for AI agents — saves and searches knowledge across sessions",
   "bin": {
@@ -67,7 +67,7 @@
     "@context-vault/core"
   ],
   "dependencies": {
-    "@context-vault/core": "^3.17.0",
+    "@context-vault/core": "^3.19.0",
     "@modelcontextprotocol/sdk": "^1.26.0",
     "adm-zip": "^0.5.16",
     "sqlite-vec": "^0.1.0"

package/scripts/validate-epipe-shutdown.mjs

@@ -0,0 +1,183 @@
+#!/usr/bin/env node
+// Regression test for mcp-epipe-graceful-shutdown.
+//
+// Spawns the built server, completes the MCP handshake, then simulates a
+// client disconnect by destroying the parent's read of child.stdout and
+// closing child.stdin. Asserts:
+//   (a) the child exits cleanly (code 0) within 3s
+//   (b) vault.db-wal is 0 bytes (WAL was checkpointed)
+//   (c) error.log has NO `write EPIPE` uncaughtException entry for this run
+
+import { spawn } from 'node:child_process';
+import {
+  existsSync,
+  mkdtempSync,
+  readFileSync,
+  rmSync,
+  statSync,
+} from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join, resolve, dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const SERVER_JS = resolve(__dirname, '..', 'dist', 'server.js');
+const EXIT_DEADLINE_MS = 3000;
+const READY_TIMEOUT_MS = 10_000;
+const RESPONSE_TIMEOUT_MS = 5000;
+
+const tmpRoot = mkdtempSync(join(tmpdir(), 'cv-epipe-'));
+const dataDir = join(tmpRoot, 'data');
+const vaultDir = join(tmpRoot, 'vault');
+
+let child = null;
+let pass = false;
+let failMsg = '';
+let stdoutBuf = '';
+let stderrBuf = '';
+
+function cleanup() {
+  if (child && child.exitCode === null) {
+    try { child.kill('SIGKILL'); } catch {}
+  }
+  try { rmSync(tmpRoot, { recursive: true, force: true }); } catch {}
+}
+
+async function waitFor(predicate, timeoutMs, label) {
+  const start = Date.now();
+  while (Date.now() - start < timeoutMs) {
+    if (predicate()) return;
+    await new Promise((r) => setTimeout(r, 25));
+  }
+  throw new Error(`timeout: ${label}`);
+}
+
+function extractJsonResponses(buffer) {
+  const results = [];
+  for (const line of buffer.split('\n')) {
+    const trimmed = line.trim();
+    if (!trimmed) continue;
+    try { results.push(JSON.parse(trimmed)); } catch {}
+  }
+  return results;
+}
+
+try {
+  if (!existsSync(SERVER_JS)) {
+    throw new Error(`server bundle missing: ${SERVER_JS} (run npm run build first)`);
+  }
+
+  child = spawn(process.execPath, [SERVER_JS], {
+    env: {
+      ...process.env,
+      CONTEXT_VAULT_DATA_DIR: dataDir,
+      CONTEXT_VAULT_VAULT_DIR: vaultDir,
+      CONTEXT_VAULT_DIR: vaultDir,
+      CONTEXT_VAULT_TELEMETRY: '0',
+    },
+    stdio: ['pipe', 'pipe', 'pipe'],
+  });
+
+  child.stdout.on('data', (chunk) => { stdoutBuf += chunk.toString('utf-8'); });
+  child.stderr.on('data', (chunk) => { stderrBuf += chunk.toString('utf-8'); });
+
+  const exitPromise = new Promise((resolveExit) => {
+    child.once('exit', (code, signal) => resolveExit({ code, signal }));
+  });
+
+  // Wait for the server to finish startup logs before sending JSON-RPC.
+  await waitFor(() => /Database:/.test(stderrBuf), READY_TIMEOUT_MS, 'server startup');
+
+  function send(msg) {
+    child.stdin.write(JSON.stringify(msg) + '\n');
+  }
+
+  send({
+    jsonrpc: '2.0',
+    id: 1,
+    method: 'initialize',
+    params: {
+      protocolVersion: '2024-11-05',
+      capabilities: {},
+      clientInfo: { name: 'epipe-validator', version: '1.0.0' },
+    },
+  });
+
+  await waitFor(
+    () => extractJsonResponses(stdoutBuf).some((r) => r.id === 1),
+    RESPONSE_TIMEOUT_MS,
+    'initialize response',
+  );
+
+  send({ jsonrpc: '2.0', method: 'notifications/initialized' });
+
+  send({ jsonrpc: '2.0', id: 2, method: 'tools/list', params: {} });
+  await waitFor(
+    () => extractJsonResponses(stdoutBuf).some((r) => r.id === 2),
+    RESPONSE_TIMEOUT_MS,
+    'tools/list response',
+  );
+
+  // Simulate client disconnect: destroy the parent's read of child.stdout so
+  // the next write from the server raises EPIPE, then send another request
+  // to force the server to attempt a write.
+  child.stdout.destroy();
+  send({ jsonrpc: '2.0', id: 3, method: 'tools/list', params: {} });
+  // Close stdin last so the server detects the full disconnect.
+  try { child.stdin.end(); } catch {}
+
+  const result = await Promise.race([
+    exitPromise,
+    new Promise((resolveTimeout) =>
+      setTimeout(() => resolveTimeout({ code: null, signal: 'deadline' }), EXIT_DEADLINE_MS),
+    ),
+  ]);
+
+  if (result.signal === 'deadline') {
+    throw new Error(`server did not exit within ${EXIT_DEADLINE_MS}ms after disconnect`);
+  }
+  if (result.code !== 0) {
+    throw new Error(`unclean exit: code=${result.code} signal=${result.signal}`);
+  }
+
+  const walPath = join(dataDir, 'vault.db-wal');
+  if (existsSync(walPath)) {
+    const size = statSync(walPath).size;
+    if (size !== 0) {
+      throw new Error(`WAL not checkpointed: ${walPath} is ${size} bytes`);
+    }
+  }
+
+  const errorLogPath = join(dataDir, 'error.log');
+  if (existsSync(errorLogPath)) {
+    const content = readFileSync(errorLogPath, 'utf-8');
+    for (const line of content.split('\n')) {
+      const trimmed = line.trim();
+      if (!trimmed) continue;
+      let entry;
+      try { entry = JSON.parse(trimmed); } catch { continue; }
+      const isUncaught = entry.error_type === 'uncaughtException';
+      const mentionsEpipe =
+        typeof entry.message === 'string' && /\bEPIPE\b/i.test(entry.message);
+      if (isUncaught && mentionsEpipe) {
+        throw new Error(`error.log contains write EPIPE uncaughtException: ${trimmed}`);
+      }
+    }
+  }
+
+  pass = true;
+  console.log('PASS: EPIPE graceful shutdown (exit 0, WAL checkpointed, no uncaught EPIPE)');
+} catch (err) {
+  failMsg = err && err.message ? err.message : String(err);
+  console.error(`FAIL: ${failMsg}`);
+  if (stderrBuf) {
+    const tail = stderrBuf.split('\n').slice(-15).join('\n');
+    if (tail.trim()) {
+      console.error('--- server stderr tail ---');
+      console.error(tail);
+    }
+  }
+} finally {
+  cleanup();
+  process.exit(pass ? 0 : 1);
+}

package/scripts/validate-sqlite-busy-retry.mjs

@@ -0,0 +1,243 @@
+#!/usr/bin/env node
+// Regression test for mcp-sqlite-busy-retry.
+//
+// Spawns the built MCP server, completes the MCP handshake, then simulates
+// a competing writer by holding a BEGIN IMMEDIATE transaction on the same
+// vault.db for ~200ms. During that window, fires a save_context tool call
+// and asserts the server retries through SQLITE_BUSY and succeeds — not
+// returning INTERNAL_ERROR.
+//
+// Pass criteria:
+//   (a) Single scenario: tool call succeeds while lock is held (returns a
+//       JSON-RPC result with no isError flag).
+//   (b) Stability: 5 independent scenarios must succeed in at least 4.
+//
+// Exit 0 on pass, non-zero on fail.
+
+import { spawn } from 'node:child_process';
+import { DatabaseSync } from 'node:sqlite';
+import {
+  existsSync,
+  mkdtempSync,
+  rmSync,
+  mkdirSync,
+} from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join, resolve, dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const SERVER_JS = resolve(__dirname, '..', 'dist', 'server.js');
+
+const READY_TIMEOUT_MS = 15_000;
+const RESPONSE_TIMEOUT_MS = 15_000;
+const LOCK_HOLD_MS = 200;
+const SCENARIOS = 5;
+const MIN_PASSES = 4;
+
+if (!existsSync(SERVER_JS)) {
+  console.error(`FAIL: server bundle missing at ${SERVER_JS}. Run: cd mcp && npm run build`);
+  process.exit(1);
+}
+
+function extractJsonResponses(buffer) {
+  const results = [];
+  for (const line of buffer.split('\n')) {
+    const trimmed = line.trim();
+    if (!trimmed) continue;
+    try { results.push(JSON.parse(trimmed)); } catch {}
+  }
+  return results;
+}
+
+async function waitFor(predicate, timeoutMs, label) {
+  const start = Date.now();
+  while (Date.now() - start < timeoutMs) {
+    if (predicate()) return;
+    await new Promise((r) => setTimeout(r, 25));
+  }
+  throw new Error(`timeout: ${label}`);
+}
+
+async function runScenario(scenarioId) {
+  const tmpRoot = mkdtempSync(join(tmpdir(), `cv-busy-${scenarioId}-`));
+  const dataDir = join(tmpRoot, 'data');
+  const vaultDir = join(tmpRoot, 'vault');
+  mkdirSync(dataDir, { recursive: true });
+  mkdirSync(vaultDir, { recursive: true });
+
+  let child = null;
+  let stdoutBuf = '';
+  let stderrBuf = '';
+  let holder = null;
+  let holderTxOpen = false;
+
+  const cleanup = () => {
+    if (holder) {
+      try {
+        if (holderTxOpen) {
+          try { holder.exec('ROLLBACK'); } catch {}
+        }
+        holder.close();
+      } catch {}
+    }
+    if (child && child.exitCode === null) {
+      try { child.kill('SIGKILL'); } catch {}
+    }
+    try { rmSync(tmpRoot, { recursive: true, force: true }); } catch {}
+  };
+
+  try {
+    child = spawn(process.execPath, [SERVER_JS], {
+      env: {
+        ...process.env,
+        CONTEXT_VAULT_DATA_DIR: dataDir,
+        CONTEXT_VAULT_VAULT_DIR: vaultDir,
+        CONTEXT_VAULT_DIR: vaultDir,
+        CONTEXT_VAULT_TELEMETRY: '0',
+      },
+      stdio: ['pipe', 'pipe', 'pipe'],
+    });
+
+    child.stdout.on('data', (chunk) => { stdoutBuf += chunk.toString('utf-8'); });
+    child.stderr.on('data', (chunk) => { stderrBuf += chunk.toString('utf-8'); });
+
+    // Wait for the server to finish startup (DB initialized)
+    await waitFor(() => /Database:/.test(stderrBuf), READY_TIMEOUT_MS, 'server startup');
+
+    const send = (msg) => { child.stdin.write(JSON.stringify(msg) + '\n'); };
+
+    send({
+      jsonrpc: '2.0', id: 1, method: 'initialize',
+      params: { protocolVersion: '2024-11-05', capabilities: {}, clientInfo: { name: 'busy-validator', version: '1.0' } },
+    });
+    await waitFor(() => extractJsonResponses(stdoutBuf).some((r) => r.id === 1),
+      RESPONSE_TIMEOUT_MS, 'initialize response');
+
+    send({ jsonrpc: '2.0', method: 'notifications/initialized' });
+
+    // Server has opened vault.db. Now we open a second connection to the same
+    // file and grab an immediate write lock. WAL keeps readers unblocked but
+    // a BEGIN IMMEDIATE holds the reserved lock — any writer (including the
+    // server) will see SQLITE_BUSY until we release.
+    const dbPath = join(dataDir, 'vault.db');
+    if (!existsSync(dbPath)) throw new Error(`vault.db not created at ${dbPath}`);
+
+    holder = new DatabaseSync(dbPath);
+    holder.exec('PRAGMA journal_mode = WAL');
+    // Do NOT set a long busy_timeout on the holder — we want it to release
+    // on demand, not wait.
+    holder.exec('BEGIN IMMEDIATE');
+    holderTxOpen = true;
+
+    // Schedule lock release after LOCK_HOLD_MS. The server's retry wrapper
+    // (50/150/500ms backoff) plus the PRAGMA busy_timeout=5000ms both give
+    // it plenty of time to retry successfully once we release.
+    const releaseTimer = setTimeout(() => {
+      try {
+        if (holderTxOpen) {
+          holder.exec('COMMIT');
+          holderTxOpen = false;
+        }
+      } catch (e) {
+        // If commit fails because the writer won a race, that's fine — our
+        // purpose (holding for LOCK_HOLD_MS) is already done.
+      }
+    }, LOCK_HOLD_MS);
+
+    const callId = 100 + scenarioId;
+    send({
+      jsonrpc: '2.0', id: callId, method: 'tools/call',
+      params: {
+        name: 'save_context',
+        arguments: {
+          kind: 'event',
+          body: `busy-retry validation scenario ${scenarioId} at ${new Date().toISOString()}`,
+          tags: ['bucket:validation', 'busy-retry-test'],
+        },
+      },
+    });
+
+    await waitFor(() => extractJsonResponses(stdoutBuf).some((r) => r.id === callId),
+      RESPONSE_TIMEOUT_MS, `tools/call response (scenario ${scenarioId})`);
+    clearTimeout(releaseTimer);
+
+    // Ensure lock is released before asserting (in case timer didn't fire)
+    if (holderTxOpen) {
+      try { holder.exec('COMMIT'); holderTxOpen = false; } catch {}
+    }
+
+    const responses = extractJsonResponses(stdoutBuf).filter((r) => r.id === callId);
+    if (responses.length === 0) throw new Error('no response received');
+    const response = responses[0];
+
+    if (response.error) {
+      throw new Error(`JSON-RPC error: ${JSON.stringify(response.error)}`);
+    }
+
+    const result = response.result;
+    if (!result) throw new Error(`empty result: ${JSON.stringify(response)}`);
+
+    // The MCP tool-result shape: { content: [...], isError?: boolean }.
+    // INTERNAL_ERROR surfaces as isError: true plus error text.
+    if (result.isError) {
+      const text = (result.content || []).map((c) => c.text || '').join(' ');
+      throw new Error(`tool returned isError: ${text.slice(0, 200)}`);
+    }
+
+    return { scenarioId, ok: true };
+  } catch (err) {
+    const msg = err && err.message ? err.message : String(err);
+    const stderrTail = stderrBuf.split('\n').slice(-10).join('\n');
+    return { scenarioId, ok: false, error: msg, stderrTail };
+  } finally {
+    cleanup();
+  }
+}
+
+async function main() {
+  console.log(`[busy-retry] Running ${SCENARIOS} scenarios (each holds DB lock for ${LOCK_HOLD_MS}ms during save_context)`);
+
+  const results = [];
+  // Run sequentially — each scenario has its own vault dir, but running in
+  // parallel would conflate lock contention signals across scenarios.
+  for (let i = 1; i <= SCENARIOS; i++) {
+    const r = await runScenario(i);
+    results.push(r);
+    if (r.ok) {
+      console.log(`  scenario ${i}: PASS`);
+    } else {
+      console.log(`  scenario ${i}: FAIL — ${r.error}`);
+      if (r.stderrTail && r.stderrTail.trim()) {
+        console.log(`    stderr tail:\n${r.stderrTail.split('\n').map(l => '      ' + l).join('\n')}`);
+      }
+    }
+  }
+
+  const passed = results.filter((r) => r.ok).length;
+  const firstPassed = results[0].ok;
+
+  console.log(`\n[busy-retry] ${passed}/${SCENARIOS} scenarios passed`);
+
+  // Assertion (a): first scenario must pass (single-shot proof that a held
+  // lock does not surface as INTERNAL_ERROR).
+  if (!firstPassed) {
+    console.error('FAIL: scenario 1 did not succeed — retry path is broken');
+    process.exit(1);
+  }
+
+  // Assertion (b): stability — at least MIN_PASSES of SCENARIOS must succeed.
+  if (passed < MIN_PASSES) {
+    console.error(`FAIL: only ${passed}/${SCENARIOS} succeeded (required ${MIN_PASSES})`);
+    process.exit(1);
+  }
+
+  console.log('PASS: SQLITE_BUSY retry wrapper absorbs transient contention');
+  process.exit(0);
+}
+
+main().catch((err) => {
+  console.error('FAIL: unhandled error');
+  console.error(err);
+  process.exit(1);
+});

package/src/register-tools.ts

@@ -18,7 +18,6 @@ import * as ingestUrl from './tools/ingest-url.js';
 import * as contextStatus from './tools/context-status.js';
 import * as clearContext from './tools/clear-context.js';
 import * as createSnapshot from './tools/create-snapshot.js';
-import * as sessionStart from './tools/session-start.js';
 import * as listBuckets from './tools/list-buckets.js';
 import * as ingestProject from './tools/ingest-project.js';
 import * as sessionEnd from './tools/session-end.js';
@@ -35,7 +34,6 @@ const toolModules = [
   contextStatus,
   clearContext,
   createSnapshot,
-  sessionStart,
   sessionEnd,
   listBuckets,
   recall,

package/src/server.ts

@@ -11,6 +11,50 @@ import { fileURLToPath } from 'node:url';
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const pkg = JSON.parse(readFileSync(join(__dirname, '..', 'package.json'), 'utf-8'));
 
+// Module-level shutdown coordination so pipe/uncaught handlers (below main())
+// can route through the graceful shutdown wired up inside main().
+let shutdownHandler: ((signal: string) => void) | null = null;
+let shutdownInProgress = false;
+
+function isPipeError(err: unknown): boolean {
+  if (!err || typeof err !== 'object') return false;
+  const e = err as { code?: string; message?: string };
+  if (e.code === 'EPIPE' || e.code === 'ERR_STREAM_DESTROYED' || e.code === 'ERR_STREAM_WRITE_AFTER_END') return true;
+  return typeof e.message === 'string' && /\bEPIPE\b/.test(e.message);
+}
+
+function handlePipeDisconnect(source: string): void {
+  if (shutdownInProgress) return;
+  if (shutdownHandler) {
+    try {
+      shutdownHandler(`EPIPE:${source}`);
+    } catch {
+      process.exit(0);
+    }
+  } else {
+    // Shutdown not wired yet (startup phase). Exit clean so WAL isn't dirtied.
+    process.exit(0);
+  }
+}
+
+// Catch broken-pipe writes to stdout/stderr so they route through graceful
+// shutdown instead of bubbling up as an `uncaughtException` that skips the
+// WAL checkpoint. Node raises EPIPE (not SIGPIPE) on pipe writes with no reader.
+process.stdout.on('error', (err) => {
+  if (isPipeError(err)) {
+    handlePipeDisconnect('stdout');
+  } else {
+    throw err;
+  }
+});
+process.stderr.on('error', (err) => {
+  if (isPipeError(err)) {
+    handlePipeDisconnect('stderr');
+  } else {
+    throw err;
+  }
+});
+
 import { resolveConfig } from '@context-vault/core/config';
 import type { LocalCtx } from './types.js';
 import { appendErrorLog } from './error-log.js';
@@ -206,7 +250,28 @@ async function main(): Promise<void> {
     }
 
     function shutdown(signal: string): void {
-      console.error(`[context-vault] Received ${signal}, shutting down...`);
+      // Idempotent: EPIPE from stdout error + uncaughtException can both fire
+      // during a single client disconnect. Second call becomes a no-op.
+      if (shutdownInProgress) return;
+      shutdownInProgress = true;
+
+      const isEpipe = signal.startsWith('EPIPE');
+      if (isEpipe) {
+        // Log a clean shutdown entry in place of the EPIPE uncaughtException
+        // that would otherwise have fired. Keeps the audit log readable.
+        appendErrorLog(config!.dataDir, {
+          timestamp: new Date().toISOString(),
+          error_type: 'EPIPE_shutdown',
+          message: `client pipe closed (${signal}); graceful shutdown`,
+          node_version: process.version,
+          platform: process.platform,
+          arch: process.arch,
+          cv_version: pkg.version,
+        });
+        console.error(`[context-vault] EPIPE shutdown: client disconnected (${signal})`);
+      } else {
+        console.error(`[context-vault] Received ${signal}, shutting down...`);
+      }
 
       if (ctx.activeOps.count > 0) {
         console.error(
@@ -232,6 +297,9 @@ async function main(): Promise<void> {
     process.on('SIGINT', () => shutdown('SIGINT'));
     process.on('SIGTERM', () => shutdown('SIGTERM'));
 
+    // Expose shutdown to module-level handlers (stdout error, uncaughtException).
+    shutdownHandler = shutdown;
+
     // RSS watchdog: kill the process if memory usage exceeds the cap.
     // Prevents runaway embedding/reindex operations from frying user systems.
     const MAX_RSS_BYTES = parseInt(process.env.CONTEXT_VAULT_MAX_RSS_MB || '1024', 10) * 1024 * 1024;
@@ -310,6 +378,13 @@ async function main(): Promise<void> {
 }
 
 process.on('uncaughtException', (err) => {
+  // EPIPE from a dead client pipe is not a crash; it's a disconnect.
+  // Route through graceful shutdown so the WAL is checkpointed.
+  if (isPipeError(err)) {
+    handlePipeDisconnect('uncaught');
+    return;
+  }
+
   const dataDir = join(homedir(), '.context-mcp');
   const logEntry = {
     timestamp: new Date().toISOString(),