agent-working-memory 0.7.17 → 0.8.5

package/src/hooks/sidecar.ts

@@ -18,7 +18,7 @@
 
 import { createServer, type IncomingMessage, type ServerResponse } from 'node:http';
 import { readFileSync, existsSync } from 'node:fs';
-import type { EngramStore } from '../storage/sqlite.js';
+import type { IEngramStore as EngramStore } from '../storage/store.js';
 import type { ConsciousState } from '../types/checkpoint.js';
 import { log, getLogPath } from '../core/logger.js';
 
@@ -211,7 +211,7 @@ export function startSidecar(deps: SidecarDeps): { close: () => void } {
           episodeId: null,
         };
 
-        store.saveCheckpoint(agentId, state);
+        await store.saveCheckpoint(agentId, state);
         log(agentId, `hook:${event}`, `auto-checkpoint files=${state.activeFiles.length} task="${state.currentTask.slice(0, 80)}"`);
 
         // On SessionEnd: run full consolidation (sleep cycle) before process dies
@@ -255,9 +255,9 @@ export function startSidecar(deps: SidecarDeps): { close: () => void } {
   });
 
   // --- Silent auto-checkpoint every 15 minutes ---
-  const autoCheckpointTimer = setInterval(() => {
+  const autoCheckpointTimer = setInterval(async () => {
     try {
-      const checkpoint = store.getCheckpoint(agentId);
+      const checkpoint = await store.getCheckpoint(agentId);
       if (!checkpoint) return; // No state to save yet
 
       // Only checkpoint if there's been activity since last auto-checkpoint
@@ -275,7 +275,7 @@ export function startSidecar(deps: SidecarDeps): { close: () => void } {
         episodeId: checkpoint.executionState?.episodeId ?? null,
       };
 
-      store.saveCheckpoint(agentId, state);
+      await store.saveCheckpoint(agentId, state);
       log(agentId, 'hook:timer', `auto-checkpoint (${Math.round(sinceActivity / 60_000)}min since activity)`);
     } catch { /* timer failure is non-fatal */ }
   }, AUTO_CHECKPOINT_INTERVAL_MS);

package/src/index.ts

@@ -1,212 +1,226 @@
-// Copyright 2026 Robert Winter / Complete Ideas
-// SPDX-License-Identifier: Apache-2.0
-import { readFileSync, copyFileSync, existsSync, mkdirSync, readdirSync, unlinkSync } from 'node:fs';
-import { resolve, dirname, basename } from 'node:path';
-import Fastify from 'fastify';
-
-// Load .env file if present (no external dependency)
-try {
-  const envPath = resolve(process.cwd(), '.env');
-  const envContent = readFileSync(envPath, 'utf-8');
-  for (const line of envContent.split('\n')) {
-    const trimmed = line.trim();
-    if (!trimmed || trimmed.startsWith('#')) continue;
-    const eqIdx = trimmed.indexOf('=');
-    if (eqIdx === -1) continue;
-    const key = trimmed.slice(0, eqIdx).trim();
-    const val = trimmed.slice(eqIdx + 1).trim().replace(/^["']|["']$/g, '');
-    if (!process.env[key]) process.env[key] = val; // Don't override existing env
-  }
-} catch { /* No .env file — that's fine */ }
-import { EngramStore } from './storage/sqlite.js';
-import { ActivationEngine } from './engine/activation.js';
-import { ConnectionEngine } from './engine/connections.js';
-import { StagingBuffer } from './engine/staging.js';
-import { EvictionEngine } from './engine/eviction.js';
-import { RetractionEngine } from './engine/retraction.js';
-import { EvalEngine } from './engine/eval.js';
-import { ConsolidationEngine } from './engine/consolidation.js';
-import { ConsolidationScheduler } from './engine/consolidation-scheduler.js';
-import { registerRoutes } from './api/routes.js';
-import { DEFAULT_AGENT_CONFIG } from './types/agent.js';
-import { getEmbedder } from './core/embeddings.js';
-import { getReranker } from './core/reranker.js';
-import { getExpander } from './core/query-expander.js';
-import { initLogger } from './core/logger.js';
-
-const PORT = parseInt(process.env.AWM_PORT ?? '8400', 10);
-const DB_PATH = process.env.AWM_DB_PATH ?? 'memory.db';
-const API_KEY = process.env.AWM_API_KEY ?? null;
-
-async function main() {
-  // Auto-backup: copy DB to backups/ on startup (cheap insurance)
-  if (existsSync(DB_PATH)) {
-    const dbDir = dirname(resolve(DB_PATH));
-    const backupDir = resolve(dbDir, 'backups');
-    mkdirSync(backupDir, { recursive: true });
-    const ts = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
-    const backupPath = resolve(backupDir, `${basename(DB_PATH, '.db')}-${ts}.db`);
-    try {
-      copyFileSync(resolve(DB_PATH), backupPath);
-      console.log(`Backup: ${backupPath}`);
-    } catch (err) {
-      console.log(`Backup skipped: ${(err as Error).message}`);
-    }
-  }
-
-  // Logger — write activity to awm.log alongside the DB
-  initLogger(DB_PATH);
-
-  // Storage
-  const store = new EngramStore(DB_PATH);
-
-  // Integrity check
-  const integrity = store.integrityCheck();
-  if (!integrity.ok) {
-    console.error(`DB integrity check FAILED: ${integrity.result}`);
-    // Close corrupt DB, restore from backup, and exit for process manager to restart
-    store.close();
-    const dbDir = dirname(resolve(DB_PATH));
-    const backupDir = resolve(dbDir, 'backups');
-    if (existsSync(backupDir)) {
-      const backups = readdirSync(backupDir)
-        .filter(f => f.endsWith('.db'))
-        .sort()
-        .reverse();
-      if (backups.length > 0) {
-        const restorePath = resolve(backupDir, backups[0]);
-        console.error(`Attempting restore from: ${restorePath}`);
-        try {
-          copyFileSync(restorePath, resolve(DB_PATH));
-          console.error('Restore complete — exiting for restart with restored DB');
-          process.exit(1);
-        } catch (restoreErr) {
-          console.error(`Restore failed: ${(restoreErr as Error).message}`);
-        }
-      }
-    }
-    console.error('No backup available — continuing with potentially corrupt DB');
-  } else {
-    console.log('  DB integrity check: ok');
-  }
-
-  // Engines
-  const activationEngine = new ActivationEngine(store);
-  const connectionEngine = new ConnectionEngine(store, activationEngine);
-  const stagingBuffer = new StagingBuffer(store, activationEngine);
-  const evictionEngine = new EvictionEngine(store);
-  const retractionEngine = new RetractionEngine(store);
-  const evalEngine = new EvalEngine(store);
-  const consolidationEngine = new ConsolidationEngine(store);
-  const consolidationScheduler = new ConsolidationScheduler(store, consolidationEngine);
-
-  // API — disable Fastify's default request logging (too noisy for hive polling)
-  // bodyLimit: 512KB to prevent Content-Length mismatch errors with large task payloads
-  const app = Fastify({ logger: false, bodyLimit: 512_000 });
-
-  // Bearer token auth — only enforced when AWM_API_KEY is explicitly set and non-empty
-  if (API_KEY && API_KEY !== 'NONE' && API_KEY.length > 1) {
-    app.addHook('onRequest', async (req, reply) => {
-      if (req.url === '/health') return; // Health check is always public
-      const bearer = req.headers.authorization;
-      const xApiKey = req.headers['x-api-key'] as string | undefined;
-      if (bearer === `Bearer ${API_KEY}` || xApiKey === API_KEY) return;
-      reply.code(401).send({ error: 'Unauthorized' });
-    });
-    console.log('API key auth enabled (AWM_API_KEY set)');
-  }
-
-  registerRoutes(app, {
-    store, activationEngine, connectionEngine,
-    evictionEngine, retractionEngine, evalEngine,
-    consolidationEngine, consolidationScheduler,
-  });
-
-  // Coordination module (opt-in via AWM_COORDINATION=true)
-  const { isCoordinationEnabled, initCoordination, stopCoordinationCleanup } = await import('./coordination/index.js');
-  if (isCoordinationEnabled()) {
-    initCoordination(app, store.getDb(), store);
-  } else {
-    console.log('  Coordination module disabled (set AWM_COORDINATION=true to enable)');
-  }
-
-  // Background tasks
-  stagingBuffer.start(DEFAULT_AGENT_CONFIG.stagingTtlMs);
-  consolidationScheduler.start();
-
-  // Periodic hot backup every 10 minutes (keep last 6 = 1hr coverage)
-  const dbDir = dirname(resolve(DB_PATH));
-  const backupDir = resolve(dbDir, 'backups');
-  mkdirSync(backupDir, { recursive: true });
-
-  // Cleanup old backups on startup (older than 2 hours)
-  try {
-    const TWO_HOURS_MS = 2 * 60 * 60 * 1000;
-    const now = Date.now();
-    for (const f of readdirSync(backupDir).filter(f => f.endsWith('.db'))) {
-      const match = f.match(/(\d{4})-(\d{2})-(\d{2})T(\d{2})-(\d{2})-(\d{2})/);
-      if (match) {
-        const fileDate = new Date(`${match[1]}-${match[2]}-${match[3]}T${match[4]}:${match[5]}:${match[6]}Z`);
-        if (now - fileDate.getTime() > TWO_HOURS_MS) {
-          unlinkSync(resolve(backupDir, f));
-        }
-      }
-    }
-  } catch { /* cleanup is non-fatal */ }
-
-  const backupTimer = setInterval(() => {
-    try {
-      const ts = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
-      const backupPath = resolve(backupDir, `${basename(DB_PATH, '.db')}-${ts}.db`);
-      store.backup(backupPath);
-      // Prune: keep only last 6 backups
-      const backups = readdirSync(backupDir).filter(f => f.endsWith('.db')).sort();
-      while (backups.length > 6) {
-        const old = backups.shift()!;
-        try { unlinkSync(resolve(backupDir, old)); } catch { /* non-fatal */ }
-      }
-    } catch (err) {
-      console.warn(`[backup] failed: ${(err as Error).message}`);
-    }
-  }, 10 * 60_000); // 10 minutes
-
-  // Pre-load ML models (downloads on first run: embeddings ~22MB, reranker ~22MB, expander ~80MB)
-  getEmbedder().catch(err => console.warn('Embedding model unavailable:', err.message));
-  getReranker().catch(err => console.warn('Reranker model unavailable:', err.message));
-  getExpander().catch(err => console.warn('Query expander model unavailable:', err.message));
-
-  // Eager slim-cache populate — pays the ~600ms first-fetch cost at startup
-  // instead of charging it to the first user recall (0.7.14+).
-  setImmediate(() => {
-    try {
-      const t0 = Date.now();
-      store.warmSlimCache();
-      const stats = store.getSlimCacheStats();
-      console.log(`  Slim cache warmed: ${stats.size} entries in ${Date.now() - t0}ms`);
-    } catch (err) {
-      console.warn(`  Slim cache warm failed: ${(err as Error).message}`);
-    }
-  });
-
-  // Start server
-  await app.listen({ port: PORT, host: '0.0.0.0' });
-  console.log(`AgentWorkingMemory v0.7.17 listening on port ${PORT}`);
-
-  // Graceful shutdown
-  const shutdown = async () => {
-    clearInterval(backupTimer);
-    await stopCoordinationCleanup();
-    consolidationScheduler.stop();
-    stagingBuffer.stop();
-    try { store.walCheckpoint(); } catch { /* non-fatal */ }
-    store.close();
-    process.exit(0);
-  };
-  process.on('SIGINT', shutdown);
-  process.on('SIGTERM', shutdown);
-}
-
-main().catch(err => {
-  console.error('Failed to start:', err);
-  process.exit(1);
-});
+// Copyright 2026 Robert Winter / Complete Ideas
+// SPDX-License-Identifier: Apache-2.0
+import { readFileSync, copyFileSync, existsSync, mkdirSync, readdirSync, unlinkSync } from 'node:fs';
+import { resolve, dirname, basename } from 'node:path';
+import Fastify from 'fastify';
+
+// Load .env file if present (no external dependency)
+try {
+  const envPath = resolve(process.cwd(), '.env');
+  const envContent = readFileSync(envPath, 'utf-8');
+  for (const line of envContent.split('\n')) {
+    const trimmed = line.trim();
+    if (!trimmed || trimmed.startsWith('#')) continue;
+    const eqIdx = trimmed.indexOf('=');
+    if (eqIdx === -1) continue;
+    const key = trimmed.slice(0, eqIdx).trim();
+    const val = trimmed.slice(eqIdx + 1).trim().replace(/^["']|["']$/g, '');
+    if (!process.env[key]) process.env[key] = val; // Don't override existing env
+  }
+} catch { /* No .env file — that's fine */ }
+import { EngramStore } from './storage/sqlite.js';
+import { ActivationEngine } from './engine/activation.js';
+import { ConnectionEngine } from './engine/connections.js';
+import { StagingBuffer } from './engine/staging.js';
+import { EvictionEngine } from './engine/eviction.js';
+import { RetractionEngine } from './engine/retraction.js';
+import { EvalEngine } from './engine/eval.js';
+import { ConsolidationEngine } from './engine/consolidation.js';
+import { ConsolidationScheduler } from './engine/consolidation-scheduler.js';
+import { registerRoutes } from './api/routes.js';
+import { DEFAULT_AGENT_CONFIG } from './types/agent.js';
+import { getEmbedder } from './core/embeddings.js';
+import { getReranker } from './core/reranker.js';
+import { getExpander } from './core/query-expander.js';
+import { initLogger } from './core/logger.js';
+import { openStore, getConfiguredBackend, type StoreBackend } from './storage/factory.js';
+import type { IEngramStore } from './storage/store.js';
+
+const PORT = parseInt(process.env.AWM_PORT ?? '8400', 10);
+const BACKEND: StoreBackend = getConfiguredBackend();
+const DB_PATH = process.env.AWM_DB_PATH ?? (BACKEND === 'pglite' ? 'memory-pglite' : 'memory.db');
+const API_KEY = process.env.AWM_API_KEY ?? null;
+
+async function main() {
+  // Auto-backup: copy DB to backups/ on startup (SQLite-only; PGlite is a
+  // directory and is backed up by snapshot/replication at the OS level).
+  if (BACKEND === 'sqlite' && existsSync(DB_PATH)) {
+    const dbDir = dirname(resolve(DB_PATH));
+    const backupDir = resolve(dbDir, 'backups');
+    mkdirSync(backupDir, { recursive: true });
+    const ts = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
+    const backupPath = resolve(backupDir, `${basename(DB_PATH, '.db')}-${ts}.db`);
+    try {
+      copyFileSync(resolve(DB_PATH), backupPath);
+      console.log(`Backup: ${backupPath}`);
+    } catch (err) {
+      console.log(`Backup skipped: ${(err as Error).message}`);
+    }
+  }
+
+  // Logger — write activity to awm.log alongside the DB
+  initLogger(DB_PATH);
+
+  // Storage — backend-agnostic open via the factory.
+  // Some routes (export, coordination health) are still SQLite-only and
+  // gracefully return 501 / skip when running on PGlite.
+  const { store: storeAny } = await openStore();
+  const store = storeAny as unknown as EngramStore;
+
+  // Integrity check — SQLite-only. PGlite has its own consistency model.
+  const integrity = BACKEND === 'sqlite'
+    ? store.integrityCheck()
+    : { ok: true, result: 'pglite: skipped' };
+  if (!integrity.ok) {
+    console.error(`DB integrity check FAILED: ${integrity.result}`);
+    // Close corrupt DB, restore from backup, and exit for process manager to restart
+    store.close();
+    const dbDir = dirname(resolve(DB_PATH));
+    const backupDir = resolve(dbDir, 'backups');
+    if (existsSync(backupDir)) {
+      const backups = readdirSync(backupDir)
+        .filter(f => f.endsWith('.db'))
+        .sort()
+        .reverse();
+      if (backups.length > 0) {
+        const restorePath = resolve(backupDir, backups[0]);
+        console.error(`Attempting restore from: ${restorePath}`);
+        try {
+          copyFileSync(restorePath, resolve(DB_PATH));
+          console.error('Restore complete — exiting for restart with restored DB');
+          process.exit(1);
+        } catch (restoreErr) {
+          console.error(`Restore failed: ${(restoreErr as Error).message}`);
+        }
+      }
+    }
+    console.error('No backup available — continuing with potentially corrupt DB');
+  } else {
+    console.log('  DB integrity check: ok');
+  }
+
+  // Engines
+  const activationEngine = new ActivationEngine(store);
+  const connectionEngine = new ConnectionEngine(store, activationEngine);
+  const stagingBuffer = new StagingBuffer(store, activationEngine);
+  const evictionEngine = new EvictionEngine(store);
+  const retractionEngine = new RetractionEngine(store);
+  const evalEngine = new EvalEngine(store);
+  const consolidationEngine = new ConsolidationEngine(store, connectionEngine);
+  const consolidationScheduler = new ConsolidationScheduler(store, consolidationEngine);
+
+  // API — disable Fastify's default request logging (too noisy for hive polling)
+  // bodyLimit: 512KB to prevent Content-Length mismatch errors with large task payloads
+  const app = Fastify({ logger: false, bodyLimit: 512_000 });
+
+  // Bearer token auth — only enforced when AWM_API_KEY is explicitly set and non-empty
+  if (API_KEY && API_KEY !== 'NONE' && API_KEY.length > 1) {
+    app.addHook('onRequest', async (req, reply) => {
+      if (req.url === '/health') return; // Health check is always public
+      const bearer = req.headers.authorization;
+      const xApiKey = req.headers['x-api-key'] as string | undefined;
+      if (bearer === `Bearer ${API_KEY}` || xApiKey === API_KEY) return;
+      reply.code(401).send({ error: 'Unauthorized' });
+    });
+    console.log('API key auth enabled (AWM_API_KEY set)');
+  }
+
+  registerRoutes(app, {
+    store, activationEngine, connectionEngine,
+    evictionEngine, retractionEngine, evalEngine,
+    consolidationEngine, consolidationScheduler,
+  });
+
+  // Coordination module (opt-in via AWM_COORDINATION=true) — SQLite-only.
+  const { isCoordinationEnabled, initCoordination, stopCoordinationCleanup } = await import('./coordination/index.js');
+  if (isCoordinationEnabled() && BACKEND === 'sqlite') {
+    initCoordination(app, store.getDb(), store);
+  } else if (isCoordinationEnabled()) {
+    console.log(`  Coordination requested but disabled — requires SQLite backend (current: ${BACKEND})`);
+  } else {
+    console.log('  Coordination module disabled (set AWM_COORDINATION=true to enable)');
+  }
+
+  // Background tasks
+  stagingBuffer.start(DEFAULT_AGENT_CONFIG.stagingTtlMs);
+  consolidationScheduler.start();
+
+  // Periodic hot backup every 10 minutes (keep last 6 = 1hr coverage)
+  const dbDir = dirname(resolve(DB_PATH));
+  const backupDir = resolve(dbDir, 'backups');
+  mkdirSync(backupDir, { recursive: true });
+
+  // Cleanup old backups on startup (older than 2 hours)
+  try {
+    const TWO_HOURS_MS = 2 * 60 * 60 * 1000;
+    const now = Date.now();
+    for (const f of readdirSync(backupDir).filter(f => f.endsWith('.db'))) {
+      const match = f.match(/(\d{4})-(\d{2})-(\d{2})T(\d{2})-(\d{2})-(\d{2})/);
+      if (match) {
+        const fileDate = new Date(`${match[1]}-${match[2]}-${match[3]}T${match[4]}:${match[5]}:${match[6]}Z`);
+        if (now - fileDate.getTime() > TWO_HOURS_MS) {
+          unlinkSync(resolve(backupDir, f));
+        }
+      }
+    }
+  } catch { /* cleanup is non-fatal */ }
+
+  // Periodic hot backup — SQLite-only. PGlite directory backups are an OS-level concern.
+  const backupTimer: ReturnType<typeof setInterval> | null = BACKEND === 'sqlite' ? setInterval(() => {
+    try {
+      const ts = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
+      const backupPath = resolve(backupDir, `${basename(DB_PATH, '.db')}-${ts}.db`);
+      store.backup(backupPath);
+      // Prune: keep only last 6 backups
+      const backups = readdirSync(backupDir).filter(f => f.endsWith('.db')).sort();
+      while (backups.length > 6) {
+        const old = backups.shift()!;
+        try { unlinkSync(resolve(backupDir, old)); } catch { /* non-fatal */ }
+      }
+    } catch (err) {
+      console.warn(`[backup] failed: ${(err as Error).message}`);
+    }
+  }, 10 * 60_000) : null;
+
+  // Pre-load ML models (downloads on first run: embeddings ~22MB, reranker ~22MB, expander ~80MB)
+  getEmbedder().catch(err => console.warn('Embedding model unavailable:', err.message));
+  getReranker().catch(err => console.warn('Reranker model unavailable:', err.message));
+  getExpander().catch(err => console.warn('Query expander model unavailable:', err.message));
+
+  if (BACKEND === 'sqlite') {
+    setImmediate(() => {
+      try {
+        const t0 = Date.now();
+        store.warmSlimCache();
+        const stats = store.getSlimCacheStats();
+        console.log(`  Slim cache warmed: ${stats.size} entries in ${Date.now() - t0}ms`);
+      } catch (err) {
+        console.warn(`  Slim cache warm failed: ${(err as Error).message}`);
+      }
+    });
+  }
+
+  // Start server
+  await app.listen({ port: PORT, host: '0.0.0.0' });
+  console.log(`AgentWorkingMemory v0.8.5 listening on port ${PORT}`);
+
+  // Graceful shutdown
+  const shutdown = async () => {
+    if (backupTimer) clearInterval(backupTimer);
+    await stopCoordinationCleanup();
+    consolidationScheduler.stop();
+    stagingBuffer.stop();
+    if (BACKEND === 'sqlite') {
+      try { store.walCheckpoint(); } catch { /* non-fatal */ }
+    }
+    try { await (store as any).close?.(); } catch { /* best-effort */ }
+    process.exit(0);
+  };
+  process.on('SIGINT', shutdown);
+  process.on('SIGTERM', shutdown);
+}
+
+main().catch(err => {
+  console.error('Failed to start:', err);
+  process.exit(1);
+});