agent-tool-forge 0.3.0 → 0.4.3

package/README.md

@@ -91,7 +91,7 @@ Then in any Claude Code session:
 - **HITL** — four levels (autonomous → paranoid), pause/resume with 5-minute TTL
 - **Verifiers** — post-response quality pipeline (warnings + flags, ACIRU ordering)
 - **Eval runner** — `node lib/index.js run --eval <path>` executes eval JSON, checks assertions, stores results in SQLite; `--record` / `--replay` for fixture-based testing
-- **Observability** — token tracking, cost estimation, per-tool metrics in SQLite
+- **Observability** — token tracking, cost estimation, per-tool metrics; chat audit log and eval history stored in Postgres when `DATABASE_URL` is set (durable across Railway/ephemeral filesystem deploys)
 - **Web component** — `<forge-chat>` drop-in chat widget (vanilla JS, zero deps)
 
 ---
@@ -103,7 +103,7 @@ The sidecar core requires only `better-sqlite3`. Additional backends are loaded
 | Package | When needed |
 |---------|-------------|
 | `redis` or `ioredis` | `conversation.store: 'redis'` or `rateLimit.enabled: true` with Redis backend |
-| `pg` | `database.type: 'postgres'` — Postgres conversation store, agent registry, and preferences |
+| `pg` | `database.type: 'postgres'` — Postgres conversation store, agent registry, preferences, eval results, chat audit log, and verifier registry |
 
 ```bash
 # Redis backend
@@ -133,7 +133,12 @@ import { makePreferenceStore } from 'tool-forge/preference-store'
 import { makeRateLimiter }     from 'tool-forge/rate-limiter'
 import { getDb }               from 'tool-forge/db'
 import { initSSE }             from 'tool-forge/sse'
-import { PostgresStore }       from 'tool-forge/postgres-store'
+import {
+  PostgresStore,
+  PostgresEvalStore,
+  PostgresChatAuditStore,
+  PostgresVerifierStore
+}                              from 'tool-forge/postgres-store'
 import { buildSidecarContext, createSidecarRouter } from 'tool-forge/forge-service'
 ```
 

package/lib/conversation-store.js

@@ -185,8 +185,14 @@ export class RedisConversationStore {
     pl.expire(msgKey, this._ttl);
     if (role === 'system' && content === '[COMPLETE]') {
       pl.sRem(ACTIVE_SET_KEY, sessionId);
+      if (userId) {
+        pl.sRem(`forge:sessions:user:${encodeURIComponent(userId)}`, sessionId);
+      }
     } else {
       pl.sAdd(ACTIVE_SET_KEY, sessionId);
+      if (userId) {
+        pl.sAdd(`forge:sessions:user:${encodeURIComponent(userId)}`, sessionId);
+      }
     }
     await pl.exec();
   }
@@ -225,7 +231,8 @@ export class RedisConversationStore {
 
   async listSessions(userId) {
     const client = await this._connect();
-    const sessionIds = await client.sMembers(ACTIVE_SET_KEY);
+    const userSetKey = `forge:sessions:user:${encodeURIComponent(userId)}`;
+    const sessionIds = await client.sMembers(userSetKey);
 
     const sessionData = await Promise.all(sessionIds.map(async (sessionId) => {
       const [firstMsgRaw, lastMsgRaw] = await Promise.all([
@@ -238,13 +245,14 @@ export class RedisConversationStore {
     const result = [];
     for (const { sessionId, firstMsgRaw, lastMsgRaw } of sessionData) {
       if (!firstMsgRaw) {
-        // stale entry — clean it up from the active set (best-effort, ignore transient errors)
+        // stale entry — clean it up from the per-user set (that's what we queried)
+        // and also from the global active set (best-effort, ignore transient errors)
+        try { await client.sRem(userSetKey, sessionId); } catch { /* ignore */ }
         try { await client.sRem(ACTIVE_SET_KEY, sessionId); } catch { /* ignore */ }
         continue;
       }
       try {
         const msg = JSON.parse(firstMsgRaw);
-        if (msg.user_id !== userId) continue;
         const last = lastMsgRaw ? JSON.parse(lastMsgRaw) : msg;
         result.push({
           sessionId,
@@ -275,6 +283,8 @@ export class RedisConversationStore {
     pl.del(`forge:conv:${sessionId}:msgs`);
     pl.sRem(ACTIVE_SET_KEY, sessionId);
     await pl.exec();
+    const userSetKey = `forge:sessions:user:${encodeURIComponent(userId)}`;
+    await client.sRem(userSetKey, sessionId);
     return true;
   }
 
@@ -321,6 +331,12 @@ export class PostgresConversationStore {
         created_at TEXT NOT NULL
       )
     `);
+    await this._pool.query(`
+      CREATE INDEX IF NOT EXISTS idx_pg_conversations_session ON conversations(session_id)
+    `);
+    await this._pool.query(`
+      CREATE INDEX IF NOT EXISTS idx_pg_conversations_user ON conversations(user_id, created_at)
+    `);
     this._tableReady = true;
   }
 

package/lib/db.js

@@ -217,62 +217,44 @@ export function getDb(dbPath) {
   const db = new Database(dbPath);
   db.exec(SCHEMA);
   // Migrate: add skipped column if it doesn't exist yet
-  try {
+  if (!(db.prepare(`SELECT COUNT(*) as n FROM pragma_table_info('eval_runs') WHERE name='skipped'`).get().n > 0)) {
     db.exec('ALTER TABLE eval_runs ADD COLUMN skipped INTEGER DEFAULT 0');
-  } catch (err) {
-    if (!err.message.includes('duplicate column name')) throw err;
   }
   // Migrate: add model column
-  try {
+  if (!(db.prepare(`SELECT COUNT(*) as n FROM pragma_table_info('eval_runs') WHERE name='model'`).get().n > 0)) {
     db.exec('ALTER TABLE eval_runs ADD COLUMN model TEXT');
-  } catch (err) {
-    if (!err.message.includes('duplicate column name')) throw err;
   }
   // Migrate: add pass_rate column
-  try {
+  if (!(db.prepare(`SELECT COUNT(*) as n FROM pragma_table_info('eval_runs') WHERE name='pass_rate'`).get().n > 0)) {
     db.exec('ALTER TABLE eval_runs ADD COLUMN pass_rate REAL');
-  } catch (err) {
-    if (!err.message.includes('duplicate column name')) throw err;
   }
   // Migrate: add sample_type column
-  try {
+  if (!(db.prepare(`SELECT COUNT(*) as n FROM pragma_table_info('eval_runs') WHERE name='sample_type'`).get().n > 0)) {
     db.exec('ALTER TABLE eval_runs ADD COLUMN sample_type TEXT');
-  } catch (err) {
-    if (!err.message.includes('duplicate column name')) throw err;
   }
   // Migrate: add agent_id to conversations
-  try {
+  if (!(db.prepare(`SELECT COUNT(*) as n FROM pragma_table_info('conversations') WHERE name='agent_id'`).get().n > 0)) {
     db.exec('ALTER TABLE conversations ADD COLUMN agent_id TEXT');
-  } catch (err) {
-    if (!err.message.includes('duplicate column name')) throw err;
   }
   // Migrate: add user_id to conversations
-  try {
+  if (!(db.prepare(`SELECT COUNT(*) as n FROM pragma_table_info('conversations') WHERE name='user_id'`).get().n > 0)) {
     db.exec('ALTER TABLE conversations ADD COLUMN user_id TEXT');
-  } catch (err) {
-    if (!err.message.includes('duplicate column name')) throw err;
   }
   // Add index for user_id lookups
   try {
     db.exec('CREATE INDEX IF NOT EXISTS idx_conversations_user ON conversations(user_id, created_at)');
   } catch { /* may already exist */ }
   // Migrate: add input_tokens to eval_run_cases
-  try {
+  if (!(db.prepare(`SELECT COUNT(*) as n FROM pragma_table_info('eval_run_cases') WHERE name='input_tokens'`).get().n > 0)) {
     db.exec('ALTER TABLE eval_run_cases ADD COLUMN input_tokens INTEGER');
-  } catch (err) {
-    if (!err.message.includes('duplicate column name')) throw err;
   }
   // Migrate: add output_tokens to eval_run_cases
-  try {
+  if (!(db.prepare(`SELECT COUNT(*) as n FROM pragma_table_info('eval_run_cases') WHERE name='output_tokens'`).get().n > 0)) {
     db.exec('ALTER TABLE eval_run_cases ADD COLUMN output_tokens INTEGER');
-  } catch (err) {
-    if (!err.message.includes('duplicate column name')) throw err;
   }
   // Migrate: add role column to verifier_registry (Phase 6 — sandbox)
-  try {
+  if (!(db.prepare(`SELECT COUNT(*) as n FROM pragma_table_info('verifier_registry') WHERE name='role'`).get().n > 0)) {
     db.exec("ALTER TABLE verifier_registry ADD COLUMN role TEXT DEFAULT 'any'");
-  } catch (err) {
-    if (!err.message.includes('duplicate column name')) throw err;
   }
   return db;
 }

package/lib/eval-runner.js

@@ -288,7 +288,7 @@ function loadEnv(projectRoot) {
  * @param {function} onProgress - called after each case: ({ done, total, caseId, passed, reason })
  * @returns {{ total, passed, failed, skipped, cases, provider, model }}
  */
-export async function runEvals(toolName, config, projectRoot, onProgress) {
+export async function runEvals(toolName, config, projectRoot, onProgress, { pgPool: injectedPool } = {}) {
   const env = loadEnv(projectRoot);
 
   // Determine provider + key.
@@ -453,32 +453,53 @@ export async function runEvals(toolName, config, projectRoot, onProgress) {
     });
   }
 
-  // Persist to SQLite
+  // Persist to Postgres (if DATABASE_URL set) or SQLite
   try {
-    const dbPath = resolve(projectRoot, config?.dbPath || 'forge.db');
-    const { getDb, insertEvalRun, insertEvalRunCases } = await import('./db.js');
-    const db = getDb(dbPath);
     const evalType = allCases.every((c) => c._evalType === 'golden') ? 'golden'
       : allCases.every((c) => c._evalType === 'labeled') ? 'labeled'
       : 'mixed';
     const ran = passed + failed;
     const passRate = ran > 0 ? passed / ran : 0;
-    const evalRunId = insertEvalRun(db, {
-      tool_name: toolName,
-      eval_type: evalType,
-      total_cases: allCases.length,
-      passed,
-      failed,
-      skipped,
-      notes: `provider:${provider} model:${model}`,
-      model,
-      pass_rate: passRate,
+    const row = {
+      tool_name: toolName, eval_type: evalType, total_cases: allCases.length,
+      passed, failed, skipped,
+      notes: `provider:${provider} model:${model}`, model, pass_rate: passRate,
       sample_type: 'targeted'
-    });
-    if (caseRows.length > 0) {
-      insertEvalRunCases(db, caseRows.map((r) => ({ ...r, eval_run_id: evalRunId })));
+    };
+
+    const dbUrl = process.env.DATABASE_URL;
+    if (dbUrl) {
+      const ownPool = !injectedPool;
+      let pool = injectedPool;
+      if (!pool) {
+        const pg = await import('pg');
+        const Pool = pg.default?.Pool ?? pg.Pool;
+        pool = new Pool({ connectionString: dbUrl });
+      }
+      try {
+        const { PostgresEvalStore } = await import('./postgres-store.js');
+        const evalStore = new PostgresEvalStore(pool);
+        const evalRunId = await evalStore.insertEvalRun(row);
+        if (caseRows.length > 0) {
+          await evalStore.insertEvalRunCases(
+            caseRows.map((r) => ({ ...r, eval_run_id: evalRunId }))
+          );
+        }
+      } finally {
+        if (ownPool) await pool.end().catch(() => {});
+      }
+    } else {
+      const dbPath = resolve(projectRoot, config?.dbPath || 'forge.db');
+      const { getDb, insertEvalRun, insertEvalRunCases } = await import('./db.js');
+      const db = getDb(dbPath);
+      const evalRunId = insertEvalRun(db, row);
+      if (caseRows.length > 0) {
+        insertEvalRunCases(db, caseRows.map((r) => ({ ...r, eval_run_id: evalRunId })));
+      }
     }
-  } catch (_) { /* db write failure is non-fatal */ }
+  } catch (err) {
+    process.stderr.write(`[eval-runner] DB write failed (non-fatal): ${err.message}\n`);
+  }
 
   return { total: allCases.length, passed, failed, skipped, cases: results, provider, model };
 }
@@ -508,34 +529,48 @@ export async function runEvalsMultiPass(toolName, config, projectRoot, options =
 
   const perModel = {};
 
-  for (const modelName of matrixNames) {
-    const mc = modelConfigForName(modelName, env);
-    if (!mc.apiKey) {
-      perModel[modelName] = { error: `No API key found for provider "${mc.provider}"` };
-      continue;
-    }
+  // Create a single shared Postgres pool for the duration of the multi-pass run
+  let sharedPool = null;
+  const dbUrl = process.env.DATABASE_URL;
+  if (dbUrl) {
+    const pg = await import('pg');
+    const Pool = pg.default?.Pool ?? pg.Pool;
+    sharedPool = new Pool({ connectionString: dbUrl });
+  }
 
-    // Build a config override for this model
-    const modelConfig = { ...config, model: modelName, models: { ...config?.models, eval: modelName } };
+  try {
+    for (const modelName of matrixNames) {
+      const mc = modelConfigForName(modelName, env);
+      if (!mc.apiKey) {
+        perModel[modelName] = { error: `No API key found for provider "${mc.provider}"` };
+        continue;
+      }
 
-    try {
-      const result = await runEvals(
-        toolName,
-        modelConfig,
-        projectRoot,
-        (progress) => onProgress?.({ model: modelName, ...progress })
-      );
-      perModel[modelName] = {
-        passed: result.passed,
-        failed: result.failed,
-        total: result.total,
-        skipped: result.skipped,
-        pass_rate: (result.passed + result.failed) > 0 ? result.passed / (result.passed + result.failed) : 0,
-        provider: result.provider
-      };
-    } catch (err) {
-      perModel[modelName] = { error: err.message };
+      // Build a config override for this model
+      const modelConfig = { ...config, model: modelName, models: { ...config?.models, eval: modelName } };
+
+      try {
+        const result = await runEvals(
+          toolName,
+          modelConfig,
+          projectRoot,
+          (progress) => onProgress?.({ model: modelName, ...progress }),
+          { pgPool: sharedPool }
+        );
+        perModel[modelName] = {
+          passed: result.passed,
+          failed: result.failed,
+          total: result.total,
+          skipped: result.skipped,
+          pass_rate: (result.passed + result.failed) > 0 ? result.passed / (result.passed + result.failed) : 0,
+          provider: result.provider
+        };
+      } catch (err) {
+        perModel[modelName] = { error: err.message };
+      }
     }
+  } finally {
+    if (sharedPool) await sharedPool.end().catch(() => {});
   }
 
   return { perModel };

package/lib/forge-service.js

@@ -47,6 +47,7 @@ import { handleAgents } from './handlers/agents.js';
 import { handleConversations } from './handlers/conversations.js';
 import { handleToolsList } from './handlers/tools-list.js';
 import { makeRateLimiter } from './rate-limiter.js';
+import { SCHEMA } from './postgres-store.js';
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const PROJECT_ROOT = resolve(__dirname, '..');
@@ -63,13 +64,14 @@ const PROJECT_ROOT = resolve(__dirname, '..');
  * @param {object} config — merged config (after mergeDefaults)
  * @param {import('better-sqlite3').Database} db
  * @param {Record<string, string>} env — environment variables
- * @returns {Promise<{ auth, promptStore, preferenceStore, conversationStore, hitlEngine, verifierRunner, agentRegistry, db, config, env, _redisClient, _pgPool }>}
+ * @returns {Promise<{ auth, promptStore, preferenceStore, conversationStore, hitlEngine, verifierRunner, agentRegistry, db, config, env, rateLimiter, configPath, evalStore, chatAuditStore, verifierStore, pgStore, _redisClient, _pgPool }>}
  */
 export async function buildSidecarContext(config, db, env = {}, opts = {}) {
   const auth = createAuth(config.auth);
 
   let redisClient = null;
   let pgPool = null;
+  let connStr = null;
   const storeType = config?.conversation?.store ?? 'sqlite';
 
   if (storeType === 'redis') {
@@ -86,7 +88,7 @@ export async function buildSidecarContext(config, db, env = {}, opts = {}) {
     const pg = await import('pg');
     const Pool = pg.default?.Pool ?? pg.Pool;
     const rawUrl = config?.database?.url;
-    let connStr = rawUrl;
+    connStr = rawUrl;
     if (rawUrl?.startsWith('${') && rawUrl.endsWith('}')) {
       const SAFE_ENV_VAR_NAME = /^[A-Z_][A-Z0-9_]*$/;
       const varName = rawUrl.slice(2, -1);
@@ -95,16 +97,35 @@ export async function buildSidecarContext(config, db, env = {}, opts = {}) {
       }
       connStr = env[varName];
     }
-    pgPool = new Pool({ connectionString: connStr ?? undefined });
+    pgPool = new Pool({
+      connectionString: connStr ?? undefined,
+      connectionTimeoutMillis: 5000,
+      idleTimeoutMillis: 30000,
+      max: 10
+    });
+    await pgPool.query(SCHEMA);  // ensure all tables exist
   }
 
   // Select store backends (Postgres if configured, SQLite otherwise)
   let promptStore, preferenceStore, agentRegistry;
-  if (pgPool && config?.database?.type === 'postgres') {
-    const { PostgresPromptStore, PostgresPreferenceStore, PostgresAgentRegistry } = await import('./postgres-store.js');
+  let evalStore = null;
+  let chatAuditStore = null;
+  let verifierStore = null;
+  let pgStore = null;
+  if (pgPool) {
+    const {
+      PostgresStore, PostgresPromptStore, PostgresPreferenceStore,
+      PostgresAgentRegistry, PostgresEvalStore, PostgresChatAuditStore,
+      PostgresVerifierStore
+    } = await import('./postgres-store.js');
     promptStore = new PostgresPromptStore(pgPool);
     preferenceStore = new PostgresPreferenceStore(pgPool, config);
     agentRegistry = new PostgresAgentRegistry(config, pgPool);
+    evalStore = new PostgresEvalStore(pgPool);
+    chatAuditStore = new PostgresChatAuditStore(pgPool);
+    verifierStore = new PostgresVerifierStore(pgPool);
+    pgStore = new PostgresStore({ connectionString: connStr ?? undefined });
+    pgStore._pool = pgPool; // reuse already-created pool
   } else {
     promptStore = makePromptStore(config, db);
     preferenceStore = makePreferenceStore(config, db);
@@ -113,7 +134,7 @@ export async function buildSidecarContext(config, db, env = {}, opts = {}) {
 
   const conversationStore = makeConversationStore(config, db, pgPool);
   const hitlEngine = makeHitlEngine(config, db, redisClient, pgPool);
-  const verifierRunner = new VerifierRunner(db, config);
+  const verifierRunner = new VerifierRunner(db, config, pgPool ?? null);
   const rateLimiter = makeRateLimiter(config, redisClient);
 
   // configPath — used by admin handler to persist overlay changes.
@@ -124,6 +145,7 @@ export async function buildSidecarContext(config, db, env = {}, opts = {}) {
   return {
     auth, promptStore, preferenceStore, conversationStore, hitlEngine, verifierRunner,
     agentRegistry, db, config, env, rateLimiter, configPath,
+    evalStore, chatAuditStore, verifierStore, pgStore,
     _redisClient: redisClient, _pgPool: pgPool
   };
 }
@@ -185,11 +207,14 @@ function serveWidgetFile(req, res, widgetDir, errorFn) {
  * @param {object} [options]
  * @param {string} [options.widgetDir] — directory for /widget/* static files (defaults to <project>/widget)
  * @param {function} [options.mcpHandler] — optional async (req, res) handler for /mcp route
+ * @param {function} [options.customRoutes] — optional async (req, res, ctx) => boolean; called before
+ *   the 404 fallback. Return true if the request was handled, false to fall through to 404.
  * @returns {function(import('http').IncomingMessage, import('http').ServerResponse): Promise<void>}
  */
 export function createSidecarRouter(ctx, options = {}) {
   const widgetDir = options.widgetDir || resolve(__dirname, '..', 'widget');
   const mcpHandler = options.mcpHandler || null;
+  const customRoutes = options.customRoutes || null;
 
   return async (req, res) => {
     const url = new URL(req.url, 'http://localhost');
@@ -245,6 +270,12 @@ export function createSidecarRouter(ctx, options = {}) {
       return;
     }
 
+    // ── Custom routes (consumer-provided) ─────────────────────────────────
+    if (customRoutes) {
+      const handled = await customRoutes(req, res, ctx);
+      if (handled) return;
+    }
+
     // ── 404 fallback ───────────────────────────────────────────────────────
     sendJson(res, 404, { error: 'not found' });
   };
@@ -527,7 +558,7 @@ function createDirectServer() {
   return server;
 }
 
-function shutdown() {
+async function shutdown() {
   // Drain waiters with 204
   for (const { res, timer } of waiters) {
     clearTimeout(timer);
@@ -535,6 +566,14 @@ function shutdown() {
   }
   waiters.length = 0;
   removeLock();
+  // Clean up Postgres pool if present
+  if (sidecarCtx?._pgPool) {
+    sidecarCtx._pgPool.end().catch(() => {});
+  }
+  // Clean up Redis client if present
+  if (sidecarCtx?._redisClient) {
+    try { await sidecarCtx._redisClient.quit(); } catch { /* non-fatal */ }
+  }
   server.close(() => process.exit(0));
   // Force-close lingering keep-alive connections (Node 18.2.0+)
   if (typeof server.closeAllConnections === 'function') {