agent-tool-forge 0.3.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -3
- package/lib/conversation-store.js +19 -3
- package/lib/db.js +9 -27
- package/lib/eval-runner.js +79 -44
- package/lib/forge-service.js +37 -7
- package/lib/handlers/chat-resume.js +66 -92
- package/lib/handlers/chat-sync.js +42 -60
- package/lib/handlers/chat.js +55 -73
- package/lib/hitl-engine.js +43 -18
- package/lib/postgres-store.js +311 -49
- package/lib/rate-limiter.js +8 -2
- package/lib/verifier-runner.js +31 -10
- package/package.json +3 -3
package/README.md
CHANGED
|
@@ -91,7 +91,7 @@ Then in any Claude Code session:
|
|
|
91
91
|
- **HITL** — four levels (autonomous → paranoid), pause/resume with 5-minute TTL
|
|
92
92
|
- **Verifiers** — post-response quality pipeline (warnings + flags, ACIRU ordering)
|
|
93
93
|
- **Eval runner** — `node lib/index.js run --eval <path>` executes eval JSON, checks assertions, stores results in SQLite; `--record` / `--replay` for fixture-based testing
|
|
94
|
-
- **Observability** — token tracking, cost estimation, per-tool metrics in
|
|
94
|
+
- **Observability** — token tracking, cost estimation, per-tool metrics; chat audit log and eval history stored in Postgres when `DATABASE_URL` is set (durable across Railway/ephemeral filesystem deploys)
|
|
95
95
|
- **Web component** — `<forge-chat>` drop-in chat widget (vanilla JS, zero deps)
|
|
96
96
|
|
|
97
97
|
---
|
|
@@ -103,7 +103,7 @@ The sidecar core requires only `better-sqlite3`. Additional backends are loaded
|
|
|
103
103
|
| Package | When needed |
|
|
104
104
|
|---------|-------------|
|
|
105
105
|
| `redis` or `ioredis` | `conversation.store: 'redis'` or `rateLimit.enabled: true` with Redis backend |
|
|
106
|
-
| `pg` | `database.type: 'postgres'` — Postgres conversation store, agent registry, and
|
|
106
|
+
| `pg` | `database.type: 'postgres'` — Postgres conversation store, agent registry, preferences, eval results, chat audit log, and verifier registry |
|
|
107
107
|
|
|
108
108
|
```bash
|
|
109
109
|
# Redis backend
|
|
@@ -133,7 +133,12 @@ import { makePreferenceStore } from 'tool-forge/preference-store'
|
|
|
133
133
|
import { makeRateLimiter } from 'tool-forge/rate-limiter'
|
|
134
134
|
import { getDb } from 'tool-forge/db'
|
|
135
135
|
import { initSSE } from 'tool-forge/sse'
|
|
136
|
-
import {
|
|
136
|
+
import {
|
|
137
|
+
PostgresStore,
|
|
138
|
+
PostgresEvalStore,
|
|
139
|
+
PostgresChatAuditStore,
|
|
140
|
+
PostgresVerifierStore
|
|
141
|
+
} from 'tool-forge/postgres-store'
|
|
137
142
|
import { buildSidecarContext, createSidecarRouter } from 'tool-forge/forge-service'
|
|
138
143
|
```
|
|
139
144
|
|
|
@@ -185,8 +185,14 @@ export class RedisConversationStore {
|
|
|
185
185
|
pl.expire(msgKey, this._ttl);
|
|
186
186
|
if (role === 'system' && content === '[COMPLETE]') {
|
|
187
187
|
pl.sRem(ACTIVE_SET_KEY, sessionId);
|
|
188
|
+
if (userId) {
|
|
189
|
+
pl.sRem(`forge:sessions:user:${encodeURIComponent(userId)}`, sessionId);
|
|
190
|
+
}
|
|
188
191
|
} else {
|
|
189
192
|
pl.sAdd(ACTIVE_SET_KEY, sessionId);
|
|
193
|
+
if (userId) {
|
|
194
|
+
pl.sAdd(`forge:sessions:user:${encodeURIComponent(userId)}`, sessionId);
|
|
195
|
+
}
|
|
190
196
|
}
|
|
191
197
|
await pl.exec();
|
|
192
198
|
}
|
|
@@ -225,7 +231,8 @@ export class RedisConversationStore {
|
|
|
225
231
|
|
|
226
232
|
async listSessions(userId) {
|
|
227
233
|
const client = await this._connect();
|
|
228
|
-
const
|
|
234
|
+
const userSetKey = `forge:sessions:user:${encodeURIComponent(userId)}`;
|
|
235
|
+
const sessionIds = await client.sMembers(userSetKey);
|
|
229
236
|
|
|
230
237
|
const sessionData = await Promise.all(sessionIds.map(async (sessionId) => {
|
|
231
238
|
const [firstMsgRaw, lastMsgRaw] = await Promise.all([
|
|
@@ -238,13 +245,14 @@ export class RedisConversationStore {
|
|
|
238
245
|
const result = [];
|
|
239
246
|
for (const { sessionId, firstMsgRaw, lastMsgRaw } of sessionData) {
|
|
240
247
|
if (!firstMsgRaw) {
|
|
241
|
-
// stale entry — clean it up from the
|
|
248
|
+
// stale entry — clean it up from the per-user set (that's what we queried)
|
|
249
|
+
// and also from the global active set (best-effort, ignore transient errors)
|
|
250
|
+
try { await client.sRem(userSetKey, sessionId); } catch { /* ignore */ }
|
|
242
251
|
try { await client.sRem(ACTIVE_SET_KEY, sessionId); } catch { /* ignore */ }
|
|
243
252
|
continue;
|
|
244
253
|
}
|
|
245
254
|
try {
|
|
246
255
|
const msg = JSON.parse(firstMsgRaw);
|
|
247
|
-
if (msg.user_id !== userId) continue;
|
|
248
256
|
const last = lastMsgRaw ? JSON.parse(lastMsgRaw) : msg;
|
|
249
257
|
result.push({
|
|
250
258
|
sessionId,
|
|
@@ -275,6 +283,8 @@ export class RedisConversationStore {
|
|
|
275
283
|
pl.del(`forge:conv:${sessionId}:msgs`);
|
|
276
284
|
pl.sRem(ACTIVE_SET_KEY, sessionId);
|
|
277
285
|
await pl.exec();
|
|
286
|
+
const userSetKey = `forge:sessions:user:${encodeURIComponent(userId)}`;
|
|
287
|
+
await client.sRem(userSetKey, sessionId);
|
|
278
288
|
return true;
|
|
279
289
|
}
|
|
280
290
|
|
|
@@ -321,6 +331,12 @@ export class PostgresConversationStore {
|
|
|
321
331
|
created_at TEXT NOT NULL
|
|
322
332
|
)
|
|
323
333
|
`);
|
|
334
|
+
await this._pool.query(`
|
|
335
|
+
CREATE INDEX IF NOT EXISTS idx_pg_conversations_session ON conversations(session_id)
|
|
336
|
+
`);
|
|
337
|
+
await this._pool.query(`
|
|
338
|
+
CREATE INDEX IF NOT EXISTS idx_pg_conversations_user ON conversations(user_id, created_at)
|
|
339
|
+
`);
|
|
324
340
|
this._tableReady = true;
|
|
325
341
|
}
|
|
326
342
|
|
package/lib/db.js
CHANGED
|
@@ -217,62 +217,44 @@ export function getDb(dbPath) {
|
|
|
217
217
|
const db = new Database(dbPath);
|
|
218
218
|
db.exec(SCHEMA);
|
|
219
219
|
// Migrate: add skipped column if it doesn't exist yet
|
|
220
|
-
|
|
220
|
+
if (!(db.prepare(`SELECT COUNT(*) as n FROM pragma_table_info('eval_runs') WHERE name='skipped'`).get().n > 0)) {
|
|
221
221
|
db.exec('ALTER TABLE eval_runs ADD COLUMN skipped INTEGER DEFAULT 0');
|
|
222
|
-
} catch (err) {
|
|
223
|
-
if (!err.message.includes('duplicate column name')) throw err;
|
|
224
222
|
}
|
|
225
223
|
// Migrate: add model column
|
|
226
|
-
|
|
224
|
+
if (!(db.prepare(`SELECT COUNT(*) as n FROM pragma_table_info('eval_runs') WHERE name='model'`).get().n > 0)) {
|
|
227
225
|
db.exec('ALTER TABLE eval_runs ADD COLUMN model TEXT');
|
|
228
|
-
} catch (err) {
|
|
229
|
-
if (!err.message.includes('duplicate column name')) throw err;
|
|
230
226
|
}
|
|
231
227
|
// Migrate: add pass_rate column
|
|
232
|
-
|
|
228
|
+
if (!(db.prepare(`SELECT COUNT(*) as n FROM pragma_table_info('eval_runs') WHERE name='pass_rate'`).get().n > 0)) {
|
|
233
229
|
db.exec('ALTER TABLE eval_runs ADD COLUMN pass_rate REAL');
|
|
234
|
-
} catch (err) {
|
|
235
|
-
if (!err.message.includes('duplicate column name')) throw err;
|
|
236
230
|
}
|
|
237
231
|
// Migrate: add sample_type column
|
|
238
|
-
|
|
232
|
+
if (!(db.prepare(`SELECT COUNT(*) as n FROM pragma_table_info('eval_runs') WHERE name='sample_type'`).get().n > 0)) {
|
|
239
233
|
db.exec('ALTER TABLE eval_runs ADD COLUMN sample_type TEXT');
|
|
240
|
-
} catch (err) {
|
|
241
|
-
if (!err.message.includes('duplicate column name')) throw err;
|
|
242
234
|
}
|
|
243
235
|
// Migrate: add agent_id to conversations
|
|
244
|
-
|
|
236
|
+
if (!(db.prepare(`SELECT COUNT(*) as n FROM pragma_table_info('conversations') WHERE name='agent_id'`).get().n > 0)) {
|
|
245
237
|
db.exec('ALTER TABLE conversations ADD COLUMN agent_id TEXT');
|
|
246
|
-
} catch (err) {
|
|
247
|
-
if (!err.message.includes('duplicate column name')) throw err;
|
|
248
238
|
}
|
|
249
239
|
// Migrate: add user_id to conversations
|
|
250
|
-
|
|
240
|
+
if (!(db.prepare(`SELECT COUNT(*) as n FROM pragma_table_info('conversations') WHERE name='user_id'`).get().n > 0)) {
|
|
251
241
|
db.exec('ALTER TABLE conversations ADD COLUMN user_id TEXT');
|
|
252
|
-
} catch (err) {
|
|
253
|
-
if (!err.message.includes('duplicate column name')) throw err;
|
|
254
242
|
}
|
|
255
243
|
// Add index for user_id lookups
|
|
256
244
|
try {
|
|
257
245
|
db.exec('CREATE INDEX IF NOT EXISTS idx_conversations_user ON conversations(user_id, created_at)');
|
|
258
246
|
} catch { /* may already exist */ }
|
|
259
247
|
// Migrate: add input_tokens to eval_run_cases
|
|
260
|
-
|
|
248
|
+
if (!(db.prepare(`SELECT COUNT(*) as n FROM pragma_table_info('eval_run_cases') WHERE name='input_tokens'`).get().n > 0)) {
|
|
261
249
|
db.exec('ALTER TABLE eval_run_cases ADD COLUMN input_tokens INTEGER');
|
|
262
|
-
} catch (err) {
|
|
263
|
-
if (!err.message.includes('duplicate column name')) throw err;
|
|
264
250
|
}
|
|
265
251
|
// Migrate: add output_tokens to eval_run_cases
|
|
266
|
-
|
|
252
|
+
if (!(db.prepare(`SELECT COUNT(*) as n FROM pragma_table_info('eval_run_cases') WHERE name='output_tokens'`).get().n > 0)) {
|
|
267
253
|
db.exec('ALTER TABLE eval_run_cases ADD COLUMN output_tokens INTEGER');
|
|
268
|
-
} catch (err) {
|
|
269
|
-
if (!err.message.includes('duplicate column name')) throw err;
|
|
270
254
|
}
|
|
271
255
|
// Migrate: add role column to verifier_registry (Phase 6 — sandbox)
|
|
272
|
-
|
|
256
|
+
if (!(db.prepare(`SELECT COUNT(*) as n FROM pragma_table_info('verifier_registry') WHERE name='role'`).get().n > 0)) {
|
|
273
257
|
db.exec("ALTER TABLE verifier_registry ADD COLUMN role TEXT DEFAULT 'any'");
|
|
274
|
-
} catch (err) {
|
|
275
|
-
if (!err.message.includes('duplicate column name')) throw err;
|
|
276
258
|
}
|
|
277
259
|
return db;
|
|
278
260
|
}
|
package/lib/eval-runner.js
CHANGED
|
@@ -288,7 +288,7 @@ function loadEnv(projectRoot) {
|
|
|
288
288
|
* @param {function} onProgress - called after each case: ({ done, total, caseId, passed, reason })
|
|
289
289
|
* @returns {{ total, passed, failed, skipped, cases, provider, model }}
|
|
290
290
|
*/
|
|
291
|
-
export async function runEvals(toolName, config, projectRoot, onProgress) {
|
|
291
|
+
export async function runEvals(toolName, config, projectRoot, onProgress, { pgPool: injectedPool } = {}) {
|
|
292
292
|
const env = loadEnv(projectRoot);
|
|
293
293
|
|
|
294
294
|
// Determine provider + key.
|
|
@@ -453,32 +453,53 @@ export async function runEvals(toolName, config, projectRoot, onProgress) {
|
|
|
453
453
|
});
|
|
454
454
|
}
|
|
455
455
|
|
|
456
|
-
// Persist to SQLite
|
|
456
|
+
// Persist to Postgres (if DATABASE_URL set) or SQLite
|
|
457
457
|
try {
|
|
458
|
-
const dbPath = resolve(projectRoot, config?.dbPath || 'forge.db');
|
|
459
|
-
const { getDb, insertEvalRun, insertEvalRunCases } = await import('./db.js');
|
|
460
|
-
const db = getDb(dbPath);
|
|
461
458
|
const evalType = allCases.every((c) => c._evalType === 'golden') ? 'golden'
|
|
462
459
|
: allCases.every((c) => c._evalType === 'labeled') ? 'labeled'
|
|
463
460
|
: 'mixed';
|
|
464
461
|
const ran = passed + failed;
|
|
465
462
|
const passRate = ran > 0 ? passed / ran : 0;
|
|
466
|
-
const
|
|
467
|
-
tool_name: toolName,
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
passed,
|
|
471
|
-
failed,
|
|
472
|
-
skipped,
|
|
473
|
-
notes: `provider:${provider} model:${model}`,
|
|
474
|
-
model,
|
|
475
|
-
pass_rate: passRate,
|
|
463
|
+
const row = {
|
|
464
|
+
tool_name: toolName, eval_type: evalType, total_cases: allCases.length,
|
|
465
|
+
passed, failed, skipped,
|
|
466
|
+
notes: `provider:${provider} model:${model}`, model, pass_rate: passRate,
|
|
476
467
|
sample_type: 'targeted'
|
|
477
|
-
}
|
|
478
|
-
|
|
479
|
-
|
|
468
|
+
};
|
|
469
|
+
|
|
470
|
+
const dbUrl = process.env.DATABASE_URL;
|
|
471
|
+
if (dbUrl) {
|
|
472
|
+
const ownPool = !injectedPool;
|
|
473
|
+
let pool = injectedPool;
|
|
474
|
+
if (!pool) {
|
|
475
|
+
const pg = await import('pg');
|
|
476
|
+
const Pool = pg.default?.Pool ?? pg.Pool;
|
|
477
|
+
pool = new Pool({ connectionString: dbUrl });
|
|
478
|
+
}
|
|
479
|
+
try {
|
|
480
|
+
const { PostgresEvalStore } = await import('./postgres-store.js');
|
|
481
|
+
const evalStore = new PostgresEvalStore(pool);
|
|
482
|
+
const evalRunId = await evalStore.insertEvalRun(row);
|
|
483
|
+
if (caseRows.length > 0) {
|
|
484
|
+
await evalStore.insertEvalRunCases(
|
|
485
|
+
caseRows.map((r) => ({ ...r, eval_run_id: evalRunId }))
|
|
486
|
+
);
|
|
487
|
+
}
|
|
488
|
+
} finally {
|
|
489
|
+
if (ownPool) await pool.end().catch(() => {});
|
|
490
|
+
}
|
|
491
|
+
} else {
|
|
492
|
+
const dbPath = resolve(projectRoot, config?.dbPath || 'forge.db');
|
|
493
|
+
const { getDb, insertEvalRun, insertEvalRunCases } = await import('./db.js');
|
|
494
|
+
const db = getDb(dbPath);
|
|
495
|
+
const evalRunId = insertEvalRun(db, row);
|
|
496
|
+
if (caseRows.length > 0) {
|
|
497
|
+
insertEvalRunCases(db, caseRows.map((r) => ({ ...r, eval_run_id: evalRunId })));
|
|
498
|
+
}
|
|
480
499
|
}
|
|
481
|
-
} catch (
|
|
500
|
+
} catch (err) {
|
|
501
|
+
process.stderr.write(`[eval-runner] DB write failed (non-fatal): ${err.message}\n`);
|
|
502
|
+
}
|
|
482
503
|
|
|
483
504
|
return { total: allCases.length, passed, failed, skipped, cases: results, provider, model };
|
|
484
505
|
}
|
|
@@ -508,34 +529,48 @@ export async function runEvalsMultiPass(toolName, config, projectRoot, options =
|
|
|
508
529
|
|
|
509
530
|
const perModel = {};
|
|
510
531
|
|
|
511
|
-
for
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
532
|
+
// Create a single shared Postgres pool for the duration of the multi-pass run
|
|
533
|
+
let sharedPool = null;
|
|
534
|
+
const dbUrl = process.env.DATABASE_URL;
|
|
535
|
+
if (dbUrl) {
|
|
536
|
+
const pg = await import('pg');
|
|
537
|
+
const Pool = pg.default?.Pool ?? pg.Pool;
|
|
538
|
+
sharedPool = new Pool({ connectionString: dbUrl });
|
|
539
|
+
}
|
|
517
540
|
|
|
518
|
-
|
|
519
|
-
const
|
|
541
|
+
try {
|
|
542
|
+
for (const modelName of matrixNames) {
|
|
543
|
+
const mc = modelConfigForName(modelName, env);
|
|
544
|
+
if (!mc.apiKey) {
|
|
545
|
+
perModel[modelName] = { error: `No API key found for provider "${mc.provider}"` };
|
|
546
|
+
continue;
|
|
547
|
+
}
|
|
520
548
|
|
|
521
|
-
|
|
522
|
-
const
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
549
|
+
// Build a config override for this model
|
|
550
|
+
const modelConfig = { ...config, model: modelName, models: { ...config?.models, eval: modelName } };
|
|
551
|
+
|
|
552
|
+
try {
|
|
553
|
+
const result = await runEvals(
|
|
554
|
+
toolName,
|
|
555
|
+
modelConfig,
|
|
556
|
+
projectRoot,
|
|
557
|
+
(progress) => onProgress?.({ model: modelName, ...progress }),
|
|
558
|
+
{ pgPool: sharedPool }
|
|
559
|
+
);
|
|
560
|
+
perModel[modelName] = {
|
|
561
|
+
passed: result.passed,
|
|
562
|
+
failed: result.failed,
|
|
563
|
+
total: result.total,
|
|
564
|
+
skipped: result.skipped,
|
|
565
|
+
pass_rate: (result.passed + result.failed) > 0 ? result.passed / (result.passed + result.failed) : 0,
|
|
566
|
+
provider: result.provider
|
|
567
|
+
};
|
|
568
|
+
} catch (err) {
|
|
569
|
+
perModel[modelName] = { error: err.message };
|
|
570
|
+
}
|
|
538
571
|
}
|
|
572
|
+
} finally {
|
|
573
|
+
if (sharedPool) await sharedPool.end().catch(() => {});
|
|
539
574
|
}
|
|
540
575
|
|
|
541
576
|
return { perModel };
|
package/lib/forge-service.js
CHANGED
|
@@ -47,6 +47,7 @@ import { handleAgents } from './handlers/agents.js';
|
|
|
47
47
|
import { handleConversations } from './handlers/conversations.js';
|
|
48
48
|
import { handleToolsList } from './handlers/tools-list.js';
|
|
49
49
|
import { makeRateLimiter } from './rate-limiter.js';
|
|
50
|
+
import { SCHEMA } from './postgres-store.js';
|
|
50
51
|
|
|
51
52
|
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
52
53
|
const PROJECT_ROOT = resolve(__dirname, '..');
|
|
@@ -63,13 +64,14 @@ const PROJECT_ROOT = resolve(__dirname, '..');
|
|
|
63
64
|
* @param {object} config — merged config (after mergeDefaults)
|
|
64
65
|
* @param {import('better-sqlite3').Database} db
|
|
65
66
|
* @param {Record<string, string>} env — environment variables
|
|
66
|
-
* @returns {Promise<{ auth, promptStore, preferenceStore, conversationStore, hitlEngine, verifierRunner, agentRegistry, db, config, env, _redisClient, _pgPool }>}
|
|
67
|
+
* @returns {Promise<{ auth, promptStore, preferenceStore, conversationStore, hitlEngine, verifierRunner, agentRegistry, db, config, env, rateLimiter, configPath, evalStore, chatAuditStore, verifierStore, pgStore, _redisClient, _pgPool }>}
|
|
67
68
|
*/
|
|
68
69
|
export async function buildSidecarContext(config, db, env = {}, opts = {}) {
|
|
69
70
|
const auth = createAuth(config.auth);
|
|
70
71
|
|
|
71
72
|
let redisClient = null;
|
|
72
73
|
let pgPool = null;
|
|
74
|
+
let connStr = null;
|
|
73
75
|
const storeType = config?.conversation?.store ?? 'sqlite';
|
|
74
76
|
|
|
75
77
|
if (storeType === 'redis') {
|
|
@@ -86,7 +88,7 @@ export async function buildSidecarContext(config, db, env = {}, opts = {}) {
|
|
|
86
88
|
const pg = await import('pg');
|
|
87
89
|
const Pool = pg.default?.Pool ?? pg.Pool;
|
|
88
90
|
const rawUrl = config?.database?.url;
|
|
89
|
-
|
|
91
|
+
connStr = rawUrl;
|
|
90
92
|
if (rawUrl?.startsWith('${') && rawUrl.endsWith('}')) {
|
|
91
93
|
const SAFE_ENV_VAR_NAME = /^[A-Z_][A-Z0-9_]*$/;
|
|
92
94
|
const varName = rawUrl.slice(2, -1);
|
|
@@ -95,16 +97,35 @@ export async function buildSidecarContext(config, db, env = {}, opts = {}) {
|
|
|
95
97
|
}
|
|
96
98
|
connStr = env[varName];
|
|
97
99
|
}
|
|
98
|
-
pgPool = new Pool({
|
|
100
|
+
pgPool = new Pool({
|
|
101
|
+
connectionString: connStr ?? undefined,
|
|
102
|
+
connectionTimeoutMillis: 5000,
|
|
103
|
+
idleTimeoutMillis: 30000,
|
|
104
|
+
max: 10
|
|
105
|
+
});
|
|
106
|
+
await pgPool.query(SCHEMA); // ensure all tables exist
|
|
99
107
|
}
|
|
100
108
|
|
|
101
109
|
// Select store backends (Postgres if configured, SQLite otherwise)
|
|
102
110
|
let promptStore, preferenceStore, agentRegistry;
|
|
103
|
-
|
|
104
|
-
|
|
111
|
+
let evalStore = null;
|
|
112
|
+
let chatAuditStore = null;
|
|
113
|
+
let verifierStore = null;
|
|
114
|
+
let pgStore = null;
|
|
115
|
+
if (pgPool) {
|
|
116
|
+
const {
|
|
117
|
+
PostgresStore, PostgresPromptStore, PostgresPreferenceStore,
|
|
118
|
+
PostgresAgentRegistry, PostgresEvalStore, PostgresChatAuditStore,
|
|
119
|
+
PostgresVerifierStore
|
|
120
|
+
} = await import('./postgres-store.js');
|
|
105
121
|
promptStore = new PostgresPromptStore(pgPool);
|
|
106
122
|
preferenceStore = new PostgresPreferenceStore(pgPool, config);
|
|
107
123
|
agentRegistry = new PostgresAgentRegistry(config, pgPool);
|
|
124
|
+
evalStore = new PostgresEvalStore(pgPool);
|
|
125
|
+
chatAuditStore = new PostgresChatAuditStore(pgPool);
|
|
126
|
+
verifierStore = new PostgresVerifierStore(pgPool);
|
|
127
|
+
pgStore = new PostgresStore({ connectionString: connStr ?? undefined });
|
|
128
|
+
pgStore._pool = pgPool; // reuse already-created pool
|
|
108
129
|
} else {
|
|
109
130
|
promptStore = makePromptStore(config, db);
|
|
110
131
|
preferenceStore = makePreferenceStore(config, db);
|
|
@@ -113,7 +134,7 @@ export async function buildSidecarContext(config, db, env = {}, opts = {}) {
|
|
|
113
134
|
|
|
114
135
|
const conversationStore = makeConversationStore(config, db, pgPool);
|
|
115
136
|
const hitlEngine = makeHitlEngine(config, db, redisClient, pgPool);
|
|
116
|
-
const verifierRunner = new VerifierRunner(db, config);
|
|
137
|
+
const verifierRunner = new VerifierRunner(db, config, pgPool ?? null);
|
|
117
138
|
const rateLimiter = makeRateLimiter(config, redisClient);
|
|
118
139
|
|
|
119
140
|
// configPath — used by admin handler to persist overlay changes.
|
|
@@ -124,6 +145,7 @@ export async function buildSidecarContext(config, db, env = {}, opts = {}) {
|
|
|
124
145
|
return {
|
|
125
146
|
auth, promptStore, preferenceStore, conversationStore, hitlEngine, verifierRunner,
|
|
126
147
|
agentRegistry, db, config, env, rateLimiter, configPath,
|
|
148
|
+
evalStore, chatAuditStore, verifierStore, pgStore,
|
|
127
149
|
_redisClient: redisClient, _pgPool: pgPool
|
|
128
150
|
};
|
|
129
151
|
}
|
|
@@ -527,7 +549,7 @@ function createDirectServer() {
|
|
|
527
549
|
return server;
|
|
528
550
|
}
|
|
529
551
|
|
|
530
|
-
function shutdown() {
|
|
552
|
+
async function shutdown() {
|
|
531
553
|
// Drain waiters with 204
|
|
532
554
|
for (const { res, timer } of waiters) {
|
|
533
555
|
clearTimeout(timer);
|
|
@@ -535,6 +557,14 @@ function shutdown() {
|
|
|
535
557
|
}
|
|
536
558
|
waiters.length = 0;
|
|
537
559
|
removeLock();
|
|
560
|
+
// Clean up Postgres pool if present
|
|
561
|
+
if (sidecarCtx?._pgPool) {
|
|
562
|
+
sidecarCtx._pgPool.end().catch(() => {});
|
|
563
|
+
}
|
|
564
|
+
// Clean up Redis client if present
|
|
565
|
+
if (sidecarCtx?._redisClient) {
|
|
566
|
+
try { await sidecarCtx._redisClient.quit(); } catch { /* non-fatal */ }
|
|
567
|
+
}
|
|
538
568
|
server.close(() => process.exit(0));
|
|
539
569
|
// Force-close lingering keep-alive connections (Node 18.2.0+)
|
|
540
570
|
if (typeof server.closeAllConnections === 'function') {
|
|
@@ -20,6 +20,16 @@ import { reactLoop } from '../react-engine.js';
|
|
|
20
20
|
import { readBody, sendJson, loadPromotedTools, extractJwt } from '../http-utils.js';
|
|
21
21
|
import { insertChatAudit } from '../db.js';
|
|
22
22
|
|
|
23
|
+
async function auditLog(ctx, row) {
|
|
24
|
+
if (ctx.chatAuditStore) {
|
|
25
|
+
await ctx.chatAuditStore.insertChatAudit(row).catch(() => {});
|
|
26
|
+
} else if (ctx.db) {
|
|
27
|
+
try { insertChatAudit(ctx.db, row); } catch { /* non-fatal */ }
|
|
28
|
+
} else {
|
|
29
|
+
process.stderr.write('[audit] No audit store available — row dropped\n');
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
|
|
23
33
|
/**
|
|
24
34
|
* @param {import('http').IncomingMessage} req
|
|
25
35
|
* @param {import('http').ServerResponse} res
|
|
@@ -41,15 +51,11 @@ export async function handleChatResume(req, res, ctx) {
|
|
|
41
51
|
// 1. Authenticate
|
|
42
52
|
const authResult = auth.authenticate(req);
|
|
43
53
|
if (!authResult.authenticated) {
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
error_message: authResult.error ?? 'Unauthorized'
|
|
50
|
-
});
|
|
51
|
-
} catch { /* non-fatal */ }
|
|
52
|
-
}
|
|
54
|
+
await auditLog(ctx, {
|
|
55
|
+
session_id: '', user_id: 'anon', route: '/agent-api/chat/resume',
|
|
56
|
+
status_code: 401, duration_ms: Date.now() - startTime,
|
|
57
|
+
error_message: authResult.error ?? 'Unauthorized'
|
|
58
|
+
});
|
|
53
59
|
sendJson(res, 401, { error: authResult.error ?? 'Unauthorized' });
|
|
54
60
|
return;
|
|
55
61
|
}
|
|
@@ -61,15 +67,11 @@ export async function handleChatResume(req, res, ctx) {
|
|
|
61
67
|
const rlResult = await ctx.rateLimiter.check(authResult.userId, '/agent-api/chat/resume');
|
|
62
68
|
if (!rlResult.allowed) {
|
|
63
69
|
res.setHeader?.('Retry-After', String(rlResult.retryAfter ?? 60));
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
error_message: 'Rate limit exceeded'
|
|
70
|
-
});
|
|
71
|
-
} catch { /* non-fatal */ }
|
|
72
|
-
}
|
|
70
|
+
await auditLog(ctx, {
|
|
71
|
+
session_id: '', user_id: userId, route: '/agent-api/chat/resume',
|
|
72
|
+
status_code: 429, duration_ms: Date.now() - startTime,
|
|
73
|
+
error_message: 'Rate limit exceeded'
|
|
74
|
+
});
|
|
73
75
|
sendJson(res, 429, { error: 'Rate limit exceeded', retryAfter: rlResult.retryAfter });
|
|
74
76
|
return;
|
|
75
77
|
}
|
|
@@ -80,28 +82,20 @@ export async function handleChatResume(req, res, ctx) {
|
|
|
80
82
|
try {
|
|
81
83
|
body = await readBody(req);
|
|
82
84
|
} catch (err) {
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
error_message: err.message
|
|
89
|
-
});
|
|
90
|
-
} catch { /* non-fatal */ }
|
|
91
|
-
}
|
|
85
|
+
await auditLog(ctx, {
|
|
86
|
+
session_id: '', user_id: userId, route: '/agent-api/chat/resume',
|
|
87
|
+
status_code: 413, duration_ms: Date.now() - startTime,
|
|
88
|
+
error_message: err.message
|
|
89
|
+
});
|
|
92
90
|
sendJson(res, 413, { error: err.message });
|
|
93
91
|
return;
|
|
94
92
|
}
|
|
95
93
|
if (!body.resumeToken) {
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
error_message: 'resumeToken is required'
|
|
102
|
-
});
|
|
103
|
-
} catch { /* non-fatal */ }
|
|
104
|
-
}
|
|
94
|
+
await auditLog(ctx, {
|
|
95
|
+
session_id: '', user_id: userId, route: '/agent-api/chat/resume',
|
|
96
|
+
status_code: 400, duration_ms: Date.now() - startTime,
|
|
97
|
+
error_message: 'resumeToken is required'
|
|
98
|
+
});
|
|
105
99
|
sendJson(res, 400, { error: 'resumeToken is required' });
|
|
106
100
|
return;
|
|
107
101
|
}
|
|
@@ -112,30 +106,22 @@ export async function handleChatResume(req, res, ctx) {
|
|
|
112
106
|
// (not resuming) is the same whether the token was valid or expired.
|
|
113
107
|
// Clients that need to distinguish "cancelled" from "token not found"
|
|
114
108
|
// should do a GET /hitl/status check before cancelling.
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
error_message: null
|
|
121
|
-
});
|
|
122
|
-
} catch { /* non-fatal */ }
|
|
123
|
-
}
|
|
109
|
+
await auditLog(ctx, {
|
|
110
|
+
session_id: '', user_id: userId, route: '/agent-api/chat/resume',
|
|
111
|
+
status_code: 200, duration_ms: Date.now() - startTime,
|
|
112
|
+
error_message: null
|
|
113
|
+
});
|
|
124
114
|
sendJson(res, 200, { message: 'Cancelled' });
|
|
125
115
|
return;
|
|
126
116
|
}
|
|
127
117
|
|
|
128
118
|
// 4. Check hitlEngine exists (only needed for actual resume)
|
|
129
119
|
if (!hitlEngine) {
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
error_message: 'HITL engine not available'
|
|
136
|
-
});
|
|
137
|
-
} catch { /* non-fatal */ }
|
|
138
|
-
}
|
|
120
|
+
await auditLog(ctx, {
|
|
121
|
+
session_id: '', user_id: userId, route: '/agent-api/chat/resume',
|
|
122
|
+
status_code: 501, duration_ms: Date.now() - startTime,
|
|
123
|
+
error_message: 'HITL engine not available'
|
|
124
|
+
});
|
|
139
125
|
sendJson(res, 501, { error: 'HITL engine not available' });
|
|
140
126
|
return;
|
|
141
127
|
}
|
|
@@ -143,15 +129,11 @@ export async function handleChatResume(req, res, ctx) {
|
|
|
143
129
|
// 5. NOW consume the pause state
|
|
144
130
|
const pausedState = await hitlEngine.resume(body.resumeToken);
|
|
145
131
|
if (!pausedState) {
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
error_message: 'Resume token not found or expired'
|
|
152
|
-
});
|
|
153
|
-
} catch { /* non-fatal */ }
|
|
154
|
-
}
|
|
132
|
+
await auditLog(ctx, {
|
|
133
|
+
session_id: '', user_id: userId, route: '/agent-api/chat/resume',
|
|
134
|
+
status_code: 404, duration_ms: Date.now() - startTime,
|
|
135
|
+
error_message: 'Resume token not found or expired'
|
|
136
|
+
});
|
|
155
137
|
sendJson(res, 404, { error: 'Resume token not found or expired' });
|
|
156
138
|
return;
|
|
157
139
|
}
|
|
@@ -164,7 +146,7 @@ export async function handleChatResume(req, res, ctx) {
|
|
|
164
146
|
|
|
165
147
|
let agent = null;
|
|
166
148
|
if (agentRegistry && pausedState.agentId) {
|
|
167
|
-
agent = agentRegistry.resolveAgent(pausedState.agentId);
|
|
149
|
+
agent = await agentRegistry.resolveAgent(pausedState.agentId);
|
|
168
150
|
// If agent no longer exists/disabled, fall back to base config (graceful degradation)
|
|
169
151
|
}
|
|
170
152
|
|
|
@@ -177,17 +159,13 @@ export async function handleChatResume(req, res, ctx) {
|
|
|
177
159
|
|
|
178
160
|
// Pre-validate API key
|
|
179
161
|
if (!effective.apiKey) {
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
error_message: `No API key configured for provider "${effective.provider}"`
|
|
188
|
-
});
|
|
189
|
-
} catch { /* non-fatal */ }
|
|
190
|
-
}
|
|
162
|
+
await auditLog(ctx, {
|
|
163
|
+
session_id: auditSessionId ?? '', user_id: userId,
|
|
164
|
+
agent_id: auditAgentId ?? null, route: '/agent-api/chat/resume',
|
|
165
|
+
status_code: 500, duration_ms: Date.now() - startTime,
|
|
166
|
+
model: auditModel ?? null,
|
|
167
|
+
error_message: `No API key configured for provider "${effective.provider}"`
|
|
168
|
+
});
|
|
191
169
|
sendJson(res, 500, {
|
|
192
170
|
error: `No API key configured for provider "${effective.provider}". Set the appropriate environment variable.`
|
|
193
171
|
});
|
|
@@ -313,22 +291,18 @@ export async function handleChatResume(req, res, ctx) {
|
|
|
313
291
|
sse.send('error', { type: 'error', message: err.message });
|
|
314
292
|
} finally {
|
|
315
293
|
sse.close();
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
error_message: auditErrorMessage ?? null
|
|
330
|
-
});
|
|
331
|
-
} catch { /* audit failure is non-fatal */ }
|
|
332
|
-
}
|
|
294
|
+
await auditLog(ctx, {
|
|
295
|
+
session_id: auditSessionId ?? '',
|
|
296
|
+
user_id: auditUserId,
|
|
297
|
+
agent_id: auditAgentId ?? null,
|
|
298
|
+
route: '/agent-api/chat/resume',
|
|
299
|
+
status_code: auditStatusCode,
|
|
300
|
+
duration_ms: Date.now() - startTime,
|
|
301
|
+
model: auditModel ?? null,
|
|
302
|
+
tool_count: auditToolCount,
|
|
303
|
+
hitl_triggered: auditHitlTriggered,
|
|
304
|
+
warnings_count: auditWarningsCount,
|
|
305
|
+
error_message: auditErrorMessage ?? null
|
|
306
|
+
});
|
|
333
307
|
}
|
|
334
308
|
}
|