agent-tool-forge 0.3.0

package/lib/verifier-runner.js

@@ -0,0 +1,338 @@
+/**
+ * VerifierRunner — post-tool-call checks in the live ReAct loop.
+ *
+ * Verifier types:
+ *   schema  — JSON Schema validation on result (basic type + required checks)
+ *   pattern — regex match/reject on result text
+ *   custom  — user-provided function loaded from verifiersDir
+ *
+ * Returns worst outcome: block > warn > pass.
+ * Block short-circuits (stops pipeline immediately).
+ */
+
+import { insertVerifierResult } from './db.js';
+import { resolve, isAbsolute } from 'path';
+
+const OUTCOME_SEVERITY = { pass: 0, warn: 1, block: 2 };
+
+export class VerifierRunner {
+  /**
+   * @param {import('better-sqlite3').Database} db
+   * @param {object} config — forge config (used for verifiersDir path + sandbox settings)
+   * @param {import('./verifier-worker-pool.js').VerifierWorkerPool} [workerPool] — optional pre-created pool
+   */
+  constructor(db, config = {}, workerPool = null) {
+    this._db = db;
+    this._config = config;
+    this._verifiers = new Map(); // toolName → verifier[]
+    this._workerPool = workerPool;
+    this._ownPool = false; // true if we created the pool internally
+  }
+
+  /**
+   * Tear down the worker pool (if we own it).
+   * Call this when the sidecar shuts down.
+   */
+  destroy() {
+    if (this._ownPool && this._workerPool) {
+      this._workerPool.destroy();
+      this._workerPool = null;
+    }
+  }
+
+  /**
+   * Register verifiers for a tool (manual/programmatic registration).
+   * @param {string} toolName
+   * @param {Array<{ name: string, type: 'schema'|'pattern'|'custom', spec: object, order?: string }>} verifiers
+   */
+  registerVerifiers(toolName, verifiers) {
+    this._verifiers.set(toolName, verifiers);
+  }
+
+  /**
+   * Load all enabled verifiers and their bindings from the DB.
+   * Builds the internal _verifiers Map keyed by tool_name.
+   * Wildcard bindings stored under '*'.
+   *
+   * For custom type: spec_json = { filePath, exportName }. Runs through worker pool if sandbox enabled.
+   * If file missing → register a warn-only stub.
+   *
+   * @param {import('better-sqlite3').Database} db
+   */
+  async loadFromDb(db) {
+    const targetDb = db || this._db;
+    if (!targetDb) return;
+
+    // Lazily create worker pool for custom verifiers if sandbox is enabled
+    // and no pool was provided to the constructor.
+    const sandboxEnabled = this._config?.verification?.sandbox !== false;
+    if (sandboxEnabled && !this._workerPool) {
+      const { VerifierWorkerPool } = await import('./verifier-worker-pool.js');
+      this._workerPool = new VerifierWorkerPool({
+        size: this._config?.verification?.workerPoolSize ?? undefined,
+        timeoutMs: this._config?.verification?.customTimeout ?? 2000,
+        maxQueueDepth: this._config?.verification?.maxQueueDepth ?? 200
+      });
+      this._ownPool = true;
+    }
+
+    // Get all enabled verifiers with their bindings
+    const allVerifiers = targetDb.prepare(
+      'SELECT * FROM verifier_registry WHERE enabled = 1 ORDER BY aciru_order ASC'
+    ).all();
+
+    const allBindings = targetDb.prepare(
+      'SELECT * FROM verifier_tool_bindings WHERE enabled = 1'
+    ).all();
+
+    // Build a map: tool_name → sorted verifier specs
+    const toolMap = new Map();
+
+    for (const binding of allBindings) {
+      const verifier = allVerifiers.find(v => v.verifier_name === binding.verifier_name);
+      if (!verifier) continue;
+
+      let spec;
+      try {
+        spec = JSON.parse(verifier.spec_json);
+      } catch (err) {
+        process.stderr.write(`[verifier-runner] Skipping verifier "${verifier.verifier_name}": malformed spec_json: ${err.message}\n`);
+        continue;
+      }
+
+      const entry = {
+        name: verifier.verifier_name,
+        type: verifier.type,
+        order: verifier.aciru_order,
+        spec
+      };
+
+      // For custom verifiers, resolve the path (sandboxed to verifiersDir)
+      if (verifier.type === 'custom' && spec.filePath) {
+        const verifiersDir = this._config?.verification?.verifiersDir;
+        const resolvedPath = isAbsolute(spec.filePath) ? spec.filePath : resolve(spec.filePath);
+        if (!verifiersDir || !resolvedPath.startsWith(resolve(verifiersDir))) {
+          entry.spec = { fn: () => ({ outcome: 'warn', message: `Custom verifier "${verifier.verifier_name}": path outside verifiersDir` }) };
+          const toolName = binding.tool_name;
+          if (!toolMap.has(toolName)) toolMap.set(toolName, []);
+          toolMap.get(toolName).push(entry);
+          continue;
+        }
+        // If sandbox mode and worker pool available, store path for pool dispatch
+        if (this._workerPool) {
+          entry.spec = {
+            ...spec,
+            resolvedPath,
+            exportName: spec.exportName || 'verify',
+            usePool: true
+          };
+        } else {
+          // Sandbox disabled — import directly (dev mode)
+          try {
+            const mod = await import(resolvedPath);
+            const fn = mod[spec.exportName || 'verify'] || mod.default;
+            entry.spec = typeof fn === 'function'
+              ? { ...spec, fn }
+              : { fn: () => ({ outcome: 'warn', message: `Custom verifier "${verifier.verifier_name}": no verify function found` }) };
+          } catch {
+            entry.spec = { fn: () => ({ outcome: 'warn', message: `Custom verifier "${verifier.verifier_name}": file not found or import failed` }) };
+          }
+        }
+        // Store verifier role for pool timeout/crash outcome
+        entry.role = verifier.role ?? 'any';
+      }
+
+      const toolName = binding.tool_name;
+      if (!toolMap.has(toolName)) toolMap.set(toolName, []);
+      toolMap.get(toolName).push(entry);
+    }
+
+    // Sort each tool's verifiers by order
+    for (const [key, verifiers] of toolMap) {
+      verifiers.sort((a, b) => a.order.localeCompare(b.order));
+      this._verifiers.set(key, verifiers);
+    }
+  }
+
+  /**
+   * Run all registered verifiers for a tool against the call result.
+   * Merges tool-specific verifiers + wildcard ('*') verifiers.
+   * Deduplicates by verifier name. Sorts by ACIRU order.
+   * Block short-circuits (returns immediately).
+   *
+   * @param {string} toolName
+   * @param {object} args — tool call input
+   * @param {object} result — tool call result ({ status, body, error })
+   * @returns {{ outcome: 'pass'|'warn'|'block', message: string|null, verifierName: string|null }}
+   */
+  async verify(toolName, args, result) {
+    const toolSpecific = this._verifiers.get(toolName) || [];
+    const wildcards = this._verifiers.get('*') || [];
+
+    // Merge and deduplicate by name
+    const seen = new Set();
+    const merged = [];
+    for (const v of [...toolSpecific, ...wildcards]) {
+      if (!seen.has(v.name)) {
+        seen.add(v.name);
+        merged.push(v);
+      }
+    }
+
+    if (merged.length === 0) {
+      return { outcome: 'pass', message: null, verifierName: null };
+    }
+
+    // Sort by order field if present
+    merged.sort((a, b) => (a.order ?? 'Z-9999').localeCompare(b.order ?? 'Z-9999'));
+
+    let worst = { outcome: 'pass', message: null, verifierName: null };
+
+    for (const v of merged) {
+      let vResult;
+      try {
+        switch (v.type) {
+          case 'schema':
+            vResult = runSchemaVerifier(v.spec, result.body);
+            break;
+          case 'pattern':
+            vResult = runPatternVerifier(v.spec, result.body);
+            break;
+          case 'custom':
+            vResult = await runCustomVerifier(v.spec, toolName, args, result, this._workerPool, v.role ?? 'any');
+            break;
+          default:
+            vResult = { outcome: 'pass', message: `Unknown verifier type: ${v.type}` };
+        }
+      } catch (err) {
+        vResult = { outcome: 'warn', message: `Verifier "${v.name}" threw: ${err.message}` };
+      }
+
+      // Block → short-circuit immediately
+      if (vResult.outcome === 'block') {
+        return { ...vResult, verifierName: v.name };
+      }
+
+      if (!(vResult.outcome in OUTCOME_SEVERITY)) {
+        vResult = { outcome: 'warn', message: `Verifier "${v.name}" returned invalid outcome: "${vResult.outcome}"` };
+      }
+      if (OUTCOME_SEVERITY[vResult.outcome] > OUTCOME_SEVERITY[worst.outcome]) {
+        worst = { ...vResult, verifierName: v.name };
+      }
+    }
+
+    return worst;
+  }
+
+  /**
+   * Log a verifier result to the verifier_results table.
+   * @param {string} sessionId
+   * @param {string} toolName
+   * @param {{ outcome: string, message: string|null, verifierName: string|null }} result
+   */
+  logResult(sessionId, toolName, result) {
+    if (!this._db) return;
+    try {
+      insertVerifierResult(this._db, {
+        session_id: sessionId,
+        tool_name: toolName,
+        verifier_name: result.verifierName ?? 'unknown',
+        outcome: result.outcome,
+        message: result.message ?? null
+      });
+    } catch { /* log failure is non-fatal */ }
+  }
+}
+
+// ── Verifier implementations ─────────────────────────────────────────────
+
+/**
+ * Schema verifier — basic type + required field checks.
+ * @param {object} schema — { required: string[], properties: { [key]: { type } } }
+ * @param {object} body — tool result body
+ * @returns {{ outcome: string, message: string|null }}
+ */
+function runSchemaVerifier(schema, body) {
+  if (!body || typeof body !== 'object') {
+    return { outcome: 'block', message: 'Result is not an object' };
+  }
+
+  // Check required fields
+  if (schema.required) {
+    for (const field of schema.required) {
+      if (!(field in body)) {
+        return { outcome: 'block', message: `Missing required field: ${field}` };
+      }
+    }
+  }
+
+  // Check types
+  if (schema.properties) {
+    for (const [key, def] of Object.entries(schema.properties)) {
+      if (key in body && def.type) {
+        const actualType = Array.isArray(body[key]) ? 'array' : typeof body[key];
+        if (actualType !== def.type) {
+          return { outcome: 'block', message: `Field "${key}" expected type "${def.type}", got "${actualType}"` };
+        }
+      }
+    }
+  }
+
+  return { outcome: 'pass', message: null };
+}
+
+/**
+ * Pattern verifier — regex match/reject on stringified result.
+ * @param {{ match?: string, reject?: string, outcome?: string }} spec
+ * @param {object} body
+ * @returns {{ outcome: string, message: string|null }}
+ */
+function runPatternVerifier(spec, body) {
+  const text = typeof body === 'string' ? body : JSON.stringify(body);
+  const outcome = spec.outcome ?? 'warn';
+
+  if (spec.reject) {
+    let regex;
+    try { regex = new RegExp(spec.reject); } catch (err) {
+      return { outcome: 'warn', message: `Invalid reject regex "${spec.reject}": ${err.message}` };
+    }
+    if (regex.test(text)) {
+      return { outcome, message: `Result matches reject pattern: ${spec.reject}` };
+    }
+  }
+
+  if (spec.match) {
+    let regex;
+    try { regex = new RegExp(spec.match); } catch (err) {
+      return { outcome: 'warn', message: `Invalid match regex "${spec.match}": ${err.message}` };
+    }
+    if (!regex.test(text)) {
+      return { outcome, message: `Result does not match required pattern: ${spec.match}` };
+    }
+  }
+
+  return { outcome: 'pass', message: null };
+}
+
+/**
+ * Custom verifier — routes through worker pool (sandboxed) or calls directly (dev mode).
+ * @param {{ fn?: Function, resolvedPath?: string, exportName?: string, usePool?: boolean }} spec
+ * @param {string} toolName
+ * @param {object} args
+ * @param {object} result
+ * @param {import('./verifier-worker-pool.js').VerifierWorkerPool|null} workerPool
+ * @param {string} role — 'read' | 'write' | 'any'
+ * @returns {Promise<{ outcome: string, message: string|null }>}
+ */
+async function runCustomVerifier(spec, toolName, args, result, workerPool, role) {
+  // Sandboxed path: dispatch to worker pool
+  if (spec.usePool && workerPool) {
+    return workerPool.run(spec.resolvedPath, spec.exportName, toolName, args, result, role);
+  }
+
+  // Direct (non-sandboxed / dev mode)
+  if (typeof spec.fn !== 'function') {
+    return { outcome: 'warn', message: 'Custom verifier has no fn function' };
+  }
+  return await spec.fn(toolName, args, result);
+}

package/lib/verifier-scanner.js

@@ -0,0 +1,70 @@
+/**
+ * Verifier Scanner — Discovers existing verifiers from barrel.
+ * Used for gap detection: tools without verifier coverage.
+ */
+
+import { readFileSync, existsSync, readdirSync } from 'fs';
+import { resolve, join } from 'path';
+
+/**
+ * Extract verifier names from barrel file.
+ * @param {string} barrelsPath
+ * @returns {string[]}
+ */
+function scanBarrel(barrelsPath) {
+  const abs = resolve(process.cwd(), barrelsPath);
+  if (!existsSync(abs)) return [];
+  const content = readFileSync(abs, 'utf-8');
+  const names = [];
+  const re = /export\s+\{\s*(\w+Verifier)\s*\}\s+from\s+['"]([^'"]+)['"]/g;
+  let m;
+  const lines = content.split('\n').filter((l) => !l.trim().startsWith('//'));
+  const text = lines.join('\n');
+  while ((m = re.exec(text))) {
+    const exportName = m[1];
+    const snake = exportName
+      .replace(/Verifier$/, '')
+      .replace(/([A-Z])/g, (c) => '_' + c.toLowerCase())
+      .replace(/^_/, '');
+    names.push(snake);
+  }
+  return names;
+}
+
+/**
+ * Scan verifier files for `name = 'snake_case'` pattern.
+ * @param {string} verifiersDir
+ * @returns {string[]}
+ */
+function scanVerifierFiles(verifiersDir) {
+  const abs = resolve(process.cwd(), verifiersDir);
+  if (!existsSync(abs)) return [];
+  const names = [];
+  const files = readdirSync(abs).filter(
+    (f) => f.endsWith('.verifier.ts') || f.endsWith('.verifier.js')
+  );
+  for (const file of files) {
+    const content = readFileSync(join(abs, file), 'utf-8');
+    const m = /name\s*=\s*['"]([^'"]+)['"]/.exec(content);
+    if (m) names.push(m[1]);
+  }
+  return names;
+}
+
+/**
+ * Get existing verifier names.
+ * @param {object} config - forge.config verification section
+ * @returns {string[]}
+ */
+export function getExistingVerifiers(config) {
+  const verifiers = [];
+  if (config?.verifiersDir) {
+    const fromFiles = scanVerifierFiles(config.verifiersDir);
+    verifiers.push(...fromFiles);
+  }
+  if (config?.barrelsFile && verifiers.length === 0) {
+    const fromBarrel = scanBarrel(config.barrelsFile);
+    verifiers.push(...fromBarrel);
+  }
+  return [...new Set(verifiers)];
+}

package/lib/verifier-worker-pool.js

@@ -0,0 +1,196 @@
+/**
+ * VerifierWorkerPool — fixed-size Worker thread pool for sandboxed custom verifier execution.
+ *
+ * Architecture (1000 verifiers × 1M sessions):
+ *   - N workers (default: min(4, cpus), configurable via poolSize)
+ *   - Module cache per worker (imported once, reused)
+ *   - Dispatch to first available idle worker
+ *   - Per-call timeout: 2000ms (configurable) — on expiry: terminate + replace worker
+ *   - Queue: max 200 pending calls (configurable); rejects with warn/block based on role
+ *   - Dead worker (crash/OOM): replaced immediately
+ *   - Timeout/crash outcome: 'write' role → 'block', 'read'|'any' → 'warn'
+ */
+
+import { Worker } from 'worker_threads';
+import { cpus } from 'os';
+import { fileURLToPath } from 'url';
+import { dirname, resolve } from 'path';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const WORKER_PATH = resolve(__dirname, 'workers', 'verifier-worker.js');
+
+export class VerifierWorkerPool {
+  /**
+   * @param {object} opts
+   * @param {number} [opts.size] — pool size (default: min(4, cpus().length))
+   * @param {number} [opts.timeoutMs=2000] — per-call timeout in ms
+   * @param {number} [opts.maxQueueDepth=200] — max pending calls before rejecting
+   */
+  constructor(opts = {}) {
+    this._size = opts.size ?? Math.min(4, cpus().length);
+    this._timeoutMs = opts.timeoutMs ?? 2000;
+    this._maxQueueDepth = opts.maxQueueDepth ?? 200;
+
+    this._workers = []; // { worker, busy, lastUsed }
+    this._callMap = new Map(); // callId → { resolve, timeoutHandle, role, entry }
+    this._queue = []; // { callArgs, role, resolve }
+    this._nextId = 1;
+    this._destroyed = false;
+
+    for (let i = 0; i < this._size; i++) {
+      this._workers.push(this._createWorkerEntry());
+    }
+  }
+
+  /** @private */
+  _createWorkerEntry() {
+    const worker = new Worker(WORKER_PATH, { type: 'module' });
+    const entry = { worker, busy: false, lastUsed: 0 };
+
+    worker.on('message', (msg) => {
+      this._handleMessage(msg, entry);
+    });
+    worker.on('error', (err) => {
+      this._handleWorkerError(entry, err);
+    });
+    worker.on('exit', (code) => {
+      if (code !== 0 && !this._destroyed) {
+        this._replaceWorker(entry);
+      }
+    });
+
+    return entry;
+  }
+
+  /** @private */
+  _handleMessage({ id, outcome, message }, entry) {
+    const pending = this._callMap.get(id);
+    if (!pending) return;
+    clearTimeout(pending.timeoutHandle);
+    this._callMap.delete(id);
+    entry.busy = false;
+    entry.lastUsed = Date.now();
+    pending.resolve({ outcome, message });
+    // Drain queue
+    this._drainQueue();
+  }
+
+  /** @private */
+  _handleWorkerError(crashedEntry, err) {
+    // Fail only the pending calls that were dispatched to the crashed worker
+    for (const [id, pending] of this._callMap) {
+      if (pending.entry !== crashedEntry) continue;
+      clearTimeout(pending.timeoutHandle);
+      this._callMap.delete(id);
+      const outcome = pending.role === 'write' ? 'block' : 'warn';
+      pending.resolve({ outcome, message: `Verifier worker crashed: ${err.message}` });
+    }
+    if (!this._destroyed) {
+      this._replaceWorker(crashedEntry);
+    }
+  }
+
+  /** @private */
+  _replaceWorker(entry) {
+    const idx = this._workers.indexOf(entry);
+    if (idx === -1) return;
+    try { entry.worker.terminate(); } catch { /* ignore */ }
+    this._workers[idx] = this._createWorkerEntry();
+    this._drainQueue();
+  }
+
+  /** @private */
+  _drainQueue() {
+    while (this._queue.length > 0) {
+      const freeEntry = this._workers.find(e => !e.busy);
+      if (!freeEntry) break;
+      const { callArgs, role, resolve } = this._queue.shift();
+      this._dispatch(freeEntry, callArgs, role, resolve);
+    }
+  }
+
+  /** @private */
+  _dispatch(entry, { id, verifierPath, exportName, toolName, args, result }, role, resolve) {
+    entry.busy = true;
+    entry.lastUsed = Date.now();
+
+    const timeoutHandle = setTimeout(() => {
+      this._callMap.delete(id);
+      const outcome = role === 'write' ? 'block' : 'warn';
+      resolve({ outcome, message: `Verifier timed out after ${this._timeoutMs}ms` });
+      // Replace the worker (stuck in user code)
+      this._replaceWorker(entry);
+    }, this._timeoutMs).unref();
+
+    this._callMap.set(id, { resolve, timeoutHandle, role, entry });
+    entry.worker.postMessage({ id, verifierPath, exportName, toolName, args, result });
+  }
+
+  /**
+   * Run a custom verifier in a worker thread.
+   *
+   * @param {string} verifierPath — absolute path to verifier module
+   * @param {string} exportName — exported function name (e.g. 'verify')
+   * @param {string} toolName
+   * @param {object} args — tool call input
+   * @param {object} result — tool call result
+   * @param {string} [role='any'] — 'read' | 'write' | 'any' (determines timeout outcome)
+   * @returns {Promise<{ outcome: 'pass'|'warn'|'block', message: string|null }>}
+   */
+  run(verifierPath, exportName, toolName, args, result, role = 'any') {
+    if (this._destroyed) {
+      const outcome = role === 'write' ? 'block' : 'warn';
+      return Promise.resolve({ outcome, message: 'Verifier pool is destroyed' });
+    }
+
+    const id = this._nextId++;
+
+    return new Promise((resolve) => {
+      const callArgs = { id, verifierPath, exportName, toolName, args, result };
+
+      // Try to dispatch immediately to a free worker
+      const freeEntry = this._workers.find(e => !e.busy);
+      if (freeEntry) {
+        this._dispatch(freeEntry, callArgs, role, resolve);
+        return;
+      }
+
+      // Queue if under limit
+      if (this._queue.length >= this._maxQueueDepth) {
+        const outcome = role === 'write' ? 'block' : 'warn';
+        resolve({ outcome, message: 'Verifier queue full — request dropped' });
+        return;
+      }
+
+      this._queue.push({ callArgs, role, resolve });
+    });
+  }
+
+  /**
+   * Tear down all workers. Outstanding calls resolve with warn/block.
+   */
+  destroy() {
+    this._destroyed = true;
+
+    // Fail all pending calls
+    for (const [id, pending] of this._callMap) {
+      clearTimeout(pending.timeoutHandle);
+      this._callMap.delete(id);
+      const outcome = pending.role === 'write' ? 'block' : 'warn';
+      pending.resolve({ outcome, message: 'Verifier pool shutting down' });
+    }
+
+    // Drain queue
+    for (const { callArgs, role, resolve } of this._queue) {
+      const outcome = role === 'write' ? 'block' : 'warn';
+      resolve({ outcome, message: 'Verifier pool shutting down' });
+    }
+    this._queue.length = 0;
+
+    // Terminate workers
+    for (const entry of this._workers) {
+      try { entry.worker.terminate(); } catch { /* ignore */ }
+    }
+    this._workers.length = 0;
+  }
+}