agent-tool-forge 0.3.0

package/lib/forge-verifier-generator.js

@@ -0,0 +1,271 @@
+/**
+ * Forge Verifier Generator — generates verifier stub files via LLM.
+ *
+ * Does NOT write files — returns content strings and computed paths so the
+ * caller (forge.js) can preview and confirm before writing.
+ *
+ * @module forge-verifier-generator
+ */
+
+import { llmTurn } from './api-client.js';
+import { inferOutputGroups, getVerifiersForGroups } from './output-groups.js';
+
+// ── Camel-case helper ──────────────────────────────────────────────────────
+
+/**
+ * Convert a snake_case identifier to camelCase.
+ * e.g. "source_attribution" → "sourceAttribution"
+ *
+ * @param {string} name
+ * @returns {string}
+ */
+function _camelCase(name) {
+  return name.replace(/_([a-z])/g, (_, c) => c.toUpperCase());
+}
+
+// ── Verifier order helper ──────────────────────────────────────────────────
+
+/**
+ * Map a verifier name to a default sort-order prefix.
+ * Known wildcard verifiers get 'A' prefix; domain-specific get 'B'.
+ *
+ * @param {string} verifierName
+ * @param {number} index - Position in the list (for uniqueness)
+ * @returns {string} e.g. 'A-0001'
+ */
+function defaultOrder(verifierName, index) {
+  const wildcardVerifiers = new Set(['source_attribution']);
+  const prefix = wildcardVerifiers.has(verifierName) ? 'A' : 'B';
+  return `${prefix}-${String(index + 1).padStart(4, '0')}`;
+}
+
+// ── Prompt builder ─────────────────────────────────────────────────────────
+
+/**
+ * Build the LLM prompt for a single verifier stub.
+ *
+ * @param {object} spec          - Tool specification
+ * @param {string} verifierName  - e.g. 'source_attribution'
+ * @param {string} orderStr      - e.g. 'A-0001'
+ * @returns {string}
+ */
+function buildVerifierPrompt(spec, verifierName, orderStr) {
+  const camelVerifier = _camelCase(verifierName);
+  const camelTool     = _camelCase(spec.name);
+
+  return `Generate a JavaScript ESM verifier stub file for the verifier named '${verifierName}'.
+This verifier applies to the tool: '${spec.name}'
+Tool description: ${spec.description}
+
+The verifier must follow this exact shape:
+
+/**
+ * ${verifierName} verifier — stub for ${spec.name}.
+ * <One-sentence description of what this verifier checks.>
+ */
+
+export const ${camelVerifier}Verifier = {
+  name: '${verifierName}',
+  order: '${orderStr}',
+  description: '<one-sentence description>',
+
+  verify(response, _toolCalls) {
+    // EXTENSION POINT: implement verification logic here
+    // Return { pass: boolean, warnings: string[], flags: string[] }
+    return { pass: true, warnings: [], flags: [] };
+  }
+};
+
+Rules:
+- The file must be a JavaScript ESM module using named export (no default export).
+- The exported const name must be: ${camelVerifier}Verifier
+- The verify() method receives (response, toolCalls) and must return { pass, warnings, flags }.
+- Include a realistic // EXTENSION POINT comment inside verify() showing what the verifier
+  would actually check for this tool's output shape (based on the tool description).
+- Do NOT implement real logic — the body after the comment should return the stub { pass: true, warnings: [], flags: [] }.
+- Add a short JSDoc comment at the top of the file.
+- No imports needed unless a standard library (e.g. path, fs) is genuinely required.
+
+Respond with ONLY the file content as plain text — no markdown fences, no prose.`;
+}
+
+// ── LLM call with retry ────────────────────────────────────────────────────
+
+/**
+ * Call the LLM to generate a single verifier stub.
+ * Retries up to MAX_RETRIES times with corrective nudges.
+ *
+ * @param {object} opts
+ * @param {object} opts.modelConfig   - { provider, apiKey, model }
+ * @param {string} opts.prompt
+ * @param {string} opts.toolName      - For error messages
+ * @param {string} opts.verifierName  - For error messages
+ * @param {number} [opts.maxRetries]
+ * @returns {Promise<string>} Raw file content
+ */
+async function callLlmForVerifier({ modelConfig, prompt, toolName, verifierName, maxRetries = 2 }) {
+  const messages = [{ role: 'user', content: prompt }];
+  let lastError;
+
+  for (let attempt = 1; attempt <= maxRetries; attempt++) {
+    let responseText;
+
+    try {
+      const turn = await llmTurn({
+        provider:  modelConfig.provider,
+        apiKey:    modelConfig.apiKey,
+        model:     modelConfig.model,
+        messages,
+        maxTokens: 2048,
+        timeoutMs: 60_000
+      });
+      responseText = turn.text;
+    } catch (err) {
+      throw new Error(
+        `LLM API call failed while generating verifier "${verifierName}" for tool "${toolName}": ${err.message}`
+      );
+    }
+
+    if (!responseText || responseText.trim() === '') {
+      lastError = new Error(
+        `LLM returned an empty response on attempt ${attempt}/${maxRetries}`
+      );
+      messages.push({ role: 'assistant', content: responseText || '' });
+      messages.push({
+        role: 'user',
+        content:
+          'Your response was empty. Please respond with the full JavaScript ESM verifier file content.'
+      });
+      continue;
+    }
+
+    // Minimal validation: the response must contain the export keyword and the camelCase verifier name
+    const camelVerifier = _camelCase(verifierName) + 'Verifier';
+    if (!responseText.includes('export') || !responseText.includes(camelVerifier)) {
+      lastError = new Error(
+        `Attempt ${attempt}/${maxRetries}: LLM response does not look like a valid verifier file ` +
+        `(missing 'export' or '${camelVerifier}')`
+      );
+      messages.push({ role: 'assistant', content: responseText });
+      messages.push({
+        role: 'user',
+        content:
+          `Your response did not look like a valid ESM verifier file — it must contain an ` +
+          `'export const ${_camelCase(verifierName)}Verifier' declaration. ` +
+          'Please provide the complete file content with no markdown or prose.'
+      });
+      continue;
+    }
+
+    // Strip markdown fences if the model wrapped the output anyway
+    // Try to extract content from a fenced block anywhere in the response
+    const fenceMatch = responseText.match(/```(?:javascript|js)?\s*\n([\s\S]*?)\n?\s*```/i);
+    const stripped = fenceMatch ? fenceMatch[1].trim() : responseText.trim();
+
+    return stripped;
+  }
+
+  throw new Error(
+    `generateVerifiers: failed to obtain valid verifier content for "${verifierName}" ` +
+    `(tool "${toolName}") after ${maxRetries} attempts. Last error: ${lastError?.message}`
+  );
+}
+
+// ── Barrel line builder ────────────────────────────────────────────────────
+
+/**
+ * Build a single ESM named re-export line for the verifier barrel file.
+ *
+ * @param {string} verifierName  - e.g. 'source_attribution'
+ * @param {string} toolName      - e.g. 'get_weather'
+ * @returns {string} e.g. "export { sourceAttributionVerifier } from './get_weather.source_attribution.verifier.js';"
+ */
+function buildBarrelLine(verifierName, toolName) {
+  const camelVerifier  = _camelCase(verifierName);
+  const fileName       = `${toolName}.${verifierName}.verifier.js`;
+  // Barrel line uses a relative path (relative to the barrel file in the same dir)
+  return `export { ${camelVerifier}Verifier } from './${fileName}';`;
+}
+
+// ── Main export ────────────────────────────────────────────────────────────
+
+/**
+ * Generate verifier stub files via LLM.
+ *
+ * Uses output-groups.js to infer which verifiers apply to the tool, then
+ * prompts the LLM to generate a stub file for each verifier.
+ *
+ * Does NOT write to disk. Returns file content and computed paths so the
+ * caller (forge.js) can preview and confirm before writing.
+ *
+ * @param {object}   opts
+ * @param {object}   opts.spec              - Tool specification
+ * @param {string}   opts.spec.name         - Snake_case tool name
+ * @param {string}   opts.spec.description  - Human-readable description
+ * @param {string[]} [opts.spec.tags]       - Used by inferOutputGroups
+ * @param {object}   opts.projectConfig     - forge.config.json contents
+ * @param {string}   opts.projectRoot       - Absolute path to project root
+ * @param {object}   opts.modelConfig       - { provider, apiKey, model }
+ * @param {string}   opts.modelConfig.provider  - 'anthropic' | 'openai'
+ * @param {string}   opts.modelConfig.apiKey
+ * @param {string}   opts.modelConfig.model
+ *
+ * @returns {Promise<{
+ *   verifierFiles: Array<{
+ *     path:       string,
+ *     content:    string,
+ *     barrelLine: string | null
+ *   }>
+ * }>}
+ *
+ * @throws {Error} If LLM returns invalid content after retries for any verifier
+ */
+export async function generateVerifiers({
+  spec,
+  projectConfig,
+  projectRoot,
+  modelConfig
+}) {
+  const verifiersDir = projectConfig?.verification?.verifiersDir || 'example/verifiers';
+
+  // Resolve absolute verifiers directory for path construction
+  const absVerifiersDir = verifiersDir.startsWith('/')
+    ? verifiersDir
+    : `${projectRoot}/${verifiersDir}`;
+
+  // ── Infer applicable verifiers from output-groups ──────────────────────
+  const outputGroups   = inferOutputGroups({ name: spec.name, tags: spec.tags, description: spec.description });
+  const verifierNames  = getVerifiersForGroups(outputGroups);
+
+  if (verifierNames.length === 0) {
+    // No matching verifiers — return empty result rather than failing
+    return { verifierFiles: [] };
+  }
+
+  // ── Generate a stub file for each verifier ─────────────────────────────
+  const verifierFiles = [];
+  const safeName = (spec.name || 'unnamed').replace(/[^a-zA-Z0-9_-]/g, '_');
+
+  for (let i = 0; i < verifierNames.length; i++) {
+    const verifierName = verifierNames[i];
+    const orderStr     = defaultOrder(verifierName, i);
+    const filePath     = `${absVerifiersDir}/${safeName}.${verifierName}.verifier.js`;
+    const barrelLine   = buildBarrelLine(verifierName, safeName);
+
+    const prompt  = buildVerifierPrompt(spec, verifierName, orderStr);
+    const content = await callLlmForVerifier({
+      modelConfig,
+      prompt,
+      toolName:     spec.name,
+      verifierName
+    });
+
+    verifierFiles.push({
+      path: filePath,
+      content,
+      barrelLine
+    });
+  }
+
+  return { verifierFiles };
+}

package/lib/handlers/admin.js

@@ -0,0 +1,151 @@
+/**
+ * Admin API — runtime config updates for app-owners.
+ *
+ * PUT /forge-admin/config/:section — update a config section
+ * GET /forge-admin/config          — read current effective config
+ *
+ * Protected by adminKey (Bearer token).
+ * Runtime overlay: in-memory Map merged on top of file config.
+ * NOT written back to forge.config.json.
+ */
+
+import { readFileSync, writeFileSync, renameSync } from 'fs';
+import { authenticateAdmin } from '../auth.js';
+import { readBody, sendJson } from '../http-utils.js';
+
+const VALID_SECTIONS = ['model', 'hitl', 'permissions', 'conversation'];
+
+// Runtime overlay — survives across requests but not restarts
+const runtimeOverlay = new Map();
+
+/**
+ * PUT /forge-admin/config/:section
+ */
+export async function handleAdminConfig(req, res, ctx) {
+  const url = new URL(req.url, 'http://localhost');
+
+  // Admin auth
+  const adminKey = ctx.config.adminKey;
+  if (!adminKey) {
+    sendJson(res, 503, { error: 'No adminKey configured' });
+    return;
+  }
+  const authResult = authenticateAdmin(req, adminKey);
+  if (!authResult.authenticated) {
+    sendJson(res, 403, { error: authResult.error ?? 'Forbidden' });
+    return;
+  }
+
+  if (req.method === 'GET') {
+    return handleAdminConfigGet(req, res, ctx);
+  }
+
+  if (req.method === 'PUT') {
+    const pathParts = url.pathname.split('/').filter(Boolean);
+    // /forge-admin/config/:section → pathParts = ['forge-admin', 'config', section]
+    const section = pathParts[2];
+
+    if (!section || !VALID_SECTIONS.includes(section)) {
+      sendJson(res, 400, { error: `Invalid section. Must be one of: ${VALID_SECTIONS.join(', ')}` });
+      return;
+    }
+
+    const body = await readBody(req);
+    runtimeOverlay.set(section, body);
+
+    // Apply overlay to live config
+    applyOverlay(ctx.config, section, body);
+
+    // Persist overlay change to forge.config.json (atomic write)
+    if (ctx.configPath) {
+      persistOverlay(ctx.configPath, section, body);
+    }
+
+    sendJson(res, 200, { ok: true, section, applied: body });
+    return;
+  }
+
+  sendJson(res, 405, { error: 'Method not allowed' });
+}
+
+/**
+ * GET /forge-admin/config
+ */
+function handleAdminConfigGet(req, res, ctx) {
+  const effective = { ...ctx.config };
+  // Merge runtime overlay
+  for (const [section, values] of runtimeOverlay) {
+    applyOverlay(effective, section, values);
+  }
+  sendJson(res, 200, effective);
+}
+
+function applyOverlay(config, section, values) {
+  switch (section) {
+    case 'model':
+      if (values.defaultModel) config.defaultModel = values.defaultModel;
+      break;
+    case 'hitl':
+      if (values.defaultHitlLevel) config.defaultHitlLevel = values.defaultHitlLevel;
+      break;
+    case 'permissions':
+      if (values.allowUserModelSelect !== undefined) config.allowUserModelSelect = values.allowUserModelSelect;
+      if (values.allowUserHitlConfig !== undefined) config.allowUserHitlConfig = values.allowUserHitlConfig;
+      break;
+    case 'conversation':
+      if (values.window) config.conversation = { ...config.conversation, window: values.window };
+      break;
+  }
+}
+
+/**
+ * Reset runtime overlay — used by tests.
+ */
+export function _resetOverlay() {
+  runtimeOverlay.clear();
+}
+
+/**
+ * Atomically persist a section update to forge.config.json.
+ * Reads the existing file (or starts from {}), applies the same field
+ * mapping as applyOverlay, writes to a .tmp file, then renames atomically.
+ * If write fails (permissions, disk full), logs a warning but does NOT throw —
+ * the runtime overlay is still active for this session.
+ *
+ * @param {string} configPath — absolute path to forge.config.json
+ * @param {string} section — config section name
+ * @param {object} values — values to merge into the section
+ */
+function persistOverlay(configPath, section, values) {
+  try {
+    let existing = {};
+    try {
+      existing = JSON.parse(readFileSync(configPath, 'utf8'));
+    } catch {
+      // File missing or unreadable — start from empty object
+    }
+
+    // Apply the same field mapping as applyOverlay
+    switch (section) {
+      case 'model':
+        if (values.defaultModel) existing.defaultModel = values.defaultModel;
+        break;
+      case 'hitl':
+        if (values.defaultHitlLevel) existing.defaultHitlLevel = values.defaultHitlLevel;
+        break;
+      case 'permissions':
+        if (values.allowUserModelSelect !== undefined) existing.allowUserModelSelect = values.allowUserModelSelect;
+        if (values.allowUserHitlConfig !== undefined) existing.allowUserHitlConfig = values.allowUserHitlConfig;
+        break;
+      case 'conversation':
+        if (values.window) existing.conversation = { ...(existing.conversation ?? {}), window: values.window };
+        break;
+    }
+
+    const tmpPath = configPath + '.tmp';
+    writeFileSync(tmpPath, JSON.stringify(existing, null, 2), 'utf8');
+    renameSync(tmpPath, configPath);
+  } catch (err) {
+    process.stderr.write(`[admin] Warning: could not persist config overlay to ${configPath}: ${err.message}\n`);
+  }
+}

package/lib/handlers/agents.js

@@ -0,0 +1,229 @@
+/**
+ * Admin Agent API — CRUD for multi-agent registry.
+ *
+ * All routes require adminKey (Bearer token).
+ *
+ * Routes:
+ *   GET    /forge-admin/agents              — list all agents
+ *   GET    /forge-admin/agents/:agentId     — get one
+ *   POST   /forge-admin/agents              — create
+ *   PUT    /forge-admin/agents/:agentId     — update
+ *   DELETE /forge-admin/agents/:agentId     — delete
+ *   POST   /forge-admin/agents/:agentId/set-default — set default
+ */
+
+import { authenticateAdmin } from '../auth.js';
+import { readBody, sendJson } from '../http-utils.js';
+
+const AGENT_ID_RE = /^[a-z0-9_-]{1,64}$/;
+const VALID_HITL_LEVELS = new Set(['autonomous', 'cautious', 'standard', 'paranoid']);
+
+/**
+ * @param {import('http').IncomingMessage} req
+ * @param {import('http').ServerResponse} res
+ * @param {object} ctx — { config, agentRegistry }
+ */
+export async function handleAgents(req, res, ctx) {
+  const { config, agentRegistry } = ctx;
+
+  // Admin auth
+  const adminKey = config.adminKey;
+  if (!adminKey) {
+    sendJson(res, 503, { error: 'No adminKey configured' });
+    return;
+  }
+  const authResult = authenticateAdmin(req, adminKey);
+  if (!authResult.authenticated) {
+    res.setHeader('WWW-Authenticate', 'Bearer');
+    sendJson(res, 401, { error: 'Unauthorized' });
+    return;
+  }
+
+  if (!agentRegistry) {
+    sendJson(res, 501, { error: 'Agent registry not initialized' });
+    return;
+  }
+
+  const url = new URL(req.url, 'http://localhost');
+  // /forge-admin/agents/:agentId/set-default or /forge-admin/agents/:agentId or /forge-admin/agents
+  const pathParts = url.pathname.split('/').filter(Boolean);
+  // pathParts: ['forge-admin', 'agents', agentId?, 'set-default'?]
+  const agentId = pathParts[2] || null;
+  const action = pathParts[3] || null;
+
+  if (agentId && !AGENT_ID_RE.test(agentId)) {
+    sendJson(res, 400, { error: 'Invalid agent ID format' });
+    return;
+  }
+
+  // POST /forge-admin/agents/:agentId/set-default
+  if (req.method === 'POST' && agentId && action === 'set-default') {
+    const existing = await agentRegistry.getAgent(agentId);
+    if (!existing) {
+      sendJson(res, 404, { error: `Agent "${agentId}" not found` });
+      return;
+    }
+    await agentRegistry.setDefault(agentId);
+    sendJson(res, 200, { ok: true, agentId });
+    return;
+  }
+
+  // GET /forge-admin/agents
+  if (req.method === 'GET' && !agentId) {
+    const agents = await agentRegistry.getAllAgents();
+    sendJson(res, 200, { agents });
+    return;
+  }
+
+  // GET /forge-admin/agents/:agentId
+  if (req.method === 'GET' && agentId) {
+    const agent = await agentRegistry.getAgent(agentId);
+    if (!agent) {
+      sendJson(res, 404, { error: `Agent "${agentId}" not found` });
+      return;
+    }
+    sendJson(res, 200, agent);
+    return;
+  }
+
+  // POST /forge-admin/agents — create
+  if (req.method === 'POST' && !agentId) {
+    const body = await readBody(req);
+    const err = validateAgentBody(body, true);
+    if (err) {
+      sendJson(res, 400, { error: err });
+      return;
+    }
+
+    // Check for duplicates
+    if (await agentRegistry.getAgent(body.id)) {
+      sendJson(res, 409, { error: `Agent "${body.id}" already exists` });
+      return;
+    }
+
+    await agentRegistry.upsertAgent(bodyToRow(body));
+    sendJson(res, 201, await agentRegistry.getAgent(body.id));
+    return;
+  }
+
+  // PUT /forge-admin/agents/:agentId — update
+  if (req.method === 'PUT' && agentId) {
+    const existing = await agentRegistry.getAgent(agentId);
+    if (!existing) {
+      sendJson(res, 404, { error: `Agent "${agentId}" not found` });
+      return;
+    }
+
+    const body = await readBody(req);
+    const err = validateAgentBody(body, false);
+    if (err) {
+      sendJson(res, 400, { error: err });
+      return;
+    }
+
+    // Merge: existing values as base, body overrides. Mark as admin-edited (seeded_from_config=0).
+    const row = bodyToRow({ ...rowToBody(existing), ...body, id: agentId, seeded_from_config: 0 });
+    await agentRegistry.upsertAgent(row);
+    sendJson(res, 200, await agentRegistry.getAgent(agentId));
+    return;
+  }
+
+  // DELETE /forge-admin/agents/:agentId
+  if (req.method === 'DELETE' && agentId) {
+    const existing = await agentRegistry.getAgent(agentId);
+    if (!existing) {
+      sendJson(res, 404, { error: `Agent "${agentId}" not found` });
+      return;
+    }
+    await agentRegistry.deleteAgent(agentId);
+    // If we deleted the default, auto-promote the first remaining enabled agent
+    if (existing.is_default) {
+      const remaining = (await agentRegistry.getAllAgents()).filter(a => a.enabled);
+      if (remaining.length > 0) {
+        await agentRegistry.setDefault(remaining[0].agent_id);
+      }
+    }
+    sendJson(res, 200, { ok: true, deleted: agentId });
+    return;
+  }
+
+  sendJson(res, 405, { error: 'Method not allowed' });
+}
+
+/**
+ * Validate an agent request body.
+ * @param {object} body
+ * @param {boolean} isCreate — if true, id and displayName are required
+ * @returns {string|null} error message or null
+ */
+function validateAgentBody(body, isCreate) {
+  if (isCreate) {
+    if (!body.id || typeof body.id !== 'string' || !AGENT_ID_RE.test(body.id)) {
+      return 'id is required and must match /^[a-z0-9_-]{1,64}$/';
+    }
+    if (!body.displayName || typeof body.displayName !== 'string') {
+      return 'displayName is required';
+    }
+  }
+  if (body.defaultHitlLevel !== undefined && !VALID_HITL_LEVELS.has(body.defaultHitlLevel)) {
+    return `defaultHitlLevel must be one of: ${[...VALID_HITL_LEVELS].join(', ')}`;
+  }
+  if (body.toolAllowlist !== undefined && body.toolAllowlist !== '*' && !Array.isArray(body.toolAllowlist)) {
+    return 'toolAllowlist must be "*" or an array of tool names';
+  }
+  if (body.maxTurns !== undefined && (typeof body.maxTurns !== 'number' || body.maxTurns < 1 || !Number.isInteger(body.maxTurns))) {
+    return 'maxTurns must be a positive integer';
+  }
+  if (body.maxTokens !== undefined && (typeof body.maxTokens !== 'number' || body.maxTokens < 1 || !Number.isInteger(body.maxTokens))) {
+    return 'maxTokens must be a positive integer';
+  }
+  return null;
+}
+
+/**
+ * Convert API request body to DB row format.
+ */
+function bodyToRow(body) {
+  return {
+    agent_id: body.id,
+    display_name: body.displayName,
+    description: body.description ?? null,
+    system_prompt: body.systemPrompt ?? null,
+    default_model: body.defaultModel ?? null,
+    default_hitl_level: body.defaultHitlLevel ?? null,
+    allow_user_model_select: body.allowUserModelSelect ? 1 : 0,
+    allow_user_hitl_config: body.allowUserHitlConfig ? 1 : 0,
+    tool_allowlist: Array.isArray(body.toolAllowlist) ? JSON.stringify(body.toolAllowlist) : (body.toolAllowlist ?? '*'),
+    max_turns: body.maxTurns ?? null,
+    max_tokens: body.maxTokens ?? null,
+    is_default: body.isDefault ? 1 : 0,
+    enabled: body.enabled !== undefined ? (body.enabled ? 1 : 0) : 1,
+    seeded_from_config: body.seeded_from_config ?? 0,
+  };
+}
+
+/**
+ * Convert DB row to API body format (for merge on update).
+ */
+function rowToBody(row) {
+  let toolAllowlist = row.tool_allowlist;
+  if (toolAllowlist && toolAllowlist !== '*') {
+    try { toolAllowlist = JSON.parse(toolAllowlist); } catch { /* keep string */ }
+  }
+  return {
+    id: row.agent_id,
+    displayName: row.display_name,
+    description: row.description,
+    systemPrompt: row.system_prompt,
+    defaultModel: row.default_model,
+    defaultHitlLevel: row.default_hitl_level,
+    allowUserModelSelect: !!row.allow_user_model_select,
+    allowUserHitlConfig: !!row.allow_user_hitl_config,
+    toolAllowlist,
+    maxTurns: row.max_turns,
+    maxTokens: row.max_tokens,
+    isDefault: !!row.is_default,
+    enabled: !!row.enabled,
+    seeded_from_config: Boolean(row.seeded_from_config)
+  };
+}