@hachej/boring-agent 0.1.0

package/dist/eval/index.js

@@ -0,0 +1,508 @@
+// src/eval/evalPrompt.ts
+import { randomUUID } from "crypto";
+
+// src/eval/types.ts
+var EvalAny = /* @__PURE__ */ Symbol.for("@hachej/boring-agent/eval/EvalAny");
+function EvalRegex(re) {
+  return { __evalRegex: typeof re === "string" ? new RegExp(re) : re };
+}
+function isEvalRegex(value) {
+  return typeof value === "object" && value !== null && "__evalRegex" in value && value.__evalRegex instanceof RegExp;
+}
+
+// src/eval/matcher.ts
+function callSatisfies(expected, actual) {
+  if (expected.tool !== actual.tool) {
+    return {
+      ok: false,
+      matchedIndex: -1,
+      reason: `tool mismatch: expected "${expected.tool}", got "${actual.tool}"`
+    };
+  }
+  if (!expected.params) {
+    return { ok: true, matchedIndex: 0 };
+  }
+  const paramsCheck = matchParams(expected.params, actual.params, expected.strict ?? false);
+  if (!paramsCheck.ok) {
+    return { ok: false, matchedIndex: -1, reason: paramsCheck.reason };
+  }
+  return { ok: true, matchedIndex: 0 };
+}
+function someCallMatches(expected, actual) {
+  const expectedList = Array.isArray(expected) ? expected : [expected];
+  for (const exp of expectedList) {
+    let matched = false;
+    let lastReason;
+    for (const cand of actual) {
+      const out = callSatisfies(exp, cand);
+      if (out.ok) {
+        matched = true;
+        break;
+      }
+      lastReason = out.reason;
+    }
+    if (!matched) {
+      return {
+        ok: false,
+        matchedIndex: -1,
+        reason: lastReason ? `no actual call matched expected ${formatExpected(exp)} \u2014 closest mismatch: ${lastReason}` : `no actual call matched expected ${formatExpected(exp)} (actual was empty)`
+      };
+    }
+  }
+  return { ok: true, matchedIndex: 0 };
+}
+function firstCallMatches(expected, actual) {
+  if (actual.length === 0) {
+    return {
+      ok: false,
+      matchedIndex: -1,
+      reason: `expectFirst: no tool calls captured (LLM answered in plain text)`
+    };
+  }
+  return callSatisfies(expected, actual[0]);
+}
+function noToolCallMatches(actual) {
+  if (actual.length === 0) {
+    return { ok: true, matchedIndex: -1 };
+  }
+  return {
+    ok: false,
+    matchedIndex: -1,
+    reason: `expectNoToolCall: ${actual.length} tool call(s) captured: ${actual.map((c) => c.tool).join(", ")}`
+  };
+}
+function matchParams(expected, actual, strict, pathPrefix = "") {
+  for (const key of Object.keys(expected)) {
+    const expVal = expected[key];
+    const actVal = actual[key];
+    const path = pathPrefix ? `${pathPrefix}.${key}` : key;
+    if (expVal === EvalAny || isEvalAnySymbol(expVal)) {
+      if (actVal === void 0) {
+        return { ok: false, reason: `${path}: missing (expected !EvalAny)` };
+      }
+      continue;
+    }
+    if (isEvalRegex(expVal)) {
+      if (typeof actVal !== "string") {
+        return {
+          ok: false,
+          reason: `${path}: expected string for !EvalRegex, got ${typeof actVal}`
+        };
+      }
+      if (!expVal.__evalRegex.test(actVal)) {
+        return {
+          ok: false,
+          reason: `${path}: ${JSON.stringify(actVal)} does not match /${expVal.__evalRegex.source}/`
+        };
+      }
+      continue;
+    }
+    if (typeof expVal === "object" && expVal !== null && !Array.isArray(expVal)) {
+      if (typeof actVal !== "object" || actVal === null || Array.isArray(actVal)) {
+        return {
+          ok: false,
+          reason: `${path}: expected object, got ${describe(actVal)}`
+        };
+      }
+      const nested = matchParams(
+        expVal,
+        actVal,
+        strict,
+        path
+      );
+      if (!nested.ok) return nested;
+      continue;
+    }
+    if (Array.isArray(expVal)) {
+      if (!Array.isArray(actVal)) {
+        return {
+          ok: false,
+          reason: `${path}: expected array, got ${describe(actVal)}`
+        };
+      }
+      if (expVal.length !== actVal.length) {
+        return {
+          ok: false,
+          reason: `${path}: array length mismatch (expected ${expVal.length}, got ${actVal.length})`
+        };
+      }
+      for (let i = 0; i < expVal.length; i++) {
+        const elemMatch = matchParams(
+          { item: expVal[i] },
+          { item: actVal[i] },
+          strict,
+          `${path}[${i}]`
+        );
+        if (!elemMatch.ok) return elemMatch;
+      }
+      continue;
+    }
+    if (expVal !== actVal) {
+      return {
+        ok: false,
+        reason: `${path}: expected ${JSON.stringify(expVal)}, got ${JSON.stringify(actVal)}`
+      };
+    }
+  }
+  if (strict) {
+    for (const key of Object.keys(actual)) {
+      if (!(key in expected)) {
+        const path = pathPrefix ? `${pathPrefix}.${key}` : key;
+        return {
+          ok: false,
+          reason: `${path}: unexpected key (strict mode)`
+        };
+      }
+    }
+  }
+  return { ok: true };
+}
+function isEvalAnySymbol(value) {
+  return value === EvalAny;
+}
+function describe(value) {
+  if (value === null) return "null";
+  if (Array.isArray(value)) return "array";
+  if (value === void 0) return "undefined";
+  return typeof value;
+}
+function formatExpected(exp) {
+  if (!exp.params) return `{tool: ${exp.tool}}`;
+  return `{tool: ${exp.tool}, params: ${safeStringify(exp.params)}}`;
+}
+function safeStringify(value) {
+  try {
+    return JSON.stringify(value, (_k, v) => {
+      if (v === EvalAny) return "<EvalAny>";
+      if (isEvalRegex(v)) return `<EvalRegex /${v.__evalRegex.source}/>`;
+      return v;
+    });
+  } catch {
+    return "<unserializable>";
+  }
+}
+
+// src/eval/evalConfig.ts
+var DEFAULT_EVAL_MODEL = { provider: "openrouter", id: "qwen/qwen3.6-plus" };
+var DEFAULT_TIMEOUT_MS = 3e4;
+var DEFAULT_SUITE_TIMEOUT_MS = 5 * 6e4;
+var DEFAULT_CONCURRENCY = 4;
+
+// src/eval/evalPrompt.ts
+async function evalAgentPrompt(opts) {
+  validateMutuallyExclusive(opts);
+  const sessionId = opts.sessionId ?? randomUUID();
+  const model = parseModelString(opts.model ?? DEFAULT_EVAL_MODEL);
+  const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+  const maxRetries = opts.retries ?? 0;
+  let attempts = 0;
+  let lastResult = null;
+  while (attempts <= maxRetries) {
+    attempts += 1;
+    let captured;
+    try {
+      captured = await runOnce(opts.app, {
+        sessionId: `${sessionId}-attempt-${attempts}`,
+        prompt: opts.prompt,
+        systemPrompt: opts.systemPrompt,
+        model,
+        timeoutMs
+      });
+    } catch (err) {
+      lastResult = {
+        ok: false,
+        actual: [],
+        text: "",
+        attempts,
+        reason: `transport: ${err instanceof Error ? err.message : String(err)}`
+      };
+      if (attempts > maxRetries) return lastResult;
+      continue;
+    }
+    const matchResult = matchExpectations(opts, captured.toolCalls);
+    lastResult = {
+      ok: matchResult.ok,
+      actual: captured.toolCalls,
+      text: captured.text,
+      usage: captured.usage,
+      attempts,
+      reason: matchResult.ok ? void 0 : matchResult.reason
+    };
+    if (matchResult.ok) return lastResult;
+  }
+  return lastResult ?? {
+    ok: false,
+    actual: [],
+    text: "",
+    attempts,
+    reason: "no attempts ran"
+  };
+}
+function validateMutuallyExclusive(opts) {
+  const set = [
+    opts.expect !== void 0,
+    opts.expectFirst !== void 0,
+    opts.expectNoToolCall === true
+  ].filter(Boolean).length;
+  if (set === 0) {
+    throw new Error(
+      "evalAgentPrompt: provide one of `expect`, `expectFirst`, or `expectNoToolCall: true`"
+    );
+  }
+  if (set > 1) {
+    throw new Error(
+      "evalAgentPrompt: `expect`, `expectFirst`, and `expectNoToolCall` are mutually exclusive"
+    );
+  }
+}
+function matchExpectations(opts, actual) {
+  if (opts.expectNoToolCall) {
+    return noToolCallMatches(actual);
+  }
+  if (opts.expectFirst) {
+    return firstCallMatches(opts.expectFirst, actual);
+  }
+  return someCallMatches(opts.expect, actual);
+}
+async function runOnce(app, opts) {
+  await app.inject({
+    method: "POST",
+    url: "/api/v1/agent/sessions",
+    payload: { id: opts.sessionId }
+  });
+  const userMessage = opts.systemPrompt ? `[SYSTEM]
+${opts.systemPrompt}
+[/SYSTEM]
+
+${opts.prompt}` : opts.prompt;
+  let res;
+  try {
+    res = await withTimeout(
+      app.inject({
+        method: "POST",
+        url: "/api/v1/agent/chat",
+        payload: {
+          sessionId: opts.sessionId,
+          message: userMessage,
+          model: opts.model
+        }
+      }),
+      opts.timeoutMs,
+      `chat request exceeded ${opts.timeoutMs}ms`
+    );
+  } finally {
+    void app.inject({
+      method: "DELETE",
+      url: `/api/v1/agent/sessions/${encodeURIComponent(opts.sessionId)}`
+    }).catch(() => {
+    });
+  }
+  if (res.statusCode !== 200) {
+    throw new Error(
+      `chat returned ${res.statusCode}: ${res.body.slice(0, 256)}`
+    );
+  }
+  return parseSseStream(res.body);
+}
+function parseSseStream(body) {
+  const toolCalls = [];
+  const textParts = [];
+  let usage;
+  for (const line of body.split("\n")) {
+    const trimmed = line.trim();
+    if (!trimmed.startsWith("data:")) continue;
+    const payload = trimmed.slice("data:".length).trim();
+    if (!payload || payload === "[DONE]") continue;
+    let chunk;
+    try {
+      chunk = JSON.parse(payload);
+    } catch {
+      continue;
+    }
+    const type = chunk.type;
+    if (type === "tool-input-available") {
+      const toolName = chunk.toolName;
+      if (typeof toolName !== "string") continue;
+      const input = chunk.input;
+      const params = input && typeof input === "object" && !Array.isArray(input) ? input : {};
+      toolCalls.push({ tool: toolName, params });
+    } else if (type === "text-delta") {
+      const delta = chunk.delta;
+      if (typeof delta === "string") textParts.push(delta);
+    } else if (type === "data-usage") {
+      const data = chunk.data;
+      if (data && typeof data === "object") {
+        const obj = data;
+        if (typeof obj.input === "number" && typeof obj.output === "number") {
+          usage = { input: obj.input, output: obj.output };
+        }
+      }
+    }
+  }
+  return { toolCalls, text: textParts.join(""), usage };
+}
+function withTimeout(p, ms, label) {
+  return new Promise((resolve, reject) => {
+    const timer = setTimeout(() => reject(new Error(label)), ms);
+    p.then(
+      (v) => {
+        clearTimeout(timer);
+        resolve(v);
+      },
+      (err) => {
+        clearTimeout(timer);
+        reject(err);
+      }
+    );
+  });
+}
+function parseModelString(input) {
+  if (typeof input === "object") return input;
+  if (input.startsWith("claude-")) return { provider: "anthropic", id: input };
+  if (input.startsWith("gpt-")) return { provider: "openai", id: input };
+  if (input.startsWith("qwen/")) return { provider: "openrouter", id: input };
+  return { provider: "anthropic", id: input };
+}
+
+// src/eval/runSuite.ts
+import { readFile } from "fs/promises";
+
+// src/eval/yamlSchema.ts
+import { parse, Schema } from "yaml";
+var evalSchema = new Schema({
+  customTags: [
+    {
+      tag: "!EvalAny",
+      // No value — `!EvalAny` alone resolves to the symbol.
+      identify: (value) => value === EvalAny,
+      resolve: () => EvalAny
+    },
+    {
+      tag: "!EvalRegex",
+      // Scalar value — `!EvalRegex "^chart:"` resolves to a matcher object.
+      identify: (value) => typeof value === "object" && value !== null && "__evalRegex" in value,
+      resolve: (str) => EvalRegex(str)
+    }
+  ]
+});
+function parseFixtureYaml(text) {
+  const parsed = parse(text, { schema: evalSchema });
+  if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
+    throw new Error(
+      `Eval fixture must be a YAML object with a "prompts" key. Got: ${describe2(parsed)}`
+    );
+  }
+  const obj = parsed;
+  if (!Array.isArray(obj.prompts)) {
+    throw new Error(`Eval fixture missing "prompts" array.`);
+  }
+  return obj;
+}
+function describe2(value) {
+  if (value === null) return "null";
+  if (value === void 0) return "undefined";
+  if (Array.isArray(value)) return "array";
+  return typeof value;
+}
+
+// src/eval/runSuite.ts
+async function runEvalSuite(opts) {
+  const fixtureText = await readFile(opts.fixturesPath, "utf8");
+  const fixture = parseFixtureYaml(fixtureText);
+  const concurrency = opts.concurrency ?? DEFAULT_CONCURRENCY;
+  const suiteTimeoutMs = opts.suiteTimeoutMs ?? DEFAULT_SUITE_TIMEOUT_MS;
+  const start = Date.now();
+  const deadline = start + suiteTimeoutMs;
+  const queue = [...fixture.prompts];
+  const results = [];
+  let bailed = false;
+  async function worker() {
+    while (queue.length > 0 && !bailed) {
+      if (Date.now() > deadline) return;
+      const prompt = queue.shift();
+      if (!prompt) return;
+      const result = await runOnePrompt(opts, fixture, prompt);
+      results.push({
+        ...result,
+        prompt: prompt.prompt,
+        expected: prompt
+      });
+      if (!result.ok && opts.bail) {
+        bailed = true;
+        return;
+      }
+    }
+  }
+  const workers = Array.from({ length: concurrency }, () => worker());
+  await Promise.race([
+    Promise.all(workers),
+    new Promise(
+      (_, reject) => setTimeout(
+        () => reject(new Error(`suite exceeded ${suiteTimeoutMs}ms`)),
+        suiteTimeoutMs
+      )
+    )
+  ]).catch((err) => {
+    results.push({
+      ok: false,
+      actual: [],
+      text: "",
+      attempts: 0,
+      reason: err instanceof Error ? err.message : String(err),
+      prompt: "(suite-level timeout)",
+      expected: { prompt: "(suite-level timeout)" }
+    });
+  });
+  const totalUsage = results.reduce(
+    (acc, r) => ({
+      input: acc.input + (r.usage?.input ?? 0),
+      output: acc.output + (r.usage?.output ?? 0)
+    }),
+    { input: 0, output: 0 }
+  );
+  const passed = results.filter((r) => r.ok).length;
+  const total = results.length;
+  const passRate = total === 0 ? 0 : passed / total;
+  return {
+    total,
+    passed,
+    failed: total - passed,
+    passRate,
+    results,
+    totalUsage,
+    totalDurationMs: Date.now() - start,
+    allPassed: total > 0 && passed === total
+  };
+}
+async function runOnePrompt(opts, fixture, p) {
+  const model = opts.model ?? p.model ?? fixture.defaults?.model ?? fixture.model;
+  const retries = p.retries ?? fixture.defaults?.retries ?? 0;
+  const timeoutMs = p.timeoutMs ?? fixture.defaults?.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+  return evalAgentPrompt({
+    app: opts.app,
+    prompt: p.prompt,
+    expect: p.expect,
+    expectFirst: p.expectFirst,
+    expectNoToolCall: p.expectNoToolCall,
+    systemPrompt: fixture.systemPrompt,
+    model,
+    retries,
+    timeoutMs
+  });
+}
+export {
+  DEFAULT_CONCURRENCY,
+  DEFAULT_EVAL_MODEL,
+  DEFAULT_SUITE_TIMEOUT_MS,
+  DEFAULT_TIMEOUT_MS,
+  EvalAny,
+  EvalRegex,
+  callSatisfies,
+  evalAgentPrompt,
+  firstCallMatches,
+  isEvalRegex,
+  noToolCallMatches,
+  parseFixtureYaml,
+  runEvalSuite,
+  someCallMatches
+};