@hachej/boring-agent 0.1.111 → 0.2.0

package/dist/eval/index.js

@@ -1,541 +0,0 @@
-// src/eval/evalPrompt.ts
-import { randomUUID } from "node:crypto";
-
-// src/eval/types.ts
-var EvalAny = /* @__PURE__ */ Symbol.for("@hachej/boring-agent/eval/EvalAny");
-function EvalRegex(re) {
-  return { __evalRegex: typeof re === "string" ? new RegExp(re) : re };
-}
-function isEvalRegex(value) {
-  return typeof value === "object" && value !== null && "__evalRegex" in value && value.__evalRegex instanceof RegExp;
-}
-
-// src/eval/matcher.ts
-function callSatisfies(expected, actual) {
-  if (expected.tool !== actual.tool) {
-    return {
-      ok: false,
-      matchedIndex: -1,
-      reason: `tool mismatch: expected "${expected.tool}", got "${actual.tool}"`
-    };
-  }
-  if (!expected.params) {
-    return { ok: true, matchedIndex: 0 };
-  }
-  const paramsCheck = matchParams(expected.params, actual.params, expected.strict ?? false);
-  if (!paramsCheck.ok) {
-    return { ok: false, matchedIndex: -1, reason: paramsCheck.reason };
-  }
-  return { ok: true, matchedIndex: 0 };
-}
-function someCallMatches(expected, actual) {
-  const expectedList = Array.isArray(expected) ? expected : [expected];
-  for (const exp of expectedList) {
-    let matched = false;
-    let lastReason;
-    for (const cand of actual) {
-      const out = callSatisfies(exp, cand);
-      if (out.ok) {
-        matched = true;
-        break;
-      }
-      lastReason = out.reason;
-    }
-    if (!matched) {
-      return {
-        ok: false,
-        matchedIndex: -1,
-        reason: lastReason ? `no actual call matched expected ${formatExpected(exp)} \u2014 closest mismatch: ${lastReason}` : `no actual call matched expected ${formatExpected(exp)} (actual was empty)`
-      };
-    }
-  }
-  return { ok: true, matchedIndex: 0 };
-}
-function firstCallMatches(expected, actual) {
-  if (actual.length === 0) {
-    return {
-      ok: false,
-      matchedIndex: -1,
-      reason: `expectFirst: no tool calls captured (LLM answered in plain text)`
-    };
-  }
-  return callSatisfies(expected, actual[0]);
-}
-function noToolCallMatches(actual) {
-  if (actual.length === 0) {
-    return { ok: true, matchedIndex: -1 };
-  }
-  return {
-    ok: false,
-    matchedIndex: -1,
-    reason: `expectNoToolCall: ${actual.length} tool call(s) captured: ${actual.map((c) => c.tool).join(", ")}`
-  };
-}
-function matchParams(expected, actual, strict, pathPrefix = "") {
-  for (const key of Object.keys(expected)) {
-    const expVal = expected[key];
-    const actVal = actual[key];
-    const path = pathPrefix ? `${pathPrefix}.${key}` : key;
-    if (expVal === EvalAny || isEvalAnySymbol(expVal)) {
-      if (actVal === void 0) {
-        return { ok: false, reason: `${path}: missing (expected !EvalAny)` };
-      }
-      continue;
-    }
-    if (isEvalRegex(expVal)) {
-      if (typeof actVal !== "string") {
-        return {
-          ok: false,
-          reason: `${path}: expected string for !EvalRegex, got ${typeof actVal}`
-        };
-      }
-      if (!expVal.__evalRegex.test(actVal)) {
-        return {
-          ok: false,
-          reason: `${path}: ${JSON.stringify(actVal)} does not match /${expVal.__evalRegex.source}/`
-        };
-      }
-      continue;
-    }
-    if (typeof expVal === "object" && expVal !== null && !Array.isArray(expVal)) {
-      if (typeof actVal !== "object" || actVal === null || Array.isArray(actVal)) {
-        return {
-          ok: false,
-          reason: `${path}: expected object, got ${describe(actVal)}`
-        };
-      }
-      const nested = matchParams(
-        expVal,
-        actVal,
-        strict,
-        path
-      );
-      if (!nested.ok) return nested;
-      continue;
-    }
-    if (Array.isArray(expVal)) {
-      if (!Array.isArray(actVal)) {
-        return {
-          ok: false,
-          reason: `${path}: expected array, got ${describe(actVal)}`
-        };
-      }
-      if (expVal.length !== actVal.length) {
-        return {
-          ok: false,
-          reason: `${path}: array length mismatch (expected ${expVal.length}, got ${actVal.length})`
-        };
-      }
-      for (let i = 0; i < expVal.length; i++) {
-        const elemMatch = matchParams(
-          { item: expVal[i] },
-          { item: actVal[i] },
-          strict,
-          `${path}[${i}]`
-        );
-        if (!elemMatch.ok) return elemMatch;
-      }
-      continue;
-    }
-    if (expVal !== actVal) {
-      return {
-        ok: false,
-        reason: `${path}: expected ${JSON.stringify(expVal)}, got ${JSON.stringify(actVal)}`
-      };
-    }
-  }
-  if (strict) {
-    for (const key of Object.keys(actual)) {
-      if (!(key in expected)) {
-        const path = pathPrefix ? `${pathPrefix}.${key}` : key;
-        return {
-          ok: false,
-          reason: `${path}: unexpected key (strict mode)`
-        };
-      }
-    }
-  }
-  return { ok: true };
-}
-function isEvalAnySymbol(value) {
-  return value === EvalAny;
-}
-function describe(value) {
-  if (value === null) return "null";
-  if (Array.isArray(value)) return "array";
-  if (value === void 0) return "undefined";
-  return typeof value;
-}
-function formatExpected(exp) {
-  if (!exp.params) return `{tool: ${exp.tool}}`;
-  return `{tool: ${exp.tool}, params: ${safeStringify(exp.params)}}`;
-}
-function safeStringify(value) {
-  try {
-    return JSON.stringify(value, (_k, v) => {
-      if (v === EvalAny) return "<EvalAny>";
-      if (isEvalRegex(v)) return `<EvalRegex /${v.__evalRegex.source}/>`;
-      return v;
-    });
-  } catch {
-    return "<unserializable>";
-  }
-}
-
-// src/eval/evalConfig.ts
-var DEFAULT_EVAL_MODEL = { provider: "openrouter", id: "qwen/qwen3.6-plus" };
-var DEFAULT_TIMEOUT_MS = 3e4;
-var DEFAULT_SUITE_TIMEOUT_MS = 5 * 6e4;
-var DEFAULT_CONCURRENCY = 4;
-
-// src/eval/evalPrompt.ts
-async function evalAgentPrompt(opts) {
-  validateMutuallyExclusive(opts);
-  const sessionId = opts.sessionId ?? randomUUID();
-  const model = parseModelString(opts.model ?? DEFAULT_EVAL_MODEL);
-  const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
-  const maxRetries = opts.retries ?? 0;
-  let attempts = 0;
-  let lastResult = null;
-  while (attempts <= maxRetries) {
-    attempts += 1;
-    let captured;
-    try {
-      captured = await runOnce(opts.app, {
-        agentTypeId: opts.agentTypeId,
-        sessionId: `${sessionId}-attempt-${attempts}`,
-        prompt: opts.prompt,
-        systemPrompt: opts.systemPrompt,
-        model,
-        timeoutMs,
-        headers: opts.headers
-      });
-    } catch (err) {
-      lastResult = {
-        ok: false,
-        actual: [],
-        text: "",
-        attempts,
-        reason: `transport: ${err instanceof Error ? err.message : String(err)}`
-      };
-      if (attempts > maxRetries) return lastResult;
-      continue;
-    }
-    if (captured.errorText) {
-      lastResult = {
-        ok: false,
-        actual: captured.toolCalls,
-        text: captured.text,
-        usage: captured.usage,
-        attempts,
-        reason: `stream: ${captured.errorText}`
-      };
-      if (attempts > maxRetries) return lastResult;
-      continue;
-    }
-    const matchResult = matchExpectations(opts, captured.toolCalls);
-    lastResult = {
-      ok: matchResult.ok,
-      actual: captured.toolCalls,
-      text: captured.text,
-      usage: captured.usage,
-      attempts,
-      reason: matchResult.ok ? void 0 : matchResult.reason
-    };
-    if (matchResult.ok) return lastResult;
-  }
-  return lastResult ?? {
-    ok: false,
-    actual: [],
-    text: "",
-    attempts,
-    reason: "no attempts ran"
-  };
-}
-function validateMutuallyExclusive(opts) {
-  const set = [
-    opts.expect !== void 0,
-    opts.expectFirst !== void 0,
-    opts.expectNoToolCall === true
-  ].filter(Boolean).length;
-  if (set === 0) {
-    throw new Error(
-      "evalAgentPrompt: provide one of `expect`, `expectFirst`, or `expectNoToolCall: true`"
-    );
-  }
-  if (set > 1) {
-    throw new Error(
-      "evalAgentPrompt: `expect`, `expectFirst`, and `expectNoToolCall` are mutually exclusive"
-    );
-  }
-}
-function matchExpectations(opts, actual) {
-  if (opts.expectNoToolCall) {
-    return noToolCallMatches(actual);
-  }
-  if (opts.expectFirst) {
-    return firstCallMatches(opts.expectFirst, actual);
-  }
-  return someCallMatches(opts.expect, actual);
-}
-async function runOnce(app, opts) {
-  const sessionsPath = `/api/v1/agents/${encodeURIComponent(opts.agentTypeId)}/sessions`;
-  const created = await app.inject({
-    method: "POST",
-    url: sessionsPath,
-    headers: opts.headers,
-    payload: { requestId: `eval-create-${opts.sessionId}`, title: opts.sessionId }
-  });
-  if (created.statusCode !== 201) {
-    throw new Error(`Agent session create returned ${created.statusCode}: ${created.body.slice(0, 256)}`);
-  }
-  const sessionId = created.json().sessionId;
-  if (typeof sessionId !== "string" || !sessionId) throw new Error("Agent session create did not return sessionId");
-  const userMessage = opts.systemPrompt ? `[SYSTEM]
-${opts.systemPrompt}
-[/SYSTEM]
-
-${opts.prompt}` : opts.prompt;
-  try {
-    const res = await withTimeout(
-      app.inject({
-        method: "POST",
-        url: `${sessionsPath}/${encodeURIComponent(sessionId)}/prompt`,
-        headers: opts.headers,
-        payload: {
-          requestId: `eval-${sessionId}`,
-          content: userMessage,
-          clientNonce: `eval-${opts.sessionId}`,
-          model: opts.model
-        }
-      }),
-      opts.timeoutMs,
-      `chat request exceeded ${opts.timeoutMs}ms`
-    );
-    if (res.statusCode !== 202 && res.statusCode !== 200) {
-      throw new Error(
-        `Pi chat prompt returned ${res.statusCode}: ${res.body.slice(0, 256)}`
-      );
-    }
-    const snapshot = await withTimeout(
-      waitForSettledState(app, opts, sessionId),
-      opts.timeoutMs,
-      `chat turn did not settle within ${opts.timeoutMs}ms`
-    );
-    return capturePiChatSnapshot(snapshot);
-  } finally {
-    void app.inject({
-      method: "DELETE",
-      url: `${sessionsPath}/${encodeURIComponent(sessionId)}`,
-      headers: opts.headers
-    }).catch(() => {
-    });
-  }
-}
-var STATE_POLL_INTERVAL_MS = 250;
-async function waitForSettledState(app, opts, sessionId) {
-  for (; ; ) {
-    const state = await app.inject({
-      method: "GET",
-      url: `/api/v1/agents/${encodeURIComponent(opts.agentTypeId)}/sessions/${encodeURIComponent(sessionId)}/state`,
-      headers: opts.headers
-    });
-    if (state.statusCode !== 200) {
-      throw new Error(`Pi chat state returned ${state.statusCode}: ${state.body.slice(0, 256)}`);
-    }
-    const envelope = JSON.parse(state.body);
-    const snapshot = envelope.state;
-    if (!snapshot || typeof snapshot !== "object" || Array.isArray(snapshot)) {
-      throw new Error("Pi chat state did not return an addressed state envelope");
-    }
-    const addressedState = snapshot;
-    if (addressedState.status !== "streaming") return addressedState;
-    await new Promise((resolve) => setTimeout(resolve, STATE_POLL_INTERVAL_MS));
-  }
-}
-function capturePiChatSnapshot(snapshot) {
-  const toolCalls = [];
-  const textParts = [];
-  const snapshotError = snapshot.error;
-  const errorText = typeof snapshotError?.message === "string" ? snapshotError.message : void 0;
-  const messages = Array.isArray(snapshot.messages) ? snapshot.messages : [];
-  for (const message of messages) {
-    if (typeof message !== "object" || message === null) continue;
-    const parts = Array.isArray(message.parts) ? message.parts : [];
-    for (const part of parts) {
-      if (typeof part !== "object" || part === null) continue;
-      const rec = part;
-      if (rec.type === "text" && typeof rec.text === "string") textParts.push(rec.text);
-      if (rec.type === "tool-call" && typeof rec.toolName === "string") {
-        toolCalls.push({ tool: rec.toolName, params: typeof rec.input === "object" && rec.input !== null ? rec.input : {} });
-      }
-    }
-  }
-  return { toolCalls, text: textParts.join(""), errorText };
-}
-function withTimeout(p, ms, label) {
-  return new Promise((resolve, reject) => {
-    const timer = setTimeout(() => reject(new Error(label)), ms);
-    p.then(
-      (v) => {
-        clearTimeout(timer);
-        resolve(v);
-      },
-      (err) => {
-        clearTimeout(timer);
-        reject(err);
-      }
-    );
-  });
-}
-function parseModelString(input) {
-  if (typeof input === "object") return input;
-  if (input.startsWith("claude-")) return { provider: "anthropic", id: input };
-  if (input.startsWith("gpt-")) return { provider: "openai", id: input };
-  if (input.startsWith("qwen/")) return { provider: "openrouter", id: input };
-  return { provider: "anthropic", id: input };
-}
-
-// src/eval/runSuite.ts
-import { readFile } from "node:fs/promises";
-
-// src/eval/yamlSchema.ts
-import { parse, Schema } from "yaml";
-var evalSchema = new Schema({
-  customTags: [
-    {
-      tag: "!EvalAny",
-      // No value — `!EvalAny` alone resolves to the symbol.
-      identify: (value) => value === EvalAny,
-      resolve: () => EvalAny
-    },
-    {
-      tag: "!EvalRegex",
-      // Scalar value — `!EvalRegex "^chart:"` resolves to a matcher object.
-      identify: (value) => typeof value === "object" && value !== null && "__evalRegex" in value,
-      resolve: (str) => EvalRegex(str)
-    }
-  ]
-});
-function parseFixtureYaml(text) {
-  const parsed = parse(text, { schema: evalSchema });
-  if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
-    throw new Error(
-      `Eval fixture must be a YAML object with a "prompts" key. Got: ${describe2(parsed)}`
-    );
-  }
-  const obj = parsed;
-  if (!Array.isArray(obj.prompts)) {
-    throw new Error(`Eval fixture missing "prompts" array.`);
-  }
-  return obj;
-}
-function describe2(value) {
-  if (value === null) return "null";
-  if (value === void 0) return "undefined";
-  if (Array.isArray(value)) return "array";
-  return typeof value;
-}
-
-// src/eval/runSuite.ts
-async function runEvalSuite(opts) {
-  const fixtureText = await readFile(opts.fixturesPath, "utf8");
-  const fixture = parseFixtureYaml(fixtureText);
-  const concurrency = opts.concurrency ?? DEFAULT_CONCURRENCY;
-  const suiteTimeoutMs = opts.suiteTimeoutMs ?? DEFAULT_SUITE_TIMEOUT_MS;
-  const start = Date.now();
-  const deadline = start + suiteTimeoutMs;
-  const queue = [...fixture.prompts];
-  const results = [];
-  let bailed = false;
-  async function worker() {
-    while (queue.length > 0 && !bailed) {
-      if (Date.now() > deadline) return;
-      const prompt = queue.shift();
-      if (!prompt) return;
-      const result = await runOnePrompt(opts, fixture, prompt);
-      results.push({
-        ...result,
-        prompt: prompt.prompt,
-        expected: prompt
-      });
-      if (!result.ok && opts.bail) {
-        bailed = true;
-        return;
-      }
-    }
-  }
-  const workers = Array.from({ length: concurrency }, () => worker());
-  await Promise.race([
-    Promise.all(workers),
-    new Promise(
-      (_, reject) => setTimeout(
-        () => reject(new Error(`suite exceeded ${suiteTimeoutMs}ms`)),
-        suiteTimeoutMs
-      )
-    )
-  ]).catch((err) => {
-    results.push({
-      ok: false,
-      actual: [],
-      text: "",
-      attempts: 0,
-      reason: err instanceof Error ? err.message : String(err),
-      prompt: "(suite-level timeout)",
-      expected: { prompt: "(suite-level timeout)" }
-    });
-  });
-  const totalUsage = results.reduce(
-    (acc, r) => ({
-      input: acc.input + (r.usage?.input ?? 0),
-      output: acc.output + (r.usage?.output ?? 0)
-    }),
-    { input: 0, output: 0 }
-  );
-  const passed = results.filter((r) => r.ok).length;
-  const total = results.length;
-  const passRate = total === 0 ? 0 : passed / total;
-  return {
-    total,
-    passed,
-    failed: total - passed,
-    passRate,
-    results,
-    totalUsage,
-    totalDurationMs: Date.now() - start,
-    allPassed: total > 0 && passed === total
-  };
-}
-async function runOnePrompt(opts, fixture, p) {
-  const model = opts.model ?? p.model ?? fixture.defaults?.model ?? fixture.model;
-  const retries = p.retries ?? fixture.defaults?.retries ?? 0;
-  const timeoutMs = p.timeoutMs ?? fixture.defaults?.timeoutMs ?? DEFAULT_TIMEOUT_MS;
-  return evalAgentPrompt({
-    app: opts.app,
-    agentTypeId: opts.agentTypeId,
-    prompt: p.prompt,
-    expect: p.expect,
-    expectFirst: p.expectFirst,
-    expectNoToolCall: p.expectNoToolCall,
-    systemPrompt: fixture.systemPrompt,
-    model,
-    retries,
-    timeoutMs
-  });
-}
-export {
-  DEFAULT_CONCURRENCY,
-  DEFAULT_EVAL_MODEL,
-  DEFAULT_SUITE_TIMEOUT_MS,
-  DEFAULT_TIMEOUT_MS,
-  EvalAny,
-  EvalRegex,
-  callSatisfies,
-  evalAgentPrompt,
-  firstCallMatches,
-  isEvalRegex,
-  noToolCallMatches,
-  parseFixtureYaml,
-  runEvalSuite,
-  someCallMatches
-};