@hachej/boring-agent 0.1.111 → 0.2.0

package/dist/eval/index.d.ts

@@ -1,247 +0,0 @@
-import { FastifyInstance } from 'fastify';
-
-/**
- * Public types for @hachej/boring-agent/eval — the eval framework.
- *
- * See packages/agent/docs/plans/AGENT_EVAL_FRAMEWORK.md for the design
- * rationale and approved contract.
- */
-
-/**
- * A captured tool call from an LLM response. Anthropic / OpenAI both
- * support emitting multiple tool calls in a single response (parallel)
- * or across turns (sequential). The framework collects ALL of them; the
- * matcher then checks for existence (default) or ordering (`expectFirst`).
- */
-interface ToolCall {
-    tool: string;
-    params: Record<string, unknown>;
-}
-/**
- * Sentinel for "must be present at this key, any value". Use in
- * `expect.params` for fields the LLM generates (uuids, timestamps).
- *
- * In YAML fixtures: `id: !EvalAny`.
- */
-declare const EvalAny: unique symbol;
-type EvalAnyType = typeof EvalAny;
-/**
- * Tagged regex matcher for string params. Use when the value is partially
- * structured (e.g. an id that must start with `chart:`) but the suffix
- * varies between runs.
- *
- * In YAML fixtures: `component: !EvalRegex "^chart:"`.
- */
-interface EvalRegexMatcher {
-    __evalRegex: RegExp;
-}
-declare function EvalRegex(re: RegExp | string): EvalRegexMatcher;
-declare function isEvalRegex(value: unknown): value is EvalRegexMatcher;
-/**
- * Expected shape of a tool call. The matcher (matcher.ts) decides if an
- * actual ToolCall satisfies it.
- *
- * - `tool`: exact name match (required).
- * - `params`: partial match by default — every key in `expect.params` must
- *   appear in the actual call's params with a matching value. Extra keys
- *   in the actual call are allowed unless `strict: true`.
- * - Values inside `params` may be the `EvalAny` sentinel or an
- *   `EvalRegexMatcher` wildcard.
- */
-interface ExpectedCall {
-    tool: string;
-    params?: Record<string, unknown>;
-    strict?: boolean;
-}
-/**
- * Per-prompt eval options. See `evalAgentPrompt(opts)` in evalPrompt.ts.
- */
-type EvalModelSelection = string | {
-    provider: string;
-    id: string;
-};
-interface EvalPromptOptions {
-    /** A FastifyInstance from createStandaloneAgentHostApp / createWorkspaceAgentApp / etc. */
-    app: FastifyInstance;
-    /** Addressed Agent owner exercised by this eval. */
-    agentTypeId: string;
-    /** User prompt sent to the agent. */
-    prompt: string;
-    /**
-     * Expected tool calls. Default: assert each ExpectedCall here matches
-     * AT LEAST ONE call in the LLM's response (any order). Mutually
-     * exclusive with `expectFirst` and `expectNoToolCall`.
-     */
-    expect?: ExpectedCall | ExpectedCall[];
-    /**
-     * Stricter alternative: assert the FIRST tool call matches.
-     * Mutually exclusive with `expect` and `expectNoToolCall`.
-     */
-    expectFirst?: ExpectedCall;
-    /**
-     * Negative assertion: NO tool was called (LLM answered in plain text).
-     * Mutually exclusive with `expect` and `expectFirst`.
-     */
-    expectNoToolCall?: boolean;
-    /**
-     * Model id. Defaults to the agent's pinned model (see evalConfig.ts).
-     * Suites typically override at the suite level (in YAML).
-     */
-    model?: EvalModelSelection;
-    /** Optional system prompt. */
-    systemPrompt?: string;
-    /** Override the chat session id (defaults to a fresh uuid). */
-    sessionId?: string;
-    /** Optional headers sent to session/chat requests (e.g. workspace scoping). */
-    headers?: Record<string, string>;
-    /** Per-call timeout in ms. Defaults to 30_000. */
-    timeoutMs?: number;
-    /** Per-prompt retry count. Default: 0. */
-    retries?: number;
-}
-interface EvalResult {
-    ok: boolean;
-    /** All tool calls the LLM made, in order. */
-    actual: ToolCall[];
-    /** Plain-text response from the LLM. */
-    text: string;
-    /** Human-readable reason on failure. */
-    reason?: string;
-    /** Tokens consumed. */
-    usage?: {
-        input: number;
-        output: number;
-    };
-    /** Number of attempts used (1 + retries on failure). */
-    attempts: number;
-}
-/**
- * Suite runner options. See `runEvalSuite(opts)` in runSuite.ts.
- */
-interface SuiteOptions {
-    app: FastifyInstance;
-    /** Addressed Agent owner exercised by every fixture. */
-    agentTypeId: string;
-    /** Path to a YAML fixture file. See AGENT_EVAL_FRAMEWORK.md for format. */
-    fixturesPath: string;
-    /** Stop on first failure. Default: false. */
-    bail?: boolean;
-    /** Concurrency. Default: 4. */
-    concurrency?: number;
-    /** Suite-level timeout in ms. Default: 5 * 60_000 (5 minutes). */
-    suiteTimeoutMs?: number;
-    /** Override per-prompt model. Useful for ad-hoc model comparison runs. */
-    model?: EvalModelSelection;
-}
-interface SuiteFixturePrompt {
-    prompt: string;
-    expect?: ExpectedCall | ExpectedCall[];
-    expectFirst?: ExpectedCall;
-    expectNoToolCall?: boolean;
-    retries?: number;
-    timeoutMs?: number;
-    model?: EvalModelSelection;
-}
-interface SuiteFixture {
-    /** Pinned model for the whole suite (each prompt may override). */
-    model?: EvalModelSelection;
-    /** System prompt prepended to every prompt in this suite. */
-    systemPrompt?: string;
-    /** Defaults applied to each prompt unless inline-overridden. */
-    defaults?: Pick<SuiteFixturePrompt, "retries" | "timeoutMs" | "model">;
-    prompts: SuiteFixturePrompt[];
-}
-interface SuiteReport {
-    total: number;
-    passed: number;
-    failed: number;
-    passRate: number;
-    results: Array<EvalResult & {
-        prompt: string;
-        expected: SuiteFixturePrompt;
-    }>;
-    totalUsage: {
-        input: number;
-        output: number;
-    };
-    totalDurationMs: number;
-    /** True iff every result is ok: true. */
-    allPassed: boolean;
-}
-
-declare function evalAgentPrompt(opts: EvalPromptOptions): Promise<EvalResult>;
-
-declare function runEvalSuite(opts: SuiteOptions): Promise<SuiteReport>;
-
-/**
- * Parse a YAML fixture file (text contents) into a `SuiteFixture`.
- *
- * Delegates to the `yaml` package with the custom-tag schema so the
- * matcher wildcards (`!EvalAny`, `!EvalRegex`) resolve into runtime
- * values matcher.ts can consume.
- *
- * Throws on invalid YAML syntax. Returns whatever shape parses — minimal
- * runtime validation here (caller asserts with the SuiteFixture type).
- */
-declare function parseFixtureYaml(text: string): SuiteFixture;
-
-/**
- * Pure matcher logic for the eval framework. No LLM, no fetch, no
- * Fastify — exercised in unit tests against fixed ToolCall[] inputs.
- *
- * The matcher implements three modes:
- * 1. someCallMatches (default `expect`) — every ExpectedCall must match
- *    at least one actual ToolCall, any order. Captures parallel /
- *    sequential tool calling without assuming an order the LLM didn't
- *    actually commit to.
- * 2. firstCallMatches (`expectFirst`) — actual[0] must match the
- *    ExpectedCall. Use only when ordering is part of the contract.
- * 3. noToolCall (`expectNoToolCall`) — actual must be empty. The LLM
- *    answered in plain text without tooling.
- *
- * Param matching is partial by default: every key in expect.params must
- * appear in actual.params with a matching value, but extra keys in
- * actual are ignored. Set `strict: true` to require exact equality (no
- * extra keys allowed).
- */
-
-interface MatchOutcome {
-    ok: boolean;
-    /** Index into `actual[]` that satisfied the expected call. -1 on failure. */
-    matchedIndex: number;
-    /** Diagnostic when ok is false. */
-    reason?: string;
-}
-/**
- * Does ONE actual call satisfy ONE expected call? Used as the inner
- * predicate by all three modes.
- */
-declare function callSatisfies(expected: ExpectedCall, actual: ToolCall): MatchOutcome;
-/**
- * "Does at least one of `actual` satisfy each entry in `expected`?"
- * Returns the index of the first match for each expected entry.
- */
-declare function someCallMatches(expected: ExpectedCall | ExpectedCall[], actual: ToolCall[]): MatchOutcome;
-declare function firstCallMatches(expected: ExpectedCall, actual: ToolCall[]): MatchOutcome;
-declare function noToolCallMatches(actual: ToolCall[]): MatchOutcome;
-
-/**
- * Default model for the eval framework.
- *
- * Pinned deliberately so model deprecation is a visible PR (changelog
- * entry + intentional bump), not a transparent update. Per-suite YAML
- * fixtures override this at the suite level so consumers upgrade at
- * their own pace.
- */
-declare const DEFAULT_EVAL_MODEL: {
-    readonly provider: "openrouter";
-    readonly id: "qwen/qwen3.6-plus";
-};
-/** Default per-call timeout in ms. */
-declare const DEFAULT_TIMEOUT_MS = 30000;
-/** Default suite-level timeout in ms (5 minutes). */
-declare const DEFAULT_SUITE_TIMEOUT_MS: number;
-/** Default concurrency for runEvalSuite. */
-declare const DEFAULT_CONCURRENCY = 4;
-
-export { DEFAULT_CONCURRENCY, DEFAULT_EVAL_MODEL, DEFAULT_SUITE_TIMEOUT_MS, DEFAULT_TIMEOUT_MS, EvalAny, type EvalAnyType, type EvalModelSelection, type EvalPromptOptions, EvalRegex, type EvalRegexMatcher, type EvalResult, type ExpectedCall, type MatchOutcome, type SuiteFixture, type SuiteFixturePrompt, type SuiteOptions, type SuiteReport, type ToolCall, callSatisfies, evalAgentPrompt, firstCallMatches, isEvalRegex, noToolCallMatches, parseFixtureYaml, runEvalSuite, someCallMatches };