@chatcops/cli 0.0.0-stage → 0.1.0

package/dist/test/assertions/index.js

@@ -0,0 +1,50 @@
+import { judge } from './judge.js';
+import { javascript } from './javascript.js';
+import { cites, leadCaptured } from './parts.js';
+import { contains } from './text.js';
+import { callsTool, noToolCall } from './tools.js';
+import { costUnder, latencyUnder, tokensUnder } from './usage.js';
+/**
+ * The checks an eval file can use, by key. Every key also works with a
+ * `not-` prefix, which passes when the check fails (`not-contains`).
+ *
+ * To add one, write it in its own file in this folder and add it here.
+ */
+export const ASSERTIONS = new Map([
+    ['contains', contains],
+    ['javascript', javascript],
+    ['judge', judge],
+    ['tokens-under', tokensUnder],
+    ['latency-under', latencyUnder],
+    ['cost-under', costUnder],
+    ['cites', cites],
+    ['lead-captured', leadCaptured],
+    ['calls-tool', callsTool],
+    ['no-tool-call', noToolCall],
+]);
+/** The base name of an assertion and whether it's negated: `not-contains` is `contains`, negated. */
+export function assertionName(assertion) {
+    const [key] = Object.keys(assertion);
+    const negated = key.startsWith('not-') && ASSERTIONS.has(key.slice(4));
+    return { key, name: negated ? key.slice(4) : key, negated };
+}
+/** Assertion keys a fixture uses that no check exists for. */
+export function unknownAssertions(assertions) {
+    return [...new Set(assertions.map(assertionName).filter(({ name }) => !ASSERTIONS.has(name)).map(({ key }) => key))];
+}
+export async function check(assertion, turn, conversation, context = { fixture: '' }) {
+    const { key, name, negated } = assertionName(assertion);
+    const run = ASSERTIONS.get(name);
+    if (!run)
+        return { assertion, pass: false, reason: `unknown assertion "${key}"` };
+    try {
+        const result = await run(assertion[key], turn, conversation, context);
+        if (!negated)
+            return { assertion, ...result };
+        return { assertion, ...result, pass: !result.pass, reason: result.pass ? `expected not: ${result.reason ?? name}` : result.reason };
+    }
+    catch (error) {
+        return { assertion, pass: false, reason: error instanceof Error ? error.message : String(error) };
+    }
+}
+//# sourceMappingURL=index.js.map

package/dist/test/assertions/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/test/assertions/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AACjD,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACnD,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAElE;;;;;GAKG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG,IAAI,GAAG,CAAyB;IACxD,CAAC,UAAU,EAAE,QAAQ,CAAC;IACtB,CAAC,YAAY,EAAE,UAAU,CAAC;IAC1B,CAAC,OAAO,EAAE,KAAK,CAAC;IAChB,CAAC,cAAc,EAAE,WAAW,CAAC;IAC7B,CAAC,eAAe,EAAE,YAAY,CAAC;IAC/B,CAAC,YAAY,EAAE,SAAS,CAAC;IACzB,CAAC,OAAO,EAAE,KAAK,CAAC;IAChB,CAAC,eAAe,EAAE,YAAY,CAAC;IAC/B,CAAC,YAAY,EAAE,SAAS,CAAC;IACzB,CAAC,cAAc,EAAE,UAAU,CAAC;CAC7B,CAAC,CAAC;AAEH,qGAAqG;AACrG,MAAM,UAAU,aAAa,CAAC,SAAoB;IAChD,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACrC,MAAM,OAAO,GAAG,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IACvE,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,OAAO,EAAE,CAAC;AAC9D,CAAC;AAED,8DAA8D;AAC9D,MAAM,UAAU,iBAAiB,CAAC,UAAuB;IACvD,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;AACvH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,KAAK,CAAC,SAAoB,EAAE,IAAgB,EAAE,YAA0B,EAAE,UAA4B,EAAE,OAAO,EAAE,EAAE,EAAE;IACzI,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,aAAa,CAAC,SAAS,CAAC,CAAC;IACxD,MAAM,GAAG,GAAG,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACjC,IAAI,CAAC,GAAG;QAAE,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,sBAAsB,GAAG,GAAG,EAAE,CAAC;IAClF,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,OAAO,CAAC,CAAC;QACtE,IAAI,CAAC,OAAO;YAAE,OAAO,EAAE,SAAS,EAAE,GAAG,MAAM,EAAE,CAAC;QAC9C,OAAO,EAAE,SAAS,EAAE,GAAG,MAAM,EAAE,IAAI,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,iBAAiB,MAAM,CAAC,MAAM,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;IACtI,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;IACpG,CAAC;AACH,CAAC"}

package/dist/test/assertions/javascript.d.ts

@@ -0,0 +1,10 @@
+import type { AssertionCheck } from '../types.js';
+/**
+ * `javascript: ./checks/mentions-plan.mjs`: loads the module (relative to the
+ * eval file) and calls its default export with `(turn, conversation)`. It may
+ * return a boolean, or `{ pass, reason?, score? }`, and may be async. A missing
+ * file, a module without a default function, or a throw inside it are failures
+ * with the file name in the reason.
+ */
+export declare const javascript: AssertionCheck;
+//# sourceMappingURL=javascript.d.ts.map

package/dist/test/assertions/javascript.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"javascript.d.ts","sourceRoot":"","sources":["../../../src/test/assertions/javascript.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,cAAc,EAAc,MAAM,aAAa,CAAC;AAc9D;;;;;;GAMG;AACH,eAAO,MAAM,UAAU,EAAE,cAgCxB,CAAC"}

package/dist/test/assertions/javascript.js

@@ -0,0 +1,51 @@
+import { basename, resolve } from 'node:path';
+import { pathToFileURL } from 'node:url';
+const cache = new Map();
+async function loadModule(path) {
+    const cached = cache.get(path);
+    if (cached)
+        return cached;
+    const mod = (await import(pathToFileURL(path).href));
+    cache.set(path, mod);
+    return mod;
+}
+/**
+ * `javascript: ./checks/mentions-plan.mjs`: loads the module (relative to the
+ * eval file) and calls its default export with `(turn, conversation)`. It may
+ * return a boolean, or `{ pass, reason?, score? }`, and may be async. A missing
+ * file, a module without a default function, or a throw inside it are failures
+ * with the file name in the reason.
+ */
+export const javascript = async (value, turn, conversation, context) => {
+    if (typeof value !== 'string' || !value.trim())
+        throw new Error('javascript takes a path to a module');
+    const dir = context.fixture ? resolve(context.fixture, '..') : process.cwd();
+    const path = resolve(dir, value);
+    const name = basename(path);
+    let mod;
+    try {
+        mod = await loadModule(path);
+    }
+    catch (error) {
+        throw new Error(`${name}: ${error instanceof Error ? error.message : String(error)}`);
+    }
+    if (typeof mod.default !== 'function') {
+        throw new Error(`${name}: the module has no default function`);
+    }
+    let result;
+    try {
+        result = await mod.default(turn, conversation);
+    }
+    catch (error) {
+        throw new Error(`${name}: ${error instanceof Error ? error.message : String(error)}`);
+    }
+    if (typeof result === 'boolean') {
+        return { pass: result, reason: result ? `${name} returned true` : `${name} returned false` };
+    }
+    if (result !== null && typeof result === 'object' && 'pass' in result && typeof result.pass === 'boolean') {
+        const { pass, reason, score } = result;
+        return { pass, ...(reason !== undefined ? { reason } : {}), ...(score !== undefined ? { score } : {}) };
+    }
+    throw new Error(`${name}: the default export must return a boolean or { pass, reason?, score? }`);
+};
+//# sourceMappingURL=javascript.js.map

package/dist/test/assertions/javascript.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"javascript.js","sourceRoot":"","sources":["../../../src/test/assertions/javascript.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC9C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAKzC,MAAM,KAAK,GAAG,IAAI,GAAG,EAA4B,CAAC;AAElD,KAAK,UAAU,UAAU,CAAC,IAAY;IACpC,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAC/B,IAAI,MAAM;QAAE,OAAO,MAAM,CAAC;IAC1B,MAAM,GAAG,GAAG,CAAC,MAAM,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAqB,CAAC;IACzE,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IACrB,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,UAAU,GAAmB,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,YAAY,EAAE,OAAO,EAAE,EAAE;IACrF,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IACvG,MAAM,GAAG,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC;IAC7E,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IACjC,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;IAE5B,IAAI,GAAqB,CAAC;IAC1B,IAAI,CAAC;QACH,GAAG,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,KAAK,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACxF,CAAC;IAED,IAAI,OAAO,GAAG,CAAC,OAAO,KAAK,UAAU,EAAE,CAAC;QACtC,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,sCAAsC,CAAC,CAAC;IACjE,CAAC;IAED,IAAI,MAAe,CAAC;IACpB,IAAI,CAAC;QACH,MAAM,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;IACjD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,KAAK,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACxF,CAAC;IAED,IAAI,OAAO,MAAM,KAAK,SAAS,EAAE,CAAC;QAChC,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,GAAG,IAAI,gBAAgB,CAAC,CAAC,CAAC,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAC/F,CAAC;IACD,IAAI,MAAM,KAAK,IAAI,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,IAAI,MAAM,IAAI,OAAQ,MAA4B,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QACjI,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,MAA4D,CAAC;QAC7F,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;IAC1G,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,yEAAyE,CAAC,CAAC;AACpG,CAAC,CAAC"}

package/dist/test/assertions/judge.d.ts

@@ -0,0 +1,8 @@
+import type { AssertionCheck } from '../types.js';
+/**
+ * `judge: "Confirms the demo without inventing a time."`, or
+ * `judge: { rubric, threshold }`: a model grades the reply against the rubric
+ * from 0 to 1, and it passes at `threshold` (default 0.7) or above.
+ */
+export declare const judge: AssertionCheck;
+//# sourceMappingURL=judge.d.ts.map

package/dist/test/assertions/judge.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"judge.d.ts","sourceRoot":"","sources":["../../../src/test/assertions/judge.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAIlD;;;;GAIG;AACH,eAAO,MAAM,KAAK,EAAE,cAWnB,CAAC"}

package/dist/test/assertions/judge.js

@@ -0,0 +1,33 @@
+const DEFAULT_THRESHOLD = 0.7;
+/**
+ * `judge: "Confirms the demo without inventing a time."`, or
+ * `judge: { rubric, threshold }`: a model grades the reply against the rubric
+ * from 0 to 1, and it passes at `threshold` (default 0.7) or above.
+ */
+export const judge = async (value, turn, conversation, context) => {
+    const { rubric, threshold } = readJudge(value);
+    if (!context.judge)
+        throw new Error('judge needs a judge model, and none is set up');
+    if (turn.error)
+        return { pass: false, reason: `the turn failed (${turn.error}), so there's nothing to grade` };
+    const verdict = await context.judge.grade(rubric, conversation);
+    return {
+        pass: verdict.score >= threshold,
+        score: verdict.score,
+        reason: `judge scored ${verdict.score.toFixed(2)} (pass at ${threshold}): ${verdict.reason}`,
+        ...(verdict.costUsd !== undefined ? { costUsd: verdict.costUsd } : {}),
+    };
+};
+function readJudge(value) {
+    if (typeof value === 'string' && value.trim())
+        return { rubric: value, threshold: DEFAULT_THRESHOLD };
+    if (value && typeof value === 'object' && !Array.isArray(value)) {
+        const { rubric, threshold = DEFAULT_THRESHOLD, ...rest } = value;
+        const extra = Object.keys(rest);
+        if (typeof rubric === 'string' && rubric.trim() && typeof threshold === 'number' && threshold >= 0 && threshold <= 1 && !extra.length) {
+            return { rubric, threshold };
+        }
+    }
+    throw new Error('judge takes a rubric, or { rubric, threshold } with a threshold from 0 to 1');
+}
+//# sourceMappingURL=judge.js.map

package/dist/test/assertions/judge.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"judge.js","sourceRoot":"","sources":["../../../src/test/assertions/judge.ts"],"names":[],"mappings":"AAEA,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAE9B;;;;GAIG;AACH,MAAM,CAAC,MAAM,KAAK,GAAmB,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,YAAY,EAAE,OAAO,EAAE,EAAE;IAChF,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;IAC/C,IAAI,CAAC,OAAO,CAAC,KAAK;QAAE,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;IACrF,IAAI,IAAI,CAAC,KAAK;QAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,oBAAoB,IAAI,CAAC,KAAK,gCAAgC,EAAE,CAAC;IAC/G,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;IAChE,OAAO;QACL,IAAI,EAAE,OAAO,CAAC,KAAK,IAAI,SAAS;QAChC,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,MAAM,EAAE,gBAAgB,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,SAAS,MAAM,OAAO,CAAC,MAAM,EAAE;QAC5F,GAAG,CAAC,OAAO,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACvE,CAAC;AACJ,CAAC,CAAC;AAEF,SAAS,SAAS,CAAC,KAAc;IAC/B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE;QAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,iBAAiB,EAAE,CAAC;IACtG,IAAI,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QAChE,MAAM,EAAE,MAAM,EAAE,SAAS,GAAG,iBAAiB,EAAE,GAAG,IAAI,EAAE,GAAG,KAAkD,CAAC;QAC9G,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAChC,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,IAAI,EAAE,IAAI,OAAO,SAAS,KAAK,QAAQ,IAAI,SAAS,IAAI,CAAC,IAAI,SAAS,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;YACtI,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;QAC/B,CAAC;IACH,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,6EAA6E,CAAC,CAAC;AACjG,CAAC"}

package/dist/test/assertions/parts.d.ts

@@ -0,0 +1,12 @@
+import type { AssertionCheck } from '../types.js';
+/**
+ * `cites: "https://acme.test/pricing*"` or a list of URL globs: each one
+ * matches a page the reply cites. `cites: true`: it cites at least one.
+ */
+export declare const cites: AssertionCheck;
+/**
+ * `lead-captured: true`: the turn captured a lead. An object, such as
+ * `lead-captured: { email: jane@acme.test }`, also has to match those fields.
+ */
+export declare const leadCaptured: AssertionCheck;
+//# sourceMappingURL=parts.d.ts.map

package/dist/test/assertions/parts.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parts.d.ts","sourceRoot":"","sources":["../../../src/test/assertions/parts.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAGlD;;;GAGG;AACH,eAAO,MAAM,KAAK,EAAE,cAWnB,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,YAAY,EAAE,cAU1B,CAAC"}

package/dist/test/assertions/parts.js

@@ -0,0 +1,39 @@
+import { globToRegExp } from '@chatcops/ingest';
+import { quoted } from './values.js';
+/**
+ * `cites: "https://acme.test/pricing*"` or a list of URL globs: each one
+ * matches a page the reply cites. `cites: true`: it cites at least one.
+ */
+export const cites = (value, turn) => {
+    const urls = turn.sources.map((source) => source.url);
+    if (value === true) {
+        return urls.length ? { pass: true, reason: `the reply cites ${quoted(urls)}` } : { pass: false, reason: 'the reply cites no pages' };
+    }
+    const globs = [value].flat();
+    if (!globs.length || globs.some((glob) => typeof glob !== 'string'))
+        throw new Error('cites takes true, a URL glob or a list of them');
+    const missing = globs.filter((glob) => !urls.some((url) => globToRegExp(glob).test(url)));
+    if (!missing.length)
+        return { pass: true, reason: `the reply cites ${quoted(globs)}` };
+    const cited = urls.length ? `it cites ${quoted(urls)}` : 'it cites no pages';
+    return { pass: false, reason: `the reply doesn't cite ${quoted(missing)} (${cited})` };
+};
+/**
+ * `lead-captured: true`: the turn captured a lead. An object, such as
+ * `lead-captured: { email: jane@acme.test }`, also has to match those fields.
+ */
+export const leadCaptured = (value, turn) => {
+    const isFields = value !== null && typeof value === 'object' && !Array.isArray(value) && Object.keys(value).length > 0;
+    if (value !== true && !isFields)
+        throw new Error('lead-captured takes true, or the fields the lead should have');
+    if (!turn.leads.length)
+        return { pass: false, reason: 'the turn captured no lead' };
+    if (!isFields)
+        return { pass: true, reason: 'the turn captured a lead' };
+    const wanted = Object.entries(value);
+    const fields = wanted.map(([key, field]) => `${key} ${JSON.stringify(field)}`).join(', ');
+    return turn.leads.some((lead) => wanted.every(([key, field]) => lead[key] === field))
+        ? { pass: true, reason: `the turn captured a lead with ${fields}` }
+        : { pass: false, reason: `no lead the turn captured has ${fields}` };
+};
+//# sourceMappingURL=parts.js.map

package/dist/test/assertions/parts.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"parts.js","sourceRoot":"","sources":["../../../src/test/assertions/parts.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAEhD,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAErC;;;GAGG;AACH,MAAM,CAAC,MAAM,KAAK,GAAmB,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;IACnD,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IACtD,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;QACnB,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,mBAAmB,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,0BAA0B,EAAE,CAAC;IACvI,CAAC;IACD,MAAM,KAAK,GAAG,CAAC,KAAK,CAAC,CAAC,IAAI,EAAc,CAAC;IACzC,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,IAAI,KAAK,QAAQ,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;IACvI,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IAC1F,IAAI,CAAC,OAAO,CAAC,MAAM;QAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,mBAAmB,MAAM,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC;IACvF,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,YAAY,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,mBAAmB,CAAC;IAC7E,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,0BAA0B,MAAM,CAAC,OAAO,CAAC,KAAK,KAAK,GAAG,EAAE,CAAC;AACzF,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,YAAY,GAAmB,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;IAC1D,MAAM,QAAQ,GAAG,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACvH,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,QAAQ;QAAE,MAAM,IAAI,KAAK,CAAC,8DAA8D,CAAC,CAAC;IACjH,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM;QAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,2BAA2B,EAAE,CAAC;IACpF,IAAI,CAAC,QAAQ;QAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,0BAA0B,EAAE,CAAC;IACzE,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,KAAgC,CAAC,CAAC;IAChE,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1F,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,CAAC;QACnF,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,iCAAiC,MAAM,EAAE,EAAE;QACnE,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,iCAAiC,MAAM,EAAE,EAAE,CAAC;AACzE,CAAC,CAAC"}

package/dist/test/assertions/text.d.ts

@@ -0,0 +1,4 @@
+import type { AssertionCheck } from '../types.js';
+/** `contains: "$19"` or `contains: [Team, "$19"]`: the reply has every one of them. */
+export declare const contains: AssertionCheck;
+//# sourceMappingURL=text.d.ts.map

package/dist/test/assertions/text.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"text.d.ts","sourceRoot":"","sources":["../../../src/test/assertions/text.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAGlD,uFAAuF;AACvF,eAAO,MAAM,QAAQ,EAAE,cAMtB,CAAC"}

package/dist/test/assertions/text.js

@@ -0,0 +1,10 @@
+import { asStrings, quoted } from './values.js';
+/** `contains: "$19"` or `contains: [Team, "$19"]`: the reply has every one of them. */
+export const contains = (value, turn) => {
+    const wanted = asStrings(value, 'contains');
+    const missing = wanted.filter((text) => !turn.text.includes(text));
+    return missing.length
+        ? { pass: false, reason: `the reply doesn't contain ${quoted(missing)}` }
+        : { pass: true, reason: `the reply contains ${quoted(wanted)}` };
+};
+//# sourceMappingURL=text.js.map

package/dist/test/assertions/text.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"text.js","sourceRoot":"","sources":["../../../src/test/assertions/text.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAEhD,uFAAuF;AACvF,MAAM,CAAC,MAAM,QAAQ,GAAmB,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;IACtD,MAAM,MAAM,GAAG,SAAS,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;IAC5C,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;IACnE,OAAO,OAAO,CAAC,MAAM;QACnB,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,6BAA6B,MAAM,CAAC,OAAO,CAAC,EAAE,EAAE;QACzE,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,sBAAsB,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;AACrE,CAAC,CAAC"}

package/dist/test/assertions/tools.d.ts

@@ -0,0 +1,13 @@
+import type { AssertionCheck } from '../types.js';
+/**
+ * `calls-tool: "lookup_order"` or `calls-tool: { name: "lookup_order", args: { id: "A-1" } }`.
+ * When `args` is supplied, matches partially against `input`: extra keys are allowed,
+ * nested objects match recursively, and arrays match exactly.
+ */
+export declare const callsTool: AssertionCheck;
+/**
+ * `no-tool-call: true`: passes only when no tools were called in the turn.
+ * `no-tool-call: "lookup_order"`: passes when that specific tool was not called.
+ */
+export declare const noToolCall: AssertionCheck;
+//# sourceMappingURL=tools.d.ts.map

package/dist/test/assertions/tools.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tools.d.ts","sourceRoot":"","sources":["../../../src/test/assertions/tools.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAmDlD;;;;GAIG;AACH,eAAO,MAAM,SAAS,EAAE,cAWvB,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,UAAU,EAAE,cAaxB,CAAC"}

package/dist/test/assertions/tools.js

@@ -0,0 +1,82 @@
+import { isDeepStrictEqual } from 'node:util';
+function isObject(value) {
+    return value !== null && typeof value === 'object' && !Array.isArray(value);
+}
+function matchesPartial(expected, actual) {
+    if (Object.is(expected, actual))
+        return true;
+    if (Array.isArray(expected))
+        return isDeepStrictEqual(expected, actual);
+    if (isObject(expected)) {
+        if (!isObject(actual))
+            return false;
+        return Object.entries(expected).every(([key, val]) => Object.hasOwn(actual, key) && matchesPartial(val, actual[key]));
+    }
+    return false;
+}
+function parseCallsTool(value) {
+    if (typeof value === 'string' && value.length > 0) {
+        return { name: value };
+    }
+    if (isObject(value)) {
+        const keys = Object.keys(value);
+        const hasValidName = typeof value.name === 'string' && value.name.length > 0;
+        const hasValidArgs = value.args === undefined || isObject(value.args);
+        const allowed = ['name', 'args'];
+        const onlyAllowedKeys = keys.length > 0 && keys.every((key) => allowed.includes(key));
+        if (hasValidName && hasValidArgs && onlyAllowedKeys) {
+            return { name: value.name, args: value.args };
+        }
+    }
+    throw new Error('calls-tool takes a tool name, or { name, args }');
+}
+function parseNoToolCall(value) {
+    if (value === true)
+        return true;
+    if (typeof value === 'string' && value.length > 0)
+        return value;
+    throw new Error('no-tool-call takes true, or a tool name');
+}
+function formatCall(call) {
+    const args = call.input === undefined ? '' : JSON.stringify(call.input);
+    return `${call.name}(${args})`;
+}
+function formatActual(calls) {
+    if (!calls.length)
+        return 'it called no tools';
+    return `it called ${calls.map(formatCall).join(', ')}`;
+}
+/**
+ * `calls-tool: "lookup_order"` or `calls-tool: { name: "lookup_order", args: { id: "A-1" } }`.
+ * When `args` is supplied, matches partially against `input`: extra keys are allowed,
+ * nested objects match recursively, and arrays match exactly.
+ */
+export const callsTool = (value, turn) => {
+    const { name, args } = parseCallsTool(value);
+    const calls = turn.toolCalls;
+    const matched = calls.some((call) => call.name === name && (args === undefined || matchesPartial(args, call.input)));
+    const actual = formatActual(calls);
+    const target = args !== undefined ? `"${name}" with ${JSON.stringify(args)}` : `"${name}"`;
+    return matched
+        ? { pass: true, reason: `the turn called ${target}` }
+        : { pass: false, reason: `the turn didn't call ${target} (${actual})` };
+};
+/**
+ * `no-tool-call: true`: passes only when no tools were called in the turn.
+ * `no-tool-call: "lookup_order"`: passes when that specific tool was not called.
+ */
+export const noToolCall = (value, turn) => {
+    const target = parseNoToolCall(value);
+    const calls = turn.toolCalls;
+    const actual = formatActual(calls);
+    if (target === true) {
+        return calls.length === 0
+            ? { pass: true, reason: 'the turn called no tools' }
+            : { pass: false, reason: `the turn called ${calls.map(formatCall).join(', ')}` };
+    }
+    const called = calls.some((call) => call.name === target);
+    return !called
+        ? { pass: true, reason: `the turn didn't call "${target}"` }
+        : { pass: false, reason: `the turn called "${target}" (${actual})` };
+};
+//# sourceMappingURL=tools.js.map

package/dist/test/assertions/tools.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tools.js","sourceRoot":"","sources":["../../../src/test/assertions/tools.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAC;AAG9C,SAAS,QAAQ,CAAC,KAAc;IAC9B,OAAO,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAC9E,CAAC;AAED,SAAS,cAAc,CAAC,QAAiB,EAAE,MAAe;IACxD,IAAI,MAAM,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,CAAC;QAAE,OAAO,IAAI,CAAC;IAC7C,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC;QAAE,OAAO,iBAAiB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IACxE,IAAI,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QACvB,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAE,OAAO,KAAK,CAAC;QACpC,OAAO,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,KAAK,CACnC,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,IAAI,cAAc,CAAC,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAC/E,CAAC;IACJ,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,cAAc,CAAC,KAAc;IACpC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClD,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;IACzB,CAAC;IACD,IAAI,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QACpB,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAChC,MAAM,YAAY,GAAG,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;QAC7E,MAAM,YAAY,GAAG,KAAK,CAAC,IAAI,KAAK,SAAS,IAAI,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACtE,MAAM,OAAO,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACjC,MAAM,eAAe,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;QACtF,IAAI,YAAY,IAAI,YAAY,IAAI,eAAe,EAAE,CAAC;YACpD,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,IAAc,EAAE,IAAI,EAAE,KAAK,CAAC,IAA2C,EAAE,CAAC;QACjG,CAAC;IACH,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;AACrE,CAAC;AAED,SAAS,eAAe,CAAC,KAAc;IACrC,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,IAAI,CAAC;IAChC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IAChE,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;AAC7D,CAAC;AAED,SAAS,UAAU,CAAC,IAAsC;IACxD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACxE,OAAO,GAAG,IAAI,CAAC,IAAI,IAAI,IAAI,GAAG,CAAC;AACjC,CAAC;AAED,SAAS,YAAY,CAAC,KAA8C;IAClE,IAAI,CAAC,KAAK,CAAC,MAAM;QAAE,OAAO,oBAAoB,CAAC;IAC/C,OAAO,aAAa,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;AACzD,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,MAAM,SAAS,GAAmB,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;IACvD,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;IAC7C,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC;IAC7B,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CACxB,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CACzF,CAAC;IACF,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;IACnC,MAAM,MAAM,GAAG,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,UAAU,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,GAAG,CAAC;IAC3F,OAAO,OAAO;QACZ,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,mBAAmB,MAAM,EAAE,EAAE;QACrD,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,wBAAwB,MAAM,KAAK,MAAM,GAAG,EAAE,CAAC;AAC5E,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,UAAU,GAAmB,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;IACxD,MAAM,MAAM,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;IACtC,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC;IAC7B,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;IACnC,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;QACpB,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;YACvB,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,0BAA0B,EAAE;YACpD,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,mBAAmB,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;IACrF,CAAC;IACD,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;IAC1D,OAAO,CAAC,MAAM;QACZ,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,yBAAyB,MAAM,GAAG,EAAE;QAC5D,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,oBAAoB,MAAM,MAAM,MAAM,GAAG,EAAE,CAAC;AACzE,CAAC,CAAC"}

package/dist/test/assertions/usage.d.ts

@@ -0,0 +1,8 @@
+import type { AssertionCheck } from '../types.js';
+/** `tokens-under: 800` (input plus output), or `tokens-under: { input: 600, output: 200 }`. */
+export declare const tokensUnder: AssertionCheck;
+/** `latency-under: 3000`: the reply finished in under 3000 ms. */
+export declare const latencyUnder: AssertionCheck;
+/** `cost-under: 0.01`: the turn cost under $0.01, from `prices` in the config. */
+export declare const costUnder: AssertionCheck;
+//# sourceMappingURL=usage.d.ts.map

package/dist/test/assertions/usage.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"usage.d.ts","sourceRoot":"","sources":["../../../src/test/assertions/usage.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAGlD,+FAA+F;AAC/F,eAAO,MAAM,WAAW,EAAE,cAazB,CAAC;AAcF,kEAAkE;AAClE,eAAO,MAAM,YAAY,EAAE,cAM1B,CAAC;AAEF,kFAAkF;AAClF,eAAO,MAAM,SAAS,EAAE,cASvB,CAAC"}

package/dist/test/assertions/usage.js

@@ -0,0 +1,52 @@
+import { dollars } from '../cost.js';
+import { asLimit } from './values.js';
+/** `tokens-under: 800` (input plus output), or `tokens-under: { input: 600, output: 200 }`. */
+export const tokensUnder = (value, turn) => {
+    const limits = readTokenLimits(value);
+    const usage = turn.usage;
+    if (!usage)
+        throw new Error('the turn has no token usage to check');
+    const checks = [
+        { label: 'tokens', limit: limits.total, used: usage.inputTokens + usage.outputTokens },
+        { label: 'input tokens', limit: limits.input, used: usage.inputTokens },
+        { label: 'output tokens', limit: limits.output, used: usage.outputTokens },
+    ].filter((check) => check.limit !== undefined);
+    const over = checks.filter((check) => check.used >= check.limit);
+    return over.length
+        ? { pass: false, reason: `the turn used ${over.map(({ label, limit, used }) => `${used} ${label}, over the ${limit} limit`).join(', ')}` }
+        : { pass: true, reason: `the turn used ${checks.map(({ label, limit, used }) => `${used} ${label} (limit ${limit})`).join(', ')}` };
+};
+function readTokenLimits(value) {
+    if (typeof value === 'number')
+        return { total: asLimit(value, 'tokens-under') };
+    const isObject = value !== null && typeof value === 'object' && !Array.isArray(value);
+    const keys = isObject ? Object.keys(value) : [];
+    if (!keys.length || keys.some((key) => key !== 'input' && key !== 'output'))
+        throw new Error('tokens-under takes a number, or { input, output }');
+    const { input, output } = value;
+    return {
+        input: input === undefined ? undefined : asLimit(input, 'tokens-under'),
+        output: output === undefined ? undefined : asLimit(output, 'tokens-under'),
+    };
+}
+/** `latency-under: 3000`: the reply finished in under 3000 ms. */
+export const latencyUnder = (value, turn) => {
+    const limit = asLimit(value, 'latency-under');
+    const took = Math.round(turn.latencyMs);
+    return turn.latencyMs < limit
+        ? { pass: true, reason: `the reply took ${took} ms (limit ${limit} ms)` }
+        : { pass: false, reason: `the reply took ${took} ms, over the ${limit} ms limit` };
+};
+/** `cost-under: 0.01`: the turn cost under $0.01, from `prices` in the config. */
+export const costUnder = (value, turn) => {
+    const limit = asLimit(value, 'cost-under');
+    if (!turn.usage)
+        throw new Error('the turn has no token usage to check');
+    if (turn.costUsd === undefined) {
+        throw new Error(`the turn has no cost: add ${turn.model ? `"${turn.model}"` : 'the model'} to \`prices\` in the config`);
+    }
+    return turn.costUsd < limit
+        ? { pass: true, reason: `the turn cost ${dollars(turn.costUsd)} (limit ${dollars(limit)})` }
+        : { pass: false, reason: `the turn cost ${dollars(turn.costUsd)}, over the ${dollars(limit)} limit` };
+};
+//# sourceMappingURL=usage.js.map

package/dist/test/assertions/usage.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"usage.js","sourceRoot":"","sources":["../../../src/test/assertions/usage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAErC,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAEtC,+FAA+F;AAC/F,MAAM,CAAC,MAAM,WAAW,GAAmB,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;IACzD,MAAM,MAAM,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;IACtC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;IACzB,IAAI,CAAC,KAAK;QAAE,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;IACpE,MAAM,MAAM,GAAG;QACb,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,CAAC,WAAW,GAAG,KAAK,CAAC,YAAY,EAAE;QACtF,EAAE,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,CAAC,WAAW,EAAE;QACvE,EAAE,KAAK,EAAE,eAAe,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,EAAE,KAAK,CAAC,YAAY,EAAE;KAC3E,CAAC,MAAM,CAAC,CAAC,KAAK,EAA2D,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC;IACxG,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC;IACjE,OAAO,IAAI,CAAC,MAAM;QAChB,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,iBAAiB,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,GAAG,IAAI,IAAI,KAAK,cAAc,KAAK,QAAQ,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE;QAC1I,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,iBAAiB,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,GAAG,IAAI,IAAI,KAAK,WAAW,KAAK,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;AACxI,CAAC,CAAC;AAEF,SAAS,eAAe,CAAC,KAAc;IACrC,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,cAAc,CAAC,EAAE,CAAC;IAChF,MAAM,QAAQ,GAAG,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IACtF,MAAM,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAChD,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,KAAK,OAAO,IAAI,GAAG,KAAK,QAAQ,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAC;IAClJ,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,KAAgC,CAAC;IAC3D,OAAO;QACL,KAAK,EAAE,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,cAAc,CAAC;QACvE,MAAM,EAAE,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,cAAc,CAAC;KAC3E,CAAC;AACJ,CAAC;AAED,kEAAkE;AAClE,MAAM,CAAC,MAAM,YAAY,GAAmB,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;IAC1D,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,EAAE,eAAe,CAAC,CAAC;IAC9C,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACxC,OAAO,IAAI,CAAC,SAAS,GAAG,KAAK;QAC3B,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,kBAAkB,IAAI,cAAc,KAAK,MAAM,EAAE;QACzE,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,kBAAkB,IAAI,iBAAiB,KAAK,WAAW,EAAE,CAAC;AACvF,CAAC,CAAC;AAEF,kFAAkF;AAClF,MAAM,CAAC,MAAM,SAAS,GAAmB,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;IACvD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,EAAE,YAAY,CAAC,CAAC;IAC3C,IAAI,CAAC,IAAI,CAAC,KAAK;QAAE,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;IACzE,IAAI,IAAI,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CAAC,6BAA6B,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,WAAW,8BAA8B,CAAC,CAAC;IAC3H,CAAC;IACD,OAAO,IAAI,CAAC,OAAO,GAAG,KAAK;QACzB,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,iBAAiB,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE;QAC5F,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,iBAAiB,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,cAAc,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;AAC1G,CAAC,CAAC"}

package/dist/test/assertions/values.d.ts

@@ -0,0 +1,8 @@
+/** Helpers for reading an assertion's value. A bad value throws, and the assertion fails with the message. */
+/** A string or a non-empty list of strings, as a list. */
+export declare function asStrings(value: unknown, name: string): string[];
+/** `"a", "b"`, for reasons. */
+export declare function quoted(texts: string[]): string;
+/** A number of 0 or more, such as a token, millisecond or dollar limit. */
+export declare function asLimit(value: unknown, name: string): number;
+//# sourceMappingURL=values.d.ts.map

package/dist/test/assertions/values.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"values.d.ts","sourceRoot":"","sources":["../../../src/test/assertions/values.ts"],"names":[],"mappings":"AAAA,8GAA8G;AAE9G,0DAA0D;AAC1D,wBAAgB,SAAS,CAAC,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAIhE;AAED,+BAA+B;AAC/B,wBAAgB,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,CAE9C;AAED,2EAA2E;AAC3E,wBAAgB,OAAO,CAAC,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAG5D"}

package/dist/test/assertions/values.js

@@ -0,0 +1,19 @@
+/** Helpers for reading an assertion's value. A bad value throws, and the assertion fails with the message. */
+/** A string or a non-empty list of strings, as a list. */
+export function asStrings(value, name) {
+    const list = Array.isArray(value) ? value : [value];
+    if (!list.length || list.some((item) => typeof item !== 'string'))
+        throw new Error(`${name} takes a string or a list of strings`);
+    return list;
+}
+/** `"a", "b"`, for reasons. */
+export function quoted(texts) {
+    return texts.map((text) => JSON.stringify(text)).join(', ');
+}
+/** A number of 0 or more, such as a token, millisecond or dollar limit. */
+export function asLimit(value, name) {
+    if (typeof value !== 'number' || !Number.isFinite(value) || value < 0)
+        throw new Error(`${name} takes a number of 0 or more`);
+    return value;
+}
+//# sourceMappingURL=values.js.map

package/dist/test/assertions/values.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"values.js","sourceRoot":"","sources":["../../../src/test/assertions/values.ts"],"names":[],"mappings":"AAAA,8GAA8G;AAE9G,0DAA0D;AAC1D,MAAM,UAAU,SAAS,CAAC,KAAc,EAAE,IAAY;IACpD,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IACpD,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,IAAI,KAAK,QAAQ,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,sCAAsC,CAAC,CAAC;IAClI,OAAO,IAAgB,CAAC;AAC1B,CAAC;AAED,+BAA+B;AAC/B,MAAM,UAAU,MAAM,CAAC,KAAe;IACpC,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC9D,CAAC;AAED,2EAA2E;AAC3E,MAAM,UAAU,OAAO,CAAC,KAAc,EAAE,IAAY;IAClD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,8BAA8B,CAAC,CAAC;IAC9H,OAAO,KAAK,CAAC;AACf,CAAC"}

package/dist/test/cassette.d.ts

@@ -0,0 +1,77 @@
+/**
+ * `live` calls the APIs. `record` calls them and saves what came back.
+ * `replay` answers every call from the saved recordings and never touches
+ * the network, so CI needs no API keys.
+ */
+export type CassetteMode = 'live' | 'record' | 'replay';
+/** One recorded HTTP call, a line of `__cassettes__/<eval file>.jsonl`. */
+export interface CassetteEntry {
+    test: string;
+    /** Hash of the method, URL and body. */
+    key: string;
+    /** Kept so a changed request can say what changed. Headers are never saved. */
+    request: {
+        method: string;
+        url: string;
+        body?: unknown;
+    };
+    /**
+     * `body` is the text, or for bodies over 4 kB (a streamed reply is one
+     * event per token) gzip in base64, with `encoding: 'gzip'`.
+     */
+    response: {
+        status: number;
+        contentType?: string;
+        body: string;
+        encoding?: 'gzip';
+    };
+}
+/** `evals/pricing.chat.yaml` records to `evals/__cassettes__/pricing.chat.jsonl`. */
+export declare function cassettePath(fixtureFile: string): string;
+/**
+ * Records and replays the HTTP calls each test makes: model steps, embedding
+ * calls, a judge's calls, tools that call APIs. Calls are matched by a hash of
+ * the request within their test, in the order they were made, so a changed
+ * prompt, model or tool list misses the recording and fails instead of
+ * passing on an old answer.
+ */
+export declare class Cassettes {
+    readonly mode: CassetteMode;
+    private readonly scope;
+    private readonly recorded;
+    private readonly fresh;
+    private readonly tallies;
+    constructor(mode: CassetteMode);
+    /**
+     * Routes `fetch` through the cassettes, for calls made inside `run()`.
+     * Install it before the config is loaded, since SDKs and embedders can keep
+     * a reference to `fetch` from when they were created. Returns the undo.
+     */
+    install(): () => void;
+    /** Reads the recordings of the eval files about to run. Replay only. */
+    load(fixtures: string[]): Promise<void>;
+    /** Runs `fn` with its HTTP calls recorded or replayed as the test's. Returns what went wrong with the cassettes. */
+    run<T>(fixture: string, test: string, fn: () => Promise<T>): Promise<{
+        value: T;
+        problems: string[];
+    }>;
+    /**
+     * Writes what `record` saw. Recordings of the tests that ran are replaced,
+     * those of tests no longer in the file are dropped, and the rest are kept,
+     * so `--record --filter` updates only the tests it ran.
+     */
+    save(fixtures: Array<{
+        path: string;
+        ran: string[];
+        all: string[];
+    }>): Promise<string[]>;
+    private handle;
+    private tally;
+    private missMessage;
+}
+/** The parts of a request that decide its answer: method, URL without secrets, and the body (parsed when it's JSON). */
+export declare function describeRequest(request: Request): Promise<CassetteEntry['request']>;
+export declare function hashRequest(request: CassetteEntry['request']): string;
+/** Where two JSON values first differ, as a path like `messages[1].content`. Undefined when they're equal. */
+export declare function firstDifference(a: unknown, b: unknown, path?: string): string | undefined;
+//# sourceMappingURL=cassette.d.ts.map

package/dist/test/cassette.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cassette.d.ts","sourceRoot":"","sources":["../../src/test/cassette.ts"],"names":[],"mappings":"AAMA;;;;GAIG;AACH,MAAM,MAAM,YAAY,GAAG,MAAM,GAAG,QAAQ,GAAG,QAAQ,CAAC;AAExD,2EAA2E;AAC3E,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,wCAAwC;IACxC,GAAG,EAAE,MAAM,CAAC;IACZ,+EAA+E;IAC/E,OAAO,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,OAAO,CAAA;KAAE,CAAC;IACzD;;;OAGG;IACH,QAAQ,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CACrF;AAmBD,qFAAqF;AACrF,wBAAgB,YAAY,CAAC,WAAW,EAAE,MAAM,GAAG,MAAM,CAExD;AAED;;;;;;GAMG;AACH,qBAAa,SAAS;IACpB,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAC;IAC5B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAkC;IACxD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA6C;IACtE,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAsC;IAC5D,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA4B;gBAExC,IAAI,EAAE,YAAY;IAI9B;;;;OAIG;IACH,OAAO,IAAI,MAAM,IAAI;IAarB,wEAAwE;IAClE,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAK7C,oHAAoH;IAC9G,GAAG,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC;QAAE,KAAK,EAAE,CAAC,CAAC;QAAC,QAAQ,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;IAM5G;;;;OAIG;IACG,IAAI,CAAC,QAAQ,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,EAAE,CAAC;QAAC,GAAG,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;YAkBhF,MAAM;IA8CpB,OAAO,CAAC,KAAK;IAUb,OAAO,CAAC,WAAW;CAUpB;AA4BD,wHAAwH;AACxH,wBAAsB,eAAe,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC,CAazF;AAED,wBAAgB,WAAW,CAAC,OAAO,EAAE,aAAa,CAAC,SAAS,CAAC,GAAG,MAAM,CAKrE;AAcD,8GAA8G;AAC9G,wBAAgB,eAAe,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,IAAI,SAAK,GAAG,MAAM,GAAG,SAAS,CAiBrF"}