@prismatic-io/lux 0.0.1

package/lib/assertions/core/resource-checks.d.ts

@@ -0,0 +1,71 @@
+import { z } from "zod";
+import { type AssertionImpl } from "../../core/index.js";
+export declare const LatencyUnderConfigSchema: z.ZodObject<{
+    id: z.ZodOptional<z.ZodString>;
+    weight: z.ZodOptional<z.ZodNumber>;
+    threshold: z.ZodOptional<z.ZodNumber>;
+    name: z.ZodOptional<z.ZodString>;
+    type: z.ZodLiteral<"latency-under">;
+    maxMs: z.ZodNumber;
+}, z.core.$strict>;
+export type LatencyUnderConfig = z.infer<typeof LatencyUnderConfigSchema>;
+export declare const latencyUnderAssertion: AssertionImpl<LatencyUnderConfig>;
+export declare const TokensUnderConfigSchema: z.ZodObject<{
+    id: z.ZodOptional<z.ZodString>;
+    weight: z.ZodOptional<z.ZodNumber>;
+    threshold: z.ZodOptional<z.ZodNumber>;
+    name: z.ZodOptional<z.ZodString>;
+    type: z.ZodLiteral<"tokens-under">;
+    max: z.ZodNumber;
+    component: z.ZodDefault<z.ZodEnum<{
+        cacheCreation: "cacheCreation";
+        cacheRead: "cacheRead";
+        input: "input";
+        output: "output";
+        total: "total";
+    }>>;
+    onUnavailable: z.ZodOptional<z.ZodEnum<{
+        fail: "fail";
+        pass: "pass";
+    }>>;
+}, z.core.$strict>;
+export type TokensUnderConfig = z.infer<typeof TokensUnderConfigSchema>;
+export declare const tokensUnderAssertion: AssertionImpl<TokensUnderConfig>;
+export declare const CostUnderConfigSchema: z.ZodObject<{
+    id: z.ZodOptional<z.ZodString>;
+    weight: z.ZodOptional<z.ZodNumber>;
+    threshold: z.ZodOptional<z.ZodNumber>;
+    name: z.ZodOptional<z.ZodString>;
+    type: z.ZodLiteral<"cost-under">;
+    maxUsd: z.ZodNumber;
+    onUnavailable: z.ZodOptional<z.ZodEnum<{
+        fail: "fail";
+        pass: "pass";
+    }>>;
+}, z.core.$strict>;
+export type CostUnderConfig = z.infer<typeof CostUnderConfigSchema>;
+export declare const costUnderAssertion: AssertionImpl<CostUnderConfig>;
+export declare const ArtifactBytesUnderConfigSchema: z.ZodObject<{
+    id: z.ZodOptional<z.ZodString>;
+    weight: z.ZodOptional<z.ZodNumber>;
+    threshold: z.ZodOptional<z.ZodNumber>;
+    name: z.ZodOptional<z.ZodString>;
+    type: z.ZodLiteral<"artifact-bytes-under">;
+    max: z.ZodNumber;
+    onUnavailable: z.ZodOptional<z.ZodEnum<{
+        fail: "fail";
+        pass: "pass";
+    }>>;
+}, z.core.$strict>;
+export type ArtifactBytesUnderConfig = z.infer<typeof ArtifactBytesUnderConfigSchema>;
+export declare const artifactBytesUnderAssertion: AssertionImpl<ArtifactBytesUnderConfig>;
+export declare const RunSucceededConfigSchema: z.ZodObject<{
+    id: z.ZodOptional<z.ZodString>;
+    weight: z.ZodOptional<z.ZodNumber>;
+    threshold: z.ZodOptional<z.ZodNumber>;
+    name: z.ZodOptional<z.ZodString>;
+    type: z.ZodLiteral<"run-succeeded">;
+}, z.core.$strict>;
+export type RunSucceededConfig = z.infer<typeof RunSucceededConfigSchema>;
+export declare const runSucceededAssertion: AssertionImpl<RunSucceededConfig>;
+//# sourceMappingURL=resource-checks.d.ts.map

package/lib/assertions/core/resource-checks.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"resource-checks.d.ts","sourceRoot":"","sources":["../../../src/assertions/core/resource-checks.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,KAAK,aAAa,EAAoC,MAAM,qBAAqB,CAAC;AAY3F,eAAO,MAAM,wBAAwB;;;;;;;kBAM1B,CAAC;AACZ,MAAM,MAAM,kBAAkB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,wBAAwB,CAAC,CAAC;AAE1E,eAAO,MAAM,qBAAqB,EAAE,aAAa,CAAC,kBAAkB,CAsBnE,CAAC;AAEF,eAAO,MAAM,uBAAuB;;;;;;;;;;;;;;;;;;kBAczB,CAAC;AACZ,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AAIxE,eAAO,MAAM,oBAAoB,EAAE,aAAa,CAAC,iBAAiB,CAqBjE,CAAC;AAEF,eAAO,MAAM,qBAAqB;;;;;;;;;;;kBAOvB,CAAC;AACZ,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC;AAEpE,eAAO,MAAM,kBAAkB,EAAE,aAAa,CAAC,eAAe,CAgB7D,CAAC;AAEF,eAAO,MAAM,8BAA8B;;;;;;;;;;;kBAOhC,CAAC;AACZ,MAAM,MAAM,wBAAwB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,8BAA8B,CAAC,CAAC;AAEtF,eAAO,MAAM,2BAA2B,EAAE,aAAa,CAAC,wBAAwB,CA0B/E,CAAC;AAMF,eAAO,MAAM,wBAAwB;;;;;;kBAK1B,CAAC;AACZ,MAAM,MAAM,kBAAkB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,wBAAwB,CAAC,CAAC;AAE1E,eAAO,MAAM,qBAAqB,EAAE,aAAa,CAAC,kBAAkB,CAanE,CAAC"}

package/lib/assertions/core/resource-checks.js

@@ -0,0 +1,159 @@
+import { z } from "zod";
+import { inspectRunArtifact, isUsageEmpty } from "../../core/index.js";
+import { baseAssertionFields as baseFields } from "./helpers.js";
+const UnavailablePolicySchema = z.enum(["pass", "fail"]);
+const unavailable = (resource, policy) => ({
+    passed: policy === "pass",
+    score: policy === "pass" ? 1 : 0,
+    comment: `${resource} unavailable`,
+    detail: { unavailable: true },
+});
+export const LatencyUnderConfigSchema = z
+    .object({
+    type: z.literal("latency-under"),
+    maxMs: z.number().positive(),
+    ...baseFields,
+})
+    .strict();
+export const latencyUnderAssertion = {
+    type: "latency-under",
+    configSchema: LatencyUnderConfigSchema,
+    evaluate: async (config, run) => {
+        const start = Date.parse(run.metadata.startedAt);
+        const end = Date.parse(run.metadata.endedAt);
+        if (Number.isNaN(start) || Number.isNaN(end)) {
+            return {
+                passed: false,
+                score: 0,
+                comment: "duration unavailable (missing timestamps)",
+            };
+        }
+        const ms = end - start;
+        const passed = ms <= config.maxMs;
+        return {
+            passed,
+            score: passed ? 1 : 0,
+            comment: `${ms}ms ${passed ? "<=" : ">"} ${config.maxMs}ms`,
+            detail: { durationMs: ms },
+        };
+    },
+};
+export const TokensUnderConfigSchema = z
+    .object({
+    type: z.literal("tokens-under"),
+    max: z.number().positive(),
+    /**
+     * `total` is uncached input + output, the historical meaning. Cache traffic
+     * is counted separately because it is billed differently and, on a warm
+     * session, dwarfs `input` by two orders of magnitude.
+     */
+    component: z.enum(["input", "output", "total", "cacheRead", "cacheCreation"]).default("total"),
+    /** Compatibility defaults to pass; optimization campaigns should require fail. */
+    onUnavailable: UnavailablePolicySchema.optional(),
+    ...baseFields,
+})
+    .strict();
+// Grades the agent under test, never the harness: an eval must not fail
+// because the persona answerer or the LLM judge got chatty.
+export const tokensUnderAssertion = {
+    type: "tokens-under",
+    configSchema: TokensUnderConfigSchema,
+    evaluate: async (config, run) => {
+        const agent = run.metadata.usage.agent;
+        // A driver that reports nothing arrives here as all-zeros. Treat that as
+        // "unknown", not as "the agent spent zero tokens" — no real agent spends
+        // zero.
+        if (isUsageEmpty(agent)) {
+            return unavailable("tokens (driver did not report)", config.onUnavailable ?? "pass");
+        }
+        const value = config.component === "total" ? agent.input + agent.output : agent[config.component];
+        const passed = value <= config.max;
+        return {
+            passed,
+            score: passed ? 1 : 0,
+            comment: `${config.component} tokens=${value} ${passed ? "<=" : ">"} ${config.max}`,
+            detail: { tokens: value },
+        };
+    },
+};
+export const CostUnderConfigSchema = z
+    .object({
+    type: z.literal("cost-under"),
+    maxUsd: z.number().positive(),
+    onUnavailable: UnavailablePolicySchema.optional(),
+    ...baseFields,
+})
+    .strict();
+export const costUnderAssertion = {
+    type: "cost-under",
+    configSchema: CostUnderConfigSchema,
+    evaluate: async (config, run) => {
+        const usd = run.metadata.cost?.usd;
+        if (usd === undefined) {
+            return unavailable("cost (driver did not report)", config.onUnavailable ?? "pass");
+        }
+        const passed = usd <= config.maxUsd;
+        return {
+            passed,
+            score: passed ? 1 : 0,
+            comment: `$${usd.toFixed(4)} ${passed ? "<=" : ">"} $${config.maxUsd.toFixed(4)}`,
+            detail: { usd },
+        };
+    },
+};
+export const ArtifactBytesUnderConfigSchema = z
+    .object({
+    type: z.literal("artifact-bytes-under"),
+    max: z.number().int().nonnegative(),
+    onUnavailable: UnavailablePolicySchema.optional(),
+    ...baseFields,
+})
+    .strict();
+export const artifactBytesUnderAssertion = {
+    type: "artifact-bytes-under",
+    configSchema: ArtifactBytesUnderConfigSchema,
+    evaluate: async (config, run) => {
+        const sizes = await Promise.all(run.artifacts.map(async (artifact) => {
+            if (artifact.size !== undefined)
+                return artifact.size;
+            const inspected = await inspectRunArtifact(run, artifact.path);
+            return inspected.stats?.isFile() ? inspected.stats.size : null;
+        }));
+        if (sizes.some((size) => size === null)) {
+            return unavailable("artifact bytes (one or more files could not be measured)", config.onUnavailable ?? "pass");
+        }
+        const bytes = sizes.reduce((sum, size) => sum + (size ?? 0), 0);
+        const passed = bytes <= config.max;
+        return {
+            passed,
+            score: passed ? 1 : 0,
+            comment: `artifact bytes=${bytes} ${passed ? "<=" : ">"} ${config.max}`,
+            detail: { bytes, files: sizes.length },
+        };
+    },
+};
+// A crashed or truncated run can still satisfy structural checks (a
+// `file-exists` staged by a fixture, say). This lets a case require the
+// agent to have finished cleanly as a first-class, graded expectation —
+// complementing the automatic `casePassed` downgrade the orchestrator does.
+export const RunSucceededConfigSchema = z
+    .object({
+    type: z.literal("run-succeeded"),
+    ...baseFields,
+})
+    .strict();
+export const runSucceededAssertion = {
+    type: "run-succeeded",
+    configSchema: RunSucceededConfigSchema,
+    evaluate: async (_config, run) => {
+        const { exitReason } = run.metadata;
+        const passed = exitReason === "done";
+        return {
+            passed,
+            score: passed ? 1 : 0,
+            comment: passed ? "run completed cleanly" : `run ended with '${exitReason}'`,
+            detail: { exitReason },
+        };
+    },
+};
+//# sourceMappingURL=resource-checks.js.map

package/lib/assertions/core/resource-checks.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"resource-checks.js","sourceRoot":"","sources":["../../../src/assertions/core/resource-checks.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAsB,kBAAkB,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAC3F,OAAO,EAAE,mBAAmB,IAAI,UAAU,EAAE,MAAM,cAAc,CAAC;AAEjE,MAAM,uBAAuB,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;AAEzD,MAAM,WAAW,GAAG,CAAC,QAAgB,EAAE,MAA+C,EAAE,EAAE,CAAC,CAAC;IAC1F,MAAM,EAAE,MAAM,KAAK,MAAM;IACzB,KAAK,EAAE,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAChC,OAAO,EAAE,GAAG,QAAQ,cAAc;IAClC,MAAM,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE;CAC9B,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,wBAAwB,GAAG,CAAC;KACtC,MAAM,CAAC;IACN,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,eAAe,CAAC;IAChC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC5B,GAAG,UAAU;CACd,CAAC;KACD,MAAM,EAAE,CAAC;AAGZ,MAAM,CAAC,MAAM,qBAAqB,GAAsC;IACtE,IAAI,EAAE,eAAe;IACrB,YAAY,EAAE,wBAAwB;IACtC,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE;QAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;QACjD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QAC7C,IAAI,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;YAC7C,OAAO;gBACL,MAAM,EAAE,KAAK;gBACb,KAAK,EAAE,CAAC;gBACR,OAAO,EAAE,2CAA2C;aACrD,CAAC;QACJ,CAAC;QACD,MAAM,EAAE,GAAG,GAAG,GAAG,KAAK,CAAC;QACvB,MAAM,MAAM,GAAG,EAAE,IAAI,MAAM,CAAC,KAAK,CAAC;QAClC,OAAO;YACL,MAAM;YACN,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACrB,OAAO,EAAE,GAAG,EAAE,MAAM,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,MAAM,CAAC,KAAK,IAAI;YAC3D,MAAM,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE;SAC3B,CAAC;IACJ,CAAC;CACF,CAAC;AAEF,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC;KACrC,MAAM,CAAC;IACN,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC;IAC/B,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC1B;;;;OAIG;IACH,SAAS,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,WAAW,EAAE,eAAe,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC;IAC9F,kFAAkF;IAClF,aAAa,EAAE,uBAAuB,CAAC,QAAQ,EAAE;IACjD,GAAG,UAAU;CACd,CAAC;KACD,MAAM,EAAE,CAAC;AAGZ,wEAAwE;AACxE,4DAA4D;AAC5D,MAAM,CAAC,MAAM,oBAAoB,GAAqC;IACpE,IAAI,EAAE,cAAc;IACpB,YAAY,EAAE,uBAAuB;IACrC,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE;QAC9B,MAAM,KAAK,GAAG,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC;QACvC,yEAAyE;QACzE,yEAAyE;QACzE,QAAQ;QACR,IAAI,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;YACxB,OAAO,WAAW,CAAC,gCAAgC,EAAE,MAAM,CAAC,aAAa,IAAI,MAAM,CAAC,CAAC;QACvF,CAAC;QACD,MAAM,KAAK,GACT,MAAM,CAAC,SAAS,KAAK,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QACtF,MAAM,MAAM,GAAG,KAAK,IAAI,MAAM,CAAC,GAAG,CAAC;QACnC,OAAO;YACL,MAAM;YACN,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACrB,OAAO,EAAE,GAAG,MAAM,CAAC,SAAS,WAAW,KAAK,IAAI,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,MAAM,CAAC,GAAG,EAAE;YACnF,MAAM,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE;SAC1B,CAAC;IACJ,CAAC;CACF,CAAC;AAEF,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC;KACnC,MAAM,CAAC;IACN,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC;IAC7B,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC7B,aAAa,EAAE,uBAAuB,CAAC,QAAQ,EAAE;IACjD,GAAG,UAAU;CACd,CAAC;KACD,MAAM,EAAE,CAAC;AAGZ,MAAM,CAAC,MAAM,kBAAkB,GAAmC;IAChE,IAAI,EAAE,YAAY;IAClB,YAAY,EAAE,qBAAqB;IACnC,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE;QAC9B,MAAM,GAAG,GAAG,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,GAAG,CAAC;QACnC,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;YACtB,OAAO,WAAW,CAAC,8BAA8B,EAAE,MAAM,CAAC,aAAa,IAAI,MAAM,CAAC,CAAC;QACrF,CAAC;QACD,MAAM,MAAM,GAAG,GAAG,IAAI,MAAM,CAAC,MAAM,CAAC;QACpC,OAAO;YACL,MAAM;YACN,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACrB,OAAO,EAAE,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,KAAK,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;YACjF,MAAM,EAAE,EAAE,GAAG,EAAE;SAChB,CAAC;IACJ,CAAC;CACF,CAAC;AAEF,MAAM,CAAC,MAAM,8BAA8B,GAAG,CAAC;KAC5C,MAAM,CAAC;IACN,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,sBAAsB,CAAC;IACvC,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE;IACnC,aAAa,EAAE,uBAAuB,CAAC,QAAQ,EAAE;IACjD,GAAG,UAAU;CACd,CAAC;KACD,MAAM,EAAE,CAAC;AAGZ,MAAM,CAAC,MAAM,2BAA2B,GAA4C;IAClF,IAAI,EAAE,sBAAsB;IAC5B,YAAY,EAAE,8BAA8B;IAC5C,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE;QAC9B,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,GAAG,CAC7B,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE;YACnC,IAAI,QAAQ,CAAC,IAAI,KAAK,SAAS;gBAAE,OAAO,QAAQ,CAAC,IAAI,CAAC;YACtD,MAAM,SAAS,GAAG,MAAM,kBAAkB,CAAC,GAAG,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC;YAC/D,OAAO,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;QACjE,CAAC,CAAC,CACH,CAAC;QACF,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;YACxC,OAAO,WAAW,CAChB,0DAA0D,EAC1D,MAAM,CAAC,aAAa,IAAI,MAAM,CAC/B,CAAC;QACJ,CAAC;QACD,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAS,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACxE,MAAM,MAAM,GAAG,KAAK,IAAI,MAAM,CAAC,GAAG,CAAC;QACnC,OAAO;YACL,MAAM;YACN,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACrB,OAAO,EAAE,kBAAkB,KAAK,IAAI,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,MAAM,CAAC,GAAG,EAAE;YACvE,MAAM,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE;SACvC,CAAC;IACJ,CAAC;CACF,CAAC;AAEF,oEAAoE;AACpE,wEAAwE;AACxE,wEAAwE;AACxE,4EAA4E;AAC5E,MAAM,CAAC,MAAM,wBAAwB,GAAG,CAAC;KACtC,MAAM,CAAC;IACN,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,eAAe,CAAC;IAChC,GAAG,UAAU;CACd,CAAC;KACD,MAAM,EAAE,CAAC;AAGZ,MAAM,CAAC,MAAM,qBAAqB,GAAsC;IACtE,IAAI,EAAE,eAAe;IACrB,YAAY,EAAE,wBAAwB;IACtC,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,EAAE;QAC/B,MAAM,EAAE,UAAU,EAAE,GAAG,GAAG,CAAC,QAAQ,CAAC;QACpC,MAAM,MAAM,GAAG,UAAU,KAAK,MAAM,CAAC;QACrC,OAAO;YACL,MAAM;YACN,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACrB,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,uBAAuB,CAAC,CAAC,CAAC,mBAAmB,UAAU,GAAG;YAC5E,MAAM,EAAE,EAAE,UAAU,EAAE;SACvB,CAAC;IACJ,CAAC;CACF,CAAC"}

package/lib/assertions/core/safe-regex.d.ts

@@ -0,0 +1,23 @@
+export type RegexEvaluationOptions = {
+    abortSignal?: AbortSignal | undefined;
+    timeoutMs?: number;
+};
+export type SafeRegexInspection = {
+    matchesEmpty: boolean;
+} | {
+    error: string;
+};
+export type SafeRegexGroup = {
+    pattern: string;
+    flags: string;
+    texts: string[];
+};
+export declare const testSafeRegex: (pattern: string, flags: string, text: string, options?: RegexEvaluationOptions) => Promise<boolean>;
+export declare const testSafeRegexSet: (patterns: string[], flags: string, text: string, options?: RegexEvaluationOptions) => Promise<boolean[]>;
+export declare const inspectSafeRegexes: (patterns: string[], flags: string, options?: RegexEvaluationOptions) => Promise<SafeRegexInspection[]>;
+export declare const testSafeRegexGroups: (groups: SafeRegexGroup[], options?: RegexEvaluationOptions) => Promise<boolean[][]>;
+export declare const countSafeRegex: (pattern: string, flags: string, text: string, stopAfter: number, options?: RegexEvaluationOptions) => Promise<{
+    count: number;
+    exact: boolean;
+}>;
+//# sourceMappingURL=safe-regex.d.ts.map

package/lib/assertions/core/safe-regex.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"safe-regex.d.ts","sourceRoot":"","sources":["../../../src/assertions/core/safe-regex.ts"],"names":[],"mappings":"AAiGA,MAAM,MAAM,sBAAsB,GAAG;IACnC,WAAW,CAAC,EAAE,WAAW,GAAG,SAAS,CAAC;IACtC,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAAE,YAAY,EAAE,OAAO,CAAA;CAAE,GAAG;IAAE,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC;AAEhF,MAAM,MAAM,cAAc,GAAG;IAC3B,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,EAAE,CAAC;CACjB,CAAC;AAgDF,eAAO,MAAM,aAAa,YACf,MAAM,SACR,MAAM,QACP,MAAM,YACF,sBAAsB,KAC/B,OAAO,CAAC,OAAO,CACkE,CAAC;AAErF,eAAO,MAAM,gBAAgB,aACjB,MAAM,EAAE,SACX,MAAM,QACP,MAAM,YACF,sBAAsB,KAC/B,OAAO,CAAC,OAAO,EAAE,CACkE,CAAC;AAEvF,eAAO,MAAM,kBAAkB,aACnB,MAAM,EAAE,SACX,MAAM,YACH,sBAAsB,KAC/B,OAAO,CAAC,mBAAmB,EAAE,CACgE,CAAC;AAEjG,eAAO,MAAM,mBAAmB,WACtB,cAAc,EAAE,YACd,sBAAsB,KAC/B,OAAO,CAAC,OAAO,EAAE,EAAE,CACsD,CAAC;AAE7E,eAAO,MAAM,cAAc,YAChB,MAAM,SACR,MAAM,QACP,MAAM,aACD,MAAM,YACP,sBAAsB,KAC/B,OAAO,CAAC;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,OAAO,CAAA;CAAE,CAIzC,CAAC"}

package/lib/assertions/core/safe-regex.js

@@ -0,0 +1,102 @@
+import { Worker } from "node:worker_threads";
+const REGEX_TIMEOUT_MS = 2_000;
+// An eval worker keeps source tests and the compiled package on the same
+// loader-independent path while still making pathological V8 regexes killable.
+const REGEX_WORKER_SOURCE = `
+  const { parentPort, workerData } = require("node:worker_threads");
+
+  const evaluate = () => {
+    if (workerData.operation === "test") {
+      return new RegExp(workerData.pattern, workerData.flags).test(workerData.text);
+    }
+    if (workerData.operation === "set") {
+      return workerData.patterns.map(
+        (pattern) => new RegExp(pattern, workerData.flags).test(workerData.text),
+      );
+    }
+    if (workerData.operation === "inspect") {
+      return workerData.patterns.map((pattern) => {
+        try {
+          return { matchesEmpty: new RegExp(pattern, workerData.flags).test("") };
+        } catch (error) {
+          return {
+            error: error instanceof Error ? error.message : String(error),
+          };
+        }
+      });
+    }
+    if (workerData.operation === "groups") {
+      return workerData.groups.map(({ pattern, flags, texts }) => {
+        const expression = new RegExp(pattern, flags);
+        return texts.map((text) => {
+          expression.lastIndex = 0;
+          return expression.test(text);
+        });
+      });
+    }
+    const flags = workerData.flags.includes("g")
+      ? workerData.flags
+      : workerData.flags + "g";
+    const expression = new RegExp(workerData.pattern, flags);
+    let count = 0;
+    for (const _match of workerData.text.matchAll(expression)) {
+      count += 1;
+      if (count >= workerData.stopAfter) return { count, exact: false };
+    }
+    return { count, exact: true };
+  };
+
+  try {
+    parentPort.postMessage({ ok: true, value: evaluate() });
+  } catch (error) {
+    parentPort.postMessage({
+      ok: false,
+      errorName: error instanceof Error ? error.name : "Error",
+      error: error instanceof Error ? error.message : String(error),
+    });
+  }
+`;
+const evaluateRegex = async (task, options = {}) => await new Promise((resolve, reject) => {
+    const worker = new Worker(REGEX_WORKER_SOURCE, {
+        eval: true,
+        workerData: task,
+    });
+    let settled = false;
+    const finish = (settle) => {
+        if (settled)
+            return;
+        settled = true;
+        clearTimeout(timeout);
+        options.abortSignal?.removeEventListener("abort", abort);
+        settle();
+        void worker.terminate();
+    };
+    const abort = () => finish(() => reject(options.abortSignal?.reason ?? new Error("regex evaluation aborted")));
+    const timeout = setTimeout(() => finish(() => reject(new Error(`regex evaluation exceeded ${options.timeoutMs ?? REGEX_TIMEOUT_MS}ms`))), options.timeoutMs ?? REGEX_TIMEOUT_MS);
+    worker.once("message", (response) => {
+        finish(() => {
+            if (response.ok)
+                resolve(response.value);
+            else {
+                const ErrorConstructor = response.errorName === "SyntaxError" ? SyntaxError : Error;
+                reject(new ErrorConstructor(`regex evaluation failed: ${response.error}`));
+            }
+        });
+    });
+    worker.once("error", (error) => finish(() => reject(error)));
+    worker.once("exit", (code) => {
+        if (code !== 0 && !settled) {
+            finish(() => reject(new Error(`regex worker exited with code ${code}`)));
+        }
+    });
+    if (options.abortSignal?.aborted)
+        abort();
+    else
+        options.abortSignal?.addEventListener("abort", abort, { once: true });
+});
+export const testSafeRegex = async (pattern, flags, text, options) => await evaluateRegex({ operation: "test", pattern, flags, text }, options);
+export const testSafeRegexSet = async (patterns, flags, text, options) => await evaluateRegex({ operation: "set", patterns, flags, text }, options);
+export const inspectSafeRegexes = async (patterns, flags, options) => await evaluateRegex({ operation: "inspect", patterns, flags }, options);
+export const testSafeRegexGroups = async (groups, options) => await evaluateRegex({ operation: "groups", groups }, options);
+export const countSafeRegex = async (pattern, flags, text, stopAfter, options) => await evaluateRegex({ operation: "count", pattern, flags, text, stopAfter }, options);
+//# sourceMappingURL=safe-regex.js.map

package/lib/assertions/core/safe-regex.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"safe-regex.js","sourceRoot":"","sources":["../../../src/assertions/core/safe-regex.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAE7C,MAAM,gBAAgB,GAAG,KAAK,CAAC;AAE/B,yEAAyE;AACzE,+EAA+E;AAC/E,MAAM,mBAAmB,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAqD3B,CAAC;AAmDF,MAAM,aAAa,GAAG,KAAK,EACzB,IAAe,EACf,OAAO,GAA2B,EAAE,EACxB,EAAE,CACd,MAAM,IAAI,OAAO,CAAI,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;IACvC,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,mBAAmB,EAAE;QAC7C,IAAI,EAAE,IAAI;QACV,UAAU,EAAE,IAAI;KACjB,CAAC,CAAC;IACH,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,MAAM,MAAM,GAAG,CAAC,MAAkB,EAAE,EAAE;QACpC,IAAI,OAAO;YAAE,OAAO;QACpB,OAAO,GAAG,IAAI,CAAC;QACf,YAAY,CAAC,OAAO,CAAC,CAAC;QACtB,OAAO,CAAC,WAAW,EAAE,mBAAmB,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QACzD,MAAM,EAAE,CAAC;QACT,KAAK,MAAM,CAAC,SAAS,EAAE,CAAC;IAC1B,CAAC,CAAC;IACF,MAAM,KAAK,GAAG,GAAG,EAAE,CACjB,MAAM,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,WAAW,EAAE,MAAM,IAAI,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC,CAAC,CAAC;IAC7F,MAAM,OAAO,GAAG,UAAU,CACxB,GAAG,EAAE,CACH,MAAM,CAAC,GAAG,EAAE,CACV,MAAM,CAAC,IAAI,KAAK,CAAC,6BAA6B,OAAO,CAAC,SAAS,IAAI,gBAAgB,IAAI,CAAC,CAAC,CAC1F,EACH,OAAO,CAAC,SAAS,IAAI,gBAAgB,CACtC,CAAC;IACF,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,QAA2B,EAAE,EAAE;QACrD,MAAM,CAAC,GAAG,EAAE;YACV,IAAI,QAAQ,CAAC,EAAE;gBAAE,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;iBACpC,CAAC;gBACJ,MAAM,gBAAgB,GAAG,QAAQ,CAAC,SAAS,KAAK,aAAa,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,KAAK,CAAC;gBACpF,MAAM,CAAC,IAAI,gBAAgB,CAAC,4BAA4B,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;YAC7E,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IACH,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAC7D,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;QAC3B,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAC3B,MAAM,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,iCAAiC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAC3E,CAAC;IACH,CAAC,CAAC,CAAC;IACH,IAAI,OAAO,CAAC,WAAW,EAAE,OAAO;QAAE,KAAK,EAAE,CAAC;;QACrC,OAAO,CAAC,WAAW,EAAE,gBAAgB,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;AAC7E,CAAC,CAAC,CAAC;AAEL,MAAM,CAAC,MAAM,aAAa,GAAG,KAAK,EAChC,OAAe,EACf,KAAa,EACb,IAAY,EACZ,OAAgC,EACd,EAAE,CACpB,MAAM,aAAa,CAAU,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,OAAO,CAAC,CAAC;AAErF,MAAM,CAAC,MAAM,gBAAgB,GAAG,KAAK,EACnC,QAAkB,EAClB,KAAa,EACb,IAAY,EACZ,OAAgC,EACZ,EAAE,CACtB,MAAM,aAAa,CAAY,EAAE,SAAS,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,OAAO,CAAC,CAAC;AAEvF,MAAM,CAAC,MAAM,kBAAkB,GAAG,KAAK,EACrC,QAAkB,EAClB,KAAa,EACb,OAAgC,EACA,EAAE,CAClC,MAAM,aAAa,CAAwB,EAAE,SAAS,EAAE,SAAS,EAAE,QAAQ,EAAE,KAAK,EAAE,EAAE,OAAO,CAAC,CAAC;AAEjG,MAAM,CAAC,MAAM,mBAAmB,GAAG,KAAK,EACtC,MAAwB,EACxB,OAAgC,EACV,EAAE,CACxB,MAAM,aAAa,CAAc,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,OAAO,CAAC,CAAC;AAE7E,MAAM,CAAC,MAAM,cAAc,GAAG,KAAK,EACjC,OAAe,EACf,KAAa,EACb,IAAY,EACZ,SAAiB,EACjB,OAAgC,EACY,EAAE,CAC9C,MAAM,aAAa,CACjB,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,EACvD,OAAO,CACR,CAAC"}

package/lib/assertions/core/tool-called-with.d.ts

@@ -0,0 +1,21 @@
+import { z } from "zod";
+import type { AssertionImpl } from "../../core/index.js";
+export declare const ToolCalledWithConfigSchema: z.ZodObject<{
+    id: z.ZodOptional<z.ZodString>;
+    weight: z.ZodOptional<z.ZodNumber>;
+    threshold: z.ZodOptional<z.ZodNumber>;
+    type: z.ZodLiteral<"tool-called-with">;
+    name: z.ZodString;
+    argsJsonPointer: z.ZodDefault<z.ZodString>;
+    argsValue: z.ZodOptional<z.ZodUnknown>;
+    argsContains: z.ZodOptional<z.ZodString>;
+    minTimes: z.ZodOptional<z.ZodNumber>;
+    by: z.ZodDefault<z.ZodEnum<{
+        any: "any";
+        main: "main";
+        subagent: "subagent";
+    }>>;
+}, z.core.$strict>;
+export type ToolCalledWithConfig = z.infer<typeof ToolCalledWithConfigSchema>;
+export declare const toolCalledWithAssertion: AssertionImpl<ToolCalledWithConfig>;
+//# sourceMappingURL=tool-called-with.d.ts.map

package/lib/assertions/core/tool-called-with.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tool-called-with.d.ts","sourceRoot":"","sources":["../../../src/assertions/core/tool-called-with.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAIzD,eAAO,MAAM,0BAA0B;;;;;;;;;;;;;;;kBAmBnC,CAAC;AACL,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,0BAA0B,CAAC,CAAC;AAiB9E,eAAO,MAAM,uBAAuB,EAAE,aAAa,CAAC,oBAAoB,CAyBvE,CAAC"}

package/lib/assertions/core/tool-called-with.js

@@ -0,0 +1,69 @@
+import { z } from "zod";
+import { matchesAgentScope } from "./event-checks.js";
+import { deepEquals, isToolCallEvent, resolvePointer, weightThresholdFields } from "./helpers.js";
+export const ToolCalledWithConfigSchema = z
+    .object({
+    type: z.literal("tool-called-with"),
+    name: z.string(),
+    /** RFC 6901 JSON pointer into the tool's input. Defaults to "" (whole input). */
+    argsJsonPointer: z.string().default(""),
+    /** Structural (key-order-insensitive) equality against the value at the JSON pointer. */
+    argsValue: z.unknown().optional(),
+    /** Substring containment check (stringifies the value at the pointer). */
+    argsContains: z.string().optional(),
+    /** How many matching calls are required. Default 1. `minTimes: 0` would pass vacuously. */
+    minTimes: z.number().int().positive().optional(),
+    /** Whose calls count: "main" agent, "subagent", or "any". */
+    by: z.enum(["any", "main", "subagent"]).default("any"),
+    ...weightThresholdFields,
+})
+    .strict()
+    .refine((c) => c.argsValue === undefined || c.argsContains === undefined, {
+    message: "set only one of argsValue or argsContains",
+});
+const matches = (value, config) => {
+    if (config.argsValue !== undefined) {
+        return deepEquals(value, config.argsValue);
+    }
+    if (config.argsContains !== undefined) {
+        // An unresolved pointer is a non-match, not an error — a later event may
+        // still genuinely match.
+        if (value === undefined)
+            return false;
+        const stringified = typeof value === "string" ? value : JSON.stringify(value);
+        return stringified.includes(config.argsContains);
+    }
+    // Both unset — match any tool call with that name.
+    return true;
+};
+export const toolCalledWithAssertion = {
+    type: "tool-called-with",
+    configSchema: ToolCalledWithConfigSchema,
+    evaluate: async (config, run) => {
+        const min = config.minTimes ?? 1;
+        let hits = 0;
+        for (const e of run.events) {
+            if (!isToolCallEvent(e))
+                continue;
+            if (!matchesAgentScope(e, config.by))
+                continue;
+            const payload = e.payload;
+            if (payload?.name !== config.name)
+                continue;
+            const argsRoot = payload.input ?? null;
+            const value = resolvePointer(argsRoot, config.argsJsonPointer);
+            if (matches(value, config))
+                hits++;
+        }
+        const passed = hits >= min;
+        const bound = passed ? `>= ${min}` : `need ${min}`;
+        const scope = (config.by ?? "any") === "any" ? "" : ` by ${config.by}`;
+        return {
+            passed,
+            score: passed ? 1 : 0,
+            comment: `tool '${config.name}' called ${hits}×${scope} with matching args (${bound})`,
+            detail: { hits },
+        };
+    },
+};
+//# sourceMappingURL=tool-called-with.js.map

package/lib/assertions/core/tool-called-with.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tool-called-with.js","sourceRoot":"","sources":["../../../src/assertions/core/tool-called-with.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,EAAE,UAAU,EAAE,eAAe,EAAE,cAAc,EAAE,qBAAqB,EAAE,MAAM,cAAc,CAAC;AAElG,MAAM,CAAC,MAAM,0BAA0B,GAAG,CAAC;KACxC,MAAM,CAAC;IACN,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,kBAAkB,CAAC;IACnC,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,iFAAiF;IACjF,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;IACvC,yFAAyF;IACzF,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IACjC,0EAA0E;IAC1E,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACnC,2FAA2F;IAC3F,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IAChD,6DAA6D;IAC7D,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;IACtD,GAAG,qBAAqB;CACzB,CAAC;KACD,MAAM,EAAE;KACR,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,SAAS,IAAI,CAAC,CAAC,YAAY,KAAK,SAAS,EAAE;IACxE,OAAO,EAAE,2CAA2C;CACrD,CAAC,CAAC;AAGL,MAAM,OAAO,GAAG,CAAC,KAAc,EAAE,MAA4B,EAAW,EAAE;IACxE,IAAI,MAAM,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;QACnC,OAAO,UAAU,CAAC,KAAK,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;IAC7C,CAAC;IACD,IAAI,MAAM,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;QACtC,yEAAyE;QACzE,yBAAyB;QACzB,IAAI,KAAK,KAAK,SAAS;YAAE,OAAO,KAAK,CAAC;QACtC,MAAM,WAAW,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QAC9E,OAAO,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;IACnD,CAAC;IACD,mDAAmD;IACnD,OAAO,IAAI,CAAC;AACd,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,uBAAuB,GAAwC;IAC1E,IAAI,EAAE,kBAAkB;IACxB,YAAY,EAAE,0BAA0B;IACxC,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE;QAC9B,MAAM,GAAG,GAAG,MAAM,CAAC,QAAQ,IAAI,CAAC,CAAC;QACjC,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,KAAK,MAAM,CAAC,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC;YAC3B,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC;gBAAE,SAAS;YAClC,IAAI,CAAC,iBAAiB,CAAC,CAAC,EAAE,MAAM,CAAC,EAAE,CAAC;gBAAE,SAAS;YAC/C,MAAM,OAAO,GAAG,CAAC,CAAC,OAA8C,CAAC;YACjE,IAAI,OAAO,EAAE,IAAI,KAAK,MAAM,CAAC,IAAI;gBAAE,SAAS;YAC5C,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,IAAI,IAAI,CAAC;YACvC,MAAM,KAAK,GAAG,cAAc,CAAC,QAAQ,EAAE,MAAM,CAAC,eAAe,CAAC,CAAC;YAC/D,IAAI,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC;gBAAE,IAAI,EAAE,CAAC;QACrC,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,IAAI,GAAG,CAAC;QAC3B,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC,QAAQ,GAAG,EAAE,CAAC;QACnD,MAAM,KAAK,GAAG,CAAC,MAAM,CAAC,EAAE,IAAI,KAAK,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,MAAM,CAAC,EAAE,EAAE,CAAC;QACvE,OAAO;YACL,MAAM;YACN,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACrB,OAAO,EAAE,SAAS,MAAM,CAAC,IAAI,YAAY,IAAI,IAAI,KAAK,wBAAwB,KAAK,GAAG;YACtF,MAAM,EAAE,EAAE,IAAI,EAAE;SACjB,CAAC;IACJ,CAAC;CACF,CAAC"}

package/lib/assertions/core/tool-calls.d.ts

@@ -0,0 +1,22 @@
+import type { Run } from "../../core/index.js";
+import { type AgentScope } from "./event-checks.js";
+/**
+ * A tool call projected from a `tool-call` progress event's payload.
+ * `parentToolUseId` is set only when a spawned subagent made the call.
+ */
+export type ToolCall = {
+    name: string;
+    input: unknown;
+    parentToolUseId?: string;
+};
+/**
+ * Every tool call the run made, in order. Reach for it inside a `predicate`
+ * assertion to grade what the agent *did*, not what it said. Pass `by` to scope
+ * to the "main" agent or a "subagent" (default "any").
+ */
+export declare const toolCalls: (run: Run, by?: AgentScope) => ToolCall[];
+/** Stringified inputs of every tool call, for substring checks in a predicate. */
+export declare const toolCallInputs: (run: Run, by?: AgentScope) => string[];
+/** Tool calls made by a spawned subagent (the driver tags these with `parentToolUseId`). */
+export declare const subagentToolCalls: (run: Run) => ToolCall[];
+//# sourceMappingURL=tool-calls.d.ts.map

package/lib/assertions/core/tool-calls.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tool-calls.d.ts","sourceRoot":"","sources":["../../../src/assertions/core/tool-calls.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,qBAAqB,CAAC;AAC/C,OAAO,EAAE,KAAK,UAAU,EAAqB,MAAM,mBAAmB,CAAC;AAGvE;;;GAGG;AACH,MAAM,MAAM,QAAQ,GAAG;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,OAAO,CAAC;IAAC,eAAe,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AAElF;;;;GAIG;AACH,eAAO,MAAM,SAAS,QAAS,GAAG,OAAM,UAAU,KAAW,QAAQ,EAKP,CAAC;AAE/D,kFAAkF;AAClF,eAAO,MAAM,cAAc,QAAS,GAAG,OAAM,UAAU,KAAW,MAAM,EACV,CAAC;AAE/D,4FAA4F;AAC5F,eAAO,MAAM,iBAAiB,QAAS,GAAG,KAAG,QAAQ,EAAgC,CAAC"}

package/lib/assertions/core/tool-calls.js

@@ -0,0 +1,17 @@
+import { matchesAgentScope } from "./event-checks.js";
+import { isToolCallEvent } from "./helpers.js";
+/**
+ * Every tool call the run made, in order. Reach for it inside a `predicate`
+ * assertion to grade what the agent *did*, not what it said. Pass `by` to scope
+ * to the "main" agent or a "subagent" (default "any").
+ */
+export const toolCalls = (run, by = "any") => run.events
+    .filter(isToolCallEvent)
+    .filter((e) => matchesAgentScope(e, by))
+    .map((e) => e.payload)
+    .filter((p) => typeof p?.name === "string");
+/** Stringified inputs of every tool call, for substring checks in a predicate. */
+export const toolCallInputs = (run, by = "any") => toolCalls(run, by).map((c) => JSON.stringify(c.input ?? {}));
+/** Tool calls made by a spawned subagent (the driver tags these with `parentToolUseId`). */
+export const subagentToolCalls = (run) => toolCalls(run, "subagent");
+//# sourceMappingURL=tool-calls.js.map

package/lib/assertions/core/tool-calls.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tool-calls.js","sourceRoot":"","sources":["../../../src/assertions/core/tool-calls.ts"],"names":[],"mappings":"AACA,OAAO,EAAmB,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AACvE,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAQ/C;;;;GAIG;AACH,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,GAAQ,EAAE,EAAE,GAAe,KAAK,EAAc,EAAE,CACxE,GAAG,CAAC,MAAM;KACP,MAAM,CAAC,eAAe,CAAC;KACvB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;KACvC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAmB,CAAC;KACjC,MAAM,CAAC,CAAC,CAAC,EAAiB,EAAE,CAAC,OAAO,CAAC,EAAE,IAAI,KAAK,QAAQ,CAAC,CAAC;AAE/D,kFAAkF;AAClF,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,GAAQ,EAAE,EAAE,GAAe,KAAK,EAAY,EAAE,CAC3E,SAAS,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC;AAE/D,4FAA4F;AAC5F,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,GAAQ,EAAc,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC"}

package/lib/assertions/core/tool-invocation.d.ts

@@ -0,0 +1,36 @@
+import { z } from "zod";
+import { type AssertionImpl } from "../../core/index.js";
+export declare const ToolInvocationConfigSchema: z.ZodObject<{
+    id: z.ZodOptional<z.ZodString>;
+    weight: z.ZodOptional<z.ZodNumber>;
+    threshold: z.ZodOptional<z.ZodNumber>;
+    type: z.ZodLiteral<"tool-invocation">;
+    name: z.ZodString;
+    result: z.ZodDefault<z.ZodEnum<{
+        any: "any";
+        error: "error";
+        success: "success";
+    }>>;
+    occurrence: z.ZodDefault<z.ZodEnum<{
+        any: "any";
+        first: "first";
+        last: "last";
+    }>>;
+    input: z.ZodOptional<z.ZodObject<{
+        jsonPointer: z.ZodDefault<z.ZodString>;
+        equals: z.ZodOptional<z.ZodUnknown>;
+        contains: z.ZodOptional<z.ZodString>;
+        matches: z.ZodOptional<z.ZodString>;
+        flags: z.ZodOptional<z.ZodString>;
+    }, z.core.$strict>>;
+    output: z.ZodOptional<z.ZodObject<{
+        jsonPointer: z.ZodDefault<z.ZodString>;
+        equals: z.ZodOptional<z.ZodUnknown>;
+        contains: z.ZodOptional<z.ZodString>;
+        matches: z.ZodOptional<z.ZodString>;
+        flags: z.ZodOptional<z.ZodString>;
+    }, z.core.$strict>>;
+}, z.core.$strict>;
+export type ToolInvocationConfig = z.infer<typeof ToolInvocationConfigSchema>;
+export declare const toolInvocationAssertion: AssertionImpl<ToolInvocationConfig>;
+//# sourceMappingURL=tool-invocation.d.ts.map

package/lib/assertions/core/tool-invocation.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tool-invocation.d.ts","sourceRoot":"","sources":["../../../src/assertions/core/tool-invocation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EACL,KAAK,aAAa,EAInB,MAAM,qBAAqB,CAAC;AA2C7B,eAAO,MAAM,0BAA0B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAW5B,CAAC;AACZ,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,0BAA0B,CAAC,CAAC;AA0C9E,eAAO,MAAM,uBAAuB,EAAE,aAAa,CAAC,oBAAoB,CA8FvE,CAAC"}