@inbrowser/agent 0.0.0-placeholder → 0.2.0

package/dist/eval/spec-framework.d.ts

@@ -0,0 +1,113 @@
+/**
+ * Success-spec framework for the eval harness.
+ *
+ * A success specification (a "spec") decides whether one captured agent
+ * run satisfied its fixture's success criterion. The framework provides
+ * three things:
+ *
+ *   1. A `SpecRegistry` plus an `createSpecRegistry()` factory.
+ *      Registration is explicit — no side-effect registration on import.
+ *      Callers build a registry, call `registerStarterSpecs()` if they
+ *      want the common helpers (see `spec-helpers.ts`), and register
+ *      their own specs on top.
+ *
+ *   2. An `evaluateSpec(registry, reference, snapshot)` evaluator that
+ *      looks the spec up by name, awaits it (specs may be async), and
+ *      returns a structured `SpecResult`. A thrown error inside the
+ *      spec body is caught and converted into `{ ok: false, error }`,
+ *      so a misbehaving spec never crashes the harness.
+ *
+ *   3. The `RunSnapshot` interface — the input every spec consumes.
+ *      Intentionally narrower than the runner's eventual `RunRecord`,
+ *      so this branch develops independently of `eval/harness-runner`.
+ *      The runner's `RunRecord` is designed to be a structural superset:
+ *      a runner caller passes its `record` straight to `evaluateSpec`.
+ *
+ * Specs read three slices of state — the final workspace (rules / code
+ * / app source), the final runtime (the most recent run summary, any
+ * uiErrors, terminal output, etc.), the full assistant text, and the
+ * trace. Anything outside those four fields is a sign the snapshot
+ * shape needs widening; do that here, not in spec bodies.
+ *
+ * Spec names follow the same `family/spec-name` kebab-case form that
+ * `validateFixture` enforces on `SuccessSpecReference.name`. Registration
+ * validates the name at registration time so typos surface immediately.
+ */
+import type { RuntimeState } from '../types/runtime.js';
+import type { TraceEvent } from '../types/trace.js';
+import type { Workspace } from '../types/workspace.js';
+import type { SuccessSpecReference } from './fixture.js';
+/**
+ * Input shape every spec consumes. Intentionally narrower than the
+ * runner's eventual `RunRecord` so this branch develops independently
+ * of `eval/harness-runner`. The runner's record is designed to be a
+ * structural superset; a runner caller can pass its record straight to
+ * `evaluateSpec`.
+ */
+export interface RunSnapshot {
+    /** Workspace state at the end of the run. */
+    finalWorkspace: Workspace;
+    /** Runtime state at the end of the run (run summary, uiErrors, ...). */
+    finalRuntime: RuntimeState;
+    /** Concatenated assistant text across the run's iterations. */
+    assistantText: string;
+    /** All trace events emitted during the run, in emission order. */
+    trace: readonly TraceEvent[];
+}
+/**
+ * Structured result returned by `evaluateSpec` and by every spec body.
+ * `ok` is the pass/fail bit. `detail` is optional structured context a
+ * report can surface (matched tokens, missing tokens, the offending
+ * trace event id, ...). `error` carries the failure reason when the
+ * spec did not run cleanly — registration miss, args validation failure,
+ * spec body threw, etc.
+ */
+export interface SpecResult {
+    ok: boolean;
+    detail?: Record<string, unknown>;
+    error?: string;
+}
+/**
+ * Spec function signature. May be sync or async; the evaluator awaits
+ * the return either way. `args` is whatever the fixture supplied in
+ * `SuccessSpecReference.args` — typed as `unknown` because every spec
+ * declares (and validates) its own arg shape.
+ */
+export type SpecFn = (snapshot: RunSnapshot, args: unknown) => SpecResult | Promise<SpecResult>;
+/**
+ * In-memory registry. Backed by a `Map`, exposed as a small object so
+ * callers do not depend on Map identity.
+ */
+export interface SpecRegistry {
+    /**
+     * Register a spec by name. Throws if the name does not match the
+     * required `family/spec-name` kebab-case form, or if the name is
+     * already registered. Throws-on-conflict is intentional: silent
+     * overwrites mask real bugs and the harness only registers specs at
+     * startup, so a throw is observable.
+     */
+    register(name: string, fn: SpecFn): void;
+    /** Returns the registered spec function, or undefined. */
+    get(name: string): SpecFn | undefined;
+    /** True iff `name` is registered. */
+    has(name: string): boolean;
+    /** All registered names, in registration order. */
+    names(): string[];
+}
+/**
+ * Create a fresh, empty spec registry. Callers register specs on it
+ * explicitly — `registerStarterSpecs()` is the common starting point
+ * for fixtures that reuse the helpers; bespoke specs are registered the
+ * same way.
+ */
+export declare function createSpecRegistry(): SpecRegistry;
+/**
+ * Resolve a `SuccessSpecReference` against a registry and evaluate it
+ * over a `RunSnapshot`. Returns a `SpecResult`. Never throws — an
+ * unregistered name, a thrown spec body, or a returned non-result is
+ * surfaced as `{ ok: false, error }`.
+ *
+ * Async specs are awaited. Sync specs are returned unchanged.
+ */
+export declare function evaluateSpec(registry: SpecRegistry, reference: SuccessSpecReference, snapshot: RunSnapshot): Promise<SpecResult>;
+//# sourceMappingURL=spec-framework.d.ts.map

package/dist/eval/spec-framework.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"spec-framework.d.ts","sourceRoot":"","sources":["../../src/eval/spec-framework.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACxD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,cAAc,CAAC;AAEzD;;;;;;GAMG;AACH,MAAM,WAAW,WAAW;IAC1B,6CAA6C;IAC7C,cAAc,EAAE,SAAS,CAAC;IAC1B,wEAAwE;IACxE,YAAY,EAAE,YAAY,CAAC;IAC3B,+DAA+D;IAC/D,aAAa,EAAE,MAAM,CAAC;IACtB,kEAAkE;IAClE,KAAK,EAAE,SAAS,UAAU,EAAE,CAAC;CAC9B;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,OAAO,CAAC;IACZ,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjC,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;;;GAKG;AACH,MAAM,MAAM,MAAM,GAAG,CAAC,QAAQ,EAAE,WAAW,EAAE,IAAI,EAAE,OAAO,KAAK,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;AAEhG;;;GAGG;AACH,MAAM,WAAW,YAAY;IAC3B;;;;;;OAMG;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,IAAI,CAAC;IACzC,0DAA0D;IAC1D,GAAG,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAAC;IACtC,qCAAqC;IACrC,GAAG,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC;IAC3B,mDAAmD;IACnD,KAAK,IAAI,MAAM,EAAE,CAAC;CACnB;AAID;;;;;GAKG;AACH,wBAAgB,kBAAkB,IAAI,YAAY,CAwBjD;AAED;;;;;;;GAOG;AACH,wBAAsB,YAAY,CAChC,QAAQ,EAAE,YAAY,EACtB,SAAS,EAAE,oBAAoB,EAC/B,QAAQ,EAAE,WAAW,GACpB,OAAO,CAAC,UAAU,CAAC,CAkBrB"}

package/dist/eval/spec-framework.js

@@ -0,0 +1,100 @@
+/**
+ * Success-spec framework for the eval harness.
+ *
+ * A success specification (a "spec") decides whether one captured agent
+ * run satisfied its fixture's success criterion. The framework provides
+ * three things:
+ *
+ *   1. A `SpecRegistry` plus an `createSpecRegistry()` factory.
+ *      Registration is explicit — no side-effect registration on import.
+ *      Callers build a registry, call `registerStarterSpecs()` if they
+ *      want the common helpers (see `spec-helpers.ts`), and register
+ *      their own specs on top.
+ *
+ *   2. An `evaluateSpec(registry, reference, snapshot)` evaluator that
+ *      looks the spec up by name, awaits it (specs may be async), and
+ *      returns a structured `SpecResult`. A thrown error inside the
+ *      spec body is caught and converted into `{ ok: false, error }`,
+ *      so a misbehaving spec never crashes the harness.
+ *
+ *   3. The `RunSnapshot` interface — the input every spec consumes.
+ *      Intentionally narrower than the runner's eventual `RunRecord`,
+ *      so this branch develops independently of `eval/harness-runner`.
+ *      The runner's `RunRecord` is designed to be a structural superset:
+ *      a runner caller passes its `record` straight to `evaluateSpec`.
+ *
+ * Specs read three slices of state — the final workspace (rules / code
+ * / app source), the final runtime (the most recent run summary, any
+ * uiErrors, terminal output, etc.), the full assistant text, and the
+ * trace. Anything outside those four fields is a sign the snapshot
+ * shape needs widening; do that here, not in spec bodies.
+ *
+ * Spec names follow the same `family/spec-name` kebab-case form that
+ * `validateFixture` enforces on `SuccessSpecReference.name`. Registration
+ * validates the name at registration time so typos surface immediately.
+ */
+const SPEC_NAME_PATTERN = /^[a-z][a-z0-9-]*\/[a-z][a-z0-9-]*$/;
+/**
+ * Create a fresh, empty spec registry. Callers register specs on it
+ * explicitly — `registerStarterSpecs()` is the common starting point
+ * for fixtures that reuse the helpers; bespoke specs are registered the
+ * same way.
+ */
+export function createSpecRegistry() {
+    const specs = new Map();
+    return {
+        register(name, fn) {
+            if (typeof name !== 'string' || !SPEC_NAME_PATTERN.test(name)) {
+                throw new Error(`spec name must match \`family/spec-name\` kebab-case, got: ${JSON.stringify(name)}`);
+            }
+            if (specs.has(name)) {
+                throw new Error(`spec already registered: ${name}`);
+            }
+            specs.set(name, fn);
+        },
+        get(name) {
+            return specs.get(name);
+        },
+        has(name) {
+            return specs.has(name);
+        },
+        names() {
+            return Array.from(specs.keys());
+        },
+    };
+}
+/**
+ * Resolve a `SuccessSpecReference` against a registry and evaluate it
+ * over a `RunSnapshot`. Returns a `SpecResult`. Never throws — an
+ * unregistered name, a thrown spec body, or a returned non-result is
+ * surfaced as `{ ok: false, error }`.
+ *
+ * Async specs are awaited. Sync specs are returned unchanged.
+ */
+export async function evaluateSpec(registry, reference, snapshot) {
+    const fn = registry.get(reference.name);
+    if (!fn) {
+        return { ok: false, error: `spec not registered: ${reference.name}` };
+    }
+    try {
+        const result = await fn(snapshot, reference.args);
+        if (!isSpecResult(result)) {
+            return {
+                ok: false,
+                error: `spec "${reference.name}" returned a non-SpecResult value`,
+            };
+        }
+        return result;
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        return { ok: false, error: `spec "${reference.name}" threw: ${message}` };
+    }
+}
+function isSpecResult(value) {
+    if (value === null || typeof value !== 'object')
+        return false;
+    const obj = value;
+    return typeof obj.ok === 'boolean';
+}
+//# sourceMappingURL=spec-framework.js.map

package/dist/eval/spec-framework.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"spec-framework.js","sourceRoot":"","sources":["../../src/eval/spec-framework.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AAoEH,MAAM,iBAAiB,GAAG,oCAAoC,CAAC;AAE/D;;;;;GAKG;AACH,MAAM,UAAU,kBAAkB;IAChC,MAAM,KAAK,GAAG,IAAI,GAAG,EAAkB,CAAC;IACxC,OAAO;QACL,QAAQ,CAAC,IAAI,EAAE,EAAE;YACf,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC9D,MAAM,IAAI,KAAK,CACb,8DAA8D,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CACrF,CAAC;YACJ,CAAC;YACD,IAAI,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBACpB,MAAM,IAAI,KAAK,CAAC,4BAA4B,IAAI,EAAE,CAAC,CAAC;YACtD,CAAC;YACD,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QACtB,CAAC;QACD,GAAG,CAAC,IAAI;YACN,OAAO,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACzB,CAAC;QACD,GAAG,CAAC,IAAI;YACN,OAAO,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACzB,CAAC;QACD,KAAK;YACH,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;QAClC,CAAC;KACF,CAAC;AACJ,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,QAAsB,EACtB,SAA+B,EAC/B,QAAqB;IAErB,MAAM,EAAE,GAAG,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IACxC,IAAI,CAAC,EAAE,EAAE,CAAC;QACR,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,wBAAwB,SAAS,CAAC,IAAI,EAAE,EAAE,CAAC;IACxE,CAAC;IACD,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,QAAQ,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC;QAClD,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,EAAE,CAAC;YAC1B,OAAO;gBACL,EAAE,EAAE,KAAK;gBACT,KAAK,EAAE,SAAS,SAAS,CAAC,IAAI,mCAAmC;aAClE,CAAC;QACJ,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACjE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,SAAS,CAAC,IAAI,YAAY,OAAO,EAAE,EAAE,CAAC;IAC5E,CAAC;AACH,CAAC;AAED,SAAS,YAAY,CAAC,KAAc;IAClC,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IAC9D,MAAM,GAAG,GAAG,KAAgC,CAAC;IAC7C,OAAO,OAAO,GAAG,CAAC,EAAE,KAAK,SAAS,CAAC;AACrC,CAAC"}

package/dist/eval/spec-helpers.d.ts

@@ -0,0 +1,245 @@
+/**
+ * Starter library of common success specs.
+ *
+ * These helpers are intentionally crude. Golden-task authors compose
+ * them through `SuccessSpecReference.args`; custom specs are written
+ * only when crudeness is not enough.
+ *
+ * Registration is explicit. Callers create a registry and call
+ * `registerStarterSpecs(registry)` to get all six helpers under their
+ * documented names. No side-effect registration on import — that way
+ * a host can choose to register a subset, swap in stricter variants,
+ * or shadow a helper without monkey-patching this module.
+ *
+ * Each helper has a `family/spec-name` kebab-case identifier that
+ * matches the form `validateFixture` enforces on
+ * `SuccessSpecReference.name`. The constants below are exported so
+ * fixture authors and reviewers have a single place to import the
+ * canonical strings from.
+ *
+ * Argument shape and behavior are documented per-spec via JSDoc.
+ * The framework treats `args` as `unknown`; each spec validates its
+ * own shape and surfaces a clear `error` if the args are malformed,
+ * rather than throwing. (The evaluator catches throws too, but a
+ * structured error message is friendlier to read.)
+ */
+import type { SpecFn, SpecRegistry } from './spec-framework.js';
+export declare const SPEC_REPORT_MENTIONS_AT_LEAST_ONE_OF = "report-mentions/at-least-one-of";
+export declare const SPEC_REPORT_MENTIONS_ALL_OF = "report-mentions/all-of";
+export declare const SPEC_TRACE_CONTAINS_TOOL_CALL_BY_NAME = "trace-contains-tool-call/by-name";
+export declare const SPEC_FINAL_RULES_INCLUDES_LITERAL = "final-rules-includes/literal";
+export declare const SPEC_FINAL_RULES_EXCLUDES_LITERAL = "final-rules-excludes/literal";
+export declare const SPEC_FINAL_RUNTIME_RUN_SUMMARY_OK = "final-runtime/run-summary-ok";
+export declare const SPEC_GAME_RULES_SIMULATOR_ACCEPTS_POSITIVE_AND_REJECTS_CHEAT = "game-rules/simulator-accepts-positive-and-rejects-cheat";
+export declare const SPEC_PYRIC_AGENTS_LINT_CLEAN_AND_RULE_REJECTS_CHEAT = "pyric-agents/lint-clean-and-rule-rejects-cheat";
+/**
+ * Args: `{ tokens: string[]; caseSensitive?: boolean }`.
+ *
+ * Passes when `assistantText` contains at least one of `tokens`.
+ * Empty / non-array `tokens` is a malformed-args error.
+ *
+ * Example fixture reference:
+ *
+ *     {
+ *       "name": "report-mentions/at-least-one-of",
+ *       "args": { "tokens": ["open-write", "missing auth check"] }
+ *     }
+ */
+export declare const reportMentionsAtLeastOneOf: SpecFn;
+/**
+ * Args: `{ tokens: string[]; caseSensitive?: boolean }`.
+ *
+ * Passes when `assistantText` contains every entry in `tokens`.
+ * Empty / non-array `tokens` is a malformed-args error.
+ *
+ * Example fixture reference:
+ *
+ *     {
+ *       "name": "report-mentions/all-of",
+ *       "args": { "tokens": ["users", "open-write", "fix"] }
+ *     }
+ */
+export declare const reportMentionsAllOf: SpecFn;
+/**
+ * Args: `{ tool: string; minCount?: number }`.
+ *
+ * Passes when the trace contains at least `minCount` (default 1)
+ * `tool_call` records emitted by the response trace for a tool named
+ * `tool`. Tool calls are read off `llm_response` trace events, which
+ * is where the agent loop records what the model asked for in each
+ * iteration.
+ *
+ * Example fixture reference:
+ *
+ *     {
+ *       "name": "trace-contains-tool-call/by-name",
+ *       "args": { "tool": "rulesSimulator", "minCount": 1 }
+ *     }
+ */
+export declare const traceContainsToolCallByName: SpecFn;
+/**
+ * Args: `{ literal: string; caseSensitive?: boolean }`.
+ *
+ * Passes when `finalWorkspace.rules` contains `literal` as a substring.
+ * `literal` must be a non-empty string.
+ *
+ * Example fixture reference:
+ *
+ *     {
+ *       "name": "final-rules-includes/literal",
+ *       "args": { "literal": "request.auth != null" }
+ *     }
+ */
+export declare const finalRulesIncludesLiteral: SpecFn;
+/**
+ * Args: `{ literal: string; caseSensitive?: boolean }`.
+ *
+ * Passes when `finalWorkspace.rules` does NOT contain `literal` as a
+ * substring. Useful for asserting that a planted antipattern has been
+ * removed.
+ *
+ * Example fixture reference:
+ *
+ *     {
+ *       "name": "final-rules-excludes/literal",
+ *       "args": { "literal": "allow write: if true" }
+ *     }
+ */
+export declare const finalRulesExcludesLiteral: SpecFn;
+/**
+ * Args: none (`undefined` or `{}`).
+ *
+ * Passes when `finalRuntime.runSummary` exists and `runSummary.ok` is
+ * true — i.e. the most recent `runCode` invocation succeeded. Fails
+ * when there is no run summary at all (the spec only makes sense for
+ * fixtures whose skill is expected to run code).
+ *
+ * Example fixture reference:
+ *
+ *     { "name": "final-runtime/run-summary-ok" }
+ */
+export declare const finalRuntimeRunSummaryOk: SpecFn;
+/**
+ * Args: `{ database?: 'firestore' | 'rtdb'; positive: PositiveArgs; cheat: CheatArgs }`.
+ *
+ * Both-direction check: the generated rules should accept a defined
+ * positive move AND reject a defined cheating attempt. The fixture
+ * supplies one side under `positive` and the other under `cheat`.
+ *
+ * V1 approximation: token-presence over `finalWorkspace.rules`. A
+ * future iteration would replace this with an actual Firestore /
+ * Realtime Database security-rules simulator call, exercising the
+ * `positive` move (expecting `allow`) and the `cheat` move
+ * (expecting `deny`). Wiring that simulator is out of scope for v1
+ * because it requires either the Firebase rules-emulator process
+ * (Node-only, slow to start, off-limits in a browser-safe surface)
+ * or a bundled WASM rules interpreter — neither of which the eval
+ * harness has today.
+ *
+ * The approximation supports two arg-shapes so it can serve both
+ * the brief's documented shape and the simulator-style shape the
+ * generative fixtures already use on disk:
+ *
+ *   1. Token shape (explicit, preferred for future-authored fixtures):
+ *        positive: { description?: string; requiredTokens: string[] }
+ *        cheat:    { description?: string; rejectionTokens: string[] }
+ *      Each token list is checked as case-sensitive substrings on
+ *      `finalWorkspace.rules`.
+ *
+ *   2. Simulator shape (used by the existing fixtures):
+ *        positive: { auth, path, op, data, expect: 'allow' }
+ *        cheat:    { auth, path, op, data, expect: 'deny' }
+ *      Tokens are derived from the simulator-side `data` and `path`
+ *      values — each string-valued leaf and the path segment are
+ *      required to appear in the rules text. The expectation field
+ *      (`allow` / `deny`) is recorded in the detail payload but is
+ *      not used by the token check; the actual accept/reject
+ *      decision is what the future simulator iteration would
+ *      validate.
+ *
+ * Passes iff every derived positive token AND every derived cheat
+ * token appears in `finalWorkspace.rules`. Otherwise returns
+ * `{ ok: false, detail: { missingPositive, missingCheat } }`.
+ *
+ * Example fixture reference (simulator shape):
+ *
+ *     {
+ *       "name": "game-rules/simulator-accepts-positive-and-rejects-cheat",
+ *       "args": {
+ *         "database": "rtdb",
+ *         "positive": { "auth": { "uid": "uidA" }, "path": "/games/g1",
+ *                       "op": "update", "data": { ... }, "expect": "allow" },
+ *         "cheat":    { "auth": { "uid": "uidB" }, "path": "/games/g1",
+ *                       "op": "update", "data": { ... }, "expect": "deny" }
+ *       }
+ *     }
+ */
+export declare const gameRulesSimulatorAcceptsPositiveAndRejectsCheat: SpecFn;
+/**
+ * Args: `{ lintToolName?: string; cheat?: CheatArgs; cheatAttempt?: CheatArgs }`.
+ *
+ * Two-part check: (1) the agent successfully called the pyric lint
+ * tool during the run, AND (2) the resulting rules text contains the
+ * tokens that should be present if the cheating attempt is
+ * structurally rejected by the rules.
+ *
+ * V1 approximation:
+ *   - Step 1 walks `snapshot.trace` for any `llm_response` event
+ *     containing a tool call whose name matches `lintToolName`
+ *     (default `lint_firestore_rules`). If absent, returns
+ *     `{ ok: false, detail: { reason: 'lint-not-called' } }`. The
+ *     pyric lint tool's success/failure is observable in the
+ *     assistant's reasoning and in the tool result, but capturing
+ *     the precise `tool_result` event shape across providers is
+ *     fragile — checking that the tool was *called* is the right
+ *     v1 signal. A future iteration would also verify the lint
+ *     tool's result was `ok: true` at the trace level.
+ *   - Step 2 checks `finalWorkspace.rules` includes every token in
+ *     the cheat's `rejectionTokens`. The cheat may be supplied
+ *     under either `cheat` (the brief's name) or `cheatAttempt`
+ *     (the name the existing fixture uses). When the cheat is
+ *     simulator-shaped (no `rejectionTokens`, only `data` + `path`),
+ *     tokens are derived from those leaves the same way as
+ *     `gameRulesSimulatorAcceptsPositiveAndRejectsCheat`.
+ *
+ * Returns `{ ok: true }` on success, or
+ * `{ ok: false, detail: { reason: 'rejection-tokens-missing', missing } }`
+ * on the second-step failure.
+ *
+ * Example fixture reference:
+ *
+ *     {
+ *       "name": "pyric-agents/lint-clean-and-rule-rejects-cheat",
+ *       "args": {
+ *         "lintToolName": "lint_firestore_rules",
+ *         "cheatAttempt": { "path": "/orders/orderA", "op": "create",
+ *                           "data": { ... }, "expect": "deny" }
+ *       }
+ *     }
+ */
+export declare const pyricAgentsLintCleanAndRuleRejectsCheat: SpecFn;
+/**
+ * Register every starter spec on a registry. Idempotency is not a
+ * design goal — calling this twice on the same registry throws (the
+ * registry rejects duplicate registrations on purpose). Callers that
+ * want a subset should call `registry.register()` themselves.
+ */
+export declare function registerStarterSpecs(registry: SpecRegistry): void;
+/**
+ * Register every custom (post-starter) spec on a registry. Sibling
+ * to `registerStarterSpecs`. Splitting the two keeps the meaning of
+ * "starter" stable as new custom specs are added.
+ */
+export declare function registerCustomSpecs(registry: SpecRegistry): void;
+/**
+ * Umbrella that registers every spec the library ships — both the
+ * starter library and the custom helpers. Equivalent to calling
+ * `registerStarterSpecs(registry)` followed by
+ * `registerCustomSpecs(registry)`.
+ */
+export declare function registerAllSpecs(registry: SpecRegistry): void;
+/** Stable list of starter spec names, in registration order. */
+export declare const STARTER_SPEC_NAMES: readonly string[];
+/** Stable list of custom spec names, in registration order. */
+export declare const CUSTOM_SPEC_NAMES: readonly string[];
+//# sourceMappingURL=spec-helpers.d.ts.map

package/dist/eval/spec-helpers.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"spec-helpers.d.ts","sourceRoot":"","sources":["../../src/eval/spec-helpers.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,OAAO,KAAK,EAAe,MAAM,EAAE,YAAY,EAAc,MAAM,qBAAqB,CAAC;AAEzF,eAAO,MAAM,oCAAoC,oCAAoC,CAAC;AACtF,eAAO,MAAM,2BAA2B,2BAA2B,CAAC;AACpE,eAAO,MAAM,qCAAqC,qCAAqC,CAAC;AACxF,eAAO,MAAM,iCAAiC,iCAAiC,CAAC;AAChF,eAAO,MAAM,iCAAiC,iCAAiC,CAAC;AAChF,eAAO,MAAM,iCAAiC,iCAAiC,CAAC;AAEhF,eAAO,MAAM,4DAA4D,4DACd,CAAC;AAC5D,eAAO,MAAM,mDAAmD,mDACd,CAAC;AAEnD;;;;;;;;;;;;GAYG;AACH,eAAO,MAAM,0BAA0B,EAAE,MAiBxC,CAAC;AAEF;;;;;;;;;;;;GAYG;AACH,eAAO,MAAM,mBAAmB,EAAE,MAiBjC,CAAC;AAEF;;;;;;;;;;;;;;;GAeG;AACH,eAAO,MAAM,2BAA2B,EAAE,MAgBzC,CAAC;AAEF;;;;;;;;;;;;GAYG;AACH,eAAO,MAAM,yBAAyB,EAAE,MAEvC,CAAC;AAEF;;;;;;;;;;;;;GAaG;AACH,eAAO,MAAM,yBAAyB,EAAE,MAEvC,CAAC;AAEF;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,wBAAwB,EAAE,MAkBtC,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsDG;AACH,eAAO,MAAM,gDAAgD,EAAE,MAoB9D,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AACH,eAAO,MAAM,uCAAuC,EAAE,MAqDrD,CAAC;AAEF;;;;;GAKG;AACH,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,YAAY,GAAG,IAAI,CAOjE;AAED;;;;GAIG;AACH,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,YAAY,GAAG,IAAI,CAShE;AAED;;;;;GAKG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,YAAY,GAAG,IAAI,CAG7D;AAED,gEAAgE;AAChE,eAAO,MAAM,kBAAkB,EAAE,SAAS,MAAM,EAO/C,CAAC;AAEF,+DAA+D;AAC/D,eAAO,MAAM,iBAAiB,EAAE,SAAS,MAAM,EAG9C,CAAC"}