@vincemakes/kiso-evals 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 kiso contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,7 @@
+# @vincemakes/kiso-evals
+
+The faux provider (a scripted model — zero keys, deterministic), the
+incident fixture library (every fixture is a real production accident
+from uooki 2026), and the cross-provider contract tests.
+
+See the repository README for the framework overview.

package/dist/faux.d.ts

@@ -0,0 +1,30 @@
+/**
+ * L7 — the faux provider: iteration's foundation (mauri ADR-0010).
+ *
+ * A provider that needs no API key and no network. A `FauxScript` is a
+ * turn-by-turn playbook: what the "model" emits. The kernel cannot tell a
+ * faux provider from a real one — which is the point: every fixture, every
+ * cross-provider matrix entry, and every regression test runs against the
+ * same loop that production runs on.
+ *
+ * Why this is the FIRST adapter: the loop is not written against a mock of
+ * the loop — it is written against a fake of the model. The model is the
+ * moving part; pinning it down pins down everything downstream (ADR-0001).
+ *
+ * The TOOL side of a fixture is fake too, but by a different mechanism: a
+ * fixture registers defineTool'd handlers that return scripted results.
+ * The loop is therefore fully isomorphic between fixture and production —
+ * no test-only branch anywhere.
+ *
+ * `seq`: the script is written WITHOUT seq (a playbook, not a recording).
+ * This adapter assigns consecutive seq per stream call; the kernel's
+ * EventLog re-asserts monotonicity across sources (ADR-0002).
+ */
+import type { Adapter } from "@vincemakes/kiso-core";
+import type { EventInput } from "@vincemakes/kiso-core";
+/** One model turn: the events it emits. Tool results live in fixture tools. */
+export interface FauxTurn {
+    readonly events: readonly EventInput[];
+}
+export type FauxScript = readonly FauxTurn[];
+export declare function createFauxProvider(script: FauxScript): Adapter;

package/dist/faux.js

@@ -0,0 +1,49 @@
+/**
+ * L7 — the faux provider: iteration's foundation (mauri ADR-0010).
+ *
+ * A provider that needs no API key and no network. A `FauxScript` is a
+ * turn-by-turn playbook: what the "model" emits. The kernel cannot tell a
+ * faux provider from a real one — which is the point: every fixture, every
+ * cross-provider matrix entry, and every regression test runs against the
+ * same loop that production runs on.
+ *
+ * Why this is the FIRST adapter: the loop is not written against a mock of
+ * the loop — it is written against a fake of the model. The model is the
+ * moving part; pinning it down pins down everything downstream (ADR-0001).
+ *
+ * The TOOL side of a fixture is fake too, but by a different mechanism: a
+ * fixture registers defineTool'd handlers that return scripted results.
+ * The loop is therefore fully isomorphic between fixture and production —
+ * no test-only branch anywhere.
+ *
+ * `seq`: the script is written WITHOUT seq (a playbook, not a recording).
+ * This adapter assigns consecutive seq per stream call; the kernel's
+ * EventLog re-asserts monotonicity across sources (ADR-0002).
+ */
+export function createFauxProvider(script) {
+    // The script is consumed turn by turn across stream() calls, mirroring a
+    // real model answering each new request with the next output. A real
+    // provider re-streams the same input on retry; the faux provider instead
+    // advances — acceptable because fixtures never combine retries with faux
+    // (retry tests use their own throwing adapter).
+    let nextTurn = 0;
+    return {
+        stream(_options) {
+            const turn = script[nextTurn];
+            nextTurn += 1;
+            return {
+                // The script is a playbook — a script MAY contain forged
+                // kernel-owned events on purpose (the trust-gate tests need
+                // them); the LOOP is the trust boundary, not the faux adapter.
+                async *[Symbol.asyncIterator]() {
+                    // An exhausted script is an empty stream: the loop sees
+                    // no tool calls and converges on `completed`.
+                    let seq = 0;
+                    for (const ev of turn?.events ?? []) {
+                        yield { ...ev, seq: seq++ };
+                    }
+                },
+            };
+        },
+    };
+}

package/dist/fixtures/compaction-regrowth.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Fixture: COMPACTION REGROWTH — repeated compaction re-archives already
+ * cleared messages, O(N²), silently overwriting original content.
+ *
+ * Incident (uooki production, 2026, video pipeline): measured 579 store_calls
+ * / 480 archive entries against a naive expectation of 44 — ~10× at 50 turns,
+ * extrapolating to 64× at 200 turns and 227× at 1000. Root cause: compaction
+ * re-archived messages whose content was already the clear marker, writing
+ * to the same revision key and overwriting the original. The fix was one
+ * line of idempotence: a marked message is never archived again.
+ *
+ * The kernel-side shape the fixture pins: `shouldClearContent` returns false
+ * for already-marked content — the predicate the compaction loop MUST apply
+ * before archiving. The fixture fails if the predicate regresses.
+ */
+import type { Fixture } from "./types.js";
+export declare const compactionRegrowth: Fixture;

package/dist/fixtures/compaction-regrowth.js

@@ -0,0 +1,46 @@
+/**
+ * Fixture: COMPACTION REGROWTH — repeated compaction re-archives already
+ * cleared messages, O(N²), silently overwriting original content.
+ *
+ * Incident (uooki production, 2026, video pipeline): measured 579 store_calls
+ * / 480 archive entries against a naive expectation of 44 — ~10× at 50 turns,
+ * extrapolating to 64× at 200 turns and 227× at 1000. Root cause: compaction
+ * re-archived messages whose content was already the clear marker, writing
+ * to the same revision key and overwriting the original. The fix was one
+ * line of idempotence: a marked message is never archived again.
+ *
+ * The kernel-side shape the fixture pins: `shouldClearContent` returns false
+ * for already-marked content — the predicate the compaction loop MUST apply
+ * before archiving. The fixture fails if the predicate regresses.
+ */
+import { CLEARED_MARKER_PREFIX, isClearedMarker, shouldClearContent } from "@vincemakes/kiso-core";
+export const compactionRegrowth = {
+    name: "compaction-regrowth",
+    incident: "uooki video pipeline: 579 store_calls vs 44 expected at 50 turns — re-archiving cleared markers overwrote originals (2026)",
+    script: [
+        {
+            events: [
+                { type: "text_start" },
+                { type: "text_delta", text: "simulating long-session compaction cycles" },
+                { type: "text_end" },
+                { type: "stop", reason: "end_turn" },
+            ],
+        },
+    ],
+    staticCheck: (events) => {
+        const violations = [];
+        // The one-line idempotence rule — present in the kernel or the
+        // fixture fails on the first regression.
+        if (shouldClearContent(CLEARED_MARKER_PREFIX + "x")) {
+            violations.push("already-cleared content must never be re-cleared");
+        }
+        if (!shouldClearContent("real tool output")) {
+            violations.push("real content must remain clearable");
+        }
+        if (!isClearedMarker(CLEARED_MARKER_PREFIX + "x")) {
+            violations.push("marker prefix detection regressed");
+        }
+        return violations;
+    },
+    requiredTerminal: ["completed"],
+};

package/dist/fixtures/index.d.ts

@@ -0,0 +1,12 @@
+import type { Fixture } from "./types.js";
+/** The accident library — every fixture is a real incident abstracted. */
+export declare const FIXTURES: readonly Fixture[];
+export * from "./types.js";
+export { terminalLies } from "./terminal-lies.js";
+export { silentToolFailure } from "./silent-tool-failure.js";
+export { compactionRegrowth } from "./compaction-regrowth.js";
+export { userAbort } from "./user-abort.js";
+export { unknownTool } from "./unknown-tool.js";
+export { permissionNegotiation } from "./permission-negotiation.js";
+export { makeAbortSignal } from "./user-abort.js";
+export { runStaticFixture, flattenScript } from "./runner.js";

package/dist/fixtures/index.js

@@ -0,0 +1,24 @@
+import { terminalLies } from "./terminal-lies.js";
+import { silentToolFailure } from "./silent-tool-failure.js";
+import { compactionRegrowth } from "./compaction-regrowth.js";
+import { userAbort } from "./user-abort.js";
+import { unknownTool } from "./unknown-tool.js";
+import { permissionNegotiation } from "./permission-negotiation.js";
+/** The accident library — every fixture is a real incident abstracted. */
+export const FIXTURES = [
+    terminalLies,
+    silentToolFailure,
+    compactionRegrowth,
+    userAbort,
+    unknownTool,
+    permissionNegotiation,
+];
+export * from "./types.js";
+export { terminalLies } from "./terminal-lies.js";
+export { silentToolFailure } from "./silent-tool-failure.js";
+export { compactionRegrowth } from "./compaction-regrowth.js";
+export { userAbort } from "./user-abort.js";
+export { unknownTool } from "./unknown-tool.js";
+export { permissionNegotiation } from "./permission-negotiation.js";
+export { makeAbortSignal } from "./user-abort.js";
+export { runStaticFixture, flattenScript } from "./runner.js";

package/dist/fixtures/permission-negotiation.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Fixture: PERMISSION NEGOTIATION — the same call shape can be denied once
+ * and allowed later; the model must see both outcomes distinctly.
+ *
+ * Incident (mauri ADR-0002): a permission system that is a yes/no gate has
+ * no memory and no upgrade path. CC's ask model allows deny-with-reason,
+ * and the reason feeds back to the model so it can adjust. The kernel's
+ * contract: each decision is a distinct tool_result — `precondition` for a
+ * refusal (the tool never ran), a normal result for the allowed retry.
+ */
+import type { Fixture } from "./types.js";
+export declare const permissionNegotiation: Fixture;

package/dist/fixtures/permission-negotiation.js

@@ -0,0 +1,47 @@
+/**
+ * Fixture: PERMISSION NEGOTIATION — the same call shape can be denied once
+ * and allowed later; the model must see both outcomes distinctly.
+ *
+ * Incident (mauri ADR-0002): a permission system that is a yes/no gate has
+ * no memory and no upgrade path. CC's ask model allows deny-with-reason,
+ * and the reason feeds back to the model so it can adjust. The kernel's
+ * contract: each decision is a distinct tool_result — `precondition` for a
+ * refusal (the tool never ran), a normal result for the allowed retry.
+ */
+export const permissionNegotiation = {
+    name: "permission-negotiation",
+    incident: "uooki GatePipeline was a message-prefix interceptor with no session memory — a denied call could not be re-attempted after user approval (mauri ADR-0002 audit)",
+    script: [
+        {
+            events: [
+                {
+                    type: "tool_call_end",
+                    callId: "c1",
+                    name: "code_execute",
+                    input: { code: "1+1" },
+                },
+                { type: "stop", reason: "tool_use" }
+            ],
+        },
+        {
+            events: [
+                {
+                    type: "tool_call_end",
+                    callId: "c2",
+                    name: "code_execute",
+                    input: { code: "1+1" },
+                },
+                { type: "stop", reason: "tool_use" }
+            ],
+        },
+        { events: [{ type: "stop", reason: "end_turn" }] },
+    ],
+    staticCheck: (events) => {
+        const calls = events.filter((e) => e.type === "tool_call_end" && e.name === "code_execute");
+        if (calls.length < 2) {
+            return ["script needs two calls of the same shape (deny then allow)"];
+        }
+        return [];
+    },
+    requiredTerminal: ["completed"],
+};

package/dist/fixtures/runner.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Fixture runner — flattens a script to its raw events and runs the static
+ * checks. Loop integration (trajectory + requiredTerminal) lands in M1.
+ */
+import type { EventInput } from "@vincemakes/kiso-core";
+import type { Fixture } from "./types.js";
+export declare function flattenScript(fixture: Fixture): readonly EventInput[];
+export interface FixtureResult {
+    readonly name: string;
+    readonly violations: readonly string[];
+}
+export declare function runStaticFixture(fixture: Fixture): FixtureResult;

package/dist/fixtures/runner.js

@@ -0,0 +1,12 @@
+/**
+ * Fixture runner — flattens a script to its raw events and runs the static
+ * checks. Loop integration (trajectory + requiredTerminal) lands in M1.
+ */
+export function flattenScript(fixture) {
+    return fixture.script.flatMap((turn) => turn.events);
+}
+export function runStaticFixture(fixture) {
+    const events = flattenScript(fixture);
+    const violations = fixture.staticCheck ? fixture.staticCheck(events) : [];
+    return { name: fixture.name, violations };
+}

package/dist/fixtures/silent-tool-failure.d.ts

@@ -0,0 +1,20 @@
+/**
+ * Fixture: SILENT TOOL FAILURE — an error result is swallowed and the model
+ * keeps narrating success.
+ *
+ * Incident (uooki production, 2026): a tool call failed with is_error=true,
+ * the model ignored it and continued with "results look great, done" —
+ * the turn ended "completed" with zero usable work. Root cause class:
+ * error results ride the same channel as successes, and nothing forced the
+ * model to acknowledge the error (CC's community loudest complaint: "reports
+ * of finished work that never happened").
+ *
+ * The kernel-side shape the fixture pins: an isError tool_result in the
+ * trajectory must carry its errorKind and remain distinguishable from
+ * success downstream — the harness can then decide (fail the turn, surface
+ * the error, re-route). The kernel never lets an error result masquerade
+ * as a clean one. The failing tool itself is scripted in the loop test
+ * (a defineTool'd web_search returning errorKind "transient").
+ */
+import type { Fixture } from "./types.js";
+export declare const silentToolFailure: Fixture;

package/dist/fixtures/silent-tool-failure.js

@@ -0,0 +1,51 @@
+/**
+ * Fixture: SILENT TOOL FAILURE — an error result is swallowed and the model
+ * keeps narrating success.
+ *
+ * Incident (uooki production, 2026): a tool call failed with is_error=true,
+ * the model ignored it and continued with "results look great, done" —
+ * the turn ended "completed" with zero usable work. Root cause class:
+ * error results ride the same channel as successes, and nothing forced the
+ * model to acknowledge the error (CC's community loudest complaint: "reports
+ * of finished work that never happened").
+ *
+ * The kernel-side shape the fixture pins: an isError tool_result in the
+ * trajectory must carry its errorKind and remain distinguishable from
+ * success downstream — the harness can then decide (fail the turn, surface
+ * the error, re-route). The kernel never lets an error result masquerade
+ * as a clean one. The failing tool itself is scripted in the loop test
+ * (a defineTool'd web_search returning errorKind "transient").
+ */
+export const silentToolFailure = {
+    name: "silent-tool-failure",
+    incident: "uooki turn: search tool returned is_error, model ignored it and reported success (2026-06)",
+    script: [
+        {
+            events: [
+                { type: "text_start" },
+                { type: "tool_call_start", callId: "c1", name: "web_search" },
+                { type: "tool_call_input_delta", callId: "c1", inputJsonDelta: '{"query": "latest"}' },
+                { type: "tool_call_end", callId: "c1", name: "web_search", input: { query: "latest" } },
+                { type: "stop", reason: "tool_use" }
+            ],
+        },
+        {
+            events: [
+                { type: "text_delta", text: "搜索完成，结果很好。" },
+                { type: "text_end" },
+                { type: "stop", reason: "end_turn" },
+            ],
+        },
+    ],
+    staticCheck: (events) => {
+        const violations = [];
+        if (!events.some((e) => e.type === "tool_call_end" && e.name === "web_search")) {
+            violations.push("script lost the failing tool call");
+        }
+        if (!events.some((e) => e.type === "text_delta" && /很好|成功|done/i.test(e.text))) {
+            violations.push("script lost the success narration that swallows the error");
+        }
+        return violations;
+    },
+    requiredTerminal: ["completed"], // the loop completes; the error must stay VISIBLE in the trajectory
+};

package/dist/fixtures/terminal-lies.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Fixture: TERMINAL LIES — the model reports done work that never landed.
+ *
+ * Incident (uooki production, 2026): a long-document turn called
+ * `document_create` twice, the chat asserted "已创建文档产物", and the
+ * Canvas stayed empty — the tool was an inline Markdown table that never
+ * reaches the canvas, and all three delivery guards were blind to it
+ * because it was absent from the producer registry. The turn was reported
+ * "completed".
+ *
+ * The kernel-side shape: text claims delivery, no `delivers` tool ran, and
+ * the terminal says "completed". The fix the fixture pins: a run whose
+ * trajectory contains a delivery claim with zero producer calls is NOT a
+ * clean `completed` — the harness's delivery tracker must see the signal.
+ */
+import type { Fixture } from "./types.js";
+export declare const terminalLies: Fixture;

package/dist/fixtures/terminal-lies.js

@@ -0,0 +1,62 @@
+/**
+ * Fixture: TERMINAL LIES — the model reports done work that never landed.
+ *
+ * Incident (uooki production, 2026): a long-document turn called
+ * `document_create` twice, the chat asserted "已创建文档产物", and the
+ * Canvas stayed empty — the tool was an inline Markdown table that never
+ * reaches the canvas, and all three delivery guards were blind to it
+ * because it was absent from the producer registry. The turn was reported
+ * "completed".
+ *
+ * The kernel-side shape: text claims delivery, no `delivers` tool ran, and
+ * the terminal says "completed". The fix the fixture pins: a run whose
+ * trajectory contains a delivery claim with zero producer calls is NOT a
+ * clean `completed` — the harness's delivery tracker must see the signal.
+ */
+import { analyzeDelivery } from "@vincemakes/kiso-core";
+export const terminalLies = {
+    name: "terminal-lies",
+    incident: "uooki long-document turn: model claimed '已创建文档产物', canvas stayed empty, turn reported completed (document_create trap, 2026-07-31)",
+    delivery: {
+        required: true,
+        producers: new Set(["create_artifact", "docx_create"]),
+    },
+    script: [
+        {
+            events: [
+                { type: "text_start" },
+                { type: "text_delta", text: "我已经生成了一份完整的调研报告，并已交付到画布上。" },
+                { type: "text_end" },
+                { type: "stop", reason: "end_turn" },
+            ],
+        },
+    ],
+    staticCheck: (events) => {
+        const violations = [];
+        const claimsDelivery = events.some((e) => e.type === "text_delta" && /交付|已生成|completed|delivered/i.test(e.text));
+        const produced = events.some((e) => e.type === "tool_call_end" && e.name === "create_artifact");
+        if (claimsDelivery && !produced) {
+            // This IS the failure shape: a claim with zero producers.
+            return [];
+        }
+        violations.push("script no longer reproduces the incident: a delivery claim must exist with zero producer calls");
+        return violations;
+    },
+    // Loop-integrated: the claim exists, the ledger says zero deliveries.
+    assert: (trajectory) => {
+        const verdict = analyzeDelivery(trajectory, {
+            required: true,
+            producers: new Set(["create_artifact", "docx_create"]),
+        });
+        const violations = [];
+        if (!verdict.claimedInText)
+            violations.push("the claim must still be in the text");
+        if (verdict.completedProducers.length !== 0)
+            violations.push("no producer may have completed");
+        if (verdict.passed) {
+            violations.push("delivery must FAIL: claimed delivery with zero completed producers");
+        }
+        return violations;
+    },
+    requiredTerminal: ["completed"], // the loop is honest; the VERDICT is the harness's
+};

package/dist/fixtures/types.d.ts

@@ -0,0 +1,42 @@
+/**
+ * Fixture — one known failure mode, recorded as a playable script + an
+ * assertion the kernel must satisfy.
+ *
+ * This is kiso's answer to "prove the loop is honest": every fixture is a
+ * real incident abstracted (uooki production, 2026). A fixture the loop
+ * cannot pass is a regression the loop must not ship. Other frameworks
+ * advertise features; kiso advertises "verifyable against known failure
+ * modes" — because CC's loudest production complaints (reports of done work
+ * that never landed, silent tool failures) are exactly these shapes.
+ *
+ * `assert` receives the FULL trajectory (all events, seq 0..N) once the
+ * loop exists. Until then, fixtures carry a `staticCheck` that runs against
+ * the raw script — the shape of the failure, before the loop is written to
+ * fail it. Loop-integrated assertions land with the loop (M1).
+ */
+import type { FauxScript } from "../faux.js";
+import type { Event } from "@vincemakes/kiso-core";
+import type { EventInput } from "@vincemakes/kiso-core";
+export interface Fixture {
+    readonly name: string;
+    /** Which real incident this abstracts. */
+    readonly incident: string;
+    readonly script: FauxScript;
+    /** Delivery expectations — runs analyzeDelivery over the trajectory. */
+    readonly delivery?: {
+        readonly required: boolean;
+        readonly producers: ReadonlySet<string>;
+    };
+    /** The loop's terminal MUST be one of these for the fixture to pass. */
+    readonly requiredTerminal?: readonly string[];
+    /**
+     * Static analysis over the script's events — the failure signature.
+     * Returns violation strings; empty = the shape is present as expected.
+     */
+    readonly staticCheck?: (events: readonly EventInput[]) => readonly string[];
+    /**
+     * Full-trajectory assertion, wired once the loop exists.
+     * Returns violation strings; empty = pass.
+     */
+    readonly assert?: (trajectory: readonly Event[]) => readonly string[];
+}

package/dist/fixtures/types.js

@@ -0,0 +1,17 @@
+/**
+ * Fixture — one known failure mode, recorded as a playable script + an
+ * assertion the kernel must satisfy.
+ *
+ * This is kiso's answer to "prove the loop is honest": every fixture is a
+ * real incident abstracted (uooki production, 2026). A fixture the loop
+ * cannot pass is a regression the loop must not ship. Other frameworks
+ * advertise features; kiso advertises "verifyable against known failure
+ * modes" — because CC's loudest production complaints (reports of done work
+ * that never landed, silent tool failures) are exactly these shapes.
+ *
+ * `assert` receives the FULL trajectory (all events, seq 0..N) once the
+ * loop exists. Until then, fixtures carry a `staticCheck` that runs against
+ * the raw script — the shape of the failure, before the loop is written to
+ * fail it. Loop-integrated assertions land with the loop (M1).
+ */
+export {};

package/dist/fixtures/unknown-tool.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Fixture: UNKNOWN TOOL — the model calls a tool that is not registered.
+ *
+ * Incident class (uooki + CC community): a tool name drifts (renamed,
+ * ghost entry, typo'd enum) and the harness either silently drops the call
+ * (agent loops forever trying to find the tool) or executes something
+ * unexpected. The kernel's answer is structural: an unregistered tool is
+ * refused with an `invalid_input` error result — the model sees the refusal,
+ * the trajectory records it, nothing silently vanishes.
+ */
+import type { Fixture } from "./types.js";
+export declare const unknownTool: Fixture;

package/dist/fixtures/unknown-tool.js

@@ -0,0 +1,35 @@
+/**
+ * Fixture: UNKNOWN TOOL — the model calls a tool that is not registered.
+ *
+ * Incident class (uooki + CC community): a tool name drifts (renamed,
+ * ghost entry, typo'd enum) and the harness either silently drops the call
+ * (agent loops forever trying to find the tool) or executes something
+ * unexpected. The kernel's answer is structural: an unregistered tool is
+ * refused with an `invalid_input` error result — the model sees the refusal,
+ * the trajectory records it, nothing silently vanishes.
+ */
+export const unknownTool = {
+    name: "unknown-tool",
+    incident: "uooki ghost-tool class: manifest listed a toolkit class name instead of the callable function name; activate_tools silently dropped it and the agent looped (2026-07)",
+    script: [
+        {
+            events: [
+                {
+                    type: "tool_call_end",
+                    callId: "c1",
+                    name: "text_to_speech_toolkit", // the ghost: a class name, not a function
+                    input: {},
+                },
+                { type: "stop", reason: "tool_use" }
+            ],
+        },
+        { events: [{ type: "stop", reason: "end_turn" }] },
+    ],
+    staticCheck: (events) => {
+        if (!events.some((e) => e.type === "tool_call_end" && e.name === "text_to_speech_toolkit")) {
+            return ["script lost the ghost tool call"];
+        }
+        return [];
+    },
+    requiredTerminal: ["completed"], // the loop completes; the refusal must be in the trajectory
+};

package/dist/fixtures/user-abort.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Fixture: USER ABORT — the user hits stop mid-run; the loop must land on an
+ * honest `aborted` terminal, not a fake `completed`.
+ *
+ * Incident (uooki production, 2026): long artifact turns had a dead
+ * interrupt window — a stop arriving before the executor registered its
+ * steering channel returned 409 and the turn kept running to "completed".
+ * The structural fix: the loop checks the signal at every phase boundary
+ * and the terminal is honest about who ended the run.
+ */
+import type { AbortSignalLike } from "@vincemakes/kiso-core";
+import type { Fixture } from "./types.js";
+/** A signal that flips to aborted after N events (test-controlled). */
+export declare function makeAbortSignal(flipAfter: number): {
+    signal: AbortSignalLike;
+    flip: () => void;
+};
+export declare const userAbort: Fixture;

package/dist/fixtures/user-abort.js

@@ -0,0 +1,55 @@
+/**
+ * Fixture: USER ABORT — the user hits stop mid-run; the loop must land on an
+ * honest `aborted` terminal, not a fake `completed`.
+ *
+ * Incident (uooki production, 2026): long artifact turns had a dead
+ * interrupt window — a stop arriving before the executor registered its
+ * steering channel returned 409 and the turn kept running to "completed".
+ * The structural fix: the loop checks the signal at every phase boundary
+ * and the terminal is honest about who ended the run.
+ */
+/** A signal that flips to aborted after N events (test-controlled). */
+export function makeAbortSignal(flipAfter) {
+    let aborted = false;
+    const listeners = [];
+    return {
+        signal: {
+            get aborted() {
+                return aborted;
+            },
+            addEventListener: (_type, listener) => {
+                listeners.push(listener);
+            },
+            removeEventListener: () => { },
+        },
+        flip: () => {
+            aborted = true;
+            for (const l of listeners)
+                l();
+        },
+    };
+}
+export const userAbort = {
+    name: "user-abort",
+    incident: "uooki long artifact turn: stop during the pre-executor window returned 409, turn ran to 'completed' (2026-07-26)",
+    script: [
+        {
+            events: [
+                { type: "text_start" },
+                { type: "text_delta", text: "working on it…" },
+                { type: "text_end" },
+                { type: "tool_call_end", callId: "c1", name: "create_artifact", input: {} },
+                { type: "stop", reason: "tool_use" }
+            ],
+        },
+        { events: [{ type: "stop", reason: "end_turn" }] },
+    ],
+    staticCheck: (events) => {
+        // The shape: a long turn that CAN be interrupted mid-tool-call.
+        if (!events.some((e) => e.type === "tool_call_end")) {
+            return ["script lost the tool call that the user interrupts"];
+        }
+        return [];
+    },
+    requiredTerminal: ["aborted"], // with the signal flipped, the loop MUST land here
+};

package/dist/index.d.ts

@@ -0,0 +1,2 @@
+export * from "./faux.js";
+export * from "./fixtures/index.js";

package/dist/index.js

@@ -0,0 +1,2 @@
+export * from "./faux.js";
+export * from "./fixtures/index.js";

package/package.json

@@ -0,0 +1,44 @@
+{
+  "name": "@vincemakes/kiso-evals",
+  "version": "0.1.0",
+  "description": "kiso evals \u2014 the faux provider, the incident fixture library, and the cross-provider contract tests.",
+  "type": "module",
+  "license": "MIT",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist",
+    "README.md",
+    "LICENSE"
+  ],
+  "scripts": {
+    "build": "tsc -p tsconfig.build.json",
+    "typecheck": "tsc -p tsconfig.json",
+    "test": "vitest run"
+  },
+  "dependencies": {
+    "@vincemakes/kiso-core": "0.1.0"
+  },
+  "devDependencies": {
+    "@anthropic-ai/sdk": "^0.115.0",
+    "@vincemakes/kiso-provider-anthropic": "0.1.0",
+    "@vincemakes/kiso-provider-openai": "0.1.0",
+    "@types/node": "^26.1.2",
+    "openai": "^7.3.0",
+    "typescript": "^5.7.2",
+    "vitest": "^3.0.0"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/vincemakes/kiso.git",
+    "directory": "packages/evals"
+  },
+  "bugs": {
+    "url": "https://github.com/vincemakes/kiso/issues"
+  },
+  "homepage": "https://github.com/vincemakes/kiso/tree/main/packages/evals#readme"
+}