@vincemakes/kiso-evals 0.42.1 → 0.42.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -2,6 +2,6 @@
2
2
 
3
3
  The faux provider (a scripted model — zero keys, deterministic), the
4
4
  incident fixture library (every fixture is a real production accident
5
- from uooki 2026), and the cross-provider contract tests.
5
+ from a production host, 2026), and the cross-provider contract tests.
6
6
 
7
7
  See the repository README for the framework overview.
package/dist/faux.d.ts CHANGED
@@ -1,5 +1,5 @@
1
1
  /**
2
- * L7 — the faux provider: iteration's foundation (mauri ADR-0010).
2
+ * L7 — the faux provider: iteration's foundation (the predecessor's ADR-0010).
3
3
  *
4
4
  * A provider that needs no API key and no network. A `FauxScript` is a
5
5
  * turn-by-turn playbook: what the "model" emits. The kernel cannot tell a
package/dist/faux.js CHANGED
@@ -1,5 +1,5 @@
1
1
  /**
2
- * L7 — the faux provider: iteration's foundation (mauri ADR-0010).
2
+ * L7 — the faux provider: iteration's foundation (the predecessor's ADR-0010).
3
3
  *
4
4
  * A provider that needs no API key and no network. A `FauxScript` is a
5
5
  * turn-by-turn playbook: what the "model" emits. The kernel cannot tell a
@@ -2,7 +2,7 @@
2
2
  * Fixture: COMPACTION REGROWTH — repeated compaction re-archives already
3
3
  * cleared messages, O(N²), silently overwriting original content.
4
4
  *
5
- * Incident (uooki production, 2026, video pipeline): measured 579 store_calls
5
+ * Incident (a production host, 2026, video pipeline): measured 579 store_calls
6
6
  * / 480 archive entries against a naive expectation of 44 — ~10× at 50 turns,
7
7
  * extrapolating to 64× at 200 turns and 227× at 1000. Root cause: compaction
8
8
  * re-archived messages whose content was already the clear marker, writing
@@ -2,7 +2,7 @@
2
2
  * Fixture: COMPACTION REGROWTH — repeated compaction re-archives already
3
3
  * cleared messages, O(N²), silently overwriting original content.
4
4
  *
5
- * Incident (uooki production, 2026, video pipeline): measured 579 store_calls
5
+ * Incident (a production host, 2026, video pipeline): measured 579 store_calls
6
6
  * / 480 archive entries against a naive expectation of 44 — ~10× at 50 turns,
7
7
  * extrapolating to 64× at 200 turns and 227× at 1000. Root cause: compaction
8
8
  * re-archived messages whose content was already the clear marker, writing
@@ -21,7 +21,7 @@
21
21
  import { projectMessages } from "@vincemakes/kiso-core";
22
22
  export const compactionRegrowth = {
23
23
  name: "compaction-regrowth",
24
- incident: "uooki video pipeline: 579 store_calls vs 44 expected at 50 turns — re-archiving cleared markers overwrote originals (2026)",
24
+ incident: "a video pipeline: 579 store_calls vs 44 expected at 50 turns — re-archiving cleared markers overwrote originals (2026)",
25
25
  script: [
26
26
  {
27
27
  events: [
@@ -2,7 +2,7 @@
2
2
  * Fixture: PERMISSION NEGOTIATION — the same call shape can be denied once
3
3
  * and allowed later; the model must see both outcomes distinctly.
4
4
  *
5
- * Incident (mauri ADR-0002): a permission system that is a yes/no gate has
5
+ * Incident (the predecessor's ADR-0002): a permission system that is a yes/no gate has
6
6
  * no memory and no upgrade path. The reference implementation's ask model
7
7
  * allows deny-with-reason,
8
8
  * and the reason feeds back to the model so it can adjust. The kernel's
@@ -2,7 +2,7 @@
2
2
  * Fixture: PERMISSION NEGOTIATION — the same call shape can be denied once
3
3
  * and allowed later; the model must see both outcomes distinctly.
4
4
  *
5
- * Incident (mauri ADR-0002): a permission system that is a yes/no gate has
5
+ * Incident (the predecessor's ADR-0002): a permission system that is a yes/no gate has
6
6
  * no memory and no upgrade path. The reference implementation's ask model
7
7
  * allows deny-with-reason,
8
8
  * and the reason feeds back to the model so it can adjust. The kernel's
@@ -11,7 +11,7 @@
11
11
  */
12
12
  export const permissionNegotiation = {
13
13
  name: "permission-negotiation",
14
- incident: "uooki GatePipeline was a message-prefix interceptor with no session memory — a denied call could not be re-attempted after user approval (mauri ADR-0002 audit)",
14
+ incident: "a gate pipeline was a message-prefix interceptor with no session memory — a denied call could not be re-attempted after user approval (the predecessor's ADR-0002 audit)",
15
15
  script: [
16
16
  {
17
17
  events: [
@@ -2,7 +2,7 @@
2
2
  * Fixture: SILENT TOOL FAILURE — an error result is swallowed and the model
3
3
  * keeps narrating success.
4
4
  *
5
- * Incident (uooki production, 2026): a tool call failed with is_error=true,
5
+ * Incident (a production host, 2026): a tool call failed with is_error=true,
6
6
  * the model ignored it and continued with "results look great, done" —
7
7
  * the turn ended "completed" with zero usable work. Root cause class:
8
8
  * error results ride the same channel as successes, and nothing forced the
@@ -2,7 +2,7 @@
2
2
  * Fixture: SILENT TOOL FAILURE — an error result is swallowed and the model
3
3
  * keeps narrating success.
4
4
  *
5
- * Incident (uooki production, 2026): a tool call failed with is_error=true,
5
+ * Incident (a production host, 2026): a tool call failed with is_error=true,
6
6
  * the model ignored it and continued with "results look great, done" —
7
7
  * the turn ended "completed" with zero usable work. Root cause class:
8
8
  * error results ride the same channel as successes, and nothing forced the
@@ -19,7 +19,7 @@
19
19
  */
20
20
  export const silentToolFailure = {
21
21
  name: "silent-tool-failure",
22
- incident: "uooki turn: search tool returned is_error, model ignored it and reported success (2026-06)",
22
+ incident: "a production turn: search tool returned is_error, model ignored it and reported success (2026-06)",
23
23
  script: [
24
24
  {
25
25
  events: [
@@ -1,7 +1,7 @@
1
1
  /**
2
2
  * Fixture: TERMINAL LIES — the model reports done work that never landed.
3
3
  *
4
- * Incident (uooki production, 2026): a long-document turn called
4
+ * Incident (a production host, 2026): a long-document turn called
5
5
  * `document_create` twice, the chat asserted "created the document artifact", and the
6
6
  * Canvas stayed empty — the tool was an inline Markdown table that never
7
7
  * reaches the canvas, and all three delivery guards were blind to it
@@ -1,7 +1,7 @@
1
1
  /**
2
2
  * Fixture: TERMINAL LIES — the model reports done work that never landed.
3
3
  *
4
- * Incident (uooki production, 2026): a long-document turn called
4
+ * Incident (a production host, 2026): a long-document turn called
5
5
  * `document_create` twice, the chat asserted "created the document artifact", and the
6
6
  * Canvas stayed empty — the tool was an inline Markdown table that never
7
7
  * reaches the canvas, and all three delivery guards were blind to it
@@ -17,7 +17,7 @@
17
17
  import { analyzeDelivery } from "../governance/delivery.js";
18
18
  export const terminalLies = {
19
19
  name: "terminal-lies",
20
- incident: "uooki long-document turn: model claimed 'created the document artifact', canvas stayed empty, turn reported completed (document_create trap, 2026-07-31)",
20
+ incident: "a long-document turn: model claimed 'created the document artifact', canvas stayed empty, turn reported completed (document_create trap, 2026-07-31)",
21
21
  delivery: {
22
22
  required: true,
23
23
  producers: new Set(["create_artifact", "docx_create"]),
@@ -3,7 +3,7 @@
3
3
  * assertion the kernel must satisfy.
4
4
  *
5
5
  * This is kiso's answer to "prove the loop is honest": every fixture is a
6
- * real incident abstracted (uooki production, 2026). A fixture the loop
6
+ * real incident abstracted (a production host, 2026). A fixture the loop
7
7
  * cannot pass is a regression the loop must not ship. Other frameworks
8
8
  * advertise features; kiso advertises "verifyable against known failure
9
9
  * modes" — because the reference implementation's loudest production
@@ -3,7 +3,7 @@
3
3
  * assertion the kernel must satisfy.
4
4
  *
5
5
  * This is kiso's answer to "prove the loop is honest": every fixture is a
6
- * real incident abstracted (uooki production, 2026). A fixture the loop
6
+ * real incident abstracted (a production host, 2026). A fixture the loop
7
7
  * cannot pass is a regression the loop must not ship. Other frameworks
8
8
  * advertise features; kiso advertises "verifyable against known failure
9
9
  * modes" — because the reference implementation's loudest production
@@ -1,7 +1,7 @@
1
1
  /**
2
2
  * Fixture: UNKNOWN TOOL — the model calls a tool that is not registered.
3
3
  *
4
- * Incident class (uooki + the reference implementation's community): a tool
4
+ * Incident class (a production host + the reference implementation's community): a tool
5
5
  * name drifts (renamed,
6
6
  * ghost entry, typo'd enum) and the harness either silently drops the call
7
7
  * (agent loops forever trying to find the tool) or executes something
@@ -1,7 +1,7 @@
1
1
  /**
2
2
  * Fixture: UNKNOWN TOOL — the model calls a tool that is not registered.
3
3
  *
4
- * Incident class (uooki + the reference implementation's community): a tool
4
+ * Incident class (a production host + the reference implementation's community): a tool
5
5
  * name drifts (renamed,
6
6
  * ghost entry, typo'd enum) and the harness either silently drops the call
7
7
  * (agent loops forever trying to find the tool) or executes something
@@ -11,7 +11,7 @@
11
11
  */
12
12
  export const unknownTool = {
13
13
  name: "unknown-tool",
14
- incident: "uooki ghost-tool class: manifest listed a toolkit class name instead of the callable function name; activate_tools silently dropped it and the agent looped (2026-07)",
14
+ incident: "a ghost-tool class: manifest listed a toolkit class name instead of the callable function name; activate_tools silently dropped it and the agent looped (2026-07)",
15
15
  script: [
16
16
  {
17
17
  events: [
@@ -2,7 +2,7 @@
2
2
  * Fixture: USER ABORT — the user hits stop mid-run; the loop must land on an
3
3
  * honest `aborted` terminal, not a fake `completed`.
4
4
  *
5
- * Incident (uooki production, 2026): long artifact turns had a dead
5
+ * Incident (a production host, 2026): long artifact turns had a dead
6
6
  * interrupt window — a stop arriving before the executor registered its
7
7
  * steering channel returned 409 and the turn kept running to "completed".
8
8
  * The structural fix: the loop checks the signal at every phase boundary
@@ -2,7 +2,7 @@
2
2
  * Fixture: USER ABORT — the user hits stop mid-run; the loop must land on an
3
3
  * honest `aborted` terminal, not a fake `completed`.
4
4
  *
5
- * Incident (uooki production, 2026): long artifact turns had a dead
5
+ * Incident (a production host, 2026): long artifact turns had a dead
6
6
  * interrupt window — a stop arriving before the executor registered its
7
7
  * steering channel returned 409 and the turn kept running to "completed".
8
8
  * The structural fix: the loop checks the signal at every phase boundary
@@ -31,7 +31,7 @@ export function makeAbortSignal(flipAfter) {
31
31
  }
32
32
  export const userAbort = {
33
33
  name: "user-abort",
34
- incident: "uooki long artifact turn: stop during the pre-executor window returned 409, turn ran to 'completed' (2026-07-26)",
34
+ incident: "a long artifact turn: stop during the pre-executor window returned 409, turn ran to 'completed' (2026-07-26)",
35
35
  script: [
36
36
  {
37
37
  events: [
@@ -1,5 +1,5 @@
1
1
  /**
2
- * Delivery truth — "done" means the ledger says so, not the model (uooki
2
+ * Delivery truth — "done" means the ledger says so, not the model (a production host
3
3
  * done_guard, 30 incidents distilled).
4
4
  *
5
5
  * EC-1 (0.13.0): this module MOVED here from packages/core/src/governance/.
@@ -1,5 +1,5 @@
1
1
  /**
2
- * Delivery truth — "done" means the ledger says so, not the model (uooki
2
+ * Delivery truth — "done" means the ledger says so, not the model (a production host
3
3
  * done_guard, 30 incidents distilled).
4
4
  *
5
5
  * EC-1 (0.13.0): this module MOVED here from packages/core/src/governance/.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@vincemakes/kiso-evals",
3
- "version": "0.42.1",
3
+ "version": "0.42.3",
4
4
  "description": "kiso evals \u2014 the faux provider, the incident fixture library, and the cross-provider contract tests.",
5
5
  "type": "module",
6
6
  "license": "MIT",
@@ -21,12 +21,12 @@
21
21
  "test": "vitest run"
22
22
  },
23
23
  "dependencies": {
24
- "@vincemakes/kiso-core": "0.42.1"
24
+ "@vincemakes/kiso-core": "0.42.3"
25
25
  },
26
26
  "devDependencies": {
27
27
  "@anthropic-ai/sdk": "^0.115.0",
28
- "@vincemakes/kiso-provider-anthropic": "0.42.1",
29
- "@vincemakes/kiso-provider-openai": "0.42.1",
28
+ "@vincemakes/kiso-provider-anthropic": "0.42.3",
29
+ "@vincemakes/kiso-provider-openai": "0.42.3",
30
30
  "@types/node": "^26.1.2",
31
31
  "openai": "^7.3.0",
32
32
  "typescript": "^5.7.2",