@skillstate/bench 2.0.0 → 2.0.2

package/README.md

@@ -0,0 +1,109 @@
+<div align="center">
+
+# @skillstate/bench
+
+**Deterministic local benchmark harness for the skillstate runtime — conversation baseline vs SKILL.state.**
+
+[![npm version](https://img.shields.io/npm/v/@skillstate/bench)](https://www.npmjs.com/package/@skillstate/bench)
+[![node](https://img.shields.io/node/v/@skillstate/bench)](https://www.npmjs.com/package/@skillstate/bench)
+[![Tests](https://img.shields.io/badge/tests-755%20passing-brightgreen)](https://github.com/vitalykuzyaev/skillstate)
+[![License: MIT](https://img.shields.io/badge/license-MIT-blue)](https://github.com/vitalykuzyaev/skillstate/blob/main/LICENSE)
+
+</div>
+
+---
+
+`@skillstate/bench` is a fully deterministic A/B harness that measures the
+O(1)/O(T) prompt-footprint claim of [`@skillstate/core`](../core) on fixed
+synthetic data: mode (b) drives the real `SkillStateRuntime` for T steps, mode
+(a) reconstructs the conversation baseline as prefix sums of those very state
+prompts. With constant-size turns the reduction is exactly `(T+1)/2`
+(paper §3.3 eq.5-7). Entry only, no bin.
+
+> **@non-paper** — this harness is *not* the paper's evaluation. The paper's
+> Table 1 / §5.2 numbers come from Gemini/Gemma runs on Warehousing tasks; this
+> measures a minimal, reproducible upper-bound A/B on our own data. Do not quote
+> these as paper results.
+
+## Installation
+
+```bash
+npm i @skillstate/core @skillstate/bench
+```
+
+Requires Node.js >= 20. TypeScript types are bundled.
+
+## Quick start
+
+Run the whole suite (from the repo root, or any installed copy):
+
+```bash
+npm run bench          # builds then runs node ./packages/bench/dist/run.js
+```
+
+Programmatically:
+
+```ts
+import { runAll, formatTable, BENCH_T_VALUES, expectedReduction } from '@skillstate/bench';
+
+const results = await runAll(BENCH_T_VALUES);  // [T=10, 50, 100, 200]
+console.log(formatTable(results));
+
+for (const r of results) {
+  console.log(r.T, r.reductionFactor.toFixed(2), expectedReduction(r.T));
+}
+```
+
+Run a single horizon with the incremental entry point:
+
+```ts
+import { runScenario } from '@skillstate/bench';
+const r = await runScenario(100);
+console.log(r.stateCumulative, r.convCumulative, r.reductionFactor);
+```
+
+## API / Exports
+
+Root path `@skillstate/bench` exports the harness plus the run entry. Importing
+the package is side-effect free — the benchmark only runs when the module is
+the process entry (`node dist/run.js`).
+
+**Harness (`harness.ts`):**
+
+- `BENCH_T_VALUES` — `readonly number[]` = `[10, 50, 100, 200]`.
+- `BENCH_SPEC` / `BENCH_OBSERVATION` / `BENCH_REASONING` / `BENCH_ACTION` /
+  `BENCH_PATCH` / `BENCH_SEED` — the fixed synthetic fixture.
+- `benchResponse(): string` / `benchObservation(): Observation`.
+- `runScenario(T): Promise<BenchResult>` — drive one horizon.
+- `runAll(horizons): Promise<BenchResult[]>`.
+- `formatTable(results): string` — human-readable text table.
+- `expectedReduction(T): number` — closed form `(T+1)/2`.
+- `BenchResult` — per-horizon outcome (all sizes in raw chars).
+
+**Run entry (`run.ts`):** `main(): Promise<BenchResult[]>` — prints the table
+and machine-readable JSON.
+
+## Notes
+
+- **Deterministic.** No RNG anywhere — a fixed spec, a fixed 64-char
+  observation, a fixed mock-LLM reply. `BENCH_SEED` documents that no seed is
+  needed.
+- **Read before quoting.** Because the spec `P` is re-sent in every
+  conversation-baseline turn, the measured savings are an **upper bound** on a
+  real baseline (which sends `P` once and re-sends cheaper turn payloads).
+- The method is the quiet `TokenTracker.compareWithBaseline` model (paper §3.3
+  eq.5), on identical data, using the paper-exact `formatPaper` prompts. All
+  metrics are raw string chars (§4.3).
+- Depends on [`@skillstate/core`](../core) for `SkillStateRuntime` and the
+  `formatPaper` prompt.
+
+## Related
+
+- Paper: [arXiv:2608.26263](https://arxiv.org/abs/2608.26263).
+- Core runtime: [`@skillstate/core`](../core).
+- Full method, tables, and limitations: [`BENCHMARK.md`](../../BENCHMARK.md).
+- Machine-readable fixture: [`tests/bench/expected.json`](../../../tests/bench/expected.json).
+
+## License
+
+[MIT](LICENSE) © 2026 Vitaly Kuzyaev

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAKA,cAAc,cAAc,CAAC;AAC7B,cAAc,UAAU,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAIA,cAAc,cAAc,CAAC;AAC7B,cAAc,UAAU,CAAC"}

package/dist/index.js

@@ -1,8 +1,7 @@
 // @skillstate/bench — deterministic local benchmark harness.
 //
-// NOTE: re-exporting `./run.js` also evaluates its top-level `main()`
-// (the `npm run bench` entry), exactly like importing the historical
-// `dist/bench/run.js` directly.
+// NOTE: re-exporting `./run.js` is side-effect free — `main` only executes
+// when `run.js` is the process entry (`node dist/run.js`), never on import.
 export * from './harness.js';
 export * from './run.js';
 //# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,6DAA6D;AAC7D,EAAE;AACF,sEAAsE;AACtE,qEAAqE;AACrE,gCAAgC;AAChC,cAAc,cAAc,CAAC;AAC7B,cAAc,UAAU,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,6DAA6D;AAC7D,EAAE;AACF,2EAA2E;AAC3E,4EAA4E;AAC5E,cAAc,cAAc,CAAC;AAC7B,cAAc,UAAU,CAAC"}

package/dist/run.d.ts

@@ -2,6 +2,10 @@
  * `npm run bench` entry point (zero-deps, runs from compiled dist).
  *
  * @non-paper — local deterministic harness CLI, not part of the paper.
+ *
+ * Importing this module is side-effect free: the benchmark only runs when
+ * this file is the process entry (`node dist/run.js`, the `npm run bench`
+ * path), never when `@skillstate/bench` is imported as a library.
  */
 import type { BenchResult } from './harness.js';
 /** Run all horizons, print the table plus machine-readable JSON. */

package/dist/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../src/run.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAEhD,oEAAoE;AACpE,wBAAsB,IAAI,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC,CAKnD"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../src/run.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAIH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAEhD,oEAAoE;AACpE,wBAAsB,IAAI,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC,CAKnD"}

package/dist/run.js

@@ -2,7 +2,12 @@
  * `npm run bench` entry point (zero-deps, runs from compiled dist).
  *
  * @non-paper — local deterministic harness CLI, not part of the paper.
+ *
+ * Importing this module is side-effect free: the benchmark only runs when
+ * this file is the process entry (`node dist/run.js`, the `npm run bench`
+ * path), never when `@skillstate/bench` is imported as a library.
  */
+import { pathToFileURL } from 'node:url';
 import { BENCH_T_VALUES, runAll, formatTable } from './harness.js';
 /** Run all horizons, print the table plus machine-readable JSON. */
 export async function main() {
@@ -11,5 +16,9 @@ export async function main() {
     console.log(JSON.stringify(results, null, 2));
     return results;
 }
-void main();
+const isEntry = process.argv[1] !== undefined &&
+    pathToFileURL(process.argv[1]).href === import.meta.url;
+if (isEntry) {
+    await main();
+}
 //# sourceMappingURL=run.js.map

package/dist/run.js.map

@@ -1 +1 @@
-{"version":3,"file":"run.js","sourceRoot":"","sources":["../src/run.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,cAAc,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAGnE,oEAAoE;AACpE,MAAM,CAAC,KAAK,UAAU,IAAI;IACxB,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,CAAC;IAC7C,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC;IAClC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAC9C,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,KAAK,IAAI,EAAE,CAAC"}
+{"version":3,"file":"run.js","sourceRoot":"","sources":["../src/run.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,cAAc,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAGnE,oEAAoE;AACpE,MAAM,CAAC,KAAK,UAAU,IAAI;IACxB,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,CAAC;IAC7C,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC;IAClC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAC9C,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,OAAO,GACX,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,SAAS;IAC7B,aAAa,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,OAAO,IAAI,CAAC,GAAG,CAAC;AAE1D,IAAI,OAAO,EAAE,CAAC;IACZ,MAAM,IAAI,EAAE,CAAC;AACf,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@skillstate/bench",
-  "version": "2.0.0",
+  "version": "2.0.2",
   "description": "Deterministic local benchmark harness for the skillstate runtime (entry only).",
   "type": "module",
   "main": "dist/index.js",
@@ -12,7 +12,9 @@
     },
     "./package.json": "./package.json"
   },
-  "files": ["dist"],
+  "files": [
+    "dist"
+  ],
   "sideEffects": false,
   "engines": {
     "node": ">=20"