@skillstate/bench 2.2.2 → 3.0.1

package/README.md

@@ -6,8 +6,7 @@
 
 [![npm version](https://img.shields.io/npm/v/@skillstate/bench)](https://www.npmjs.com/package/@skillstate/bench)
 [![node](https://img.shields.io/node/v/@skillstate/bench)](https://www.npmjs.com/package/@skillstate/bench)
-[![Tests](https://img.shields.io/badge/tests-873%20passing-brightgreen)](https://github.com/vitalykuzyaev/skillstate)
-[![License: MIT](https://img.shields.io/badge/license-MIT-blue)](https://github.com/vitalykuzyaev/skillstate/blob/main/LICENSE)
+[![License: MIT](https://img.shields.io/badge/license-MIT-blue)](https://github.com/vitkuz573/skillstate/blob/main/LICENSE)
 
 </div>
 

package/dist/ab/engagement.d.ts

@@ -0,0 +1,70 @@
+/**
+ * Proving the integration was actually engaged.
+ *
+ * ── The single most important check in the harness ────────────────────────
+ *
+ * The previous experiment produced a confident "39% saving" from a run in
+ * which the instrumented arm never wrote the state file once. The model was
+ * never asked to use the tools, or asked and declined; either way the plugin
+ * was decoration, and the token delta was a sample of how the model varies
+ * between two identical runs.
+ *
+ * Nothing in the token stream can distinguish those cases. Both arms produce
+ * perfectly ordinary messages. The only evidence that a difference was caused
+ * by the integration is that the integration left a mark — and the mark is
+ * the state file.
+ *
+ * So engagement is measured by DIFFING the state file, not by asking the
+ * model. A model that reports having used the tools is not evidence; bytes
+ * on disk are.
+ *
+ * @non-paper — measurement infrastructure, not the paper's evaluation.
+ */
+import type { EngagementEvidence } from './record.js';
+/** One observed transition of the state file. */
+export interface StateSample {
+    /** Which trial the sample belongs to. */
+    readonly trial: number;
+    /**
+     * Point in the run at which the file was read: 0 before the first turn,
+     * n after the last.
+     */
+    readonly step: number;
+    /** Raw file content, or `null` when no file existed. */
+    readonly content: string | null;
+    /** True when a `state_patch` from the response sink caused this sample. */
+    readonly fromSink?: boolean;
+}
+/** The result of watching a run's state file. */
+export interface EngagementReport {
+    readonly evidence: EngagementEvidence;
+    /** Samples whose content differed from the previous one, in order. */
+    readonly transitions: readonly StateSample[];
+}
+/**
+ * Reduce a run's state samples to engagement evidence.
+ *
+ * A "write" is a change in content, not a call to the write function. A sink
+ * that re-writes the same bytes has changed nothing, and counting it would
+ * let a retry loop manufacture the appearance of engagement — the mirror
+ * image of the bug this module exists to catch.
+ *
+ * The first sample is the baseline, so a file that already existed at step 0
+ * is not itself evidence of a write. That distinction matters for a resumed
+ * session, where `hadStateAtStep0` is true and a naive diff would report
+ * engagement even if the model never touched anything.
+ */
+export declare function assessEngagement(samples: readonly StateSample[]): EngagementReport;
+/** Attach sink rejection counts to evidence produced by {@link assessEngagement}. */
+export declare function withRejections(evidence: EngagementEvidence, rejections: number, firstRejection?: string): EngagementEvidence;
+/**
+ * Whether a set of state samples is sufficient to judge engagement.
+ *
+ * One sample cannot: it is equally consistent with "the file never changed"
+ * and with "the file changed and was not sampled". The harness treats an
+ * under-sampled run as unmeasured rather than as inert, because reporting
+ * "inert" for a run nobody watched would be a false accusation of the
+ * integration, which is the one error this harness must never make.
+ */
+export declare function isWitnessed(samples: readonly StateSample[]): boolean;
+//# sourceMappingURL=engagement.d.ts.map

package/dist/ab/engagement.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"engagement.d.ts","sourceRoot":"","sources":["../../src/ab/engagement.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAEtD,iDAAiD;AACjD,MAAM,WAAW,WAAW;IAC1B,yCAAyC;IACzC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB;;;OAGG;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,wDAAwD;IACxD,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,2EAA2E;IAC3E,QAAQ,CAAC,QAAQ,CAAC,EAAE,OAAO,CAAC;CAC7B;AAED,iDAAiD;AACjD,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,QAAQ,EAAE,kBAAkB,CAAC;IACtC,sEAAsE;IACtE,QAAQ,CAAC,WAAW,EAAE,SAAS,WAAW,EAAE,CAAC;CAC9C;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,SAAS,WAAW,EAAE,GAAG,gBAAgB,CAqClF;AAED,qFAAqF;AACrF,wBAAgB,cAAc,CAC5B,QAAQ,EAAE,kBAAkB,EAC5B,UAAU,EAAE,MAAM,EAClB,cAAc,CAAC,EAAE,MAAM,GACtB,kBAAkB,CAMpB;AAED;;;;;;;;GAQG;AACH,wBAAgB,WAAW,CAAC,OAAO,EAAE,SAAS,WAAW,EAAE,GAAG,OAAO,CAWpE"}

package/dist/ab/engagement.js

@@ -0,0 +1,102 @@
+/**
+ * Proving the integration was actually engaged.
+ *
+ * ── The single most important check in the harness ────────────────────────
+ *
+ * The previous experiment produced a confident "39% saving" from a run in
+ * which the instrumented arm never wrote the state file once. The model was
+ * never asked to use the tools, or asked and declined; either way the plugin
+ * was decoration, and the token delta was a sample of how the model varies
+ * between two identical runs.
+ *
+ * Nothing in the token stream can distinguish those cases. Both arms produce
+ * perfectly ordinary messages. The only evidence that a difference was caused
+ * by the integration is that the integration left a mark — and the mark is
+ * the state file.
+ *
+ * So engagement is measured by DIFFING the state file, not by asking the
+ * model. A model that reports having used the tools is not evidence; bytes
+ * on disk are.
+ *
+ * @non-paper — measurement infrastructure, not the paper's evaluation.
+ */
+/**
+ * Reduce a run's state samples to engagement evidence.
+ *
+ * A "write" is a change in content, not a call to the write function. A sink
+ * that re-writes the same bytes has changed nothing, and counting it would
+ * let a retry loop manufacture the appearance of engagement — the mirror
+ * image of the bug this module exists to catch.
+ *
+ * The first sample is the baseline, so a file that already existed at step 0
+ * is not itself evidence of a write. That distinction matters for a resumed
+ * session, where `hadStateAtStep0` is true and a naive diff would report
+ * engagement even if the model never touched anything.
+ */
+export function assessEngagement(samples) {
+    const ordered = [...samples].sort((a, b) => a.trial - b.trial || a.step - b.step);
+    const transitions = [];
+    let previous = null;
+    let writes = 0;
+    let sinkWrites = 0;
+    // Tracked separately from `previous` because `previous === null` is a
+    // legitimate first value (no state file yet) and cannot double as the
+    // "is this the baseline sample" marker.
+    let isBaseline = true;
+    for (const sample of ordered) {
+        if (isBaseline) {
+            isBaseline = false;
+            previous = sample.content;
+            continue;
+        }
+        if (sample.content === previous)
+            continue;
+        transitions.push(sample);
+        writes += 1;
+        if (sample.fromSink === true)
+            sinkWrites += 1;
+        previous = sample.content;
+    }
+    const first = ordered[0];
+    return {
+        evidence: {
+            engaged: writes > 0,
+            writes,
+            sinkWrites,
+            rejections: 0,
+            hadStateAtStep0: (first?.content ?? null) !== null,
+        },
+        transitions,
+    };
+}
+/** Attach sink rejection counts to evidence produced by {@link assessEngagement}. */
+export function withRejections(evidence, rejections, firstRejection) {
+    return {
+        ...evidence,
+        rejections,
+        ...(firstRejection === undefined ? {} : { firstRejection }),
+    };
+}
+/**
+ * Whether a set of state samples is sufficient to judge engagement.
+ *
+ * One sample cannot: it is equally consistent with "the file never changed"
+ * and with "the file changed and was not sampled". The harness treats an
+ * under-sampled run as unmeasured rather than as inert, because reporting
+ * "inert" for a run nobody watched would be a false accusation of the
+ * integration, which is the one error this harness must never make.
+ */
+export function isWitnessed(samples) {
+    if (samples.length === 0)
+        return false;
+    const byTrial = new Map();
+    for (const sample of samples) {
+        const steps = byTrial.get(sample.trial) ?? [];
+        steps.push(sample.step);
+        byTrial.set(sample.trial, steps);
+    }
+    // Every trial needs at least two samples — a before and an after — or a
+    // change between them is unobservable.
+    return [...byTrial.values()].every((steps) => steps.length >= 2);
+}
+//# sourceMappingURL=engagement.js.map

package/dist/ab/engagement.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"engagement.js","sourceRoot":"","sources":["../../src/ab/engagement.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AA0BH;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAA+B;IAC9D,MAAM,OAAO,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC,IAAI,CAC/B,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAC/C,CAAC;IACF,MAAM,WAAW,GAAkB,EAAE,CAAC;IACtC,IAAI,QAAQ,GAAkB,IAAI,CAAC;IACnC,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,sEAAsE;IACtE,sEAAsE;IACtE,wCAAwC;IACxC,IAAI,UAAU,GAAG,IAAI,CAAC;IAEtB,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,UAAU,EAAE,CAAC;YACf,UAAU,GAAG,KAAK,CAAC;YACnB,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC;YAC1B,SAAS;QACX,CAAC;QACD,IAAI,MAAM,CAAC,OAAO,KAAK,QAAQ;YAAE,SAAS;QAC1C,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACzB,MAAM,IAAI,CAAC,CAAC;QACZ,IAAI,MAAM,CAAC,QAAQ,KAAK,IAAI;YAAE,UAAU,IAAI,CAAC,CAAC;QAC9C,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC;IAC5B,CAAC;IAED,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;IACzB,OAAO;QACL,QAAQ,EAAE;YACR,OAAO,EAAE,MAAM,GAAG,CAAC;YACnB,MAAM;YACN,UAAU;YACV,UAAU,EAAE,CAAC;YACb,eAAe,EAAE,CAAC,KAAK,EAAE,OAAO,IAAI,IAAI,CAAC,KAAK,IAAI;SACnD;QACD,WAAW;KACZ,CAAC;AACJ,CAAC;AAED,qFAAqF;AACrF,MAAM,UAAU,cAAc,CAC5B,QAA4B,EAC5B,UAAkB,EAClB,cAAuB;IAEvB,OAAO;QACL,GAAG,QAAQ;QACX,UAAU;QACV,GAAG,CAAC,cAAc,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,CAAC;KAC5D,CAAC;AACJ,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,WAAW,CAAC,OAA+B;IACzD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACvC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAoB,CAAC;IAC5C,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QAC9C,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACxB,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;IACnC,CAAC;IACD,wEAAwE;IACxE,uCAAuC;IACvC,OAAO,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC;AACnE,CAAC"}

package/dist/ab/index.d.ts

@@ -0,0 +1,28 @@
+/**
+ * The A/B harness: a run record per trial, and gates on the result.
+ *
+ * Everything here is either pure or injected. The live driver
+ * (`run-live.ts`) supplies the I/O; the gates in `verdict.ts` and the
+ * statistics in `stats-core.ts` never touch a filesystem or a clock, which
+ * is what allows them to be tested against the real historical numbers.
+ *
+ * @non-paper — measurement infrastructure, not the paper's evaluation.
+ */
+export { ARM_IDS, CONTROL_ENGAGEMENT, buildArmRecord, isInstrumented, promptTokens, } from './record.js';
+export type { ArmId, ArmRecord, EngagementEvidence, RunRecord, TokenUsage, WorkEvidence, } from './record.js';
+export { MIN_EFFECT_IN_MADS, describe, effectSize, mad, median, relativeMad, } from './stats-core.js';
+export type { Distribution, EffectSize } from './stats-core.js';
+export { MIN_TRIALS, runExperiment } from './verdict.js';
+export type { ExperimentOptions, GateFailure, Verdict, VerdictResult, } from './verdict.js';
+export { assessEngagement, isWitnessed, withRejections } from './engagement.js';
+export type { EngagementReport, StateSample } from './engagement.js';
+export { NO_USAGE, resolveSessionUsage, sessionUsage } from './usage.js';
+export type { HostMessageRow, UsageFailure, UsageOutcome, UsageReader, } from './usage.js';
+export { formatArmTable, formatVerdict } from './report.js';
+export { serverSessionUsage, serverUsageReader } from './opencode-usage.js';
+export type { ServerFetcher } from './opencode-usage.js';
+export { DEFAULT_BOUNDED_PROMPT_TOKENS, DEFAULT_CACHE_READ_DISCOUNT, assessReconstruction, promptTokensOf, reconstruct, } from './replay.js';
+export type { HostSession, ReconstructOptions, ReconstructResult, StepUsage, } from './replay.js';
+export { breakEvenPromptTokens, formatSurvey, survey } from './survey.js';
+export type { Survey, SurveyOptions } from './survey.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/ab/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/ab/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EACL,OAAO,EACP,kBAAkB,EAClB,cAAc,EACd,cAAc,EACd,YAAY,GACb,MAAM,aAAa,CAAC;AACrB,YAAY,EACV,KAAK,EACL,SAAS,EACT,kBAAkB,EAClB,SAAS,EACT,UAAU,EACV,YAAY,GACb,MAAM,aAAa,CAAC;AAErB,OAAO,EACL,kBAAkB,EAClB,QAAQ,EACR,UAAU,EACV,GAAG,EACH,MAAM,EACN,WAAW,GACZ,MAAM,iBAAiB,CAAC;AACzB,YAAY,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAEhE,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AACzD,YAAY,EACV,iBAAiB,EACjB,WAAW,EACX,OAAO,EACP,aAAa,GACd,MAAM,cAAc,CAAC;AAEtB,OAAO,EAAE,gBAAgB,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAChF,YAAY,EAAE,gBAAgB,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAErE,OAAO,EAAE,QAAQ,EAAE,mBAAmB,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AACzE,YAAY,EACV,cAAc,EACd,YAAY,EACZ,YAAY,EACZ,WAAW,GACZ,MAAM,YAAY,CAAC;AAEpB,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAE5D,OAAO,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAC5E,YAAY,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEzD,OAAO,EACL,6BAA6B,EAC7B,2BAA2B,EAC3B,oBAAoB,EACpB,cAAc,EACd,WAAW,GACZ,MAAM,aAAa,CAAC;AACrB,YAAY,EACV,WAAW,EACX,kBAAkB,EAClB,iBAAiB,EACjB,SAAS,GACV,MAAM,aAAa,CAAC;AAErB,OAAO,EAAE,qBAAqB,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAC1E,YAAY,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC"}

package/dist/ab/index.js

@@ -0,0 +1,20 @@
+/**
+ * The A/B harness: a run record per trial, and gates on the result.
+ *
+ * Everything here is either pure or injected. The live driver
+ * (`run-live.ts`) supplies the I/O; the gates in `verdict.ts` and the
+ * statistics in `stats-core.ts` never touch a filesystem or a clock, which
+ * is what allows them to be tested against the real historical numbers.
+ *
+ * @non-paper — measurement infrastructure, not the paper's evaluation.
+ */
+export { ARM_IDS, CONTROL_ENGAGEMENT, buildArmRecord, isInstrumented, promptTokens, } from './record.js';
+export { MIN_EFFECT_IN_MADS, describe, effectSize, mad, median, relativeMad, } from './stats-core.js';
+export { MIN_TRIALS, runExperiment } from './verdict.js';
+export { assessEngagement, isWitnessed, withRejections } from './engagement.js';
+export { NO_USAGE, resolveSessionUsage, sessionUsage } from './usage.js';
+export { formatArmTable, formatVerdict } from './report.js';
+export { serverSessionUsage, serverUsageReader } from './opencode-usage.js';
+export { DEFAULT_BOUNDED_PROMPT_TOKENS, DEFAULT_CACHE_READ_DISCOUNT, assessReconstruction, promptTokensOf, reconstruct, } from './replay.js';
+export { breakEvenPromptTokens, formatSurvey, survey } from './survey.js';
+//# sourceMappingURL=index.js.map

package/dist/ab/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/ab/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EACL,OAAO,EACP,kBAAkB,EAClB,cAAc,EACd,cAAc,EACd,YAAY,GACb,MAAM,aAAa,CAAC;AAUrB,OAAO,EACL,kBAAkB,EAClB,QAAQ,EACR,UAAU,EACV,GAAG,EACH,MAAM,EACN,WAAW,GACZ,MAAM,iBAAiB,CAAC;AAGzB,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAQzD,OAAO,EAAE,gBAAgB,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAGhF,OAAO,EAAE,QAAQ,EAAE,mBAAmB,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAQzE,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAE5D,OAAO,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAG5E,OAAO,EACL,6BAA6B,EAC7B,2BAA2B,EAC3B,oBAAoB,EACpB,cAAc,EACd,WAAW,GACZ,MAAM,aAAa,CAAC;AAQrB,OAAO,EAAE,qBAAqB,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC"}

package/dist/ab/opencode-usage.d.ts

@@ -0,0 +1,52 @@
+/**
+ * Reading token usage out of an OpenCode server's own store.
+ *
+ * ── Why read the host's store instead of parsing its output ──────────────
+ *
+ * The obvious way to measure a host integration is to run the host and read
+ * what it prints. That fails in three ways that matter here:
+ *
+ * 1. **Quoting the measurement to the measured.** The numbers come from the
+ *    run that is being measured, through a reporting path the run itself
+ *    controls. The previous A/B took its figures exactly that way, and one of
+ *    its "arms" turned out to be two readings of a single session.
+ * 2. **Unreliability under the conditions that matter.** A run that ends on an
+ *    error prints its error, not its totals — which is exactly the run whose
+ *    cost most needs checking.
+ * 3. **Irreproducibility.** Once a run's output is gone, its cost cannot be
+ *    re-derived. A store keyed by session id can be re-read later, which means
+ *    a disputed measurement can be settled without re-running anything.
+ *
+ * The store is OpenCode's own accounting, so it is not our reconstruction
+ * either. That makes it a third-party number rather than a self-report, which
+ * is a meaningfully better position than either alternative.
+ *
+ * @non-paper — measurement infrastructure, not the paper's evaluation.
+ */
+import { NO_USAGE } from './usage.js';
+import type { HostMessageRow, UsageOutcome, UsageReader } from './usage.js';
+/** The HTTP seam onto a running OpenCode server. */
+export interface ServerFetcher {
+    /**
+     * `GET <baseUrl><path>`, resolving the parsed JSON body.
+     *
+     * Rejects on a non-2xx response so a 404 cannot be mistaken for a session
+     * with no messages — the first is a mistake in the request, the second is a
+     * fact about the run, and they need different fixes.
+     */
+    get(path: string): Promise<unknown>;
+}
+/**
+ * Build a {@link UsageReader} over an OpenCode server.
+ *
+ * `baseUrl` is the server root, e.g. `http://localhost:4096`. Messages are
+ * requested per session so the reader never has to hold the whole store in
+ * memory, and a session with thousands of messages costs one request rather
+ * than a full scan.
+ */
+export declare function serverUsageReader(baseUrl: string, fetchJson: ServerFetcher['get']): UsageReader;
+/** Resolve a session's token spend over HTTP. */
+export declare function serverSessionUsage(baseUrl: string, fetchJson: ServerFetcher['get'], sessionID: string): Promise<UsageOutcome>;
+export { NO_USAGE };
+export type { HostMessageRow, UsageOutcome, UsageReader };
+//# sourceMappingURL=opencode-usage.d.ts.map

package/dist/ab/opencode-usage.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"opencode-usage.d.ts","sourceRoot":"","sources":["../../src/ab/opencode-usage.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,OAAO,EAAE,QAAQ,EAAuB,MAAM,YAAY,CAAC;AAC3D,OAAO,KAAK,EACV,cAAc,EACd,YAAY,EACZ,WAAW,EACZ,MAAM,YAAY,CAAC;AA6CpB,oDAAoD;AACpD,MAAM,WAAW,aAAa;IAC5B;;;;;;OAMG;IACH,GAAG,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CACrC;AAMD;;;;;;;GAOG;AACH,wBAAgB,iBAAiB,CAC/B,OAAO,EAAE,MAAM,EACf,SAAS,EAAE,aAAa,CAAC,KAAK,CAAC,GAC9B,WAAW,CAqBb;AAED,iDAAiD;AACjD,wBAAgB,kBAAkB,CAChC,OAAO,EAAE,MAAM,EACf,SAAS,EAAE,aAAa,CAAC,KAAK,CAAC,EAC/B,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,YAAY,CAAC,CAEvB;AAED,OAAO,EAAE,QAAQ,EAAE,CAAC;AACpB,YAAY,EAAE,cAAc,EAAE,YAAY,EAAE,WAAW,EAAE,CAAC"}

package/dist/ab/opencode-usage.js

@@ -0,0 +1,85 @@
+/**
+ * Reading token usage out of an OpenCode server's own store.
+ *
+ * ── Why read the host's store instead of parsing its output ──────────────
+ *
+ * The obvious way to measure a host integration is to run the host and read
+ * what it prints. That fails in three ways that matter here:
+ *
+ * 1. **Quoting the measurement to the measured.** The numbers come from the
+ *    run that is being measured, through a reporting path the run itself
+ *    controls. The previous A/B took its figures exactly that way, and one of
+ *    its "arms" turned out to be two readings of a single session.
+ * 2. **Unreliability under the conditions that matter.** A run that ends on an
+ *    error prints its error, not its totals — which is exactly the run whose
+ *    cost most needs checking.
+ * 3. **Irreproducibility.** Once a run's output is gone, its cost cannot be
+ *    re-derived. A store keyed by session id can be re-read later, which means
+ *    a disputed measurement can be settled without re-running anything.
+ *
+ * The store is OpenCode's own accounting, so it is not our reconstruction
+ * either. That makes it a third-party number rather than a self-report, which
+ * is a meaningfully better position than either alternative.
+ *
+ * @non-paper — measurement infrastructure, not the paper's evaluation.
+ */
+import { NO_USAGE, resolveSessionUsage } from './usage.js';
+function asNumber(value) {
+    return typeof value === 'number' && Number.isFinite(value) ? value : 0;
+}
+/**
+ * Convert one `message.data` blob into a row.
+ *
+ * `reasoning` is folded into `output` deliberately: reasoning tokens are
+ * generated tokens, they are billed as such, and excluding them would make a
+ * thinking-heavy run look cheaper than it was.
+ */
+function toRow(sessionID, created, data) {
+    const record = (typeof data === 'object' && data !== null ? data : {});
+    const tokens = record.tokens;
+    if (tokens === undefined) {
+        return { sessionID, created, role: 'unknown', tokens: null };
+    }
+    const usage = {
+        input: asNumber(tokens.input),
+        cacheRead: asNumber(tokens.cache?.read),
+        cacheWrite: asNumber(tokens.cache?.write),
+        output: asNumber(tokens.output) + asNumber(tokens.reasoning),
+    };
+    return {
+        sessionID,
+        created,
+        role: typeof record.role === 'string' ? record.role : 'unknown',
+        tokens: usage,
+    };
+}
+function asArray(value) {
+    return Array.isArray(value) ? value : [];
+}
+/**
+ * Build a {@link UsageReader} over an OpenCode server.
+ *
+ * `baseUrl` is the server root, e.g. `http://localhost:4096`. Messages are
+ * requested per session so the reader never has to hold the whole store in
+ * memory, and a session with thousands of messages costs one request rather
+ * than a full scan.
+ */
+export function serverUsageReader(baseUrl, fetchJson) {
+    const root = baseUrl.replace(/\/+$/, '');
+    return {
+        async messagesFor(sessionID) {
+            const body = await fetchJson(`${root}/api/session/${encodeURIComponent(sessionID)}/message`);
+            return asArray(body).map((entry) => {
+                const record = (typeof entry === 'object' && entry !== null ? entry : {});
+                const info = (typeof record.info === 'object' && record.info !== null ? record.info : {});
+                return toRow(typeof info.sessionID === 'string' ? info.sessionID : sessionID, asNumber(info.time?.created ?? record.created), record.info);
+            });
+        },
+    };
+}
+/** Resolve a session's token spend over HTTP. */
+export function serverSessionUsage(baseUrl, fetchJson, sessionID) {
+    return resolveSessionUsage(serverUsageReader(baseUrl, fetchJson), sessionID);
+}
+export { NO_USAGE };
+//# sourceMappingURL=opencode-usage.js.map

package/dist/ab/opencode-usage.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"opencode-usage.js","sourceRoot":"","sources":["../../src/ab/opencode-usage.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,OAAO,EAAE,QAAQ,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC;AAmB3D,SAAS,QAAQ,CAAC,KAAc;IAC9B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AACzE,CAAC;AAED;;;;;;GAMG;AACH,SAAS,KAAK,CAAC,SAAiB,EAAE,OAAe,EAAE,IAAa;IAC9D,MAAM,MAAM,GAAG,CAAC,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAoB,CAAC;IAC1F,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;IAC7B,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QACzB,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;IAC/D,CAAC;IACD,MAAM,KAAK,GAAe;QACxB,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC;QAC7B,SAAS,EAAE,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC;QACvC,UAAU,EAAE,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC;QACzC,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC;KAC7D,CAAC;IACF,OAAO;QACL,SAAS;QACT,OAAO;QACP,IAAI,EAAE,OAAO,MAAM,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS;QAC/D,MAAM,EAAE,KAAK;KACd,CAAC;AACJ,CAAC;AAcD,SAAS,OAAO,CAAC,KAAc;IAC7B,OAAO,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;AAC3C,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,iBAAiB,CAC/B,OAAe,EACf,SAA+B;IAE/B,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IACzC,OAAO;QACL,KAAK,CAAC,WAAW,CAAC,SAAiB;YACjC,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,GAAG,IAAI,gBAAgB,kBAAkB,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;YAC7F,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;gBACjC,MAAM,MAAM,GAAG,CACb,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAChB,CAAC;gBAC3C,MAAM,IAAI,GAAG,CAAC,OAAO,MAAM,CAAC,IAAI,KAAK,QAAQ,IAAI,MAAM,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAGvF,CAAC;gBACF,OAAO,KAAK,CACV,OAAO,IAAI,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,EAC/D,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC,EAC9C,MAAM,CAAC,IAAI,CACZ,CAAC;YACJ,CAAC,CAAC,CAAC;QACL,CAAC;KACF,CAAC;AACJ,CAAC;AAED,iDAAiD;AACjD,MAAM,UAAU,kBAAkB,CAChC,OAAe,EACf,SAA+B,EAC/B,SAAiB;IAEjB,OAAO,mBAAmB,CAAC,iBAAiB,CAAC,OAAO,EAAE,SAAS,CAAC,EAAE,SAAS,CAAC,CAAC;AAC/E,CAAC;AAED,OAAO,EAAE,QAAQ,EAAE,CAAC"}

package/dist/ab/record.d.ts

@@ -0,0 +1,173 @@
+/**
+ * The shape of one A/B run record.
+ *
+ * ── Why a record and not a number ────────────────────────────────────────
+ *
+ * The previous A/B produced a number — "39% saved" — from a run in which
+ * the instrumented arm never once touched the state file. The number was
+ * arithmetically correct and scientifically meaningless, and nothing in the
+ * pipeline could tell the difference, because the pipeline carried a number
+ * instead of a record.
+ *
+ * A record keeps the things that decide whether a comparison means anything:
+ * what the task was, what each arm spent, and — the part that was missing —
+ * **whether the instrumentation was engaged at all**. Gates in `verdict.ts`
+ * read these fields; none of them is derivable from token counts.
+ *
+ * @non-paper — measurement infrastructure for OUR host integration. Not
+ * part of the paper's own evaluation.
+ */
+/** Which integration an arm ran under. */
+export type ArmId = 'plain' | 'notes' | 'paper';
+/** Every arm, in the order an experiment should run them. */
+export declare const ARM_IDS: readonly ArmId[];
+/**
+ * Whether an arm has the skillstate integration switched ON.
+ *
+ * `plain` is the control: no plugin, no state, no system fragment. The other
+ * two carry the integration in its two modes.
+ */
+export declare function isInstrumented(arm: ArmId): boolean;
+/** Token spend, split by billing class. */
+export interface TokenUsage {
+    /** Fresh, uncached input tokens. */
+    readonly input: number;
+    /** Input tokens served from the prompt cache. */
+    readonly cacheRead: number;
+    /** Cache writes caused by this run. */
+    readonly cacheWrite: number;
+    /** Generated tokens, including reasoning. */
+    readonly output: number;
+}
+/**
+ * Tokens that constitute PROMPT cost.
+ *
+ * `input + cacheRead` is what the model had to be shown, whether it was
+ * billed as fresh or served from cache. `cacheWrite` is excluded because it
+ * is an artefact of a cold cache rather than of prompt size, and folding it
+ * in would make a run that happened to miss the cache look like an expensive
+ * one. `output` is excluded because the paper's claim is about prompt
+ * economy, not generation economy.
+ */
+export declare function promptTokens(usage: TokenUsage): number;
+/**
+ * How the state file behaved during one run.
+ *
+ * This is the field whose absence made the old experiment unreadable. A run
+ * where `engaged` is false measured a plugin that did nothing, and its token
+ * count is a sample of model variance, not of the integration.
+ */
+export interface EngagementEvidence {
+    /**
+     * Did the state file change at any point during the run?
+     *
+     * Computed by comparing the file's bytes before the first turn and after
+     * the last, NOT by asking the model whether it called a tool. A model
+     * reporting "I used the state" is not evidence; a changed file is.
+     */
+    readonly engaged: boolean;
+    /** How many times the state file's content changed during the run. */
+    readonly writes: number;
+    /** How many of those writes came from the paper-mode response sink. */
+    readonly sinkWrites: number;
+    /** How many times a response was rejected and never reached the state. */
+    readonly rejections: number;
+    /** First rejection reason, for diagnostics. Absent when none occurred. */
+    readonly firstRejection?: string;
+    /** True when the arm found a state file already present at step 0. */
+    readonly hadStateAtStep0: boolean;
+}
+/** Engagement for a run whose arm carries no integration. */
+export declare const CONTROL_ENGAGEMENT: EngagementEvidence;
+/**
+ * What an arm actually produced, used to decide whether the two arms did
+ * comparable work.
+ */
+export interface WorkEvidence {
+    /**
+     * Content hash of the task's declared artifact, e.g. the `AUDIT.md` the
+     * task asks for. Equal hashes across arms is the strongest available
+     * check that both arms finished the same thing.
+     */
+    readonly artifactDigest: string | null;
+    /** Number of model turns the run took. */
+    readonly turns: number;
+    /** Tool calls made, across all tools. */
+    readonly toolCalls: number;
+}
+/** One arm of one trial, executed once. */
+export interface RunRecord {
+    /** Which arm this was. */
+    readonly arm: ArmId;
+    /** Trial index, 0-based. Multiple trials per arm are what make variance visible. */
+    readonly trial: number;
+    /** The task both arms were given. */
+    readonly task: string;
+    /** Provider and model id, e.g. `openai/gpt-5`. Equal across arms or the run is void. */
+    readonly model: string;
+    /** Host version, e.g. `2.0.19`. Equal across arms or the run is void. */
+    readonly hostVersion: string;
+    /** Host session id, so the run can be re-read from the host's own store. */
+    readonly sessionID: string;
+    /** Token spend for the run. */
+    readonly usage: TokenUsage;
+    /** What the state file did during the run. */
+    readonly engagement: EngagementEvidence;
+    /** What the run produced. */
+    readonly work: WorkEvidence;
+    /** Wall-clock milliseconds for the run. */
+    readonly durationMs: number;
+    /** True when the run completed without a provider or host error. */
+    readonly completed: boolean;
+    /**
+     * Whether the run produced the correct answer, when the experiment recorded
+     * it.
+     *
+     * Optional because a trial file written before this field existed has none,
+     * and because a task with no checkable answer has none to give. What is NOT
+     * optional is its absence counting as a pass — hence the outcome gate, which
+     * refuses an experiment that measured cost and not the work.
+     */
+    readonly outcome?: Outcome;
+    /** Failure reason when `completed` is false. */
+    readonly error?: string;
+}
+/**
+ * What the run was supposed to produce, and whether it did.
+ *
+ * Separate from `completed`: a run can complete perfectly and still be wrong.
+ * That distinction is the whole reason this type exists — "it finished" and
+ * "it did the work" are different claims, and conflating them is how an
+ * experiment reports a saving for a run that answered nothing.
+ */
+export interface Outcome {
+    readonly correct: boolean;
+    /** What the run reported, for a human reading the record. */
+    readonly reported?: string;
+}
+/** One arm's results across every trial it ran. */
+export interface ArmRecord {
+    readonly arm: ArmId;
+    readonly task: string;
+    readonly model: string;
+    readonly hostVersion: string;
+    readonly runs: readonly RunRecord[];
+}
+/**
+ * Build an {@link ArmRecord} from its runs, checking they are commensurable.
+ *
+ * A trial index must be unique within an arm: two runs labelled `trial: 0`
+ * are two samples of the same cell, and silently averaging them would hide
+ * exactly the variance the harness exists to surface.
+ *
+ * Returns a typed failure rather than throwing — a malformed arm is a
+ * harness bug, and the caller must be able to report it as one.
+ */
+export declare function buildArmRecord(arm: ArmId, runs: readonly RunRecord[]): {
+    ok: true;
+    record: ArmRecord;
+} | {
+    ok: false;
+    reason: string;
+};
+//# sourceMappingURL=record.d.ts.map

package/dist/ab/record.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"record.d.ts","sourceRoot":"","sources":["../../src/ab/record.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,0CAA0C;AAC1C,MAAM,MAAM,KAAK,GAAG,OAAO,GAAG,OAAO,GAAG,OAAO,CAAC;AAEhD,6DAA6D;AAC7D,eAAO,MAAM,OAAO,EAAE,SAAS,KAAK,EAAgC,CAAC;AAErE;;;;;GAKG;AACH,wBAAgB,cAAc,CAAC,GAAG,EAAE,KAAK,GAAG,OAAO,CAElD;AAED,2CAA2C;AAC3C,MAAM,WAAW,UAAU;IACzB,oCAAoC;IACpC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,iDAAiD;IACjD,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,uCAAuC;IACvC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,6CAA6C;IAC7C,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED;;;;;;;;;GASG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE,UAAU,GAAG,MAAM,CAEtD;AAED;;;;;;GAMG;AACH,MAAM,WAAW,kBAAkB;IACjC;;;;;;OAMG;IACH,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,sEAAsE;IACtE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,uEAAuE;IACvE,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,0EAA0E;IAC1E,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,0EAA0E;IAC1E,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;IACjC,sEAAsE;IACtE,QAAQ,CAAC,eAAe,EAAE,OAAO,CAAC;CACnC;AAED,6DAA6D;AAC7D,eAAO,MAAM,kBAAkB,EAAE,kBAMhC,CAAC;AAEF;;;GAGG;AACH,MAAM,WAAW,YAAY;IAC3B;;;;OAIG;IACH,QAAQ,CAAC,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IACvC,0CAA0C;IAC1C,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,yCAAyC;IACzC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B;AAED,2CAA2C;AAC3C,MAAM,WAAW,SAAS;IACxB,0BAA0B;IAC1B,QAAQ,CAAC,GAAG,EAAE,KAAK,CAAC;IACpB,oFAAoF;IACpF,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,qCAAqC;IACrC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,wFAAwF;IACxF,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,yEAAyE;IACzE,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,4EAA4E;IAC5E,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,+BAA+B;IAC/B,QAAQ,CAAC,KAAK,EAAE,UAAU,CAAC;IAC3B,8CAA8C;IAC9C,QAAQ,CAAC,UAAU,EAAE,kBAAkB,CAAC;IACxC,6BAA6B;IAC7B,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAC;IAC5B,2CAA2C;IAC3C,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,oEAAoE;IACpE,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC;IAC5B;;;;;;;;OAQG;IACH,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC;IAC3B,gDAAgD;IAChD,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;CACzB;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,OAAO;IACtB,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,6DAA6D;IAC7D,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED,mDAAmD;AACnD,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,GAAG,EAAE,KAAK,CAAC;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,IAAI,EAAE,SAAS,SAAS,EAAE,CAAC;CACrC;AAED;;;;;;;;;GASG;AACH,wBAAgB,cAAc,CAC5B,GAAG,EAAE,KAAK,EACV,IAAI,EAAE,SAAS,SAAS,EAAE,GACzB;IAAE,EAAE,EAAE,IAAI,CAAC;IAAC,MAAM,EAAE,SAAS,CAAA;CAAE,GAAG;IAAE,EAAE,EAAE,KAAK,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAoCjE"}