@dsh-enhanced/assistant-evaluation 0.1.24 → 0.1.31

package/lib/benchmark/runner.js

@@ -0,0 +1,119 @@
+import { acceptanceCanonicalJson } from '@dsh-enhanced/task-acceptance-contract';
+import { benchmarkAssert, benchmarkHash, benchmarkObject, benchmarkSchedule, benchmarkSnapshot, parseBenchmarkMetrics, parseBenchmarkPlan, } from './schema.js';
+import { BenchmarkStore } from './store.js';
+export const emptyBenchmarkMetrics = () => ({
+    inputTokens: null, outputTokens: null, costUsdMicros: null, toolCalls: null,
+    rework: null, interventions: null, latencyMs: null,
+});
+function observation(input, request) {
+    const raw = benchmarkObject(benchmarkSnapshot(input), ['versions', 'inputDigest', 'acceptanceDigest', 'verdict', 'metrics', 'evidenceDigest', 'quiescent']);
+    benchmarkAssert(acceptanceCanonicalJson(raw.versions) === acceptanceCanonicalJson(request.variant.versions)
+        && raw.inputDigest === request.task.inputDigest && raw.acceptanceDigest === request.task.acceptanceDigest, 'benchmark observation changed frozen inputs or versions');
+    benchmarkHash(raw.evidenceDigest);
+    benchmarkAssert(['achieved', 'not-achieved', 'unknown'].includes(raw.verdict) && typeof raw.quiescent === 'boolean', 'invalid benchmark observation');
+    parseBenchmarkMetrics(raw.metrics);
+    return raw;
+}
+/** Serial by design: a cell must settle before another can use the Host resources. */
+export async function runBenchmark(store, input, executor, signal) {
+    const plan = parseBenchmarkPlan(input);
+    store.create(plan);
+    for (const cell of benchmarkSchedule(plan)) {
+        if (signal?.aborted)
+            break;
+        const recorded = store.results(plan.id);
+        // Unknown transport/lifecycle means old work may still be running. Never launch its successor.
+        if (recorded.some(result => result.status === 'unknown'))
+            break;
+        if (recorded.some(result => result.cell.id === cell.id))
+            continue;
+        const startedAt = Date.now();
+        if (!store.start(plan.id, cell, startedAt))
+            continue;
+        const controller = new AbortController();
+        let timer;
+        let removeAbort;
+        const request = Object.freeze({
+            planId: plan.id, dataset: plan.dataset, cell,
+            task: plan.cases.find(task => task.id === cell.caseId),
+            variant: plan.variants.find(variant => variant.id === cell.variantId),
+            budget: plan.budget, signal: controller.signal,
+        });
+        let result;
+        const unknown = (reason, metrics = emptyBenchmarkMetrics()) => ({
+            cell, status: 'unknown', verdict: 'unknown', metrics, evidenceDigest: null,
+            reason, startedAt, completedAt: Math.max(startedAt, Date.now()),
+        });
+        const startedMonotonic = performance.now();
+        try {
+            const stop = new Promise(resolve => {
+                const abort = () => { controller.abort(); resolve({ kind: 'interrupted' }); };
+                signal?.addEventListener('abort', abort, { once: true });
+                removeAbort = () => signal?.removeEventListener('abort', abort);
+                if (signal?.aborted)
+                    abort();
+                timer = setTimeout(() => { controller.abort(); resolve({ kind: 'timeout' }); }, plan.budget.durationMs);
+            });
+            const execution = Promise.resolve().then(async () => {
+                if (controller.signal.aborted)
+                    return { kind: 'interrupted' };
+                return { kind: 'observation', value: await executor.execute(request) };
+            }).catch(() => ({ kind: 'error' }));
+            const outcome = await Promise.race([execution, stop]);
+            const elapsed = Math.ceil(performance.now() - startedMonotonic);
+            if (outcome.kind !== 'observation') {
+                result = unknown(outcome.kind === 'error' ? 'adapter-error' : outcome.kind);
+            }
+            else if (signal?.aborted) {
+                controller.abort();
+                result = unknown('interrupted');
+            }
+            else {
+                try {
+                    const value = observation(outcome.value, request);
+                    const metrics = { ...value.metrics, latencyMs: elapsed };
+                    const budgetKeys = ['inputTokens', 'outputTokens', 'toolCalls'];
+                    const missingCost = plan.budget.costUsdMicros !== null && metrics.costUsdMicros === null;
+                    const exceededCost = plan.budget.costUsdMicros !== null && metrics.costUsdMicros !== null && metrics.costUsdMicros > plan.budget.costUsdMicros;
+                    if (!value.quiescent)
+                        result = unknown('not-quiescent', metrics);
+                    else if (missingCost || budgetKeys.some(key => metrics[key] === null))
+                        result = unknown('invalid-observation', metrics);
+                    else if (exceededCost || elapsed > plan.budget.durationMs || budgetKeys.some(key => metrics[key] > plan.budget[key]))
+                        result = unknown('budget-exceeded', metrics);
+                    else
+                        result = {
+                            cell, status: 'completed', verdict: value.verdict, metrics,
+                            evidenceDigest: value.evidenceDigest, reason: 'verified',
+                            startedAt, completedAt: Math.max(startedAt, Date.now()),
+                        };
+                }
+                catch {
+                    result = unknown('invalid-observation');
+                }
+            }
+        }
+        finally {
+            if (timer !== undefined)
+                clearTimeout(timer);
+            removeAbort?.();
+        }
+        if (result.status === 'unknown') {
+            controller.abort();
+            try {
+                const captured = executor.failure?.(request, result.reason);
+                if (captured !== undefined) {
+                    const value = observation(captured, request);
+                    benchmarkAssert(value.verdict === 'unknown', 'failure snapshot cannot upgrade a result');
+                    result = { ...result, metrics: { ...value.metrics, latencyMs: result.metrics.latencyMs }, evidenceDigest: value.evidenceDigest };
+                }
+            }
+            catch { /* The original unknown result remains authoritative if diagnostics cannot be saved. */ }
+        }
+        store.finish(plan.id, result);
+        if (result.status === 'unknown')
+            break;
+    }
+    return store.results(plan.id);
+}
+//# sourceMappingURL=runner.js.map

package/lib/benchmark/runner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../src/benchmark/runner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,uBAAuB,EAAE,MAAM,wCAAwC,CAAA;AAChF,OAAO,EACL,eAAe,EAAE,aAAa,EAAE,eAAe,EAAE,iBAAiB,EAAE,iBAAiB,EACrF,qBAAqB,EAAE,kBAAkB,GAC1C,MAAM,aAAa,CAAA;AACpB,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAA;AA2B3C,MAAM,CAAC,MAAM,qBAAqB,GAAG,GAAqB,EAAE,CAAC,CAAC;IAC5D,WAAW,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI;IAC3E,MAAM,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI;CACnD,CAAC,CAAA;AAEF,SAAS,WAAW,CAAC,KAAc,EAAE,OAAkC;IACrE,MAAM,GAAG,GAAG,eAAe,CAAC,iBAAiB,CAAC,KAAK,CAAC,EAAE,CAAC,UAAU,EAAE,aAAa,EAAE,kBAAkB,EAAE,SAAS,EAAE,SAAS,EAAE,gBAAgB,EAAE,WAAW,CAAC,CAAC,CAAA;IAC3J,eAAe,CAAC,uBAAuB,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,uBAAuB,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC;WACtG,GAAG,CAAC,WAAW,KAAK,OAAO,CAAC,IAAI,CAAC,WAAW,IAAI,GAAG,CAAC,gBAAgB,KAAK,OAAO,CAAC,IAAI,CAAC,gBAAgB,EAC3G,yDAAyD,CAAC,CAAA;IAC1D,aAAa,CAAC,GAAG,CAAC,cAAc,CAAC,CAAA;IACjC,eAAe,CAAC,CAAC,UAAU,EAAE,cAAc,EAAE,SAAS,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAiB,CAAC,IAAI,OAAO,GAAG,CAAC,SAAS,KAAK,SAAS,EAAE,+BAA+B,CAAC,CAAA;IAC/J,qBAAqB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;IAClC,OAAO,GAAsC,CAAA;AAC/C,CAAC;AAED,sFAAsF;AACtF,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,KAAqB,EAAE,KAAoB,EAAE,QAA2B,EAAE,MAAoB;IAE9F,MAAM,IAAI,GAAG,kBAAkB,CAAC,KAAK,CAAC,CAAA;IACtC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;IAClB,KAAK,MAAM,IAAI,IAAI,iBAAiB,CAAC,IAAI,CAAC,EAAE,CAAC;QAC3C,IAAI,MAAM,EAAE,OAAO;YAAE,MAAK;QAC1B,MAAM,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;QACvC,+FAA+F;QAC/F,IAAI,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,KAAK,SAAS,CAAC;YAAE,MAAK;QAC/D,IAAI,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;YAAE,SAAQ;QACjE,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QAC5B,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,EAAE,SAAS,CAAC;YAAE,SAAQ;QACpD,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAA;QACxC,IAAI,KAAgD,CAAA;QACpD,IAAI,WAAqC,CAAA;QACzC,MAAM,OAAO,GAA8B,MAAM,CAAC,MAAM,CAAC;YACvD,MAAM,EAAE,IAAI,CAAC,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,IAAI;YAC5C,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,KAAK,IAAI,CAAC,MAAM,CAAE;YACvD,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,EAAE,KAAK,IAAI,CAAC,SAAS,CAAE;YACtE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM;SAC/C,CAAC,CAAA;QACF,IAAI,MAAuB,CAAA;QAC3B,MAAM,OAAO,GAAG,CAAC,MAAiC,EAAE,OAAO,GAAG,qBAAqB,EAAE,EAAmB,EAAE,CAAC,CAAC;YAC1G,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,cAAc,EAAE,IAAI;YAC1E,MAAM,EAAE,SAAS,EAAE,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC;SAChE,CAAC,CAAA;QACF,MAAM,gBAAgB,GAAG,WAAW,CAAC,GAAG,EAAE,CAAA;QAC1C,IAAI,CAAC;YAEH,MAAM,IAAI,GAAG,IAAI,OAAO,CAAU,OAAO,CAAC,EAAE;gBAC1C,MAAM,KAAK,GAAG,GAAS,EAAE,GAAG,UAAU,CAAC,KAAK,EAAE,CAAC,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,CAAC,CAAA,CAAC,CAAC,CAAA;gBAClF,MAAM,EAAE,gBAAgB,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAA;gBACxD,WAAW,GAAG,GAAG,EAAE,CAAC,MAAM,EAAE,mBAAmB,CAAC,OAAO,EAAE,KAAK,CAAC,CAAA;gBAC/D,IAAI,MAAM,EAAE,OAAO;oBAAE,KAAK,EAAE,CAAA;gBAC5B,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,GAAG,UAAU,CAAC,KAAK,EAAE,CAAC,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAA,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAA;YACxG,CAAC,CAAC,CAAA;YACF,MAAM,SAAS,GAAqB,OAAO,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;gBACpE,IAAI,UAAU,CAAC,MAAM,CAAC,OAAO;oBAAE,OAAO,EAAE,IAAI,EAAE,aAAa,EAAW,CAAA;gBACtE,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,EAAW,CAAA;YACjF,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAY,CAAA,CAAC,CAAA;YAC5C,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,CAAA;YACrD,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,gBAAgB,CAAC,CAAA;YAC/D,IAAI,OAAO,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC;gBACnC,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;YAC7E,CAAC;iBAAM,IAAI,MAAM,EAAE,OAAO,EAAE,CAAC;gBAC3B,UAAU,CAAC,KAAK,EAAE,CAAA;gBAClB,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,CAAA;YACjC,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC;oBACH,MAAM,KAAK,GAAG,WAAW,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAA;oBACjD,MAAM,OAAO,GAAG,EAAE,GAAG,KAAK,CAAC,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,CAAA;oBACxD,MAAM,UAAU,GAAG,CAAC,aAAa,EAAE,cAAc,EAAE,WAAW,CAAU,CAAA;oBACxE,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,aAAa,KAAK,IAAI,IAAI,OAAO,CAAC,aAAa,KAAK,IAAI,CAAA;oBACxF,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,aAAa,KAAK,IAAI,IAAI,OAAO,CAAC,aAAa,KAAK,IAAI,IAAI,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,aAAa,CAAA;oBAC9I,IAAI,CAAC,KAAK,CAAC,SAAS;wBAAE,MAAM,GAAG,OAAO,CAAC,eAAe,EAAE,OAAO,CAAC,CAAA;yBAC3D,IAAI,WAAW,IAAI,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC;wBAAE,MAAM,GAAG,OAAO,CAAC,qBAAqB,EAAE,OAAO,CAAC,CAAA;yBAClH,IAAI,YAAY,IAAI,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,UAAU,IAAI,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAE,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;wBAAE,MAAM,GAAG,OAAO,CAAC,iBAAiB,EAAE,OAAO,CAAC,CAAA;;wBAC9J,MAAM,GAAG;4BACZ,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,OAAO;4BAC1D,cAAc,EAAE,KAAK,CAAC,cAAc,EAAE,MAAM,EAAE,UAAU;4BACxD,SAAS,EAAE,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC;yBACxD,CAAA;gBACH,CAAC;gBAAC,MAAM,CAAC;oBAAC,MAAM,GAAG,OAAO,CAAC,qBAAqB,CAAC,CAAA;gBAAC,CAAC;YACrD,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,IAAI,KAAK,KAAK,SAAS;gBAAE,YAAY,CAAC,KAAK,CAAC,CAAA;YAC5C,WAAW,EAAE,EAAE,CAAA;QACjB,CAAC;QACD,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAChC,UAAU,CAAC,KAAK,EAAE,CAAA;YAClB,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,MAAwD,CAAC,CAAA;gBAC7G,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;oBAC3B,MAAM,KAAK,GAAG,WAAW,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAA;oBAC5C,eAAe,CAAC,KAAK,CAAC,OAAO,KAAK,SAAS,EAAE,0CAA0C,CAAC,CAAA;oBACxF,MAAM,GAAG,EAAE,GAAG,MAAM,EAAE,OAAO,EAAE,EAAE,GAAG,KAAK,CAAC,OAAO,EAAE,SAAS,EAAE,MAAM,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,cAAc,EAAE,KAAK,CAAC,cAAc,EAAE,CAAA;gBAClI,CAAC;YACH,CAAC;YAAC,MAAM,CAAC,CAAC,uFAAuF,CAAC,CAAC;QACrG,CAAC;QACD,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,MAAM,CAAC,CAAA;QAC7B,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS;YAAE,MAAK;IACxC,CAAC;IACD,OAAO,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;AAC/B,CAAC"}

package/lib/benchmark/schema.d.ts

@@ -0,0 +1,20 @@
+import type { BenchmarkCell, BenchmarkMetrics, BenchmarkPlan, BenchmarkResult } from './types.js';
+export declare class BenchmarkError extends Error {
+    constructor(message: string);
+}
+export declare function benchmarkAssert(condition: unknown, message: string): asserts condition;
+/** Reject getters/non-enumerable fields before taking a detached bounded JSON snapshot. */
+export declare function benchmarkSnapshot<T>(value: T): T;
+export declare function benchmarkObject(value: unknown, keys: readonly string[]): Record<string, unknown>;
+export declare function benchmarkInteger(value: unknown, min?: number, max?: number): asserts value is number;
+export declare function benchmarkHash(value: unknown): asserts value is string;
+export declare const benchmarkVersionKeys: readonly ["model", "prompt", "skills", "tools", "policy", "runtime"];
+export declare const benchmarkMetricKeys: readonly ["inputTokens", "outputTokens", "costUsdMicros", "toolCalls", "rework", "interventions", "latencyMs"];
+export declare function parseBenchmarkPlan(value: unknown): Readonly<BenchmarkPlan>;
+export declare function benchmarkPlanDigest(plan: BenchmarkPlan): string;
+export declare function benchmarkSchedule(input: BenchmarkPlan): readonly BenchmarkCell[];
+export declare function parseBenchmarkMetrics(value: unknown): BenchmarkMetrics;
+/** Compile identity validation once for a batch rather than hashing every schedule per result. */
+export declare function benchmarkResultParser(input: BenchmarkPlan): (value: unknown) => BenchmarkResult;
+export declare function parseBenchmarkResult(plan: BenchmarkPlan, value: unknown): BenchmarkResult;
+//# sourceMappingURL=schema.d.ts.map

package/lib/benchmark/schema.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../src/benchmark/schema.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,aAAa,EAAE,gBAAgB,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,YAAY,CAAA;AAEjG,qBAAa,cAAe,SAAQ,KAAK;gBAC3B,OAAO,EAAE,MAAM;CAC5B;AACD,wBAAgB,eAAe,CAAC,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,CAEtF;AACD,2FAA2F;AAC3F,wBAAgB,iBAAiB,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,GAAG,CAAC,CAyBhD;AACD,wBAAgB,eAAe,CAAC,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAKhG;AACD,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,OAAO,EAAE,GAAG,SAAI,EAAE,GAAG,SAA0B,GAAG,OAAO,CAAC,KAAK,IAAI,MAAM,CAEhH;AACD,wBAAgB,aAAa,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAAC,KAAK,IAAI,MAAM,CAErE;AAID,eAAO,MAAM,oBAAoB,sEAAuE,CAAA;AACxG,eAAO,MAAM,mBAAmB,gHAAiH,CAAA;AAEjJ,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,OAAO,GAAG,QAAQ,CAAC,aAAa,CAAC,CA0D1E;AAED,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,aAAa,GAAG,MAAM,CAAsD;AAEtH,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,aAAa,GAAG,SAAS,aAAa,EAAE,CAahF;AAED,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,OAAO,GAAG,gBAAgB,CAItE;AA0BD,kGAAkG;AAClG,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,aAAa,GAAG,CAAC,KAAK,EAAE,OAAO,KAAK,eAAe,CAI/F;AAED,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,aAAa,EAAE,KAAK,EAAE,OAAO,GAAG,eAAe,CAEzF"}

package/lib/benchmark/schema.js

@@ -0,0 +1,183 @@
+import { acceptanceCanonicalJson, acceptanceDigest } from '@dsh-enhanced/task-acceptance-contract';
+import { benchmarkDomains } from './types.js';
+export class BenchmarkError extends Error {
+    constructor(message) { super(message); this.name = 'BenchmarkError'; }
+}
+export function benchmarkAssert(condition, message) {
+    if (!condition)
+        throw new BenchmarkError(message);
+}
+/** Reject getters/non-enumerable fields before taking a detached bounded JSON snapshot. */
+export function benchmarkSnapshot(value) {
+    const seen = new Set();
+    let nodes = 0;
+    const inspect = (entry, depth) => {
+        benchmarkAssert(++nodes <= 4096 && depth <= 16, 'benchmark data too complex');
+        if (entry === null || typeof entry !== 'object')
+            return;
+        benchmarkAssert(!seen.has(entry), 'cyclic benchmark data');
+        seen.add(entry);
+        for (const key of Reflect.ownKeys(entry)) {
+            if (Array.isArray(entry) && key === 'length')
+                continue;
+            const descriptor = Object.getOwnPropertyDescriptor(entry, key);
+            benchmarkAssert(typeof key === 'string' && descriptor.enumerable && 'value' in descriptor, 'benchmark data must be plain enumerable data');
+            inspect(descriptor.value, depth + 1);
+        }
+        seen.delete(entry);
+    };
+    inspect(value, 0);
+    const copy = JSON.parse(acceptanceCanonicalJson(value));
+    const freeze = (entry) => {
+        if (entry === null || typeof entry !== 'object')
+            return;
+        for (const child of Object.values(entry))
+            freeze(child);
+        Object.freeze(entry);
+    };
+    freeze(copy);
+    return copy;
+}
+export function benchmarkObject(value, keys) {
+    benchmarkAssert(value !== null && typeof value === 'object' && !Array.isArray(value), 'expected benchmark object');
+    const object = value;
+    benchmarkAssert(Object.keys(object).sort().join('|') === [...keys].sort().join('|'), 'unexpected benchmark fields');
+    return object;
+}
+export function benchmarkInteger(value, min = 0, max = Number.MAX_SAFE_INTEGER) {
+    benchmarkAssert(typeof value === 'number' && Number.isSafeInteger(value) && value >= min && value <= max, 'invalid benchmark integer');
+}
+export function benchmarkHash(value) {
+    benchmarkAssert(typeof value === 'string' && /^[a-f0-9]{64}$/u.test(value), 'expected SHA-256 benchmark digest');
+}
+function id(value) {
+    benchmarkAssert(typeof value === 'string' && /^[a-zA-Z0-9][a-zA-Z0-9._-]{0,127}$/u.test(value), 'invalid benchmark identifier');
+}
+export const benchmarkVersionKeys = ['model', 'prompt', 'skills', 'tools', 'policy', 'runtime'];
+export const benchmarkMetricKeys = ['inputTokens', 'outputTokens', 'costUsdMicros', 'toolCalls', 'rework', 'interventions', 'latencyMs'];
+export function parseBenchmarkPlan(value) {
+    const input = benchmarkSnapshot(value);
+    const plan = benchmarkObject(input, ['schemaVersion', 'id', 'dataset', 'comparison', 'cases', 'variants', 'budget', 'repeats', 'seed']);
+    benchmarkAssert(plan.schemaVersion === 1, 'unsupported benchmark schema');
+    id(plan.id);
+    benchmarkAssert(plan.comparison === 'capability' || plan.comparison === 'model', 'invalid comparison');
+    const dataset = benchmarkObject(plan.dataset, ['id', 'version', 'digest', 'split']);
+    id(dataset.id);
+    id(dataset.version);
+    benchmarkHash(dataset.digest);
+    benchmarkAssert(dataset.split === 'development' || dataset.split === 'holdout', 'invalid dataset split');
+    benchmarkAssert(Array.isArray(plan.cases) && plan.cases.length >= 1 && plan.cases.length <= 100, 'expected 1..100 benchmark cases');
+    const caseIds = new Set();
+    const inputDigests = new Set();
+    for (const raw of plan.cases) {
+        const entry = benchmarkObject(raw, ['id', 'domain', 'inputDigest', 'acceptanceDigest']);
+        id(entry.id);
+        benchmarkHash(entry.inputDigest);
+        benchmarkHash(entry.acceptanceDigest);
+        benchmarkAssert(benchmarkDomains.includes(entry.domain) && !caseIds.has(entry.id), 'invalid or duplicate benchmark case');
+        benchmarkAssert(!inputDigests.has(entry.inputDigest), 'duplicate task input must be a repeat, not a separate task');
+        caseIds.add(entry.id);
+        inputDigests.add(entry.inputDigest);
+    }
+    benchmarkAssert(Array.isArray(plan.variants) && plan.variants.length >= 2 && plan.variants.length <= 8, 'expected 2..8 variants');
+    const variantIds = new Set();
+    for (const raw of plan.variants) {
+        const variant = benchmarkObject(raw, ['id', 'role', 'versions', 'features']);
+        id(variant.id);
+        benchmarkAssert(!variantIds.has(variant.id) && ['baseline', 'candidate', 'ablation'].includes(variant.role), 'invalid or duplicate variant');
+        variantIds.add(variant.id);
+        const versions = benchmarkObject(variant.versions, benchmarkVersionKeys);
+        Object.values(versions).forEach(benchmarkHash);
+        const features = benchmarkObject(variant.features, ['memory', 'planning', 'review', 'growth']);
+        benchmarkAssert(Object.values(features).every(flag => typeof flag === 'boolean'), 'invalid feature flags');
+    }
+    const parsed = input;
+    benchmarkAssert(parsed.variants.filter(variant => variant.role === 'baseline').length === 1, 'exactly one baseline required');
+    const baseline = parsed.variants.find(variant => variant.role === 'baseline');
+    for (const variant of parsed.variants) {
+        if (parsed.comparison === 'capability') {
+            benchmarkAssert(variant.versions.model === baseline.versions.model, 'capability comparisons require the same model configuration');
+        }
+        else {
+            benchmarkAssert(benchmarkVersionKeys.filter(key => key !== 'model').every(key => variant.versions[key] === baseline.versions[key])
+                && acceptanceCanonicalJson(variant.features) === acceptanceCanonicalJson(baseline.features), 'model comparisons must change only model configuration');
+        }
+    }
+    const candidates = parsed.variants.filter(variant => variant.role === 'candidate');
+    for (const variant of parsed.variants.filter(variant => variant.role === 'ablation')) {
+        benchmarkAssert(parsed.comparison === 'capability' && candidates.length === 1, 'ablations require exactly one capability candidate');
+        const candidate = candidates[0];
+        benchmarkAssert(acceptanceCanonicalJson(candidate.versions) === acceptanceCanonicalJson(variant.versions), 'ablations must preserve candidate versions');
+        const keys = ['memory', 'planning', 'review', 'growth'];
+        const differences = keys.filter(key => variant.features[key] !== candidate.features[key]);
+        benchmarkAssert(differences.length === 1 && candidate.features[differences[0]] && !variant.features[differences[0]], 'ablation must disable exactly one candidate feature');
+    }
+    const budget = benchmarkObject(plan.budget, ['durationMs', 'inputTokens', 'outputTokens', 'costUsdMicros', 'toolCalls']);
+    benchmarkInteger(budget.durationMs, 1, 86_400_000);
+    for (const key of ['inputTokens', 'outputTokens', 'toolCalls'])
+        benchmarkInteger(budget[key], 0, 1_000_000_000);
+    if (budget.costUsdMicros !== null)
+        benchmarkInteger(budget.costUsdMicros, 0, 1_000_000_000);
+    benchmarkInteger(plan.repeats, 2, 20);
+    benchmarkInteger(plan.seed, 0, 0xffffffff);
+    return parsed;
+}
+export function benchmarkPlanDigest(plan) { return acceptanceDigest(parseBenchmarkPlan(plan)); }
+export function benchmarkSchedule(input) {
+    const plan = parseBenchmarkPlan(input);
+    const cells = [];
+    for (let repeat = 0; repeat < plan.repeats; repeat++) {
+        plan.cases.forEach((task, caseIndex) => {
+            const seed = Number.parseInt(acceptanceDigest([plan.seed, task.id, repeat]).slice(0, 8), 16);
+            for (let index = 0; index < plan.variants.length; index++) {
+                const variant = plan.variants[(index + repeat + caseIndex) % plan.variants.length];
+                cells.push({ id: acceptanceDigest([plan.id, task.id, variant.id, repeat]), caseId: task.id, variantId: variant.id, repeat, seed });
+            }
+        });
+    }
+    return Object.freeze(cells.map(cell => Object.freeze(cell)));
+}
+export function parseBenchmarkMetrics(value) {
+    const metrics = benchmarkObject(benchmarkSnapshot(value), benchmarkMetricKeys);
+    for (const entry of Object.values(metrics))
+        if (entry !== null)
+            benchmarkInteger(entry, 0, 1_000_000_000_000);
+    return metrics;
+}
+function parseResult(plan, expectedCells, value) {
+    const raw = benchmarkObject(benchmarkSnapshot(value), ['cell', 'status', 'verdict', 'metrics', 'evidenceDigest', 'reason', 'startedAt', 'completedAt']);
+    const cell = benchmarkObject(raw.cell, ['id', 'caseId', 'variantId', 'repeat', 'seed']);
+    const expected = expectedCells.get(cell.id);
+    benchmarkAssert(expected && acceptanceCanonicalJson(expected) === acceptanceCanonicalJson(cell), 'result does not match planned cell');
+    benchmarkAssert(raw.status === 'completed' || raw.status === 'unknown', 'invalid result status');
+    benchmarkAssert(['achieved', 'not-achieved', 'unknown'].includes(raw.verdict), 'invalid verdict');
+    benchmarkAssert(['verified', 'adapter-error', 'interrupted', 'timeout', 'invalid-observation', 'budget-exceeded', 'not-quiescent'].includes(raw.reason), 'invalid result reason');
+    benchmarkAssert(raw.status === 'completed' ? raw.reason === 'verified' : raw.verdict === 'unknown' && raw.reason !== 'verified', 'inconsistent result status');
+    if (raw.evidenceDigest !== null)
+        benchmarkHash(raw.evidenceDigest);
+    if (raw.status === 'completed')
+        benchmarkHash(raw.evidenceDigest);
+    const metrics = parseBenchmarkMetrics(raw.metrics);
+    if (raw.status === 'completed') {
+        for (const key of ['inputTokens', 'outputTokens', 'toolCalls']) {
+            benchmarkAssert(metrics[key] !== null && metrics[key] <= plan.budget[key], 'completed result lacks in-budget measurement');
+        }
+        if (plan.budget.costUsdMicros !== null)
+            benchmarkAssert(metrics.costUsdMicros !== null && metrics.costUsdMicros <= plan.budget.costUsdMicros, 'completed result lacks in-budget cost measurement');
+        benchmarkAssert(metrics.latencyMs !== null && metrics.latencyMs <= plan.budget.durationMs, 'completed result lacks in-budget latency');
+    }
+    benchmarkInteger(raw.startedAt);
+    benchmarkInteger(raw.completedAt);
+    benchmarkAssert(raw.completedAt >= raw.startedAt, 'result ends before start');
+    return raw;
+}
+/** Compile identity validation once for a batch rather than hashing every schedule per result. */
+export function benchmarkResultParser(input) {
+    const plan = parseBenchmarkPlan(input);
+    const cells = new Map(benchmarkSchedule(plan).map(cell => [cell.id, cell]));
+    return value => parseResult(plan, cells, value);
+}
+export function parseBenchmarkResult(plan, value) {
+    return benchmarkResultParser(plan)(value);
+}
+//# sourceMappingURL=schema.js.map

package/lib/benchmark/schema.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../src/benchmark/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,uBAAuB,EAAE,gBAAgB,EAAE,MAAM,wCAAwC,CAAA;AAClG,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAA;AAG7C,MAAM,OAAO,cAAe,SAAQ,KAAK;IACvC,YAAY,OAAe,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,GAAG,gBAAgB,CAAA,CAAC,CAAC;CAC9E;AACD,MAAM,UAAU,eAAe,CAAC,SAAkB,EAAE,OAAe;IACjE,IAAI,CAAC,SAAS;QAAE,MAAM,IAAI,cAAc,CAAC,OAAO,CAAC,CAAA;AACnD,CAAC;AACD,2FAA2F;AAC3F,MAAM,UAAU,iBAAiB,CAAI,KAAQ;IAC3C,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;IAC9B,IAAI,KAAK,GAAG,CAAC,CAAA;IACb,MAAM,OAAO,GAAG,CAAC,KAAc,EAAE,KAAa,EAAQ,EAAE;QACtD,eAAe,CAAC,EAAE,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI,EAAE,EAAE,4BAA4B,CAAC,CAAA;QAC7E,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ;YAAE,OAAM;QACvD,eAAe,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,uBAAuB,CAAC,CAAA;QAC1D,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;QACf,KAAK,MAAM,GAAG,IAAI,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YACzC,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,GAAG,KAAK,QAAQ;gBAAE,SAAQ;YACtD,MAAM,UAAU,GAAG,MAAM,CAAC,wBAAwB,CAAC,KAAK,EAAE,GAAG,CAAE,CAAA;YAC/D,eAAe,CAAC,OAAO,GAAG,KAAK,QAAQ,IAAI,UAAU,CAAC,UAAU,IAAI,OAAO,IAAI,UAAU,EAAE,8CAA8C,CAAC,CAAA;YAC1I,OAAO,CAAC,UAAU,CAAC,KAAK,EAAE,KAAK,GAAG,CAAC,CAAC,CAAA;QACtC,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;IACpB,CAAC,CAAA;IACD,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAA;IACjB,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,uBAAuB,CAAC,KAAK,CAAC,CAAM,CAAA;IAC5D,MAAM,MAAM,GAAG,CAAC,KAAc,EAAQ,EAAE;QACtC,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ;YAAE,OAAM;QACvD,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC;YAAE,MAAM,CAAC,KAAK,CAAC,CAAA;QACvD,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;IACtB,CAAC,CAAA;IACD,MAAM,CAAC,IAAI,CAAC,CAAA;IACZ,OAAO,IAAI,CAAA;AACb,CAAC;AACD,MAAM,UAAU,eAAe,CAAC,KAAc,EAAE,IAAuB;IACrE,eAAe,CAAC,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,2BAA2B,CAAC,CAAA;IAClH,MAAM,MAAM,GAAG,KAAgC,CAAA;IAC/C,eAAe,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,6BAA6B,CAAC,CAAA;IACnH,OAAO,MAAM,CAAA;AACf,CAAC;AACD,MAAM,UAAU,gBAAgB,CAAC,KAAc,EAAE,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,MAAM,CAAC,gBAAgB;IACrF,eAAe,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,GAAG,IAAI,KAAK,IAAI,GAAG,EAAE,2BAA2B,CAAC,CAAA;AACxI,CAAC;AACD,MAAM,UAAU,aAAa,CAAC,KAAc;IAC1C,eAAe,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,iBAAiB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,mCAAmC,CAAC,CAAA;AAClH,CAAC;AACD,SAAS,EAAE,CAAC,KAAc;IACxB,eAAe,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,qCAAqC,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,8BAA8B,CAAC,CAAA;AACjI,CAAC;AACD,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,CAAU,CAAA;AACxG,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,aAAa,EAAE,cAAc,EAAE,eAAe,EAAE,WAAW,EAAE,QAAQ,EAAE,eAAe,EAAE,WAAW,CAAU,CAAA;AAEjJ,MAAM,UAAU,kBAAkB,CAAC,KAAc;IAC/C,MAAM,KAAK,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAA;IACtC,MAAM,IAAI,GAAG,eAAe,CAAC,KAAK,EAAE,CAAC,eAAe,EAAE,IAAI,EAAE,SAAS,EAAE,YAAY,EAAE,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC,CAAA;IACvI,eAAe,CAAC,IAAI,CAAC,aAAa,KAAK,CAAC,EAAE,8BAA8B,CAAC,CAAA;IACzE,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IACX,eAAe,CAAC,IAAI,CAAC,UAAU,KAAK,YAAY,IAAI,IAAI,CAAC,UAAU,KAAK,OAAO,EAAE,oBAAoB,CAAC,CAAA;IACtG,MAAM,OAAO,GAAG,eAAe,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAA;IACnF,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAAC,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAAC,aAAa,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;IAClE,eAAe,CAAC,OAAO,CAAC,KAAK,KAAK,aAAa,IAAI,OAAO,CAAC,KAAK,KAAK,SAAS,EAAE,uBAAuB,CAAC,CAAA;IACxG,eAAe,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,GAAG,EAAE,iCAAiC,CAAC,CAAA;IACnI,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAA;IACjC,MAAM,YAAY,GAAG,IAAI,GAAG,EAAU,CAAA;IACtC,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;QAC7B,MAAM,KAAK,GAAG,eAAe,CAAC,GAAG,EAAE,CAAC,IAAI,EAAE,QAAQ,EAAE,aAAa,EAAE,kBAAkB,CAAC,CAAC,CAAA;QACvF,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAAC,aAAa,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;QAAC,aAAa,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAA;QACrF,eAAe,CAAC,gBAAgB,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAyC,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,qCAAqC,CAAC,CAAA;QAC5J,eAAe,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE,4DAA4D,CAAC,CAAA;QACnH,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;QACrB,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,CAAA;IACrC,CAAC;IACD,eAAe,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,IAAI,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,IAAI,CAAC,EAAE,wBAAwB,CAAC,CAAA;IACjI,MAAM,UAAU,GAAG,IAAI,GAAG,EAAU,CAAA;IACpC,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;QAChC,MAAM,OAAO,GAAG,eAAe,CAAC,GAAG,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC,CAAA;QAC5E,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;QACd,eAAe,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,IAAI,CAAC,UAAU,EAAE,WAAW,EAAE,UAAU,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAc,CAAC,EAAE,8BAA8B,CAAC,CAAA;QACtJ,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;QAC1B,MAAM,QAAQ,GAAG,eAAe,CAAC,OAAO,CAAC,QAAQ,EAAE,oBAAoB,CAAC,CAAA;QACxE,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,aAAa,CAAC,CAAA;QAC9C,MAAM,QAAQ,GAAG,eAAe,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,UAAU,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAA;QAC9F,eAAe,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,IAAI,KAAK,SAAS,CAAC,EAAE,uBAAuB,CAAC,CAAA;IAC5G,CAAC;IACD,MAAM,MAAM,GAAG,KAAsB,CAAA;IACrC,eAAe,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,+BAA+B,CAAC,CAAA;IAC7H,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,IAAI,KAAK,UAAU,CAAE,CAAA;IAC9E,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;QACtC,IAAI,MAAM,CAAC,UAAU,KAAK,YAAY,EAAE,CAAC;YACvC,eAAe,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,KAAK,QAAQ,CAAC,QAAQ,CAAC,KAAK,EAAE,6DAA6D,CAAC,CAAA;QACpI,CAAC;aAAM,CAAC;YACN,eAAe,CAAC,oBAAoB,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,OAAO,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;mBAC7H,uBAAuB,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,uBAAuB,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,wDAAwD,CAAC,CAAA;QAC1J,CAAC;IACH,CAAC;IACD,MAAM,UAAU,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,IAAI,KAAK,WAAW,CAAC,CAAA;IAClF,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,IAAI,KAAK,UAAU,CAAC,EAAE,CAAC;QACrF,eAAe,CAAC,MAAM,CAAC,UAAU,KAAK,YAAY,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,oDAAoD,CAAC,CAAA;QACpI,MAAM,SAAS,GAAG,UAAU,CAAC,CAAC,CAAE,CAAA;QAChC,eAAe,CAAC,uBAAuB,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,uBAAuB,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,4CAA4C,CAAC,CAAA;QACxJ,MAAM,IAAI,GAAG,CAAC,QAAQ,EAAE,UAAU,EAAE,QAAQ,EAAE,QAAQ,CAAU,CAAA;QAChE,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAA;QACzF,eAAe,CAAC,WAAW,CAAC,MAAM,KAAK,CAAC,IAAI,SAAS,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAE,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAE,CAAC,EAAE,qDAAqD,CAAC,CAAA;IAC/K,CAAC;IACD,MAAM,MAAM,GAAG,eAAe,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,YAAY,EAAE,aAAa,EAAE,cAAc,EAAE,eAAe,EAAE,WAAW,CAAC,CAAC,CAAA;IACxH,gBAAgB,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,EAAE,UAAU,CAAC,CAAA;IAClD,KAAK,MAAM,GAAG,IAAI,CAAC,aAAa,EAAE,cAAc,EAAE,WAAW,CAAC;QAAE,gBAAgB,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,aAAa,CAAC,CAAA;IAC/G,IAAI,MAAM,CAAC,aAAa,KAAK,IAAI;QAAE,gBAAgB,CAAC,MAAM,CAAC,aAAa,EAAE,CAAC,EAAE,aAAa,CAAC,CAAA;IAC3F,gBAAgB,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;IAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,EAAE,UAAU,CAAC,CAAA;IACjF,OAAO,MAAM,CAAA;AACf,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,IAAmB,IAAY,OAAO,gBAAgB,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAA,CAAC,CAAC;AAEtH,MAAM,UAAU,iBAAiB,CAAC,KAAoB;IACpD,MAAM,IAAI,GAAG,kBAAkB,CAAC,KAAK,CAAC,CAAA;IACtC,MAAM,KAAK,GAAoB,EAAE,CAAA;IACjC,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,CAAC;QACrD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,SAAS,EAAE,EAAE;YACrC,MAAM,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;YAC5F,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;gBAC1D,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,KAAK,GAAG,MAAM,GAAG,SAAS,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAE,CAAA;gBACnF,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,gBAAgB,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,OAAO,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;YACpI,CAAC;QACH,CAAC,CAAC,CAAA;IACJ,CAAC;IACD,OAAO,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;AAC9D,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,KAAc;IAClD,MAAM,OAAO,GAAG,eAAe,CAAC,iBAAiB,CAAC,KAAK,CAAC,EAAE,mBAAmB,CAAC,CAAA;IAC9E,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC;QAAE,IAAI,KAAK,KAAK,IAAI;YAAE,gBAAgB,CAAC,KAAK,EAAE,CAAC,EAAE,iBAAiB,CAAC,CAAA;IAC7G,OAAO,OAAsC,CAAA;AAC/C,CAAC;AAED,SAAS,WAAW,CAAC,IAAmB,EAAE,aAAiD,EAAE,KAAc;IACzG,MAAM,GAAG,GAAG,eAAe,CAAC,iBAAiB,CAAC,KAAK,CAAC,EAAE,CAAC,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,SAAS,EAAE,gBAAgB,EAAE,QAAQ,EAAE,WAAW,EAAE,aAAa,CAAC,CAAC,CAAA;IACvJ,MAAM,IAAI,GAAG,eAAe,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAA;IACvF,MAAM,QAAQ,GAAG,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,EAAY,CAAC,CAAA;IACrD,eAAe,CAAC,QAAQ,IAAI,uBAAuB,CAAC,QAAQ,CAAC,KAAK,uBAAuB,CAAC,IAAI,CAAC,EAAE,oCAAoC,CAAC,CAAA;IACtI,eAAe,CAAC,GAAG,CAAC,MAAM,KAAK,WAAW,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS,EAAE,uBAAuB,CAAC,CAAA;IAChG,eAAe,CAAC,CAAC,UAAU,EAAE,cAAc,EAAE,SAAS,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAiB,CAAC,EAAE,iBAAiB,CAAC,CAAA;IAC3G,eAAe,CAAC,CAAC,UAAU,EAAE,eAAe,EAAE,aAAa,EAAE,SAAS,EAAE,qBAAqB,EAAE,iBAAiB,EAAE,eAAe,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAgB,CAAC,EAAE,uBAAuB,CAAC,CAAA;IAC3L,eAAe,CAAC,GAAG,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,KAAK,SAAS,IAAI,GAAG,CAAC,MAAM,KAAK,UAAU,EAAE,4BAA4B,CAAC,CAAA;IAC9J,IAAI,GAAG,CAAC,cAAc,KAAK,IAAI;QAAE,aAAa,CAAC,GAAG,CAAC,cAAc,CAAC,CAAA;IAClE,IAAI,GAAG,CAAC,MAAM,KAAK,WAAW;QAAE,aAAa,CAAC,GAAG,CAAC,cAAc,CAAC,CAAA;IACjE,MAAM,OAAO,GAAG,qBAAqB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;IAClD,IAAI,GAAG,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;QAC/B,KAAK,MAAM,GAAG,IAAI,CAAC,aAAa,EAAE,cAAc,EAAE,WAAW,CAAU,EAAE,CAAC;YACxE,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,IAAI,OAAO,CAAC,GAAG,CAAE,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,8CAA8C,CAAC,CAAA;QAC7H,CAAC;QACD,IAAI,IAAI,CAAC,MAAM,CAAC,aAAa,KAAK,IAAI;YAAE,eAAe,CAAC,OAAO,CAAC,aAAa,KAAK,IAAI,IAAI,OAAO,CAAC,aAAa,IAAI,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,mDAAmD,CAAC,CAAA;QAClM,eAAe,CAAC,OAAO,CAAC,SAAS,KAAK,IAAI,IAAI,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,0CAA0C,CAAC,CAAA;IACxI,CAAC;IACD,gBAAgB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IAAC,gBAAgB,CAAC,GAAG,CAAC,WAAW,CAAC,CAAA;IAClE,eAAe,CAAC,GAAG,CAAC,WAAW,IAAI,GAAG,CAAC,SAAS,EAAE,0BAA0B,CAAC,CAAA;IAC7E,OAAO,GAAiC,CAAA;AAC1C,CAAC;AAED,kGAAkG;AAClG,MAAM,UAAU,qBAAqB,CAAC,KAAoB;IACxD,MAAM,IAAI,GAAG,kBAAkB,CAAC,KAAK,CAAC,CAAA;IACtC,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC,CAAC,CAAA;IAC3E,OAAO,KAAK,CAAC,EAAE,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,CAAA;AACjD,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,IAAmB,EAAE,KAAc;IACtE,OAAO,qBAAqB,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAA;AAC3C,CAAC"}

package/lib/benchmark/store.d.ts

@@ -0,0 +1,18 @@
+import type { BenchmarkCell, BenchmarkPlan, BenchmarkResult } from './types.js';
+export interface BenchmarkStoreStatus {
+    readonly results: readonly BenchmarkResult[];
+    readonly runningCell: BenchmarkCell | null;
+}
+export declare class BenchmarkStore {
+    #private;
+    constructor(path: string);
+    create(input: BenchmarkPlan): void;
+    plan(id: string): Readonly<BenchmarkPlan>;
+    results(id: string): readonly BenchmarkResult[];
+    status(id: string): Readonly<BenchmarkStoreStatus>;
+    start(id: string, input: BenchmarkCell, now: number): boolean;
+    finish(id: string, input: BenchmarkResult): void;
+    interrupt(id: string, now: number): void;
+    close(): void;
+}
+//# sourceMappingURL=store.d.ts.map

package/lib/benchmark/store.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../../src/benchmark/store.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,aAAa,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,YAAY,CAAA;AAa/E,MAAM,WAAW,oBAAoB;IACnC,QAAQ,CAAC,OAAO,EAAE,SAAS,eAAe,EAAE,CAAA;IAC5C,QAAQ,CAAC,WAAW,EAAE,aAAa,GAAG,IAAI,CAAA;CAC3C;AA0GD,qBAAa,cAAc;;gBAIb,IAAI,EAAE,MAAM;IA0ExB,MAAM,CAAC,KAAK,EAAE,aAAa,GAAG,IAAI;IAgBlC,IAAI,CAAC,EAAE,EAAE,MAAM,GAAG,QAAQ,CAAC,aAAa,CAAC;IAEzC,OAAO,CAAC,EAAE,EAAE,MAAM,GAAG,SAAS,eAAe,EAAE;IAE/C,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,QAAQ,CAAC,oBAAoB,CAAC;IAmBlD,KAAK,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,aAAa,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO;IA0B7D,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,eAAe,GAAG,IAAI;IAoBhD,SAAS,CAAC,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,IAAI;IAoBxC,KAAK,IAAI,IAAI;CACd"}