@llm4ts/flow 0.1.0

package/dist/Bench.js

@@ -0,0 +1,272 @@
+import * as Effect from "effect/Effect";
+import * as Ref from "effect/Ref";
+import * as Schema from "effect/Schema";
+import * as Stream from "effect/Stream";
+import { TokenUsage } from "@llm4ts/core/Models";
+import { estimateCostUsd } from "./PriceList.js";
+export class BenchTokens extends Schema.Class("BenchTokens")({
+    prompt: Schema.Int,
+    completion: Schema.Int,
+    cached: Schema.Int
+}) {
+    get total() {
+        return this.prompt + this.completion;
+    }
+    add(that) {
+        return new BenchTokens({
+            prompt: this.prompt + that.prompt,
+            completion: this.completion + that.completion,
+            cached: this.cached + that.cached
+        });
+    }
+}
+export const emptyBenchTokens = () => BenchTokens.make({ prompt: 0, completion: 0, cached: 0 });
+export const benchTokensFromUsage = (usage) => BenchTokens.make({
+    prompt: usage.prompt,
+    completion: usage.completion,
+    cached: usage.cached ?? 0
+});
+export class BenchCounters extends Schema.Class("BenchCounters")({
+    flakyRetries: Schema.Int,
+    transientRetries: Schema.Int,
+    autoResumes: Schema.Int,
+    turnLimitTrips: Schema.Int,
+    shrinks: Schema.Int
+}) {
+    get selfHealing() {
+        return this.flakyRetries + this.transientRetries + this.autoResumes + this.shrinks;
+    }
+}
+export const emptyBenchCounters = () => BenchCounters.make({
+    flakyRetries: 0,
+    transientRetries: 0,
+    autoResumes: 0,
+    turnLimitTrips: 0,
+    shrinks: 0
+});
+export class BenchPhase extends Schema.Class("BenchPhase")({
+    name: Schema.String,
+    ms: Schema.Int,
+    tokens: BenchTokens,
+    costUsd: Schema.optionalKey(Schema.Number),
+    counters: BenchCounters,
+    detail: Schema.Record(Schema.String, Schema.String)
+}) {
+}
+export class BenchScore extends Schema.Class("BenchScore")({
+    name: Schema.String,
+    score: Schema.Int,
+    max: Schema.Int,
+    reasoning: Schema.String
+}) {
+}
+export class BenchJudge extends Schema.Class("BenchJudge")({
+    provider: Schema.String,
+    model: Schema.String,
+    fixed: Schema.Boolean
+}) {
+}
+export class BenchMachine extends Schema.Class("BenchMachine")({
+    hostname: Schema.String,
+    os: Schema.String,
+    arch: Schema.String,
+    cores: Schema.Int,
+    memoryGb: Schema.Number,
+    runtime: Schema.String
+}) {
+}
+export class BenchQuality extends Schema.Class("BenchQuality")({
+    buildPassed: Schema.optionalKey(Schema.Boolean),
+    testsRun: Schema.optionalKey(Schema.Int),
+    testsFailed: Schema.optionalKey(Schema.Int),
+    coveragePct: Schema.optionalKey(Schema.Number),
+    featureFiles: Schema.Int,
+    scenarios: Schema.Int,
+    malformedFeatures: Schema.Int,
+    specFiles: Schema.Int,
+    programs: Schema.Int,
+    sourceFiles: Schema.Int,
+    sourceLines: Schema.Int,
+    testFiles: Schema.Int,
+    testLines: Schema.Int,
+    deterministicFindings: Schema.Int,
+    judgeFindings: Schema.Int,
+    gateCleared: Schema.optionalKey(Schema.Boolean),
+    gateRounds: Schema.optionalKey(Schema.Int),
+    gateOpenIssues: Schema.optionalKey(Schema.Int),
+    judgeScores: Schema.Array(BenchScore)
+}) {
+}
+export class BenchRecord extends Schema.Class("BenchRecord")({
+    schemaVersion: Schema.Int,
+    runId: Schema.String,
+    label: Schema.optionalKey(Schema.String),
+    startedAt: Schema.String,
+    finishedAt: Schema.String,
+    provider: Schema.String,
+    modelRequested: Schema.String,
+    modelsServed: Schema.Array(Schema.String),
+    judge: BenchJudge,
+    toolVersion: Schema.optionalKey(Schema.String),
+    llm4tsVersion: Schema.String,
+    pack: Schema.String,
+    fixtureFingerprint: Schema.String,
+    machine: BenchMachine,
+    outcome: Schema.String,
+    resumed: Schema.optionalKey(Schema.Boolean),
+    totalMs: Schema.Int,
+    phases: Schema.Array(BenchPhase),
+    quality: Schema.optionalKey(BenchQuality)
+}) {
+    get completed() {
+        return this.outcome === "completed";
+    }
+    get totalTokens() {
+        return this.phases.reduce((sum, phase) => sum + phase.tokens.total + phase.tokens.cached, 0);
+    }
+    get totalCostUsd() {
+        const values = this.phases.flatMap((phase) => phase.costUsd === undefined ? [] : [phase.costUsd]);
+        return values.length === 0 ? undefined : values.reduce((sum, value) => sum + value, 0);
+    }
+    get selfHealing() {
+        return this.phases.reduce((sum, phase) => sum + phase.counters.selfHealing, 0);
+    }
+}
+export const CurrentBenchSchema = 1;
+export const BenchUnstaged = "(none)";
+export const emptyBenchObservation = () => ({
+    openStages: [],
+    cells: new Map(),
+    cellKeys: new Map(),
+    counters: new Map(),
+    modelsServed: new Set()
+});
+const currentStage = (observation) => observation.openStages[observation.openStages.length - 1] ?? BenchUnstaged;
+const dropFirst = (values, value) => {
+    const index = values.indexOf(value);
+    return index < 0 ? values : [...values.slice(0, index), ...values.slice(index + 1)];
+};
+const classifyNotice = (message, counters) => {
+    if (message.startsWith("⟳ flaky")) {
+        return new BenchCounters({
+            ...counters,
+            flakyRetries: counters.flakyRetries + 1
+        });
+    }
+    if (message.startsWith("⟳ transient")) {
+        return new BenchCounters({
+            ...counters,
+            transientRetries: counters.transientRetries + 1
+        });
+    }
+    if (message.startsWith("↻ auto-resume")) {
+        return new BenchCounters({
+            ...counters,
+            autoResumes: counters.autoResumes + 1
+        });
+    }
+    if (message.includes("turn limit hit")) {
+        return new BenchCounters({
+            ...counters,
+            turnLimitTrips: counters.turnLimitTrips + 1
+        });
+    }
+    if (message.includes("shrinking to")) {
+        return new BenchCounters({
+            ...counters,
+            shrinks: counters.shrinks + 1
+        });
+    }
+    return undefined;
+};
+export const observeBench = (observation, event) => {
+    switch (event._tag) {
+        case "StageStarted":
+            return {
+                ...observation,
+                openStages: [event.stage, ...observation.openStages]
+            };
+        case "StageCompleted":
+        case "StageFailed":
+            return {
+                ...observation,
+                openStages: dropFirst(observation.openStages, event.stage)
+            };
+        case "TokensUsed": {
+            const stage = currentStage(observation);
+            const model = event.model ?? "";
+            const key = JSON.stringify([stage, model]);
+            const cells = new Map(observation.cells);
+            cells.set(key, (cells.get(key) ?? emptyBenchTokens()).add(benchTokensFromUsage(event.usage)));
+            return {
+                ...observation,
+                cells,
+                cellKeys: new Map(observation.cellKeys).set(key, [stage, model]),
+                modelsServed: event.model === undefined
+                    ? observation.modelsServed
+                    : new Set([...observation.modelsServed, event.model])
+            };
+        }
+        case "Info": {
+            const stage = currentStage(observation);
+            const current = observation.counters.get(stage) ?? emptyBenchCounters();
+            const next = classifyNotice(event.message, current);
+            return next === undefined
+                ? observation
+                : {
+                    ...observation,
+                    counters: new Map(observation.counters).set(stage, next)
+                };
+        }
+        default:
+            return observation;
+    }
+};
+export const stageBenchTokens = (observation, stage) => [...observation.cells.entries()].reduce((total, [key, tokens]) => {
+    const parts = observation.cellKeys.get(key);
+    return parts?.[0] === stage ? total.add(tokens) : total;
+}, emptyBenchTokens());
+export const stageBenchCounters = (observation, stage) => observation.counters.get(stage) ?? emptyBenchCounters();
+export const makeBenchPhase = (observation, stage, ms, detail = {}) => {
+    const costs = [...observation.cells.entries()].flatMap(([key, tokens]) => {
+        const parts = observation.cellKeys.get(key);
+        if (parts?.[0] !== stage) {
+            return [];
+        }
+        const cost = estimateCostUsd(parts[1], TokenUsage.make({
+            prompt: tokens.prompt,
+            completion: tokens.completion,
+            total: tokens.total,
+            ...(tokens.cached <= 0 ? {} : { cached: tokens.cached })
+        }));
+        return cost === undefined ? [] : [cost];
+    });
+    return BenchPhase.make({
+        name: stage,
+        ms,
+        tokens: stageBenchTokens(observation, stage),
+        counters: stageBenchCounters(observation, stage),
+        detail,
+        ...(costs.length === 0 ? {} : { costUsd: costs.reduce((sum, cost) => sum + cost, 0) })
+    });
+};
+const isHub = (events) => "subscribe" in events && "publishedCount" in events && "stream" in events;
+export const makeBenchTap = Effect.fn("@llm4ts/flow/Bench.makeTap")(function* (events) {
+    const observation = yield* Ref.make(emptyBenchObservation());
+    const consumed = yield* Ref.make(0);
+    if (!isHub(events)) {
+        return {
+            get: Ref.get(observation),
+            reset: Ref.set(observation, emptyBenchObservation())
+        };
+    }
+    const subscription = yield* events.subscribe;
+    const baseline = yield* events.publishedCount;
+    yield* Stream.fromSubscription(subscription).pipe(Stream.runForEach((event) => Ref.update(observation, (current) => observeBench(current, event)).pipe(Effect.andThen(Ref.update(consumed, (count) => count + 1)))), Effect.forkScoped);
+    const drain = Effect.suspend(() => Effect.all([events.publishedCount, Ref.get(consumed)]).pipe(Effect.flatMap(([published, seen]) => seen >= published - baseline ? Effect.void : Effect.yieldNow.pipe(Effect.andThen(drain)))));
+    return {
+        get: drain.pipe(Effect.andThen(Ref.get(observation))),
+        reset: drain.pipe(Effect.andThen(Ref.set(observation, emptyBenchObservation())))
+    };
+});
+//# sourceMappingURL=Bench.js.map

package/dist/Bench.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"Bench.js","sourceRoot":"","sources":["../src/Bench.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,KAAK,GAAG,MAAM,YAAY,CAAA;AACjC,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AAEvC,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAA;AAEhD,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAA;AAEhD,MAAM,OAAO,WAAY,SAAQ,MAAM,CAAC,KAAK,CAAc,aAAa,CAAC,CAAC;IACxE,MAAM,EAAE,MAAM,CAAC,GAAG;IAClB,UAAU,EAAE,MAAM,CAAC,GAAG;IACtB,MAAM,EAAE,MAAM,CAAC,GAAG;CACnB,CAAC;IACA,IAAI,KAAK;QACP,OAAO,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,UAAU,CAAA;IACtC,CAAC;IAED,GAAG,CAAC,IAAiB;QACnB,OAAO,IAAI,WAAW,CAAC;YACrB,MAAM,EAAE,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM;YACjC,UAAU,EAAE,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU;YAC7C,MAAM,EAAE,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM;SAClC,CAAC,CAAA;IACJ,CAAC;CACF;AAED,MAAM,CAAC,MAAM,gBAAgB,GAAG,GAAgB,EAAE,CAChD,WAAW,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC,CAAA;AAE3D,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,KAAiB,EAAe,EAAE,CACrE,WAAW,CAAC,IAAI,CAAC;IACf,MAAM,EAAE,KAAK,CAAC,MAAM;IACpB,UAAU,EAAE,KAAK,CAAC,UAAU;IAC5B,MAAM,EAAE,KAAK,CAAC,MAAM,IAAI,CAAC;CAC1B,CAAC,CAAA;AAEJ,MAAM,OAAO,aAAc,SAAQ,MAAM,CAAC,KAAK,CAAgB,eAAe,CAAC,CAAC;IAC9E,YAAY,EAAE,MAAM,CAAC,GAAG;IACxB,gBAAgB,EAAE,MAAM,CAAC,GAAG;IAC5B,WAAW,EAAE,MAAM,CAAC,GAAG;IACvB,cAAc,EAAE,MAAM,CAAC,GAAG;IAC1B,OAAO,EAAE,MAAM,CAAC,GAAG;CACpB,CAAC;IACA,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,OAAO,CAAA;IACpF,CAAC;CACF;AAED,MAAM,CAAC,MAAM,kBAAkB,GAAG,GAAkB,EAAE,CACpD,aAAa,CAAC,IAAI,CAAC;IACjB,YAAY,EAAE,CAAC;IACf,gBAAgB,EAAE,CAAC;IACnB,WAAW,EAAE,CAAC;IACd,cAAc,EAAE,CAAC;IACjB,OAAO,EAAE,CAAC;CACX,CAAC,CAAA;AAEJ,MAAM,OAAO,UAAW,SAAQ,MAAM,CAAC,KAAK,CAAa,YAAY,CAAC,CAAC;IACrE,IAAI,EAAE,MAAM,CAAC,MAAM;IACnB,EAAE,EAAE,MAAM,CAAC,GAAG;IACd,MAAM,EAAE,WAAW;IACnB,OAAO,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC;IAC1C,QAAQ,EAAE,aAAa;IACvB,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC;CACpD,CAAC;CAAG;AAEL,MAAM,OAAO,UAAW,SAAQ,MAAM,CAAC,KAAK,CAAa,YAAY,CAAC,CAAC;IACrE,IAAI,EAAE,MAAM,CAAC,MAAM;IACnB,KAAK,EAAE,MAAM,CAAC,GAAG;IACjB,GAAG,EAAE,MAAM,CAAC,GAAG;IACf,SAAS,EAAE,MAAM,CAAC,MAAM;CACzB,CAAC;CAAG;AAEL,MAAM,OAAO,UAAW,SAAQ,MAAM,CAAC,KAAK,CAAa,YAAY,CAAC,CAAC;IACrE,QAAQ,EAAE,MAAM,CAAC,MAAM;IACvB,KAAK,EAAE,MAAM,CAAC,MAAM;IACpB,KAAK,EAAE,MAAM,CAAC,OAAO;CACtB,CAAC;CAAG;AAEL,MAAM,OAAO,YAAa,SAAQ,MAAM,CAAC,KAAK,CAAe,cAAc,CAAC,CAAC;IAC3E,QAAQ,EAAE,MAAM,CAAC,MAAM;IACvB,EAAE,EAAE,MAAM,CAAC,MAAM;IACjB,IAAI,EAAE,MAAM,CAAC,MAAM;IACnB,KAAK,EAAE,MAAM,CAAC,GAAG;IACjB,QAAQ,EAAE,MAAM,CAAC,MAAM;IACvB,OAAO,EAAE,MAAM,CAAC,MAAM;CACvB,CAAC;CAAG;AAEL,MAAM,OAAO,YAAa,SAAQ,MAAM,CAAC,KAAK,CAAe,cAAc,CAAC,CAAC;IAC3E,WAAW,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC;IAC/C,QAAQ,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC;IACxC,WAAW,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC;IAC3C,WAAW,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC;IAC9C,YAAY,EAAE,MAAM,CAAC,GAAG;IACxB,SAAS,EAAE,MAAM,CAAC,GAAG;IACrB,iBAAiB,EAAE,MAAM,CAAC,GAAG;IAC7B,SAAS,EAAE,MAAM,CAAC,GAAG;IACrB,QAAQ,EAAE,MAAM,CAAC,GAAG;IACpB,WAAW,EAAE,MAAM,CAAC,GAAG;IACvB,WAAW,EAAE,MAAM,CAAC,GAAG;IACvB,SAAS,EAAE,MAAM,CAAC,GAAG;IACrB,SAAS,EAAE,MAAM,CAAC,GAAG;IACrB,qBAAqB,EAAE,MAAM,CAAC,GAAG;IACjC,aAAa,EAAE,MAAM,CAAC,GAAG;IACzB,WAAW,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC;IAC/C,UAAU,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC;IAC1C,cAAc,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC;IAC9C,WAAW,EAAE,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC;CACtC,CAAC;CAAG;AAEL,MAAM,OAAO,WAAY,SAAQ,MAAM,CAAC,KAAK,CAAc,aAAa,CAAC,CAAC;IACxE,aAAa,EAAE,MAAM,CAAC,GAAG;IACzB,KAAK,EAAE,MAAM,CAAC,MAAM;IACpB,KAAK,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC;IACxC,SAAS,EAAE,MAAM,CAAC,MAAM;IACxB,UAAU,EAAE,MAAM,CAAC,MAAM;IACzB,QAAQ,EAAE,MAAM,CAAC,MAAM;IACvB,cAAc,EAAE,MAAM,CAAC,MAAM;IAC7B,YAAY,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;IACzC,KAAK,EAAE,UAAU;IACjB,WAAW,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC;IAC9C,aAAa,EAAE,MAAM,CAAC,MAAM;IAC5B,IAAI,EAAE,MAAM,CAAC,MAAM;IACnB,kBAAkB,EAAE,MAAM,CAAC,MAAM;IACjC,OAAO,EAAE,YAAY;IACrB,OAAO,EAAE,MAAM,CAAC,MAAM;IACtB,OAAO,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC;IAC3C,OAAO,EAAE,MAAM,CAAC,GAAG;IACnB,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC;IAChC,OAAO,EAAE,MAAM,CAAC,WAAW,CAAC,YAAY,CAAC;CAC1C,CAAC;IACA,IAAI,SAAS;QACX,OAAO,IAAI,CAAC,OAAO,KAAK,WAAW,CAAA;IACrC,CAAC;IAED,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,CAAC,MAAM,CAAC,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAA;IAC9F,CAAC;IAED,IAAI,YAAY;QACd,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAC3C,KAAK,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CACnD,CAAA;QACD,OAAO,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,CAAA;IACxF,CAAC;IAED,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC,CAAA;IAChF,CAAC;CACF;AAED,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,CAAA;AACnC,MAAM,CAAC,MAAM,aAAa,GAAG,QAAQ,CAAA;AAUrC,MAAM,CAAC,MAAM,qBAAqB,GAAG,GAAqB,EAAE,CAAC,CAAC;IAC5D,UAAU,EAAE,EAAE;IACd,KAAK,EAAE,IAAI,GAAG,EAAE;IAChB,QAAQ,EAAE,IAAI,GAAG,EAAE;IACnB,QAAQ,EAAE,IAAI,GAAG,EAAE;IACnB,YAAY,EAAE,IAAI,GAAG,EAAE;CACxB,CAAC,CAAA;AAEF,MAAM,YAAY,GAAG,CAAC,WAA6B,EAAU,EAAE,CAC7D,WAAW,CAAC,UAAU,CAAC,WAAW,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,aAAa,CAAA;AAE5E,MAAM,SAAS,GAAG,CAAC,MAA6B,EAAE,KAAa,EAAyB,EAAE;IACxF,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;IACnC,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAA;AACrF,CAAC,CAAA;AAED,MAAM,cAAc,GAAG,CAAC,OAAe,EAAE,QAAuB,EAA6B,EAAE;IAC7F,IAAI,OAAO,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAClC,OAAO,IAAI,aAAa,CAAC;YACvB,GAAG,QAAQ;YACX,YAAY,EAAE,QAAQ,CAAC,YAAY,GAAG,CAAC;SACxC,CAAC,CAAA;IACJ,CAAC;IACD,IAAI,OAAO,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC;QACtC,OAAO,IAAI,aAAa,CAAC;YACvB,GAAG,QAAQ;YACX,gBAAgB,EAAE,QAAQ,CAAC,gBAAgB,GAAG,CAAC;SAChD,CAAC,CAAA;IACJ,CAAC;IACD,IAAI,OAAO,CAAC,UAAU,CAAC,eAAe,CAAC,EAAE,CAAC;QACxC,OAAO,IAAI,aAAa,CAAC;YACvB,GAAG,QAAQ;YACX,WAAW,EAAE,QAAQ,CAAC,WAAW,GAAG,CAAC;SACtC,CAAC,CAAA;IACJ,CAAC;IACD,IAAI,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,CAAC;QACvC,OAAO,IAAI,aAAa,CAAC;YACvB,GAAG,QAAQ;YACX,cAAc,EAAE,QAAQ,CAAC,cAAc,GAAG,CAAC;SAC5C,CAAC,CAAA;IACJ,CAAC;IACD,IAAI,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,EAAE,CAAC;QACrC,OAAO,IAAI,aAAa,CAAC;YACvB,GAAG,QAAQ;YACX,OAAO,EAAE,QAAQ,CAAC,OAAO,GAAG,CAAC;SAC9B,CAAC,CAAA;IACJ,CAAC;IACD,OAAO,SAAS,CAAA;AAClB,CAAC,CAAA;AAED,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,WAA6B,EAAE,KAAgB,EAAoB,EAAE;IAChG,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;QACnB,KAAK,cAAc;YACjB,OAAO;gBACL,GAAG,WAAW;gBACd,UAAU,EAAE,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,WAAW,CAAC,UAAU,CAAC;aACrD,CAAA;QACH,KAAK,gBAAgB,CAAC;QACtB,KAAK,aAAa;YAChB,OAAO;gBACL,GAAG,WAAW;gBACd,UAAU,EAAE,SAAS,CAAC,WAAW,CAAC,UAAU,EAAE,KAAK,CAAC,KAAK,CAAC;aAC3D,CAAA;QACH,KAAK,YAAY,CAAC,CAAC,CAAC;YAClB,MAAM,KAAK,GAAG,YAAY,CAAC,WAAW,CAAC,CAAA;YACvC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,IAAI,EAAE,CAAA;YAC/B,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAA;YAC1C,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,KAAK,CAAC,CAAA;YACxC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,gBAAgB,EAAE,CAAC,CAAC,GAAG,CAAC,oBAAoB,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;YAC7F,OAAO;gBACL,GAAG,WAAW;gBACd,KAAK;gBACL,QAAQ,EAAE,IAAI,GAAG,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;gBAChE,YAAY,EACV,KAAK,CAAC,KAAK,KAAK,SAAS;oBACvB,CAAC,CAAC,WAAW,CAAC,YAAY;oBAC1B,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,WAAW,CAAC,YAAY,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;aAC1D,CAAA;QACH,CAAC;QACD,KAAK,MAAM,CAAC,CAAC,CAAC;YACZ,MAAM,KAAK,GAAG,YAAY,CAAC,WAAW,CAAC,CAAA;YACvC,MAAM,OAAO,GAAG,WAAW,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,kBAAkB,EAAE,CAAA;YACvE,MAAM,IAAI,GAAG,cAAc,CAAC,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;YACnD,OAAO,IAAI,KAAK,SAAS;gBACvB,CAAC,CAAC,WAAW;gBACb,CAAC,CAAC;oBACE,GAAG,WAAW;oBACd,QAAQ,EAAE,IAAI,GAAG,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC;iBACzD,CAAA;QACP,CAAC;QACD;YACE,OAAO,WAAW,CAAA;IACtB,CAAC;AACH,CAAC,CAAA;AAED,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,WAA6B,EAAE,KAAa,EAAe,EAAE,CAC5F,CAAC,GAAG,WAAW,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,CAAC,GAAG,EAAE,MAAM,CAAC,EAAE,EAAE;IAC/D,MAAM,KAAK,GAAG,WAAW,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;IAC3C,OAAO,KAAK,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAA;AACzD,CAAC,EAAE,gBAAgB,EAAE,CAAC,CAAA;AAExB,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,WAA6B,EAAE,KAAa,EAAiB,EAAE,CAChG,WAAW,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,kBAAkB,EAAE,CAAA;AAEzD,MAAM,CAAC,MAAM,cAAc,GAAG,CAC5B,WAA6B,EAC7B,KAAa,EACb,EAAU,EACV,SAA2C,EAAE,EACjC,EAAE;IACd,MAAM,KAAK,GAAG,CAAC,GAAG,WAAW,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,MAAM,CAAC,EAAE,EAAE;QACvE,MAAM,KAAK,GAAG,WAAW,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QAC3C,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,EAAE,CAAC;YACzB,OAAO,EAAE,CAAA;QACX,CAAC;QACD,MAAM,IAAI,GAAG,eAAe,CAC1B,KAAK,CAAC,CAAC,CAAC,EACR,UAAU,CAAC,IAAI,CAAC;YACd,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,GAAG,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC;SACzD,CAAC,CACH,CAAA;QACD,OAAO,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;IACzC,CAAC,CAAC,CAAA;IACF,OAAO,UAAU,CAAC,IAAI,CAAC;QACrB,IAAI,EAAE,KAAK;QACX,EAAE;QACF,MAAM,EAAE,gBAAgB,CAAC,WAAW,EAAE,KAAK,CAAC;QAC5C,QAAQ,EAAE,kBAAkB,CAAC,WAAW,EAAE,KAAK,CAAC;QAChD,MAAM;QACN,GAAG,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;KACvF,CAAC,CAAA;AACJ,CAAC,CAAA;AAED,MAAM,KAAK,GAAG,CAAC,MAAuB,EAA0B,EAAE,CAChE,WAAW,IAAI,MAAM,IAAI,gBAAgB,IAAI,MAAM,IAAI,QAAQ,IAAI,MAAM,CAAA;AAO3E,MAAM,CAAC,MAAM,YAAY,GAAG,MAAM,CAAC,EAAE,CAAC,4BAA4B,CAAC,CAAC,QAAQ,CAAC,EAC3E,MAAuB;IAEvB,MAAM,WAAW,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,qBAAqB,EAAE,CAAC,CAAA;IAC5D,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACnC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;QACnB,OAAO;YACL,GAAG,EAAE,GAAG,CAAC,GAAG,CAAC,WAAW,CAAC;YACzB,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE,qBAAqB,EAAE,CAAC;SACrD,CAAA;IACH,CAAC;IACD,MAAM,YAAY,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,SAAS,CAAA;IAC5C,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,cAAc,CAAA;IAC7C,KAAK,CAAC,CAAC,MAAM,CAAC,gBAAgB,CAAC,YAAY,CAAC,CAAC,IAAI,CAC/C,MAAM,CAAC,UAAU,CAAC,CAAC,KAAK,EAAE,EAAE,CAC1B,GAAG,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,YAAY,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,CACrE,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAC3D,CACF,EACD,MAAM,CAAC,UAAU,CAClB,CAAA;IACD,MAAM,KAAK,GAAwB,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,CACrD,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,cAAc,EAAE,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CACzD,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,EAAE,IAAI,CAAC,EAAE,EAAE,CACnC,IAAI,IAAI,SAAS,GAAG,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CACzF,CACF,CACF,CAAA;IACD,OAAO;QACL,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC;QACrD,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE,qBAAqB,EAAE,CAAC,CAAC,CAAC;KACjF,CAAA;AACH,CAAC,CAAC,CAAA"}

package/dist/BenchReport.d.ts

@@ -0,0 +1,9 @@
+import * as Effect from "effect/Effect";
+import { BenchRecord } from "./Bench.ts";
+import { PlanParseError, UnsupportedSchemaVersion } from "./FlowError.ts";
+import type { PlainFileStoreShape } from "./Persistence.ts";
+export declare const median: (values: ReadonlyArray<number>) => number;
+export declare const renderBenchReport: (records: ReadonlyArray<BenchRecord>, projectPrograms?: number) => string;
+export declare const appendBenchRecord: (files: PlainFileStoreShape, path: string, record: BenchRecord) => Effect.Effect<void, import("./FlowError.ts").PersistenceError | PlanParseError | UnsupportedSchemaVersion | import("./FlowError.ts").WorkspacePathError | import("./FlowError.ts").WorkspaceLimitError | import("./FlowError.ts").WorkspaceIoError | import("./FlowError.ts").FlowAborted | import("./FlowError.ts").ProcessError | import("./FlowError.ts").FlowLlmError | import("./FlowError.ts").FlowCapabilityDenied | import("./FlowError.ts").BudgetExceeded, never>;
+export declare const loadBenchRecords: (files: PlainFileStoreShape, path: string) => Effect.Effect<readonly BenchRecord[], import("./FlowError.ts").PersistenceError | PlanParseError | UnsupportedSchemaVersion | import("./FlowError.ts").WorkspacePathError | import("./FlowError.ts").WorkspaceLimitError | import("./FlowError.ts").WorkspaceIoError | import("./FlowError.ts").FlowAborted | import("./FlowError.ts").ProcessError | import("./FlowError.ts").FlowLlmError | import("./FlowError.ts").FlowCapabilityDenied | import("./FlowError.ts").BudgetExceeded, never>;
+//# sourceMappingURL=BenchReport.d.ts.map

package/dist/BenchReport.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"BenchReport.d.ts","sourceRoot":"","sources":["../src/BenchReport.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AAEvC,OAAO,EAAE,WAAW,EAAuC,MAAM,YAAY,CAAA;AAC7E,OAAO,EAAE,cAAc,EAAE,wBAAwB,EAAkB,MAAM,gBAAgB,CAAA;AACzF,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAA;AAI3D,eAAO,MAAM,MAAM,GAAI,QAAQ,aAAa,CAAC,MAAM,CAAC,KAAG,MAStD,CAAA;AA4WD,eAAO,MAAM,iBAAiB,GAC5B,SAAS,aAAa,CAAC,WAAW,CAAC,EACnC,kBAAkB,MAAM,KACvB,MAUF,CAAA;AAID,eAAO,MAAM,iBAAiB,ghBAa5B,CAAA;AAEF,eAAO,MAAM,gBAAgB,6gBAiC3B,CAAA"}

package/dist/BenchReport.js

@@ -0,0 +1,360 @@
+import * as Effect from "effect/Effect";
+import * as Schema from "effect/Schema";
+import { BenchRecord, CurrentBenchSchema } from "./Bench.js";
+import { PlanParseError, UnsupportedSchemaVersion } from "./FlowError.js";
+import { estimateCostUsd } from "./PriceList.js";
+import { TokenUsage } from "@llm4ts/core/Models";
+export const median = (values) => {
+    if (values.length === 0) {
+        return 0;
+    }
+    const sorted = [...values].sort((left, right) => left - right);
+    const middle = Math.floor(sorted.length / 2);
+    return sorted.length % 2 === 1
+        ? (sorted[middle] ?? 0)
+        : ((sorted[middle - 1] ?? 0) + (sorted[middle] ?? 0)) / 2;
+};
+const formatDuration = (milliseconds) => {
+    if (milliseconds >= 3_600_000) {
+        return `${(milliseconds / 3_600_000).toFixed(1)}h`;
+    }
+    if (milliseconds >= 60_000) {
+        return `${(milliseconds / 60_000).toFixed(1)}m`;
+    }
+    return `${(milliseconds / 1_000).toFixed(1)}s`;
+};
+const formatTokens = (tokens) => {
+    if (Math.abs(tokens) >= 1_000_000) {
+        return `${(tokens / 1_000_000).toFixed(1)}M`;
+    }
+    if (Math.abs(tokens) >= 1_000) {
+        return `${(tokens / 1_000).toFixed(1)}k`;
+    }
+    return Math.round(tokens).toString();
+};
+const formatInteger = (value) => Math.round(value).toString();
+const formatPercent = (value) => `${value.toFixed(1)}%`;
+const formatBoolean = (value) => (value >= 0.5 ? "yes" : "no");
+const spread = (records, metric) => {
+    const values = records.flatMap((record) => {
+        const value = metric.value(record);
+        return value === undefined ? [] : [value];
+    });
+    if (values.length === 0) {
+        return "–";
+    }
+    const center = median(values);
+    const base = metric.format(center);
+    return values.length > 1 && Math.min(...values) !== Math.max(...values)
+        ? `${base} (${metric.format(Math.min(...values))}–${metric.format(Math.max(...values))})`
+        : base;
+};
+const judgePercent = (record) => {
+    const scores = record.quality?.judgeScores ?? [];
+    const maximum = scores.reduce((sum, score) => sum + score.max, 0);
+    return maximum <= 0
+        ? undefined
+        : (scores.reduce((sum, score) => sum + score.score, 0) / maximum) * 100;
+};
+const totalOutputTokens = (record) => record.phases.reduce((sum, phase) => sum + phase.tokens.completion, 0);
+const totalCachedTokens = (record) => record.phases.reduce((sum, phase) => sum + phase.tokens.cached, 0);
+const headlineMetrics = (judgesComparable) => [
+    {
+        label: "Total time",
+        direction: "lower",
+        value: (record) => record.totalMs,
+        format: formatDuration
+    },
+    {
+        label: "Total tokens",
+        direction: "lower",
+        value: (record) => record.totalTokens,
+        format: formatTokens
+    },
+    {
+        label: "Output tokens",
+        direction: "lower",
+        value: totalOutputTokens,
+        format: formatTokens
+    },
+    {
+        label: "Cached reads",
+        direction: "neutral",
+        value: totalCachedTokens,
+        format: formatTokens
+    },
+    {
+        label: "Self-healing actions",
+        direction: "lower",
+        value: (record) => record.selfHealing,
+        format: formatInteger
+    },
+    {
+        label: "Gate cleared",
+        direction: "higher",
+        value: (record) => record.quality?.gateCleared === undefined ? undefined : record.quality.gateCleared ? 1 : 0,
+        format: formatBoolean,
+        gap: formatInteger
+    },
+    {
+        label: "Build passed",
+        direction: "higher",
+        value: (record) => record.quality?.buildPassed === undefined ? undefined : record.quality.buildPassed ? 1 : 0,
+        format: formatBoolean,
+        gap: formatInteger
+    },
+    {
+        label: "Coverage %",
+        direction: "higher",
+        value: (record) => record.quality?.coveragePct,
+        format: formatPercent
+    },
+    {
+        label: "Judge score",
+        direction: judgesComparable ? "higher" : "neutral",
+        value: judgePercent,
+        format: formatPercent
+    }
+];
+const gapCell = (metric, values) => {
+    if (values.length < 2) {
+        return "";
+    }
+    const low = Math.min(...values);
+    const high = Math.max(...values);
+    const difference = high - low;
+    const percent = low === 0 ? 100 : (difference / Math.abs(low)) * 100;
+    return `+${(metric.gap ?? metric.format)(difference)} (+${percent.toFixed(1)}%)`;
+};
+const metricRow = (metric, providers, byProvider) => {
+    const centers = providers.map((provider) => {
+        const values = (byProvider.get(provider) ?? []).flatMap((record) => {
+            const value = metric.value(record);
+            return value === undefined ? [] : [value];
+        });
+        return values.length === 0 ? undefined : median(values);
+    });
+    const comparable = centers.filter((value) => value !== undefined);
+    const best = metric.direction === "neutral" || comparable.length < 2
+        ? undefined
+        : metric.direction === "lower"
+            ? Math.min(...comparable)
+            : Math.max(...comparable);
+    const worst = metric.direction === "neutral" || comparable.length < 2
+        ? undefined
+        : metric.direction === "lower"
+            ? Math.max(...comparable)
+            : Math.min(...comparable);
+    const cells = providers.map((provider, index) => {
+        const value = centers[index];
+        const display = spread(byProvider.get(provider) ?? [], metric);
+        if (value === undefined || best === undefined || best === worst) {
+            return display;
+        }
+        return value === best ? `**${display} ✅**` : value === worst ? `${display} ⚠️` : display;
+    });
+    return `| ${metric.label} | ${cells.join(" | ")} | ${gapCell(metric, comparable)} |`;
+};
+const costForPhase = (record, phase) => {
+    if (phase.costUsd !== undefined) {
+        return [phase.costUsd, false];
+    }
+    if (phase.tokens.prompt === 0 && phase.tokens.completion === 0 && phase.tokens.cached === 0) {
+        return undefined;
+    }
+    const cost = estimateCostUsd(record.modelRequested, TokenUsage.make({
+        prompt: phase.tokens.prompt,
+        completion: phase.tokens.completion,
+        total: phase.tokens.total,
+        ...(phase.tokens.cached <= 0 ? {} : { cached: phase.tokens.cached })
+    }));
+    return cost === undefined ? undefined : [cost, true];
+};
+const costForRecord = (record) => {
+    const values = record.phases.flatMap((phase) => {
+        const cost = costForPhase(record, phase);
+        return cost === undefined ? [] : [cost];
+    });
+    return values.length === 0
+        ? undefined
+        : [values.reduce((sum, [cost]) => sum + cost, 0), values.some(([, estimated]) => estimated)];
+};
+const costRow = (providers, byProvider, select) => {
+    const cells = providers.map((provider) => {
+        const values = (byProvider.get(provider) ?? []).flatMap((record) => {
+            const value = select(record);
+            return value === undefined ? [] : [value];
+        });
+        if (values.length === 0) {
+            return "–";
+        }
+        const suffix = values.some(([, estimated]) => estimated) ? "*" : "";
+        const costs = values.map(([cost]) => cost);
+        return `$${median(costs).toFixed(2)}${suffix}`;
+    });
+    return `| Est. cost (USD) | ${cells.join(" | ")} |  |`;
+};
+const projection = (programs, providers, byProvider) => {
+    const rows = providers.map((provider) => {
+        const records = byProvider.get(provider) ?? [];
+        const perProgramTokens = records.flatMap((record) => {
+            const count = record.quality?.programs ?? 0;
+            return count <= 0 ? [] : [record.totalTokens / count];
+        });
+        const perProgramMs = records.flatMap((record) => {
+            const count = record.quality?.programs ?? 0;
+            return count <= 0 ? [] : [record.totalMs / count];
+        });
+        if (perProgramTokens.length === 0 || perProgramMs.length === 0) {
+            return `| ${provider} | – | – | – |`;
+        }
+        const tokens = median(perProgramTokens);
+        const milliseconds = median(perProgramMs);
+        return `| ${provider} | ${formatTokens(tokens)} | ${formatTokens(tokens * programs)} | ${formatDuration(milliseconds * programs)} |`;
+    });
+    return [
+        `## Projection @ ${programs} programs`,
+        "",
+        "| Provider | Tokens/program | Projected tokens | Projected time |",
+        "| -------- | -------------- | ---------------- | -------------- |",
+        ...rows,
+        "",
+        "_Linear extrapolation from recorded medians; not a delivery forecast._"
+    ].join("\n");
+};
+const renderSection = (records, projectPrograms) => {
+    const providers = [...new Set(records.map((record) => record.provider))].sort();
+    const byProvider = new Map(providers.map((provider) => [
+        provider,
+        records.filter((record) => record.provider === provider)
+    ]));
+    const first = records[0];
+    if (first === undefined) {
+        return "";
+    }
+    const judgesComparable = new Set(records.map((record) => `${record.judge.provider}\0${record.judge.model}`)).size === 1;
+    const judgeLine = judgesComparable
+        ? `Judge: ${first.judge.provider}/${first.judge.model} — same examiner for all runs`
+        : "Judges: self-graded — scores NOT cross-comparable";
+    const header = `| Metric | ${providers.join(" | ")} | Gap |`;
+    const separator = `| ------ | ${providers.map(() => "------").join(" | ")} | --- |`;
+    const outcomes = `| Outcome | ${providers
+        .map((provider) => {
+        const group = byProvider.get(provider) ?? [];
+        return group.some((record) => record.resumed === true)
+            ? `${group[0]?.outcome ?? "unknown"} ♻`
+            : (group[0]?.outcome ?? "unknown");
+    })
+        .join(" | ")} |  |`;
+    const phaseNames = [
+        ...new Set(records.flatMap((record) => record.phases.map((phase) => phase.name)))
+    ];
+    const phaseSections = phaseNames.map((phaseName) => {
+        const duration = {
+            label: "Duration",
+            direction: "lower",
+            value: (record) => record.phases.find((phase) => phase.name === phaseName)?.ms,
+            format: formatDuration
+        };
+        const tokens = {
+            label: "Tokens",
+            direction: "lower",
+            value: (record) => {
+                const phase = record.phases.find((item) => item.name === phaseName);
+                return phase === undefined ? undefined : phase.tokens.total + phase.tokens.cached;
+            },
+            format: formatTokens
+        };
+        return [
+            `### ${phaseName}`,
+            "",
+            header,
+            separator,
+            metricRow(duration, providers, byProvider),
+            metricRow(tokens, providers, byProvider),
+            costRow(providers, byProvider, (record) => {
+                const phase = record.phases.find((item) => item.name === phaseName);
+                return phase === undefined ? undefined : costForPhase(record, phase);
+            })
+        ].join("\n");
+    });
+    const failures = records.filter((record) => !record.completed);
+    const estimated = records.some((record) => costForRecord(record)?.[1] === true);
+    const extras = [
+        ...(failures.length === 0
+            ? []
+            : [
+                "## Failures",
+                ...failures.map((record) => `- ${record.provider} ${record.runId}: ${record.outcome}`)
+            ]),
+        ...(projectPrograms === undefined ? [] : [projection(projectPrograms, providers, byProvider)]),
+        ...(records.some((record) => record.resumed === true)
+            ? [
+                "_♻ resumed run: duration and tokens include only work after resume and are not fresh-run comparable._"
+            ]
+            : []),
+        ...(estimated
+            ? ["_\\* Cost estimated at report time from recorded tokens × the requested model._"]
+            : []),
+        `_Machines: ${[...new Set(records.map((record) => record.machine.hostname))].sort().join(", ")}._`
+    ];
+    return [
+        `# Benchmark — ${first.pack} @ ${first.fixtureFingerprint.slice(0, 8)}`,
+        "",
+        `_${judgeLine}_`,
+        "",
+        "## Headline",
+        "",
+        header,
+        separator,
+        outcomes,
+        ...headlineMetrics(judgesComparable).map((metric) => metricRow(metric, providers, byProvider)),
+        costRow(providers, byProvider, costForRecord),
+        "",
+        "## Phases",
+        "",
+        ...phaseSections,
+        ...extras
+    ].join("\n");
+};
+export const renderBenchReport = (records, projectPrograms) => {
+    const groups = new Map();
+    for (const record of records) {
+        const key = `${record.pack}\0${record.fixtureFingerprint}`;
+        groups.set(key, [...(groups.get(key) ?? []), record]);
+    }
+    return [...groups.entries()]
+        .sort(([left], [right]) => left.localeCompare(right))
+        .map(([, group]) => renderSection(group, projectPrograms))
+        .join("\n\n");
+};
+const benchCodec = Schema.fromJsonString(BenchRecord);
+export const appendBenchRecord = Effect.fn("@llm4ts/flow/BenchReport.append")(function* (files, path, record) {
+    const encoded = yield* Schema.encodeEffect(benchCodec)(record).pipe(Effect.mapError((error) => PlanParseError.make({
+        message: `failed to encode benchmark record: ${String(error)}`
+    })));
+    yield* files.append(path, `${encoded}\n`);
+});
+export const loadBenchRecords = Effect.fn("@llm4ts/flow/BenchReport.load")(function* (files, path) {
+    const contents = yield* files.read(path);
+    if (contents === undefined) {
+        return [];
+    }
+    const records = yield* Effect.forEach(contents
+        .split(/\r?\n/u)
+        .map((line) => line.trim())
+        .filter((line) => line.length > 0), (line, index) => Schema.decodeUnknownEffect(benchCodec)(line).pipe(Effect.mapError((error) => PlanParseError.make({
+        message: `malformed benchmark record at ${path}:${index + 1}: ${String(error)}`
+    }))), { concurrency: 1 });
+    for (const record of records) {
+        if (record.schemaVersion !== CurrentBenchSchema) {
+            return yield* UnsupportedSchemaVersion.make({
+                path,
+                expected: CurrentBenchSchema,
+                actual: record.schemaVersion
+            });
+        }
+    }
+    return records;
+});
+//# sourceMappingURL=BenchReport.js.map