@trazum/core 1.9.0 → 1.25.0

package/dist/conversation.d.ts

@@ -0,0 +1,121 @@
+import type { PricingCatalogue } from './pricing.js';
+import type { UsageRecord } from './usage.js';
+/**
+ * What re-sending the conversation costs.
+ *
+ * ## The line nothing was watching
+ *
+ * A chat or agent workload sends the whole conversation back on every turn. Turn
+ * one is a system prompt and a question; turn twenty is a system prompt and
+ * nineteen previous exchanges and a question. The input grows linearly with the
+ * turn count, and on an agent bill that growth is routinely the largest single
+ * line — larger than the prompt, larger than the answers.
+ *
+ * Nothing in this package could see it. A prompt file shows the system prompt and
+ * not the history. A total shows the sum and not the shape. Even `profile` reported
+ * "input is 71% of this bill" without being able to say that most of that input was
+ * the same sentences, sent again.
+ *
+ * ## What it will and will not claim
+ *
+ * The honest figure is a **ceiling**, and the token half of it is exact: the
+ * input tokens beyond every turn being the size of the session's smallest turn.
+ * That quantity is order-independent — the first-seen anchor made the identical
+ * workload vanish when exported newest-first — and it is immune to billing
+ * rates: a cost-based anchor charged an ordinary 5-minute-TTL agent 77.5%
+ * "growth" on a conversation that never grew, because a cache-miss turn costs
+ * 12.5x a cache-hit turn of the same size. The dollars are that token share of
+ * what the session actually spent, at its own blended rate.
+ *
+ * It is a ceiling and not a saving because part of that growth is the user's own
+ * new messages, which nobody can truncate away, and this module cannot tell those
+ * apart from re-sent history — it sees counts, not content. Reporting the ceiling
+ * as an opportunity would be the flattering direction; reporting nothing because
+ * the exact split is unknowable would be worse. So it reports the bound and says
+ * what it is.
+ *
+ * ## The session key never leaves this module
+ *
+ * A session identifier is somebody's conversation, and in a real log it is often an
+ * account id, a ticket number or an email. It is used to group calls and count
+ * turns; **no figure reported anywhere carries it**, and every result is aggregated
+ * per label. The promise that a usage log handed to Trazum contains no content is
+ * only worth something if nothing identifying comes back out either.
+ */
+/** How one label-and-model slice grows across a conversation. */
+export interface ConversationGrowth {
+    label: string;
+    model: string;
+    modelName: string;
+    /** How many distinct conversations were seen. Never which ones. */
+    sessions: number;
+    calls: number;
+    /** Mean input tokens on the smallest turn of a conversation. */
+    minTurnTokens: number;
+    /** Mean input tokens on the largest turn. */
+    maxTurnTokens: number;
+    /** Turns in the longest conversation seen. */
+    longestSession: number;
+    /** Input-side spend: plain input, cache reads and cache writes. */
+    inputUsd: number;
+    /** What that would have been if every turn had cost what its cheapest turn did. */
+    flatUsd: number;
+    /**
+     * `inputUsd - flatUsd`. **A ceiling on what removing conversation growth could
+     * be worth, not a saving** — part of it is the user's own new messages.
+     */
+    growthUsd: number;
+    /** `growthUsd` as a fraction of the whole bill in the log. */
+    shareOfBill: number;
+}
+export interface ConversationOptions {
+    catalogue: PricingCatalogue;
+    on?: Date;
+    /**
+     * Slices whose growth is below this share of the bill are dropped, and slices
+     * shorter than `minTurns` never count as conversations at all.
+     */
+    minShare?: number;
+    /**
+     * Conversations shorter than this are ignored.
+     *
+     * Two turns is not a conversation, it is a retry — and a workload that never
+     * exceeds two turns has no growth to measure, so including it would put a row on
+     * screen whose figure is arithmetic noise. Default 3.
+     */
+    minTurns?: number;
+}
+/**
+ * Measures what conversation growth costs, from records that carry a session.
+ *
+ * Records without one are skipped rather than lumped together: calls from
+ * different conversations pushed into a single bucket would report a turn count
+ * that is really a call count, and a growth figure derived from it would be
+ * arithmetic performed on a fiction.
+ *
+ * Takes records rather than a report because turn order is the whole measurement,
+ * and a breakdown has already thrown it away.
+ */
+export interface ConversationTracker {
+    /** Feed one parsed record. Records without a session are ignored. */
+    add(record: UsageRecord): void;
+    /** The finished measurement, once the whole bill is known. */
+    finish(totalUsd: number): ConversationGrowth[];
+}
+/**
+ * An accumulator, so a profile can measure this in the pass it already makes.
+ *
+ * The alternative was holding every record to hand to a pure function afterwards,
+ * and a usage log is measured in megabytes — a profile that needs the whole file in
+ * memory to answer one question is a profile that stops working on the logs most
+ * worth reading. What this holds is bounded by the number of **conversations**, and
+ * only ever four numbers each.
+ */
+export declare function createConversationTracker(options: ConversationOptions): ConversationTracker;
+/**
+ * The same measurement over a list of records, for a caller holding one already.
+ *
+ * `profileUsage` uses the tracker instead, so it never has to keep the log.
+ */
+export declare function conversationGrowth(records: readonly UsageRecord[], totalUsd: number, options: ConversationOptions): ConversationGrowth[];
+//# sourceMappingURL=conversation.d.ts.map

package/dist/conversation.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"conversation.d.ts","sourceRoot":"","sources":["../src/conversation.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAE9C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AAEH,iEAAiE;AACjE,MAAM,WAAW,kBAAkB;IACjC,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,mEAAmE;IACnE,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,gEAAgE;IAChE,aAAa,EAAE,MAAM,CAAC;IACtB,6CAA6C;IAC7C,aAAa,EAAE,MAAM,CAAC;IACtB,8CAA8C;IAC9C,cAAc,EAAE,MAAM,CAAC;IACvB,mEAAmE;IACnE,QAAQ,EAAE,MAAM,CAAC;IACjB,mFAAmF;IACnF,OAAO,EAAE,MAAM,CAAC;IAChB;;;OAGG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB,8DAA8D;IAC9D,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,mBAAmB;IAClC,SAAS,EAAE,gBAAgB,CAAC;IAC5B,EAAE,CAAC,EAAE,IAAI,CAAC;IACV;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;;;;OAMG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AA6CD;;;;;;;;;;GAUG;AACH,MAAM,WAAW,mBAAmB;IAClC,qEAAqE;IACrE,GAAG,CAAC,MAAM,EAAE,WAAW,GAAG,IAAI,CAAC;IAC/B,8DAA8D;IAC9D,MAAM,CAAC,QAAQ,EAAE,MAAM,GAAG,kBAAkB,EAAE,CAAC;CAChD;AAED;;;;;;;;GAQG;AACH,wBAAgB,yBAAyB,CAAC,OAAO,EAAE,mBAAmB,GAAG,mBAAmB,CA8H3F;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAChC,OAAO,EAAE,SAAS,WAAW,EAAE,EAC/B,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,mBAAmB,GAC3B,kBAAkB,EAAE,CAItB"}

package/dist/conversation.js

@@ -0,0 +1,157 @@
+import { effectivePricing, multipliersFor } from './pricing.js';
+import { UNLABELLED } from './usage.js';
+/** Input-side cost of one call at its own model's rates. */
+function inputCostOf(record, catalogue, on) {
+    const model = catalogue.byId.get(record.model);
+    if (!model)
+        return null;
+    const { inputPerMTok } = effectivePricing(model, on);
+    const rates = multipliersFor(model);
+    const per = (tokens, rate) => (tokens / 1_000_000) * rate;
+    return (per(record.inputTokens, inputPerMTok) +
+        per(record.cacheReadTokens, inputPerMTok * rates.cacheRead) +
+        per(record.cacheWrite5mTokens, inputPerMTok * rates.cacheWrite5m) +
+        per(record.cacheWrite1hTokens, inputPerMTok * rates.cacheWrite1h));
+}
+/** Every input-side token of one call, whatever rate it was billed at. */
+const inputTokensOf = (r) => r.inputTokens + r.cacheReadTokens + r.cacheWrite5mTokens + r.cacheWrite1hTokens;
+/**
+ * An accumulator, so a profile can measure this in the pass it already makes.
+ *
+ * The alternative was holding every record to hand to a pure function afterwards,
+ * and a usage log is measured in megabytes — a profile that needs the whole file in
+ * memory to answer one question is a profile that stops working on the logs most
+ * worth reading. What this holds is bounded by the number of **conversations**, and
+ * only ever four numbers each.
+ */
+export function createConversationTracker(options) {
+    const { catalogue, on = new Date(), minShare = 0.01, minTurns = 3 } = options;
+    // Keyed on the pair, then on the session inside it. A newline cannot occur in a
+    // model id, and both halves are trimmed strings.
+    const slices = new Map();
+    const add = (record) => {
+        if (record.session === null)
+            return;
+        const cost = inputCostOf(record, catalogue, on);
+        // An unpriced model contributes no dollars anywhere else either; including it
+        // here would report growth of zero on a workload that grew.
+        if (cost === null)
+            return;
+        const sliceKey = `${record.label ?? UNLABELLED}\n${record.model}`;
+        let sessions = slices.get(sliceKey);
+        if (!sessions) {
+            sessions = new Map();
+            slices.set(sliceKey, sessions);
+        }
+        const tokens = inputTokensOf(record);
+        const existing = sessions.get(record.session);
+        if (!existing) {
+            sessions.set(record.session, {
+                turns: 1,
+                minTokens: tokens,
+                maxTokens: tokens,
+                totalTokens: tokens,
+                totalUsd: cost,
+            });
+            return;
+        }
+        existing.turns += 1;
+        existing.minTokens = Math.min(existing.minTokens, tokens);
+        existing.maxTokens = Math.max(existing.maxTokens, tokens);
+        existing.totalTokens += tokens;
+        existing.totalUsd += cost;
+    };
+    const finish = (totalUsd) => {
+        const out = [];
+        for (const [sliceKey, sessions] of slices) {
+            const split = sliceKey.indexOf('\n');
+            const label = sliceKey.slice(0, split);
+            const modelId = sliceKey.slice(split + 1);
+            const model = catalogue.byId.get(modelId);
+            if (!model)
+                continue;
+            /**
+             * Only conversations long enough to have grown. A two-turn session has one
+             * step of growth and is as likely to be a retry, and averaging it in drags the
+             * measured shape towards flat — understating the real thing, which is the
+             * direction that flatters.
+             */
+            const long = [...sessions.values()].filter((s) => s.turns >= minTurns);
+            if (long.length === 0)
+                continue;
+            let inputUsd = 0;
+            let flatUsd = 0;
+            let minTokens = 0;
+            let maxTokens = 0;
+            let calls = 0;
+            let longestSession = 0;
+            for (const session of long) {
+                inputUsd += session.totalUsd;
+                /**
+                 * The growth is measured in tokens — `totalTokens - minTokens·turns`,
+                 * which is exact, order-independent, and zero for a conversation whose
+                 * turns never change size however each one was billed. The money is that
+                 * token share of what the session actually spent: pricing the excess at
+                 * any single rate would either overstate it (full input rate, when most
+                 * re-sent history is cache-read cheap) or move with billing noise (the
+                 * cheapest turn's rate, which is what mis-billed a flat cached agent).
+                 */
+                const flatTokens = session.minTokens * session.turns;
+                flatUsd +=
+                    session.totalTokens > 0
+                        ? session.totalUsd * (flatTokens / session.totalTokens)
+                        : session.totalUsd;
+                minTokens += session.minTokens;
+                maxTokens += session.maxTokens;
+                calls += session.turns;
+                longestSession = Math.max(longestSession, session.turns);
+            }
+            const growthUsd = inputUsd - flatUsd;
+            const shareOfBill = totalUsd > 0 ? growthUsd / totalUsd : 0;
+            /**
+             * Below the attention threshold — **and that covers shrinking conversations
+             * too**, because a negative share is below any threshold at or above zero.
+             *
+             * There was a separate `growthUsd <= 0` check here. No mutation could break
+             * it: every case it caught, this one caught first. A guard nothing can
+             * distinguish is not defence in depth, it is a second place for the intent to
+             * drift from the code, so the intent lives in this comment instead.
+             *
+             * The case it was written for is ordinary: an opening turn carrying an
+             * attachment or a retrieved document is bigger than everything after it, and
+             * reporting that as "conversation growth" would be a negative ceiling
+             * presented as an opportunity.
+             */
+            if (shareOfBill < minShare)
+                continue;
+            out.push({
+                label,
+                model: modelId,
+                modelName: model.displayName,
+                sessions: long.length,
+                calls,
+                minTurnTokens: minTokens / long.length,
+                maxTurnTokens: maxTokens / long.length,
+                longestSession,
+                inputUsd,
+                flatUsd,
+                growthUsd,
+                shareOfBill,
+            });
+        }
+        return out.sort((a, b) => b.growthUsd - a.growthUsd);
+    };
+    return { add, finish };
+}
+/**
+ * The same measurement over a list of records, for a caller holding one already.
+ *
+ * `profileUsage` uses the tracker instead, so it never has to keep the log.
+ */
+export function conversationGrowth(records, totalUsd, options) {
+    const tracker = createConversationTracker(options);
+    for (const record of records)
+        tracker.add(record);
+    return tracker.finish(totalUsd);
+}
+//# sourceMappingURL=conversation.js.map

package/dist/conversation.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"conversation.js","sourceRoot":"","sources":["../src/conversation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAChE,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AA4FxC,4DAA4D;AAC5D,SAAS,WAAW,CAAC,MAAmB,EAAE,SAA2B,EAAE,EAAQ;IAC7E,MAAM,KAAK,GAAG,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC/C,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IACxB,MAAM,EAAE,YAAY,EAAE,GAAG,gBAAgB,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IACrD,MAAM,KAAK,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;IACpC,MAAM,GAAG,GAAG,CAAC,MAAc,EAAE,IAAY,EAAU,EAAE,CAAC,CAAC,MAAM,GAAG,SAAS,CAAC,GAAG,IAAI,CAAC;IAClF,OAAO,CACL,GAAG,CAAC,MAAM,CAAC,WAAW,EAAE,YAAY,CAAC;QACrC,GAAG,CAAC,MAAM,CAAC,eAAe,EAAE,YAAY,GAAG,KAAK,CAAC,SAAS,CAAC;QAC3D,GAAG,CAAC,MAAM,CAAC,kBAAkB,EAAE,YAAY,GAAG,KAAK,CAAC,YAAY,CAAC;QACjE,GAAG,CAAC,MAAM,CAAC,kBAAkB,EAAE,YAAY,GAAG,KAAK,CAAC,YAAY,CAAC,CAClE,CAAC;AACJ,CAAC;AAED,0EAA0E;AAC1E,MAAM,aAAa,GAAG,CAAC,CAAc,EAAU,EAAE,CAC/C,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,eAAe,GAAG,CAAC,CAAC,kBAAkB,GAAG,CAAC,CAAC,kBAAkB,CAAC;AA4ClF;;;;;;;;GAQG;AACH,MAAM,UAAU,yBAAyB,CAAC,OAA4B;IACpE,MAAM,EAAE,SAAS,EAAE,EAAE,GAAG,IAAI,IAAI,EAAE,EAAE,QAAQ,GAAG,IAAI,EAAE,QAAQ,GAAG,CAAC,EAAE,GAAG,OAAO,CAAC;IAE9E,gFAAgF;IAChF,iDAAiD;IACjD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAgC,CAAC;IAEvD,MAAM,GAAG,GAAG,CAAC,MAAmB,EAAQ,EAAE;QACxC,IAAI,MAAM,CAAC,OAAO,KAAK,IAAI;YAAE,OAAO;QACpC,MAAM,IAAI,GAAG,WAAW,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC,CAAC;QAChD,8EAA8E;QAC9E,4DAA4D;QAC5D,IAAI,IAAI,KAAK,IAAI;YAAE,OAAO;QAE1B,MAAM,QAAQ,GAAG,GAAG,MAAM,CAAC,KAAK,IAAI,UAAU,KAAK,MAAM,CAAC,KAAK,EAAE,CAAC;QAClE,IAAI,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACpC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,QAAQ,GAAG,IAAI,GAAG,EAAE,CAAC;YACrB,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QACjC,CAAC;QAED,MAAM,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;QACrC,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAC9C,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,EAAE;gBAC3B,KAAK,EAAE,CAAC;gBACR,SAAS,EAAE,MAAM;gBACjB,SAAS,EAAE,MAAM;gBACjB,WAAW,EAAE,MAAM;gBACnB,QAAQ,EAAE,IAAI;aACf,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QACD,QAAQ,CAAC,KAAK,IAAI,CAAC,CAAC;QACpB,QAAQ,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAC1D,QAAQ,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAC1D,QAAQ,CAAC,WAAW,IAAI,MAAM,CAAC;QAC/B,QAAQ,CAAC,QAAQ,IAAI,IAAI,CAAC;IAC5B,CAAC,CAAC;IAEF,MAAM,MAAM,GAAG,CAAC,QAAgB,EAAwB,EAAE;QAC1D,MAAM,GAAG,GAAyB,EAAE,CAAC;QAErC,KAAK,MAAM,CAAC,QAAQ,EAAE,QAAQ,CAAC,IAAI,MAAM,EAAE,CAAC;YAC1C,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YACrC,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;YACvC,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;YAC1C,MAAM,KAAK,GAAG,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YAC1C,IAAI,CAAC,KAAK;gBAAE,SAAS;YAErB;;;;;eAKG;YACH,MAAM,IAAI,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,QAAQ,CAAC,CAAC;YACvE,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS;YAEhC,IAAI,QAAQ,GAAG,CAAC,CAAC;YACjB,IAAI,OAAO,GAAG,CAAC,CAAC;YAChB,IAAI,SAAS,GAAG,CAAC,CAAC;YAClB,IAAI,SAAS,GAAG,CAAC,CAAC;YAClB,IAAI,KAAK,GAAG,CAAC,CAAC;YACd,IAAI,cAAc,GAAG,CAAC,CAAC;YAEvB,KAAK,MAAM,OAAO,IAAI,IAAI,EAAE,CAAC;gBAC3B,QAAQ,IAAI,OAAO,CAAC,QAAQ,CAAC;gBAC7B;;;;;;;;mBAQG;gBACH,MAAM,UAAU,GAAG,OAAO,CAAC,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC;gBACrD,OAAO;oBACL,OAAO,CAAC,WAAW,GAAG,CAAC;wBACrB,CAAC,CAAC,OAAO,CAAC,QAAQ,GAAG,CAAC,UAAU,GAAG,OAAO,CAAC,WAAW,CAAC;wBACvD,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC;gBACvB,SAAS,IAAI,OAAO,CAAC,SAAS,CAAC;gBAC/B,SAAS,IAAI,OAAO,CAAC,SAAS,CAAC;gBAC/B,KAAK,IAAI,OAAO,CAAC,KAAK,CAAC;gBACvB,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,cAAc,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;YAC3D,CAAC;YAED,MAAM,SAAS,GAAG,QAAQ,GAAG,OAAO,CAAC;YACrC,MAAM,WAAW,GAAG,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;YAC5D;;;;;;;;;;;;;eAaG;YACH,IAAI,WAAW,GAAG,QAAQ;gBAAE,SAAS;YAErC,GAAG,CAAC,IAAI,CAAC;gBACP,KAAK;gBACL,KAAK,EAAE,OAAO;gBACd,SAAS,EAAE,KAAK,CAAC,WAAW;gBAC5B,QAAQ,EAAE,IAAI,CAAC,MAAM;gBACrB,KAAK;gBACL,aAAa,EAAE,SAAS,GAAG,IAAI,CAAC,MAAM;gBACtC,aAAa,EAAE,SAAS,GAAG,IAAI,CAAC,MAAM;gBACtC,cAAc;gBACd,QAAQ;gBACR,OAAO;gBACP,SAAS;gBACT,WAAW;aACZ,CAAC,CAAC;QACL,CAAC;QAEC,OAAO,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC;IACvD,CAAC,CAAC;IAEF,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC;AACzB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,kBAAkB,CAChC,OAA+B,EAC/B,QAAgB,EAChB,OAA4B;IAE5B,MAAM,OAAO,GAAG,yBAAyB,CAAC,OAAO,CAAC,CAAC;IACnD,KAAK,MAAM,MAAM,IAAI,OAAO;QAAE,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAClD,OAAO,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;AAClC,CAAC"}

package/dist/csv.d.ts

@@ -0,0 +1,61 @@
+import type { UsageProfileReport } from './usage.js';
+/**
+ * The profile as a spreadsheet.
+ *
+ * ## Why a file format is a feature
+ *
+ * The terminal report is read once and closed. The people who decide what a
+ * workload is allowed to cost live in spreadsheets, and handing them a
+ * screenshot of a terminal is how a finding stops at the person who ran the
+ * command. `--json` is for machines; this is for the pivot table that gets
+ * shown to whoever signs off the bill.
+ *
+ * ## One row per label and model, and no total row
+ *
+ * `byLabelAndModel` is the grouping a decision is actually made at — routing
+ * `classify` to a cheaper model is a question about one label's calls to one
+ * model — so it is the grain of the file.
+ *
+ * **There is deliberately no TOTAL row.** A total inside a data file is the
+ * oldest spreadsheet trap there is: somebody sums the column, the total row is
+ * included, and every figure downstream is exactly twice what it should be.
+ * The sum of this file is the bill, and a spreadsheet can compute it.
+ *
+ * ## Unpriced models get empty cells, never zeros
+ *
+ * A model the catalogue does not know has real tokens and unknown dollars.
+ * Writing `0` there would be a claim — that those calls were free — and it
+ * would survive into every chart built on the file. An empty cell is the
+ * absence it actually is, and spreadsheets already know how to skip one.
+ */
+/** Columns, in order. Exported so a test can pin the header rather than a string. */
+export declare const PROFILE_CSV_COLUMNS: readonly ['label', 'model', 'calls', 'input_tokens', 'cache_read_tokens', 'cache_write_tokens', 'output_tokens', 'input_usd', 'cache_read_usd', 'cache_write_usd', 'output_usd', 'total_usd'];
+export interface ProfileCsvOptions {
+    /** What to call the bucket for calls carrying no label. */
+    unlabelled: string;
+    /**
+     * Which table to write.
+     *
+     * `slice` is one row per label and model — the grain a routing or budget
+     * decision is made at. `day` and `hour` are the time series, which is what
+     * a spreadsheet gets asked to chart; keeping them behind a choice rather
+     * than in extra columns means every file has one row shape, and a
+     * spreadsheet that has to filter before it can sum is a spreadsheet
+     * somebody sums wrong.
+     */
+    shape?: ProfileCsvShape;
+}
+export type ProfileCsvShape = 'slice' | 'day' | 'hour';
+/** Columns for the per-day series. */
+export declare const PROFILE_CSV_DAY_COLUMNS: readonly ['day', 'usd', 'calls', 'top_label', 'top_label_usd'];
+/** Columns for the per-hour-of-UTC-day series. */
+export declare const PROFILE_CSV_HOUR_COLUMNS: readonly ['hour_utc', 'usd', 'calls'];
+/**
+ * The report as CSV text, one row per label and model.
+ *
+ * Rows arrive in the report's own order — largest bill first — because a
+ * spreadsheet can re-sort and a reader opening the file should see the
+ * expensive workload at the top either way.
+ */
+export declare function profileToCsv(report: UsageProfileReport, options: ProfileCsvOptions): string;
+//# sourceMappingURL=csv.d.ts.map

package/dist/csv.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"csv.d.ts","sourceRoot":"","sources":["../src/csv.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAErD;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAEH,qFAAqF;AACrF,eAAO,MAAM,mBAAmB,YAC9B,OAAO,EACP,OAAO,EACP,OAAO,EACP,cAAc,EACd,mBAAmB,EACnB,oBAAoB,EACpB,eAAe,EACf,WAAW,EACX,gBAAgB,EAChB,iBAAiB,EACjB,YAAY,EACZ,WAAW,CACH,CAAC;AAwBX,MAAM,WAAW,iBAAiB;IAChC,2DAA2D;IAC3D,UAAU,EAAE,MAAM,CAAC;IACnB;;;;;;;;;OASG;IACH,KAAK,CAAC,EAAE,eAAe,CAAC;CACzB;AAED,MAAM,MAAM,eAAe,GAAG,OAAO,GAAG,KAAK,GAAG,MAAM,CAAC;AAEvD,sCAAsC;AACtC,eAAO,MAAM,uBAAuB,YAAI,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,WAAW,EAAE,eAAe,CAAU,CAAC;AAEtG,kDAAkD;AAClD,eAAO,MAAM,wBAAwB,YAAI,UAAU,EAAE,KAAK,EAAE,OAAO,CAAU,CAAC;AAE9E;;;;;;GAMG;AACH,wBAAgB,YAAY,CAAC,MAAM,EAAE,kBAAkB,EAAE,OAAO,EAAE,iBAAiB,GAAG,MAAM,CAiF3F"}

package/dist/csv.js

@@ -0,0 +1,149 @@
+import { UNLABELLED } from './usage.js';
+/**
+ * The profile as a spreadsheet.
+ *
+ * ## Why a file format is a feature
+ *
+ * The terminal report is read once and closed. The people who decide what a
+ * workload is allowed to cost live in spreadsheets, and handing them a
+ * screenshot of a terminal is how a finding stops at the person who ran the
+ * command. `--json` is for machines; this is for the pivot table that gets
+ * shown to whoever signs off the bill.
+ *
+ * ## One row per label and model, and no total row
+ *
+ * `byLabelAndModel` is the grouping a decision is actually made at — routing
+ * `classify` to a cheaper model is a question about one label's calls to one
+ * model — so it is the grain of the file.
+ *
+ * **There is deliberately no TOTAL row.** A total inside a data file is the
+ * oldest spreadsheet trap there is: somebody sums the column, the total row is
+ * included, and every figure downstream is exactly twice what it should be.
+ * The sum of this file is the bill, and a spreadsheet can compute it.
+ *
+ * ## Unpriced models get empty cells, never zeros
+ *
+ * A model the catalogue does not know has real tokens and unknown dollars.
+ * Writing `0` there would be a claim — that those calls were free — and it
+ * would survive into every chart built on the file. An empty cell is the
+ * absence it actually is, and spreadsheets already know how to skip one.
+ */
+/** Columns, in order. Exported so a test can pin the header rather than a string. */
+export const PROFILE_CSV_COLUMNS = [
+    'label',
+    'model',
+    'calls',
+    'input_tokens',
+    'cache_read_tokens',
+    'cache_write_tokens',
+    'output_tokens',
+    'input_usd',
+    'cache_read_usd',
+    'cache_write_usd',
+    'output_usd',
+    'total_usd',
+];
+/**
+ * One CSV field, RFC 4180.
+ *
+ * Labels are arbitrary strings out of somebody's log: a label containing a
+ * comma would shift every column after it, and one containing a quote would
+ * break the row it sits in. Both are quoted here rather than sanitised,
+ * because changing the label would make the file disagree with every other
+ * rendering about what the workload is called.
+ *
+ * A leading `=`, `+`, `-` or `@` is prefixed with an apostrophe: those are
+ * how a spreadsheet is told a cell is a formula, and a label out of a log is
+ * data. This is the one place a value is altered, and it is altered to stop
+ * a log from executing anything when the file is opened.
+ */
+function field(value) {
+    const guarded = /^[=+\-@\t\r]/.test(value) ? `'${value}` : value;
+    return /[",\n\r]/.test(guarded) ? `"${guarded.replace(/"/g, '""')}"` : guarded;
+}
+/** A dollar figure with enough places to survive being summed. */
+const usd = (value) => value.toFixed(6);
+/** Columns for the per-day series. */
+export const PROFILE_CSV_DAY_COLUMNS = ['day', 'usd', 'calls', 'top_label', 'top_label_usd'];
+/** Columns for the per-hour-of-UTC-day series. */
+export const PROFILE_CSV_HOUR_COLUMNS = ['hour_utc', 'usd', 'calls'];
+/**
+ * The report as CSV text, one row per label and model.
+ *
+ * Rows arrive in the report's own order — largest bill first — because a
+ * spreadsheet can re-sort and a reader opening the file should see the
+ * expensive workload at the top either way.
+ */
+export function profileToCsv(report, options) {
+    if (options.shape === 'day') {
+        const rows = [PROFILE_CSV_DAY_COLUMNS.join(',')];
+        for (const day of report.spendByDay) {
+            rows.push([
+                day.day,
+                usd(day.usd),
+                String(day.calls),
+                // A day whose calls carried no label at all has no top label, and an
+                // empty cell is that absence. Naming the unlabelled bucket here
+                // would claim a label the log never carried.
+                day.topLabel === null
+                    ? ''
+                    : field(day.topLabel === UNLABELLED ? options.unlabelled : day.topLabel),
+                day.topLabel === null ? '' : usd(day.topLabelUsd),
+            ].join(','));
+        }
+        return `${rows.join('\n')}\n`;
+    }
+    if (options.shape === 'hour') {
+        const rows = [PROFILE_CSV_HOUR_COLUMNS.join(',')];
+        for (const hour of report.spendByHour) {
+            rows.push([String(hour.hour), usd(hour.usd), String(hour.calls)].join(','));
+        }
+        return `${rows.join('\n')}\n`;
+    }
+    const rows = [PROFILE_CSV_COLUMNS.join(',')];
+    for (const { label, model, breakdown } of report.byLabelAndModel) {
+        rows.push([
+            field(label === UNLABELLED ? options.unlabelled : label),
+            field(model),
+            String(breakdown.calls),
+            String(breakdown.inputTokens),
+            String(breakdown.cacheReadTokens),
+            String(breakdown.cacheWriteTokens),
+            String(breakdown.outputTokens),
+            usd(breakdown.inputUsd),
+            usd(breakdown.cacheReadUsd),
+            usd(breakdown.cacheWriteUsd),
+            usd(breakdown.outputUsd),
+            usd(breakdown.totalUsd),
+        ].join(','));
+    }
+    /**
+     * The unpriced calls, with their tokens and no dollars.
+     *
+     * They are absent from `byLabelAndModel` — which holds what could be priced
+     * — and leaving them out of the file entirely would make its token columns
+     * disagree with the log. `byModel` keeps them, so they are recovered from
+     * there, with empty dollar cells rather than zeros.
+     */
+    for (const model of report.unpricedModels) {
+        const row = report.byModel.find((entry) => entry.model === model);
+        if (!row)
+            continue;
+        rows.push([
+            field(options.unlabelled),
+            field(model),
+            String(row.breakdown.calls),
+            String(row.breakdown.inputTokens),
+            String(row.breakdown.cacheReadTokens),
+            String(row.breakdown.cacheWriteTokens),
+            String(row.breakdown.outputTokens),
+            '',
+            '',
+            '',
+            '',
+            '',
+        ].join(','));
+    }
+    return `${rows.join('\n')}\n`;
+}
+//# sourceMappingURL=csv.js.map

package/dist/csv.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"csv.js","sourceRoot":"","sources":["../src/csv.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAGxC;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAEH,qFAAqF;AACrF,MAAM,CAAC,MAAM,mBAAmB,GAAG;IACjC,OAAO;IACP,OAAO;IACP,OAAO;IACP,cAAc;IACd,mBAAmB;IACnB,oBAAoB;IACpB,eAAe;IACf,WAAW;IACX,gBAAgB;IAChB,iBAAiB;IACjB,YAAY;IACZ,WAAW;CACH,CAAC;AAEX;;;;;;;;;;;;;GAaG;AACH,SAAS,KAAK,CAAC,KAAa;IAC1B,MAAM,OAAO,GAAG,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;IACjE,OAAO,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC;AACjF,CAAC;AAED,kEAAkE;AAClE,MAAM,GAAG,GAAG,CAAC,KAAa,EAAU,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;AAoBxD,sCAAsC;AACtC,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,WAAW,EAAE,eAAe,CAAU,CAAC;AAEtG,kDAAkD;AAClD,MAAM,CAAC,MAAM,wBAAwB,GAAG,CAAC,UAAU,EAAE,KAAK,EAAE,OAAO,CAAU,CAAC;AAE9E;;;;;;GAMG;AACH,MAAM,UAAU,YAAY,CAAC,MAA0B,EAAE,OAA0B;IACjF,IAAI,OAAO,CAAC,KAAK,KAAK,KAAK,EAAE,CAAC;QAC5B,MAAM,IAAI,GAAa,CAAC,uBAAuB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QAC3D,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;YACpC,IAAI,CAAC,IAAI,CACP;gBACE,GAAG,CAAC,GAAG;gBACP,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC;gBACZ,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC;gBACjB,qEAAqE;gBACrE,gEAAgE;gBAChE,6CAA6C;gBAC7C,GAAG,CAAC,QAAQ,KAAK,IAAI;oBACnB,CAAC,CAAC,EAAE;oBACJ,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,KAAK,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC;gBAC1E,GAAG,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,CAAC;aAClD,CAAC,IAAI,CAAC,GAAG,CAAC,CACZ,CAAC;QACJ,CAAC;QACD,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;IAChC,CAAC;IAED,IAAI,OAAO,CAAC,KAAK,KAAK,MAAM,EAAE,CAAC;QAC7B,MAAM,IAAI,GAAa,CAAC,wBAAwB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QAC5D,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;YACtC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QAC9E,CAAC;QACD,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;IAChC,CAAC;IAED,MAAM,IAAI,GAAa,CAAC,mBAAmB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IAEvD,KAAK,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,MAAM,CAAC,eAAe,EAAE,CAAC;QACjE,IAAI,CAAC,IAAI,CACP;YACE,KAAK,CAAC,KAAK,KAAK,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC;YACxD,KAAK,CAAC,KAAK,CAAC;YACZ,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC;YACvB,MAAM,CAAC,SAAS,CAAC,WAAW,CAAC;YAC7B,MAAM,CAAC,SAAS,CAAC,eAAe,CAAC;YACjC,MAAM,CAAC,SAAS,CAAC,gBAAgB,CAAC;YAClC,MAAM,CAAC,SAAS,CAAC,YAAY,CAAC;YAC9B,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC;YACvB,GAAG,CAAC,SAAS,CAAC,YAAY,CAAC;YAC3B,GAAG,CAAC,SAAS,CAAC,aAAa,CAAC;YAC5B,GAAG,CAAC,SAAS,CAAC,SAAS,CAAC;YACxB,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC;SACxB,CAAC,IAAI,CAAC,GAAG,CAAC,CACZ,CAAC;IACJ,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,cAAc,EAAE,CAAC;QAC1C,MAAM,GAAG,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,KAAK,CAAC,CAAC;QAClE,IAAI,CAAC,GAAG;YAAE,SAAS;QACnB,IAAI,CAAC,IAAI,CACP;YACE,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC;YACzB,KAAK,CAAC,KAAK,CAAC;YACZ,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC;YAC3B,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,WAAW,CAAC;YACjC,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,eAAe,CAAC;YACrC,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,gBAAgB,CAAC;YACtC,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,YAAY,CAAC;YAClC,EAAE;YACF,EAAE;YACF,EAAE;YACF,EAAE;YACF,EAAE;SACH,CAAC,IAAI,CAAC,GAAG,CAAC,CACZ,CAAC;IACJ,CAAC;IAED,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;AAChC,CAAC"}

package/dist/evaluate.d.ts

@@ -35,6 +35,16 @@ export type EvalVerdict = 'indistinguishable' | 'within-noise' | 'diverges' | 'i
 export interface EvalReport {
     provider: string;
     model: string;
+    /**
+     * The model the candidate answer came from.
+     *
+     * Equal to `model` on the ordinary comparison — same model, two prompts. It
+     * differs when the question is the other one: **same prompt, two models**, which
+     * is what a routing decision is. `profile` prices that route exactly and can say
+     * nothing at all about whether the cheaper model still does the job; this is the
+     * measurement that can.
+     */
+    candidateModel: string;
     cases: EvalCase[];
     /** Mean agreement of the original prompt with itself. The yardstick. */
     selfAgreement: number;
@@ -51,6 +61,20 @@ export interface EvaluateOptions {
      * call already paid for.
      */
     concurrency?: number;
+    /**
+     * Where the candidate answer comes from. Defaults to `provider`.
+     *
+     * This is the whole routing axis, and it needed no new yardstick. The baseline
+     * prompt is still run **twice on the original model** to measure that model's own
+     * variance, and the candidate is still judged against it — so the question
+     * becomes "does the cheaper model agree with the expensive one more closely than
+     * the expensive one agrees with itself?", which is the honest form of "is this
+     * route safe".
+     *
+     * A verdict built any other way would be a threshold somebody picked. This one is
+     * the model's own noise floor, measured on the same cases in the same run.
+     */
+    candidateProvider?: LlmProvider;
 }
 /**
  * Builds the prompt for one case.

package/dist/evaluate.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"evaluate.d.ts","sourceRoot":"","sources":["../src/evaluate.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAE9C;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,MAAM,WAAW,QAAQ;IACvB,4CAA4C;IAC5C,KAAK,EAAE,MAAM,CAAC;IACd,2EAA2E;IAC3E,QAAQ,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC3B,qCAAqC;IACrC,SAAS,EAAE,MAAM,CAAC;IAClB,yDAAyD;IACzD,cAAc,EAAE,MAAM,CAAC;IACvB,2EAA2E;IAC3E,eAAe,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,MAAM,WAAW,GAAG,mBAAmB,GAAG,cAAc,GAAG,UAAU,GAAG,cAAc,CAAC;AAE7F,MAAM,WAAW,UAAU;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,QAAQ,EAAE,CAAC;IAClB,wEAAwE;IACxE,aAAa,EAAE,MAAM,CAAC;IACtB,oEAAoE;IACpE,cAAc,EAAE,MAAM,CAAC;IACvB,OAAO,EAAE,WAAW,CAAC;IACrB,kEAAkE;IAClE,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,eAAe;IAC9B;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;;;;;GAMG;AACH,wBAAgB,UAAU,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAMhE;AAED,0CAA0C;AAC1C;;;;;;;;GAQG;AACH,wBAAgB,SAAS,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,CAKtD;AAKD;;;;;;;;;;;GAWG;AACH,wBAAgB,UAAU,CAAC,aAAa,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,GAAG,WAAW,CAOrF;AAED,sEAAsE;AACtE,wBAAsB,MAAM,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC,CAc3F;AAED;;;;;;;GAOG;AACH,wBAAsB,QAAQ,CAC5B,cAAc,EAAE,MAAM,EACtB,eAAe,EAAE,MAAM,EACvB,MAAM,EAAE,SAAS,MAAM,EAAE,EACzB,QAAQ,EAAE,WAAW,EACrB,OAAO,GAAE,eAAoB,GAC5B,OAAO,CAAC,UAAU,CAAC,CAsCrB"}
+{"version":3,"file":"evaluate.d.ts","sourceRoot":"","sources":["../src/evaluate.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAE9C;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,MAAM,WAAW,QAAQ;IACvB,4CAA4C;IAC5C,KAAK,EAAE,MAAM,CAAC;IACd,2EAA2E;IAC3E,QAAQ,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC3B,qCAAqC;IACrC,SAAS,EAAE,MAAM,CAAC;IAClB,yDAAyD;IACzD,cAAc,EAAE,MAAM,CAAC;IACvB,2EAA2E;IAC3E,eAAe,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,MAAM,WAAW,GAAG,mBAAmB,GAAG,cAAc,GAAG,UAAU,GAAG,cAAc,CAAC;AAE7F,MAAM,WAAW,UAAU;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd;;;;;;;;OAQG;IACH,cAAc,EAAE,MAAM,CAAC;IACvB,KAAK,EAAE,QAAQ,EAAE,CAAC;IAClB,wEAAwE;IACxE,aAAa,EAAE,MAAM,CAAC;IACtB,oEAAoE;IACpE,cAAc,EAAE,MAAM,CAAC;IACvB,OAAO,EAAE,WAAW,CAAC;IACrB,kEAAkE;IAClE,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,eAAe;IAC9B;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;;;;;;;;;OAYG;IACH,iBAAiB,CAAC,EAAE,WAAW,CAAC;CACjC;AAED;;;;;;GAMG;AACH,wBAAgB,UAAU,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAMhE;AAED,0CAA0C;AAC1C;;;;;;;;GAQG;AACH,wBAAgB,SAAS,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,CAKtD;AAKD;;;;;;;;;;;GAWG;AACH,wBAAgB,UAAU,CAAC,aAAa,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,GAAG,WAAW,CAOrF;AAED,sEAAsE;AACtE,wBAAsB,MAAM,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC,CAc3F;AAED;;;;;;;GAOG;AACH,wBAAsB,QAAQ,CAC5B,cAAc,EAAE,MAAM,EACtB,eAAe,EAAE,MAAM,EACvB,MAAM,EAAE,SAAS,MAAM,EAAE,EACzB,QAAQ,EAAE,WAAW,EACrB,OAAO,GAAE,eAAoB,GAC5B,OAAO,CAAC,UAAU,CAAC,CA4CrB"}

package/dist/evaluate.js

@@ -79,14 +79,16 @@ export async function pooled(tasks, limit) {
  */
 export async function evaluate(originalPrompt, optimizedPrompt, inputs, provider, options = {}) {
     const concurrency = Math.max(1, options.concurrency ?? 3);
-    const run = (prompt, input) => provider.complete({ system: fillPrompt(prompt, input), user: input });
+    const candidate = options.candidateProvider ?? provider;
+    const run = (prompt, input, on = provider) => on.complete({ system: fillPrompt(prompt, input), user: input });
     const cases = await pooled(inputs.map((input) => async () => {
         // Sequential within a case: the two baseline runs exist to measure the
         // model's variance, and issuing them together invites a provider to
         // serve one from a cache and report a variance of zero.
         const baselineA = await run(originalPrompt, input);
         const baselineB = await run(originalPrompt, input);
-        const optimized = await run(optimizedPrompt, input);
+        // On `candidate`, which is `provider` unless a route is being measured.
+        const optimized = await run(optimizedPrompt, input, candidate);
         return {
             input,
             baseline: [baselineA, baselineB],
@@ -100,6 +102,7 @@ export async function evaluate(originalPrompt, optimizedPrompt, inputs, provider
     return {
         provider: provider.name,
         model: provider.model,
+        candidateModel: candidate.model,
         cases,
         selfAgreement,
         crossAgreement,

package/dist/evaluate.js.map

@@ -1 +1 @@
-{"version":3,"file":"evaluate.js","sourceRoot":"","sources":["../src/evaluate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AA6D/D;;;;;;GAMG;AACH,MAAM,UAAU,UAAU,CAAC,MAAc,EAAE,KAAa;IACtD,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,CACtC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,WAAW,IAAI,CAAC,CAAC,UAAU,KAAK,aAAa,CAChE,CAAC;IACF,IAAI,CAAC,WAAW;QAAE,OAAO,GAAG,MAAM,CAAC,OAAO,EAAE,OAAO,KAAK,EAAE,CAAC;IAC3D,OAAO,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;AACjD,CAAC;AAED,0CAA0C;AAC1C;;;;;;;;GAQG;AACH,MAAM,UAAU,SAAS,CAAC,CAAS,EAAE,CAAS;IAC5C,MAAM,IAAI,GAAG,mBAAmB,CAAC,CAAC,CAAC,CAAC;IACpC,MAAM,KAAK,GAAG,mBAAmB,CAAC,CAAC,CAAC,CAAC;IACrC,IAAI,IAAI,KAAK,KAAK;QAAE,OAAO,CAAC,CAAC;IAC7B,OAAO,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;AAC9B,CAAC;AAED,MAAM,IAAI,GAAG,CAAC,MAAgB,EAAU,EAAE,CACxC,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;AAElF;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,UAAU,CAAC,aAAqB,EAAE,cAAsB;IACtE,IAAI,cAAc,IAAI,KAAK;QAAE,OAAO,mBAAmB,CAAC;IACxD,IAAI,aAAa,GAAG,GAAG;QAAE,OAAO,cAAc,CAAC;IAC/C,0EAA0E;IAC1E,kCAAkC;IAClC,IAAI,cAAc,IAAI,aAAa,GAAG,IAAI;QAAE,OAAO,cAAc,CAAC;IAClE,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,sEAAsE;AACtE,MAAM,CAAC,KAAK,UAAU,MAAM,CAAI,KAA8B,EAAE,KAAa;IAC3E,MAAM,OAAO,GAAG,IAAI,KAAK,CAAI,KAAK,CAAC,MAAM,CAAC,CAAC;IAC3C,IAAI,IAAI,GAAG,CAAC,CAAC;IAEb,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,KAAK,IAAI,EAAE;QAC/E,SAAS,CAAC;YACR,MAAM,KAAK,GAAG,IAAI,EAAE,CAAC;YACrB,IAAI,KAAK,IAAI,KAAK,CAAC,MAAM;gBAAE,OAAO;YAClC,OAAO,CAAC,KAAK,CAAC,GAAG,MAAM,KAAK,CAAC,KAAK,CAAE,EAAE,CAAC;QACzC,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAC3B,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,cAAsB,EACtB,eAAuB,EACvB,MAAyB,EACzB,QAAqB,EACrB,OAAO,GAAoB,EAAE;IAE7B,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;IAE1D,MAAM,GAAG,GAAG,CAAC,MAAc,EAAE,KAAa,EAAmB,EAAE,CAC7D,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IAExE,MAAM,KAAK,GAAG,MAAM,MAAM,CACxB,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,IAAuB,EAAE;QAClD,uEAAuE;QACvE,oEAAoE;QACpE,wDAAwD;QACxD,MAAM,SAAS,GAAG,MAAM,GAAG,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC;QACnD,MAAM,SAAS,GAAG,MAAM,GAAG,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC;QACnD,MAAM,SAAS,GAAG,MAAM,GAAG,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC;QAEpD,OAAO;YACL,KAAK;YACL,QAAQ,EAAE,CAAC,SAAS,EAAE,SAAS,CAAC;YAChC,SAAS;YACT,cAAc,EAAE,SAAS,CAAC,SAAS,EAAE,SAAS,CAAC;YAC/C,eAAe,EAAE,SAAS,CAAC,SAAS,EAAE,SAAS,CAAC;SACjD,CAAC;IACJ,CAAC,CAAC,EACF,WAAW,CACZ,CAAC;IAEF,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC;IAC/D,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC;IAEjE,OAAO;QACL,QAAQ,EAAE,QAAQ,CAAC,IAAI;QACvB,KAAK,EAAE,QAAQ,CAAC,KAAK;QACrB,KAAK;QACL,aAAa;QACb,cAAc;QACd,OAAO,EAAE,UAAU,CAAC,aAAa,EAAE,cAAc,CAAC;QAClD,SAAS,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC;KAC5B,CAAC;AACJ,CAAC"}
+{"version":3,"file":"evaluate.js","sourceRoot":"","sources":["../src/evaluate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAqF/D;;;;;;GAMG;AACH,MAAM,UAAU,UAAU,CAAC,MAAc,EAAE,KAAa;IACtD,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,CACtC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,WAAW,IAAI,CAAC,CAAC,UAAU,KAAK,aAAa,CAChE,CAAC;IACF,IAAI,CAAC,WAAW;QAAE,OAAO,GAAG,MAAM,CAAC,OAAO,EAAE,OAAO,KAAK,EAAE,CAAC;IAC3D,OAAO,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;AACjD,CAAC;AAED,0CAA0C;AAC1C;;;;;;;;GAQG;AACH,MAAM,UAAU,SAAS,CAAC,CAAS,EAAE,CAAS;IAC5C,MAAM,IAAI,GAAG,mBAAmB,CAAC,CAAC,CAAC,CAAC;IACpC,MAAM,KAAK,GAAG,mBAAmB,CAAC,CAAC,CAAC,CAAC;IACrC,IAAI,IAAI,KAAK,KAAK;QAAE,OAAO,CAAC,CAAC;IAC7B,OAAO,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;AAC9B,CAAC;AAED,MAAM,IAAI,GAAG,CAAC,MAAgB,EAAU,EAAE,CACxC,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;AAElF;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,UAAU,CAAC,aAAqB,EAAE,cAAsB;IACtE,IAAI,cAAc,IAAI,KAAK;QAAE,OAAO,mBAAmB,CAAC;IACxD,IAAI,aAAa,GAAG,GAAG;QAAE,OAAO,cAAc,CAAC;IAC/C,0EAA0E;IAC1E,kCAAkC;IAClC,IAAI,cAAc,IAAI,aAAa,GAAG,IAAI;QAAE,OAAO,cAAc,CAAC;IAClE,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,sEAAsE;AACtE,MAAM,CAAC,KAAK,UAAU,MAAM,CAAI,KAA8B,EAAE,KAAa;IAC3E,MAAM,OAAO,GAAG,IAAI,KAAK,CAAI,KAAK,CAAC,MAAM,CAAC,CAAC;IAC3C,IAAI,IAAI,GAAG,CAAC,CAAC;IAEb,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,KAAK,IAAI,EAAE;QAC/E,SAAS,CAAC;YACR,MAAM,KAAK,GAAG,IAAI,EAAE,CAAC;YACrB,IAAI,KAAK,IAAI,KAAK,CAAC,MAAM;gBAAE,OAAO;YAClC,OAAO,CAAC,KAAK,CAAC,GAAG,MAAM,KAAK,CAAC,KAAK,CAAE,EAAE,CAAC;QACzC,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAC3B,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,cAAsB,EACtB,eAAuB,EACvB,MAAyB,EACzB,QAAqB,EACrB,OAAO,GAAoB,EAAE;IAE7B,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;IAC1D,MAAM,SAAS,GAAG,OAAO,CAAC,iBAAiB,IAAI,QAAQ,CAAC;IAExD,MAAM,GAAG,GAAG,CACV,MAAc,EACd,KAAa,EACb,EAAE,GAAgB,QAAQ,EACT,EAAE,CAAC,EAAE,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IAEtF,MAAM,KAAK,GAAG,MAAM,MAAM,CACxB,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,IAAuB,EAAE;QAClD,uEAAuE;QACvE,oEAAoE;QACpE,wDAAwD;QACxD,MAAM,SAAS,GAAG,MAAM,GAAG,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC;QACnD,MAAM,SAAS,GAAG,MAAM,GAAG,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC;QACnD,wEAAwE;QACxE,MAAM,SAAS,GAAG,MAAM,GAAG,CAAC,eAAe,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC;QAE/D,OAAO;YACL,KAAK;YACL,QAAQ,EAAE,CAAC,SAAS,EAAE,SAAS,CAAC;YAChC,SAAS;YACT,cAAc,EAAE,SAAS,CAAC,SAAS,EAAE,SAAS,CAAC;YAC/C,eAAe,EAAE,SAAS,CAAC,SAAS,EAAE,SAAS,CAAC;SACjD,CAAC;IACJ,CAAC,CAAC,EACF,WAAW,CACZ,CAAC;IAEF,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC;IAC/D,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC;IAEjE,OAAO;QACL,QAAQ,EAAE,QAAQ,CAAC,IAAI;QACvB,KAAK,EAAE,QAAQ,CAAC,KAAK;QACrB,cAAc,EAAE,SAAS,CAAC,KAAK;QAC/B,KAAK;QACL,aAAa;QACb,cAAc;QACd,OAAO,EAAE,UAAU,CAAC,aAAa,EAAE,cAAc,CAAC;QAClD,SAAS,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC;KAC5B,CAAC;AACJ,CAAC"}

package/dist/i18n/en.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"en.d.ts","sourceRoot":"","sources":["../../src/i18n/en.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAI/C,iFAAiF;AACjF,eAAO,MAAM,EAAE,EAAE,YA0OhB,CAAC"}
+{"version":3,"file":"en.d.ts","sourceRoot":"","sources":["../../src/i18n/en.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAI/C,iFAAiF;AACjF,eAAO,MAAM,EAAE,EAAE,YAuPhB,CAAC"}