jevkit-drift 0.1.0

package/README.md

@@ -0,0 +1,70 @@
+# jevkit-drift
+
+TypeSafe's docs warn that `jev-latest` moves when a new version ships, and that
+confidence thresholds tuned against one version do not automatically hold on the
+next. `jevkit-drift` answers the question that warning implies: for the requests
+you actually care about, what changed?
+
+> Unofficial and unaffiliated with TypeSafe.
+
+```bash
+npm install jevkit-drift
+```
+
+## Two kinds of change
+
+A **flip** is a changed decision. Your code takes a different branch. This is
+what breaks things.
+
+A **shift** is movement in the probability distribution with the same decision
+still on top. Harmless on its own, but it is what walks an answer toward a
+threshold, so a large shift is an early warning before anything has flipped.
+
+Both are reported separately, because conflating them is how a drift report ends
+up either too noisy to read or too quiet to help.
+
+## Use
+
+```ts
+import { readRecords, writeRecords } from "jevkit-core";
+import { compareSets, replay } from "jevkit-drift";
+
+const baseline = readRecords("golden.jevl");
+const candidate = await replay(baseline, (s, q) =>
+  client.systemOne(s, q, { model: "jev-1.14.0" }),
+);
+writeRecords("candidate.jevl", candidate);
+
+const report = compareSets(baseline, candidate);
+console.log(report.summary());
+
+for (const [requestId, delta] of report.flips) {
+  console.log(requestId, delta.describe());
+}
+```
+
+## CLI
+
+```bash
+npx jevkit-drift golden.jevl candidate.jevl
+npx jevkit-drift golden.jevl candidate.jevl --max-flips 3
+npx jevkit-drift golden.jevl candidate.jevl --max-shift 0.15
+npx jevkit-drift golden.jevl candidate.jevl --format json
+```
+
+Exit codes: `0` within tolerance, `1` drifted, `2` bad usage.
+
+## How records are paired
+
+On `request_id`, the digest of state plus questions with the model deliberately
+excluded. That is the whole reason the record format keeps two digests: the full
+`id` changes when the model changes, so it cannot pair a baseline with its
+replay, while `request_id` can.
+
+Change the state or a question and the records stop pairing, which is correct.
+They are no longer the same request, and comparing them would be meaningless.
+The CLI says so rather than reporting zero drift.
+
+## License
+
+MIT

package/dist/cli.d.ts

@@ -0,0 +1,7 @@
+#!/usr/bin/env node
+/** `jevkit-drift` command line interface. */
+export declare const EXIT_OK = 0;
+export declare const EXIT_DRIFTED = 1;
+export declare const EXIT_USAGE = 2;
+export declare function main(argv?: string[]): number;
+//# sourceMappingURL=cli.d.ts.map

package/dist/cli.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA,6CAA6C;AAM7C,eAAO,MAAM,OAAO,IAAI,CAAC;AACzB,eAAO,MAAM,YAAY,IAAI,CAAC;AAC9B,eAAO,MAAM,UAAU,IAAI,CAAC;AAY5B,wBAAgB,IAAI,CAAC,IAAI,GAAE,MAAM,EAA0B,GAAG,MAAM,CAoEnE"}

package/dist/cli.js

@@ -0,0 +1,104 @@
+#!/usr/bin/env node
+/** `jevkit-drift` command line interface. */
+import { RecordFormatError, readRecords } from "jevkit-core";
+import { compareSets } from "./compare.js";
+export const EXIT_OK = 0;
+export const EXIT_DRIFTED = 1;
+export const EXIT_USAGE = 2;
+const USAGE = `usage: jevkit-drift <baseline.jevl> <candidate.jevl> [options]
+
+Compare two .jevl runs of the same requests and report which decisions flipped.
+Never calls the API.
+
+  --max-flips N    tolerated number of flipped decisions (default 0)
+  --max-shift F    fail if any distribution moves more than this (0..1)
+  --format text|json
+  --quiet          only print the summary`;
+export function main(argv = process.argv.slice(2)) {
+    const files = [];
+    let maxFlips = 0;
+    let maxShift = null;
+    let format = "text";
+    let quiet = false;
+    for (let i = 0; i < argv.length; i++) {
+        const arg = argv[i];
+        switch (arg) {
+            case "--max-flips":
+                maxFlips = Number(argv[++i]);
+                break;
+            case "--max-shift":
+                maxShift = Number(argv[++i]);
+                break;
+            case "--format": {
+                const v = argv[++i];
+                if (v !== "text" && v !== "json") {
+                    console.error("jevkit-drift: --format expects text or json");
+                    return EXIT_USAGE;
+                }
+                format = v;
+                break;
+            }
+            case "--quiet":
+                quiet = true;
+                break;
+            case "-h":
+            case "--help":
+                console.log(USAGE);
+                return EXIT_OK;
+            default:
+                if (arg.startsWith("--")) {
+                    console.error(`jevkit-drift: unknown option ${arg}`);
+                    return EXIT_USAGE;
+                }
+                files.push(arg);
+        }
+    }
+    if (files.length !== 2) {
+        console.error(USAGE);
+        return EXIT_USAGE;
+    }
+    let before, after;
+    try {
+        before = readRecords(files[0]);
+        after = readRecords(files[1]);
+    }
+    catch (err) {
+        const message = err instanceof RecordFormatError ? err.message : err.message;
+        console.error(`jevkit-drift: ${message}`);
+        return EXIT_USAGE;
+    }
+    let report;
+    try {
+        report = compareSets(before, after);
+    }
+    catch (err) {
+        console.error(`jevkit-drift: ${err.message}`);
+        return EXIT_USAGE;
+    }
+    if (format === "json") {
+        console.log(JSON.stringify(report.toJSON(), null, 2));
+    }
+    else {
+        console.log(report.summary());
+        if (report.flips.length && !quiet) {
+            console.log("\nflipped decisions:");
+            for (const [requestId, delta] of report.flips) {
+                console.log(`  ${requestId.slice(7, 19)}  ${delta.describe()}`);
+            }
+        }
+    }
+    if (!report.deltas.length && (before.length || after.length)) {
+        console.error("jevkit-drift: no requests matched between the two files. Records pair on state " +
+            "and questions, so a change to either makes them incomparable.");
+        return EXIT_USAGE;
+    }
+    if (report.flips.length > maxFlips)
+        return EXIT_DRIFTED;
+    if (maxShift !== null && report.maxShift > maxShift)
+        return EXIT_DRIFTED;
+    return EXIT_OK;
+}
+const invokedDirectly = process.argv[1] !== undefined && import.meta.url === `file://${process.argv[1]}`;
+if (invokedDirectly)
+    process.exit(main());
+//# sourceMappingURL=cli.js.map

package/dist/cli.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA,6CAA6C;AAE7C,OAAO,EAAE,iBAAiB,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAE7D,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAE3C,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,CAAC;AACzB,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,CAAC;AAC9B,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,CAAC;AAE5B,MAAM,KAAK,GAAG;;;;;;;;0CAQ4B,CAAC;AAE3C,MAAM,UAAU,IAAI,CAAC,OAAiB,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;IACzD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,QAAQ,GAAkB,IAAI,CAAC;IACnC,IAAI,MAAM,GAAoB,MAAM,CAAC;IACrC,IAAI,KAAK,GAAG,KAAK,CAAC;IAElB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAE,CAAC;QACrB,QAAQ,GAAG,EAAE,CAAC;YACZ,KAAK,aAAa;gBAAE,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;gBAAC,MAAM;YACxD,KAAK,aAAa;gBAAE,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;gBAAC,MAAM;YACxD,KAAK,UAAU,CAAC,CAAC,CAAC;gBAChB,MAAM,CAAC,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;gBACpB,IAAI,CAAC,KAAK,MAAM,IAAI,CAAC,KAAK,MAAM,EAAE,CAAC;oBAAC,OAAO,CAAC,KAAK,CAAC,6CAA6C,CAAC,CAAC;oBAAC,OAAO,UAAU,CAAC;gBAAC,CAAC;gBACtH,MAAM,GAAG,CAAC,CAAC;gBAAC,MAAM;YACpB,CAAC;YACD,KAAK,SAAS;gBAAE,KAAK,GAAG,IAAI,CAAC;gBAAC,MAAM;YACpC,KAAK,IAAI,CAAC;YAAC,KAAK,QAAQ;gBAAE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;gBAAC,OAAO,OAAO,CAAC;YAC7D;gBACE,IAAI,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;oBAAC,OAAO,CAAC,KAAK,CAAC,gCAAgC,GAAG,EAAE,CAAC,CAAC;oBAAC,OAAO,UAAU,CAAC;gBAAC,CAAC;gBACtG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACpB,CAAC;IACH,CAAC;IAED,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAAC,OAAO,UAAU,CAAC;IAAC,CAAC;IAEpE,IAAI,MAAM,EAAE,KAAK,CAAC;IAClB,IAAI,CAAC;QACH,MAAM,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,CAAC;QAChC,KAAK,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,CAAC;IACjC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,OAAO,GAAG,GAAG,YAAY,iBAAiB,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAE,GAAa,CAAC,OAAO,CAAC;QACxF,OAAO,CAAC,KAAK,CAAC,iBAAiB,OAAO,EAAE,CAAC,CAAC;QAC1C,OAAO,UAAU,CAAC;IACpB,CAAC;IAED,IAAI,MAAM,CAAC;IACX,IAAI,CAAC;QACH,MAAM,GAAG,WAAW,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IACtC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,CAAC,KAAK,CAAC,iBAAkB,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC;QACzD,OAAO,UAAU,CAAC;IACpB,CAAC;IAED,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;QACtB,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IACxD,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;QAC9B,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;YAClC,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;YACpC,KAAK,MAAM,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;gBAC9C,OAAO,CAAC,GAAG,CAAC,KAAK,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,KAAK,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YAClE,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;QAC7D,OAAO,CAAC,KAAK,CACX,iFAAiF;YACjF,+DAA+D,CAChE,CAAC;QACF,OAAO,UAAU,CAAC;IACpB,CAAC;IAED,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,QAAQ;QAAE,OAAO,YAAY,CAAC;IACxD,IAAI,QAAQ,KAAK,IAAI,IAAI,MAAM,CAAC,QAAQ,GAAG,QAAQ;QAAE,OAAO,YAAY,CAAC;IACzE,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,eAAe,GACnB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,UAAU,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;AACnF,IAAI,eAAe;IAAE,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC"}

package/dist/compare.d.ts

@@ -0,0 +1,94 @@
+/**
+ * Comparing two answers to the same request.
+ *
+ * TypeSafe's docs warn that `jev-latest` moves when a new version ships, and
+ * that confidence thresholds tuned against one version do not automatically
+ * hold on the next. This module answers the question that warning implies: for
+ * a set of requests you already care about, what actually changed?
+ *
+ * Two kinds of change are tracked separately, because they mean different
+ * things:
+ *
+ * - A **flip** is a changed decision. Your code takes a different branch. This
+ *   is what breaks things.
+ * - A **shift** is movement in the probability distribution with the same
+ *   decision on top. Harmless alone, but it is what moves an answer toward a
+ *   threshold, so a large shift is an early warning before anything flips.
+ */
+import { type Answer, type Record as JevlRecord } from "jevkit-core";
+/**
+ * Total variation distance between two distributions, on 0..1.
+ *
+ * Outcomes present in one distribution and not the other count in full, which
+ * is what makes this meaningful when a new model version changes the option
+ * set. Distributions are not renormalized: if the API returns something that
+ * does not sum to 1, that is reported rather than hidden.
+ */
+export declare function totalVariation(a: Record<string, number>, b: Record<string, number>): number;
+export declare class QuestionDelta {
+    readonly questionId: string;
+    readonly type: string;
+    readonly before: unknown;
+    readonly after: unknown;
+    readonly distributionShift: number;
+    readonly confidenceBefore: number | null;
+    readonly confidenceAfter: number | null;
+    readonly scoreBefore: number | null;
+    readonly scoreAfter: number | null;
+    constructor(questionId: string, type: string, before: unknown, after: unknown, distributionShift: number, confidenceBefore: number | null, confidenceAfter: number | null, scoreBefore?: number | null, scoreAfter?: number | null);
+    /** The selected outcome changed. */
+    get flipped(): boolean;
+    get confidenceDelta(): number | null;
+    describe(): string;
+}
+export declare class RecordDelta {
+    readonly requestId: string;
+    readonly modelBefore: string;
+    readonly modelAfter: string;
+    readonly questions: QuestionDelta[];
+    constructor(requestId: string, modelBefore: string, modelAfter: string, questions?: QuestionDelta[]);
+    get flips(): QuestionDelta[];
+    get maxShift(): number;
+}
+export declare function compareAnswers(qid: string, before: Answer, after: Answer): QuestionDelta;
+export declare function compareRecords(before: JevlRecord, after: JevlRecord): RecordDelta;
+export declare class DriftReport {
+    readonly deltas: RecordDelta[];
+    readonly unmatchedBefore: string[];
+    readonly unmatchedAfter: string[];
+    constructor(deltas?: RecordDelta[], unmatchedBefore?: string[], unmatchedAfter?: string[]);
+    get totalQuestions(): number;
+    get flips(): Array<[string, QuestionDelta]>;
+    get flipRate(): number;
+    get maxShift(): number;
+    get meanShift(): number;
+    models(): [Set<string>, Set<string>];
+    summary(): string;
+    toJSON(): {
+        requests: number;
+        questions: number;
+        flips: number;
+        flipRate: number;
+        meanShift: number;
+        maxShift: number;
+        unmatchedBefore: string[];
+        unmatchedAfter: string[];
+        details: {
+            requestId: string;
+            questionId: string;
+            before: unknown;
+            after: unknown;
+            distributionShift: number;
+            confidenceDelta: number | null;
+        }[];
+    };
+}
+/**
+ * Pair two sets of records by request id and compare each pair.
+ *
+ * Records pair on `requestId`, the digest of state plus questions with the
+ * model excluded, which is exactly what makes a baseline comparable to its
+ * replay on a different model version.
+ */
+export declare function compareSets(before: JevlRecord[], after: JevlRecord[]): DriftReport;
+//# sourceMappingURL=compare.d.ts.map

package/dist/compare.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"compare.d.ts","sourceRoot":"","sources":["../src/compare.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EAAE,KAAK,MAAM,EAAE,KAAK,MAAM,IAAI,UAAU,EAAgB,MAAM,aAAa,CAAC;AAEnF;;;;;;;GAOG;AACH,wBAAgB,cAAc,CAC5B,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EACzB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GACxB,MAAM,CAKR;AAED,qBAAa,aAAa;IAEtB,QAAQ,CAAC,UAAU,EAAE,MAAM;IAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM;IACrB,QAAQ,CAAC,MAAM,EAAE,OAAO;IACxB,QAAQ,CAAC,KAAK,EAAE,OAAO;IACvB,QAAQ,CAAC,iBAAiB,EAAE,MAAM;IAClC,QAAQ,CAAC,gBAAgB,EAAE,MAAM,GAAG,IAAI;IACxC,QAAQ,CAAC,eAAe,EAAE,MAAM,GAAG,IAAI;IACvC,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI;IACnC,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;gBARzB,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,OAAO,EACf,KAAK,EAAE,OAAO,EACd,iBAAiB,EAAE,MAAM,EACzB,gBAAgB,EAAE,MAAM,GAAG,IAAI,EAC/B,eAAe,EAAE,MAAM,GAAG,IAAI,EAC9B,WAAW,GAAE,MAAM,GAAG,IAAW,EACjC,UAAU,GAAE,MAAM,GAAG,IAAW;IAG3C,oCAAoC;IACpC,IAAI,OAAO,IAAI,OAAO,CAErB;IAED,IAAI,eAAe,IAAI,MAAM,GAAG,IAAI,CAGnC;IAED,QAAQ,IAAI,MAAM;CAenB;AAED,qBAAa,WAAW;IAEpB,QAAQ,CAAC,SAAS,EAAE,MAAM;IAC1B,QAAQ,CAAC,WAAW,EAAE,MAAM;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM;IAC3B,QAAQ,CAAC,SAAS,EAAE,aAAa,EAAE;gBAH1B,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,SAAS,GAAE,aAAa,EAAO;IAG1C,IAAI,KAAK,IAAI,aAAa,EAAE,CAE3B;IAED,IAAI,QAAQ,IAAI,MAAM,CAErB;CACF;AAED,wBAAgB,cAAc,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,aAAa,CAkBxF;AAED,wBAAgB,cAAc,CAAC,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,UAAU,GAAG,WAAW,CA2BjF;AAED,qBAAa,WAAW;IAEpB,QAAQ,CAAC,MAAM,EAAE,WAAW,EAAE;IAC9B,QAAQ,CAAC,eAAe,EAAE,MAAM,EAAE;IAClC,QAAQ,CAAC,cAAc,EAAE,MAAM,EAAE;gBAFxB,MAAM,GAAE,WAAW,EAAO,EAC1B,eAAe,GAAE,MAAM,EAAO,EAC9B,cAAc,GAAE,MAAM,EAAO;IAGxC,IAAI,cAAc,IAAI,MAAM,CAE3B;IAED,IAAI,KAAK,IAAI,KAAK,CAAC,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC,CAE1C;IAED,IAAI,QAAQ,IAAI,MAAM,CAGrB;IAED,IAAI,QAAQ,IAAI,MAAM,CAErB;IAED,IAAI,SAAS,IAAI,MAAM,CAGtB;IAED,MAAM,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAOpC,OAAO,IAAI,MAAM;IAwBjB,MAAM;;;;;;;;;;;;;;;;;;CAoBP;AAED;;;;;;GAMG;AACH,wBAAgB,WAAW,CAAC,MAAM,EAAE,UAAU,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,WAAW,CASlF"}

package/dist/compare.js

@@ -0,0 +1,209 @@
+/**
+ * Comparing two answers to the same request.
+ *
+ * TypeSafe's docs warn that `jev-latest` moves when a new version ships, and
+ * that confidence thresholds tuned against one version do not automatically
+ * hold on the next. This module answers the question that warning implies: for
+ * a set of requests you already care about, what actually changed?
+ *
+ * Two kinds of change are tracked separately, because they mean different
+ * things:
+ *
+ * - A **flip** is a changed decision. Your code takes a different branch. This
+ *   is what breaks things.
+ * - A **shift** is movement in the probability distribution with the same
+ *   decision on top. Harmless alone, but it is what moves an answer toward a
+ *   threshold, so a large shift is an early warning before anything flips.
+ */
+import { parseAnswers } from "jevkit-core";
+/**
+ * Total variation distance between two distributions, on 0..1.
+ *
+ * Outcomes present in one distribution and not the other count in full, which
+ * is what makes this meaningful when a new model version changes the option
+ * set. Distributions are not renormalized: if the API returns something that
+ * does not sum to 1, that is reported rather than hidden.
+ */
+export function totalVariation(a, b) {
+    const keys = new Set([...Object.keys(a), ...Object.keys(b)]);
+    let sum = 0;
+    for (const k of keys)
+        sum += Math.abs((a[k] ?? 0) - (b[k] ?? 0));
+    return sum / 2;
+}
+export class QuestionDelta {
+    questionId;
+    type;
+    before;
+    after;
+    distributionShift;
+    confidenceBefore;
+    confidenceAfter;
+    scoreBefore;
+    scoreAfter;
+    constructor(questionId, type, before, after, distributionShift, confidenceBefore, confidenceAfter, scoreBefore = null, scoreAfter = null) {
+        this.questionId = questionId;
+        this.type = type;
+        this.before = before;
+        this.after = after;
+        this.distributionShift = distributionShift;
+        this.confidenceBefore = confidenceBefore;
+        this.confidenceAfter = confidenceAfter;
+        this.scoreBefore = scoreBefore;
+        this.scoreAfter = scoreAfter;
+    }
+    /** The selected outcome changed. */
+    get flipped() {
+        return this.before !== this.after;
+    }
+    get confidenceDelta() {
+        if (this.confidenceBefore === null || this.confidenceAfter === null)
+            return null;
+        return this.confidenceAfter - this.confidenceBefore;
+    }
+    describe() {
+        const head = this.flipped
+            ? `${this.questionId}: FLIP ${JSON.stringify(this.before)} -> ${JSON.stringify(this.after)}`
+            : `${this.questionId}: stable (${JSON.stringify(this.before)})`;
+        const parts = [`shift=${this.distributionShift.toFixed(3)}`];
+        const delta = this.confidenceDelta;
+        if (delta !== null) {
+            const sign = delta >= 0 ? "+" : "";
+            parts.push(`conf ${this.confidenceBefore.toFixed(3)} -> ${this.confidenceAfter.toFixed(3)} ` +
+                `(${sign}${delta.toFixed(3)})`);
+        }
+        return `${head}  [${parts.join(", ")}]`;
+    }
+}
+export class RecordDelta {
+    requestId;
+    modelBefore;
+    modelAfter;
+    questions;
+    constructor(requestId, modelBefore, modelAfter, questions = []) {
+        this.requestId = requestId;
+        this.modelBefore = modelBefore;
+        this.modelAfter = modelAfter;
+        this.questions = questions;
+    }
+    get flips() {
+        return this.questions.filter((q) => q.flipped);
+    }
+    get maxShift() {
+        return this.questions.reduce((m, q) => Math.max(m, q.distributionShift), 0);
+    }
+}
+export function compareAnswers(qid, before, after) {
+    if (before.type !== after.type) {
+        throw new Error(`question "${qid}": type changed from "${before.type}" to "${after.type}". ` +
+            `These are not the same question and cannot be compared.`);
+    }
+    return new QuestionDelta(qid, before.type, before.predicted(), after.predicted(), totalVariation(before.probabilities, after.probabilities), before.confidence, after.confidence, before.score, after.score);
+}
+export function compareRecords(before, after) {
+    if (before.requestId !== after.requestId) {
+        throw new Error(`records describe different requests and cannot be compared ` +
+            `(${before.requestId} vs ${after.requestId})`);
+    }
+    const old = parseAnswers(before.answers);
+    const fresh = parseAnswers(after.answers);
+    const missing = Object.keys(old).filter((k) => !(k in fresh)).sort();
+    const added = Object.keys(fresh).filter((k) => !(k in old)).sort();
+    if (missing.length || added.length) {
+        const detail = [];
+        if (missing.length)
+            detail.push(`missing in the new run: ${missing.join(", ")}`);
+        if (added.length)
+            detail.push(`only in the new run: ${added.join(", ")}`);
+        throw new Error(`answer sets differ for request ${before.requestId}: ${detail.join("; ")}`);
+    }
+    return new RecordDelta(before.requestId, before.model, after.model, Object.keys(old)
+        .sort()
+        .map((qid) => compareAnswers(qid, old[qid], fresh[qid])));
+}
+export class DriftReport {
+    deltas;
+    unmatchedBefore;
+    unmatchedAfter;
+    constructor(deltas = [], unmatchedBefore = [], unmatchedAfter = []) {
+        this.deltas = deltas;
+        this.unmatchedBefore = unmatchedBefore;
+        this.unmatchedAfter = unmatchedAfter;
+    }
+    get totalQuestions() {
+        return this.deltas.reduce((n, d) => n + d.questions.length, 0);
+    }
+    get flips() {
+        return this.deltas.flatMap((d) => d.flips.map((q) => [d.requestId, q]));
+    }
+    get flipRate() {
+        const total = this.totalQuestions;
+        return total ? this.flips.length / total : 0;
+    }
+    get maxShift() {
+        return this.deltas.reduce((m, d) => Math.max(m, d.maxShift), 0);
+    }
+    get meanShift() {
+        const shifts = this.deltas.flatMap((d) => d.questions.map((q) => q.distributionShift));
+        return shifts.length ? shifts.reduce((a, b) => a + b, 0) / shifts.length : 0;
+    }
+    models() {
+        return [
+            new Set(this.deltas.map((d) => d.modelBefore)),
+            new Set(this.deltas.map((d) => d.modelAfter)),
+        ];
+    }
+    summary() {
+        const [before, after] = this.models();
+        const lines = [
+            `compared ${this.deltas.length} request(s), ${this.totalQuestions} question(s)`,
+            `  ${[...before].sort().join(", ") || "?"} -> ${[...after].sort().join(", ") || "?"}`,
+            `  flips:      ${this.flips.length} (${(this.flipRate * 100).toFixed(1)}%)`,
+            `  mean shift: ${this.meanShift.toFixed(4)}`,
+            `  max shift:  ${this.maxShift.toFixed(4)}`,
+        ];
+        if (this.unmatchedBefore.length) {
+            lines.push(`  ${this.unmatchedBefore.length} record(s) in the baseline had no counterpart ` +
+                `and were not compared`);
+        }
+        if (this.unmatchedAfter.length) {
+            lines.push(`  ${this.unmatchedAfter.length} record(s) in the new run had no counterpart ` +
+                `and were not compared`);
+        }
+        return lines.join("\n");
+    }
+    toJSON() {
+        return {
+            requests: this.deltas.length,
+            questions: this.totalQuestions,
+            flips: this.flips.length,
+            flipRate: this.flipRate,
+            meanShift: this.meanShift,
+            maxShift: this.maxShift,
+            unmatchedBefore: this.unmatchedBefore,
+            unmatchedAfter: this.unmatchedAfter,
+            details: this.flips.map(([rid, q]) => ({
+                requestId: rid,
+                questionId: q.questionId,
+                before: q.before,
+                after: q.after,
+                distributionShift: q.distributionShift,
+                confidenceDelta: q.confidenceDelta,
+            })),
+        };
+    }
+}
+/**
+ * Pair two sets of records by request id and compare each pair.
+ *
+ * Records pair on `requestId`, the digest of state plus questions with the
+ * model excluded, which is exactly what makes a baseline comparable to its
+ * replay on a different model version.
+ */
+export function compareSets(before, after) {
+    const old = new Map(before.map((r) => [r.requestId, r]));
+    const fresh = new Map(after.map((r) => [r.requestId, r]));
+    const shared = [...old.keys()].filter((k) => fresh.has(k)).sort();
+    return new DriftReport(shared.map((rid) => compareRecords(old.get(rid), fresh.get(rid))), [...old.keys()].filter((k) => !fresh.has(k)).sort(), [...fresh.keys()].filter((k) => !old.has(k)).sort());
+}
+//# sourceMappingURL=compare.js.map

package/dist/compare.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"compare.js","sourceRoot":"","sources":["../src/compare.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EAA0C,YAAY,EAAE,MAAM,aAAa,CAAC;AAEnF;;;;;;;GAOG;AACH,MAAM,UAAU,cAAc,CAC5B,CAAyB,EACzB,CAAyB;IAEzB,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7D,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,KAAK,MAAM,CAAC,IAAI,IAAI;QAAE,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjE,OAAO,GAAG,GAAG,CAAC,CAAC;AACjB,CAAC;AAED,MAAM,OAAO,aAAa;IAEb;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IATX,YACW,UAAkB,EAClB,IAAY,EACZ,MAAe,EACf,KAAc,EACd,iBAAyB,EACzB,gBAA+B,EAC/B,eAA8B,EAC9B,cAA6B,IAAI,EACjC,aAA4B,IAAI;QARhC,eAAU,GAAV,UAAU,CAAQ;QAClB,SAAI,GAAJ,IAAI,CAAQ;QACZ,WAAM,GAAN,MAAM,CAAS;QACf,UAAK,GAAL,KAAK,CAAS;QACd,sBAAiB,GAAjB,iBAAiB,CAAQ;QACzB,qBAAgB,GAAhB,gBAAgB,CAAe;QAC/B,oBAAe,GAAf,eAAe,CAAe;QAC9B,gBAAW,GAAX,WAAW,CAAsB;QACjC,eAAU,GAAV,UAAU,CAAsB;IACxC,CAAC;IAEJ,oCAAoC;IACpC,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,KAAK,CAAC;IACpC,CAAC;IAED,IAAI,eAAe;QACjB,IAAI,IAAI,CAAC,gBAAgB,KAAK,IAAI,IAAI,IAAI,CAAC,eAAe,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC;QACjF,OAAO,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,gBAAgB,CAAC;IACtD,CAAC;IAED,QAAQ;QACN,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO;YACvB,CAAC,CAAC,GAAG,IAAI,CAAC,UAAU,UAAU,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE;YAC5F,CAAC,CAAC,GAAG,IAAI,CAAC,UAAU,aAAa,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC;QAClE,MAAM,KAAK,GAAG,CAAC,SAAS,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC7D,MAAM,KAAK,GAAG,IAAI,CAAC,eAAe,CAAC;QACnC,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACnB,MAAM,IAAI,GAAG,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;YACnC,KAAK,CAAC,IAAI,CACR,QAAQ,IAAI,CAAC,gBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,eAAgB,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG;gBACjF,IAAI,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CACjC,CAAC;QACJ,CAAC;QACD,OAAO,GAAG,IAAI,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;IAC1C,CAAC;CACF;AAED,MAAM,OAAO,WAAW;IAEX;IACA;IACA;IACA;IAJX,YACW,SAAiB,EACjB,WAAmB,EACnB,UAAkB,EAClB,YAA6B,EAAE;QAH/B,cAAS,GAAT,SAAS,CAAQ;QACjB,gBAAW,GAAX,WAAW,CAAQ;QACnB,eAAU,GAAV,UAAU,CAAQ;QAClB,cAAS,GAAT,SAAS,CAAsB;IACvC,CAAC;IAEJ,IAAI,KAAK;QACP,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;IACjD,CAAC;IAED,IAAI,QAAQ;QACV,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC,CAAC;IAC9E,CAAC;CACF;AAED,MAAM,UAAU,cAAc,CAAC,GAAW,EAAE,MAAc,EAAE,KAAa;IACvE,IAAI,MAAM,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CACb,aAAa,GAAG,yBAAyB,MAAM,CAAC,IAAI,SAAS,KAAK,CAAC,IAAI,KAAK;YAC1E,yDAAyD,CAC5D,CAAC;IACJ,CAAC;IACD,OAAO,IAAI,aAAa,CACtB,GAAG,EACH,MAAM,CAAC,IAAI,EACX,MAAM,CAAC,SAAS,EAAE,EAClB,KAAK,CAAC,SAAS,EAAE,EACjB,cAAc,CAAC,MAAM,CAAC,aAAa,EAAE,KAAK,CAAC,aAAa,CAAC,EACzD,MAAM,CAAC,UAAU,EACjB,KAAK,CAAC,UAAU,EAChB,MAAM,CAAC,KAAK,EACZ,KAAK,CAAC,KAAK,CACZ,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,MAAkB,EAAE,KAAiB;IAClE,IAAI,MAAM,CAAC,SAAS,KAAK,KAAK,CAAC,SAAS,EAAE,CAAC;QACzC,MAAM,IAAI,KAAK,CACb,6DAA6D;YAC3D,IAAI,MAAM,CAAC,SAAS,OAAO,KAAK,CAAC,SAAS,GAAG,CAChD,CAAC;IACJ,CAAC;IACD,MAAM,GAAG,GAAG,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACzC,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAE1C,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACrE,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACnE,IAAI,OAAO,CAAC,MAAM,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QACnC,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,IAAI,OAAO,CAAC,MAAM;YAAE,MAAM,CAAC,IAAI,CAAC,2BAA2B,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACjF,IAAI,KAAK,CAAC,MAAM;YAAE,MAAM,CAAC,IAAI,CAAC,wBAAwB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC1E,MAAM,IAAI,KAAK,CAAC,kCAAkC,MAAM,CAAC,SAAS,KAAK,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC9F,CAAC;IAED,OAAO,IAAI,WAAW,CACpB,MAAM,CAAC,SAAS,EAChB,MAAM,CAAC,KAAK,EACZ,KAAK,CAAC,KAAK,EACX,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC;SACb,IAAI,EAAE;SACN,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,cAAc,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,CAAE,EAAE,KAAK,CAAC,GAAG,CAAE,CAAC,CAAC,CAC7D,CAAC;AACJ,CAAC;AAED,MAAM,OAAO,WAAW;IAEX;IACA;IACA;IAHX,YACW,SAAwB,EAAE,EAC1B,kBAA4B,EAAE,EAC9B,iBAA2B,EAAE;QAF7B,WAAM,GAAN,MAAM,CAAoB;QAC1B,oBAAe,GAAf,eAAe,CAAe;QAC9B,mBAAc,GAAd,cAAc,CAAe;IACrC,CAAC;IAEJ,IAAI,cAAc;QAChB,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACjE,CAAC;IAED,IAAI,KAAK;QACP,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAA4B,CAAC,CAAC,CAAC;IACrG,CAAC;IAED,IAAI,QAAQ;QACV,MAAM,KAAK,GAAG,IAAI,CAAC,cAAc,CAAC;QAClC,OAAO,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/C,CAAC;IAED,IAAI,QAAQ;QACV,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;IAClE,CAAC;IAED,IAAI,SAAS;QACX,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC;QACvF,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/E,CAAC;IAED,MAAM;QACJ,OAAO;YACL,IAAI,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;YAC9C,IAAI,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;SAC9C,CAAC;IACJ,CAAC;IAED,OAAO;QACL,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QACtC,MAAM,KAAK,GAAG;YACZ,YAAY,IAAI,CAAC,MAAM,CAAC,MAAM,gBAAgB,IAAI,CAAC,cAAc,cAAc;YAC/E,KAAK,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,EAAE;YACrF,iBAAiB,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI;YAC3E,iBAAiB,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;YAC5C,iBAAiB,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;SAC5C,CAAC;QACF,IAAI,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,CAAC;YAChC,KAAK,CAAC,IAAI,CACR,KAAK,IAAI,CAAC,eAAe,CAAC,MAAM,gDAAgD;gBAC9E,uBAAuB,CAC1B,CAAC;QACJ,CAAC;QACD,IAAI,IAAI,CAAC,cAAc,CAAC,MAAM,EAAE,CAAC;YAC/B,KAAK,CAAC,IAAI,CACR,KAAK,IAAI,CAAC,cAAc,CAAC,MAAM,+CAA+C;gBAC5E,uBAAuB,CAC1B,CAAC;QACJ,CAAC;QACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;IAED,MAAM;QACJ,OAAO;YACL,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM;YAC5B,SAAS,EAAE,IAAI,CAAC,cAAc;YAC9B,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM;YACxB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,eAAe,EAAE,IAAI,CAAC,eAAe;YACrC,cAAc,EAAE,IAAI,CAAC,cAAc;YACnC,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACrC,SAAS,EAAE,GAAG;gBACd,UAAU,EAAE,CAAC,CAAC,UAAU;gBACxB,MAAM,EAAE,CAAC,CAAC,MAAM;gBAChB,KAAK,EAAE,CAAC,CAAC,KAAK;gBACd,iBAAiB,EAAE,CAAC,CAAC,iBAAiB;gBACtC,eAAe,EAAE,CAAC,CAAC,eAAe;aACnC,CAAC,CAAC;SACJ,CAAC;IACJ,CAAC;CACF;AAED;;;;;;GAMG;AACH,MAAM,UAAU,WAAW,CAAC,MAAoB,EAAE,KAAmB;IACnE,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IACzD,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1D,MAAM,MAAM,GAAG,CAAC,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAClE,OAAO,IAAI,WAAW,CACpB,MAAM,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAE,EAAE,KAAK,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC,CAAC,EACnE,CAAC,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,EACnD,CAAC,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CACpD,CAAC;AACJ,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Detect when a new Jev model version changes decisions you depend on.
+ *
+ * TypeSafe's docs warn that `jev-latest` moves and that tuned confidence
+ * thresholds move with it. This package replays a golden set against a new
+ * version and reports what actually changed, separating flipped decisions from
+ * probability shifts that have not flipped anything yet.
+ */
+export { DriftReport, QuestionDelta, RecordDelta, compareAnswers, compareRecords, compareSets, totalVariation, } from "./compare.js";
+export { replay, type SystemOneCallable } from "./replay.js";
+export declare const VERSION = "0.1.0";
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EACL,WAAW,EAAE,aAAa,EAAE,WAAW,EAAE,cAAc,EAAE,cAAc,EACvE,WAAW,EAAE,cAAc,GAC5B,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,MAAM,EAAE,KAAK,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAE7D,eAAO,MAAM,OAAO,UAAU,CAAC"}

package/dist/index.js

@@ -0,0 +1,12 @@
+/**
+ * Detect when a new Jev model version changes decisions you depend on.
+ *
+ * TypeSafe's docs warn that `jev-latest` moves and that tuned confidence
+ * thresholds move with it. This package replays a golden set against a new
+ * version and reports what actually changed, separating flipped decisions from
+ * probability shifts that have not flipped anything yet.
+ */
+export { DriftReport, QuestionDelta, RecordDelta, compareAnswers, compareRecords, compareSets, totalVariation, } from "./compare.js";
+export { replay } from "./replay.js";
+export const VERSION = "0.1.0";
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EACL,WAAW,EAAE,aAAa,EAAE,WAAW,EAAE,cAAc,EAAE,cAAc,EACvE,WAAW,EAAE,cAAc,GAC5B,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,MAAM,EAA0B,MAAM,aAAa,CAAC;AAE7D,MAAM,CAAC,MAAM,OAAO,GAAG,OAAO,CAAC"}

package/dist/replay.d.ts

@@ -0,0 +1,14 @@
+/** Replaying a golden set against a model. */
+import { Record as JevlRecord } from "jevkit-core";
+export type SystemOneCallable = (state: unknown, questions: Record<string, unknown>) => Promise<unknown> | unknown;
+/**
+ * Re-send each record's request and return the new answers as new records.
+ *
+ * `systemOne` is any callable taking `(state, questions)`, which is the shape
+ * both official SDKs already expose. Labels and tags carry over from the
+ * baseline so a replayed set stays scoreable.
+ */
+export declare function replay(records: Iterable<JevlRecord>, systemOne: SystemOneCallable, options?: {
+    tags?: string[];
+}): Promise<JevlRecord[]>;
+//# sourceMappingURL=replay.d.ts.map

package/dist/replay.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"replay.d.ts","sourceRoot":"","sources":["../src/replay.ts"],"names":[],"mappings":"AAAA,8CAA8C;AAE9C,OAAO,EAAE,MAAM,IAAI,UAAU,EAAE,MAAM,aAAa,CAAC;AAEnD,MAAM,MAAM,iBAAiB,GAAG,CAC9B,KAAK,EAAE,OAAO,EACd,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAC/B,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC;AAyBhC;;;;;;GAMG;AACH,wBAAsB,MAAM,CAC1B,OAAO,EAAE,QAAQ,CAAC,UAAU,CAAC,EAC7B,SAAS,EAAE,iBAAiB,EAC5B,OAAO,GAAE;IAAE,IAAI,CAAC,EAAE,MAAM,EAAE,CAAA;CAAO,GAChC,OAAO,CAAC,UAAU,EAAE,CAAC,CAoBvB"}

package/dist/replay.js

@@ -0,0 +1,52 @@
+/** Replaying a golden set against a model. */
+import { Record as JevlRecord } from "jevkit-core";
+function plain(value) {
+    if (value === null || value === undefined)
+        return value;
+    if (typeof value !== "object")
+        return value;
+    if (Array.isArray(value))
+        return value;
+    // Class instances from an SDK become plain objects; own enumerable keys only.
+    return { ...value };
+}
+function extract(response) {
+    const obj = (response ?? {});
+    const answers = (obj["answers"] ?? {});
+    const out = {};
+    for (const [qid, answer] of Object.entries(answers))
+        out[qid] = plain(answer);
+    const usage = obj["usage"];
+    return {
+        model: String(obj["model"] ?? ""),
+        answers: out,
+        usage: usage === undefined || usage === null ? null : plain(usage),
+    };
+}
+/**
+ * Re-send each record's request and return the new answers as new records.
+ *
+ * `systemOne` is any callable taking `(state, questions)`, which is the shape
+ * both official SDKs already expose. Labels and tags carry over from the
+ * baseline so a replayed set stays scoreable.
+ */
+export async function replay(records, systemOne, options = {}) {
+    const extraTags = options.tags ?? [];
+    const out = [];
+    for (const record of records) {
+        const response = await systemOne(record.state, record.questions);
+        const { model, answers, usage } = extract(response);
+        out.push(new JevlRecord({
+            model: model || record.model,
+            state: record.state,
+            questions: record.questions,
+            answers,
+            usage,
+            label: record.label,
+            tags: [...record.tags, ...extraTags],
+            meta: { ...record.meta, replayed_from: record.id },
+        }));
+    }
+    return out;
+}
+//# sourceMappingURL=replay.js.map

package/dist/replay.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"replay.js","sourceRoot":"","sources":["../src/replay.ts"],"names":[],"mappings":"AAAA,8CAA8C;AAE9C,OAAO,EAAE,MAAM,IAAI,UAAU,EAAE,MAAM,aAAa,CAAC;AASnD,SAAS,KAAK,CAAC,KAAc;IAC3B,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,KAAK,CAAC;IACxD,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IAC5C,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACvC,8EAA8E;IAC9E,OAAO,EAAE,GAAI,KAAiC,EAAE,CAAC;AACnD,CAAC;AAED,SAAS,OAAO,CAAC,QAAiB;IAChC,MAAM,GAAG,GAAG,CAAC,QAAQ,IAAI,EAAE,CAA4B,CAAC;IACxD,MAAM,OAAO,GAAG,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,EAAE,CAA4B,CAAC;IAClE,MAAM,GAAG,GAA4B,EAAE,CAAC;IACxC,KAAK,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC;QAAE,GAAG,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;IAC9E,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,CAAC;IAC3B,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QACjC,OAAO,EAAE,GAAG;QACZ,KAAK,EAAE,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAE,KAAK,CAAC,KAAK,CAA6B;KAChG,CAAC;AACJ,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,MAAM,CAC1B,OAA6B,EAC7B,SAA4B,EAC5B,UAA+B,EAAE;IAEjC,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC;IACrC,MAAM,GAAG,GAAiB,EAAE,CAAC;IAC7B,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;QACjE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;QACpD,GAAG,CAAC,IAAI,CACN,IAAI,UAAU,CAAC;YACb,KAAK,EAAE,KAAK,IAAI,MAAM,CAAC,KAAK;YAC5B,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,OAAO;YACP,KAAK;YACL,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,IAAI,EAAE,CAAC,GAAG,MAAM,CAAC,IAAI,EAAE,GAAG,SAAS,CAAC;YACpC,IAAI,EAAE,EAAE,GAAG,MAAM,CAAC,IAAI,EAAE,aAAa,EAAE,MAAM,CAAC,EAAE,EAAE;SACnD,CAAC,CACH,CAAC;IACJ,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC"}

package/package.json

@@ -0,0 +1,43 @@
+{
+  "name": "jevkit-drift",
+  "version": "0.1.0",
+  "description": "Detect when a new TypeSafe Jev model version flips decisions your code depends on. Replays a golden set and diffs the answers.",
+  "keywords": [
+    "jev",
+    "typesafe",
+    "system-one",
+    "drift",
+    "regression",
+    "testing"
+  ],
+  "license": "MIT",
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    }
+  },
+  "bin": {
+    "jevkit-drift": "./dist/cli.js"
+  },
+  "files": [
+    "dist",
+    "README.md"
+  ],
+  "engines": {
+    "node": ">=18"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/pjdurden/jevkit-js.git"
+  },
+  "dependencies": {
+    "jevkit-core": "^0.2.0"
+  },
+  "scripts": {
+    "build": "tsc -b"
+  }
+}