modelrig 0.0.1 → 0.2.0

package/dist/matrix/aggregate.js

@@ -0,0 +1,186 @@
+"use strict";
+/**
+ * Matrix aggregation job — Phase 4 WS6 (plan §4). Pure fold: telemetry
+ * rows → priors cells keyed (fingerprint_cell × model), each carrying
+ * outcome stats. Writes go to priors_cells ONLY (deny-all RLS, no UI, no
+ * cross-account use — standing decision 7; the gate stays off).
+ *
+ * Contribution discipline (spec WS6): the flag decides whether a cell may
+ * ever aggregate beyond its own account — respected here in code even
+ * though today's deployment is single-tenant. Non-contributing sources
+ * produce "account"-scoped cells, full stop.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.aggregationScope = aggregationScope;
+exports.aggregatePairwiseFromTelemetry = aggregatePairwiseFromTelemetry;
+exports.aggregatePriors = aggregatePriors;
+exports.writePriorsCells = writePriorsCells;
+/**
+ * Whether these rows may become community knowledge.
+ *
+ * One function so the two-veto rule has a single home and a name, rather than
+ * living as a `&&` inside a ternary inside a map.
+ */
+function aggregationScope(opts) {
+    const posture = opts.telemetryPosture ?? "pure_router";
+    return opts.contribution && posture === "optimized" ? "community" : "account";
+}
+/** A2: fold paired replay outcomes from telemetry. A pair exists where the
+ * SAME captureId was replayed through two different models within a cell.
+ * Sample outcome = did ANY attempt of the latest replay run for that
+ * (capture, model) conform — sample-level, retry-inclusive, matching the
+ * bake-off's own conformance notion. Single-source deployment folds
+ * sources: 1; cross-account distinct counts arrive with the community job. */
+function aggregatePairwiseFromTelemetry(rows) {
+    const samples = new Map();
+    for (const row of rows) {
+        if (row.fingerprintCell === null || row.provider === "search")
+            continue;
+        let tags;
+        try {
+            tags = JSON.parse(row.tagsJson);
+        }
+        catch {
+            continue;
+        }
+        const sampleTag = tags.replay_capture;
+        const replayId = tags.replay;
+        if (typeof sampleTag !== "string" || typeof replayId !== "string")
+            continue;
+        const captureId = sampleTag.slice(replayId.length + 1);
+        if (captureId === "")
+            continue;
+        const modelKey = `${row.provider}/${row.model}`;
+        const key = `${row.fingerprintCell}|${captureId}|${modelKey}`;
+        const entry = samples.get(key) ?? {
+            key: { cell: row.fingerprintCell, captureId, modelKey },
+            runs: new Map(),
+        };
+        const run = entry.runs.get(replayId) ?? { replayId, latestTs: row.ts, conform: false };
+        if (row.ts > run.latestTs)
+            run.latestTs = row.ts;
+        if (row.failureClass === null)
+            run.conform = true;
+        entry.runs.set(replayId, run);
+        samples.set(key, entry);
+    }
+    // Latest run per sample → (cell, capture) → model outcomes.
+    const byCapture = new Map(); // cell|capture → model → conform
+    for (const { key, runs } of samples.values()) {
+        const latest = [...runs.values()].sort((a, b) => (a.latestTs < b.latestTs ? 1 : -1))[0];
+        const captureKey = `${key.cell}|${key.captureId}`;
+        const models = byCapture.get(captureKey) ?? new Map();
+        models.set(key.modelKey, latest.conform);
+        byCapture.set(captureKey, models);
+    }
+    const out = new Map();
+    function record(cell, a, b, outcome) {
+        const byModel = out.get(cell) ?? new Map();
+        const opponents = byModel.get(a) ?? new Map();
+        const prev = opponents.get(b) ?? { wins: 0, losses: 0, ties: 0, samples: 0, sources: 1 };
+        opponents.set(b, {
+            wins: prev.wins + (outcome === "win" ? 1 : 0),
+            losses: prev.losses + (outcome === "loss" ? 1 : 0),
+            ties: prev.ties + (outcome === "tie" ? 1 : 0),
+            samples: prev.samples + 1,
+            sources: 1,
+        });
+        byModel.set(a, opponents);
+        out.set(cell, byModel);
+    }
+    for (const [captureKey, models] of byCapture) {
+        if (models.size < 2)
+            continue;
+        const cell = captureKey.slice(0, captureKey.indexOf("|"));
+        const entries = [...models.entries()].sort(([a], [b]) => a.localeCompare(b));
+        for (let i = 0; i < entries.length; i += 1) {
+            for (let j = i + 1; j < entries.length; j += 1) {
+                const [modelA, conformA] = entries[i];
+                const [modelB, conformB] = entries[j];
+                if (conformA === conformB) {
+                    record(cell, modelA, modelB, "tie");
+                    record(cell, modelB, modelA, "tie");
+                }
+                else if (conformA) {
+                    record(cell, modelA, modelB, "win");
+                    record(cell, modelB, modelA, "loss");
+                }
+                else {
+                    record(cell, modelA, modelB, "loss");
+                    record(cell, modelB, modelA, "win");
+                }
+            }
+        }
+    }
+    return out;
+}
+function aggregatePriors(rows, opts) {
+    const byCell = new Map();
+    for (const row of rows) {
+        if (row.fingerprintCell === null)
+            continue;
+        const key = `${row.fingerprintCell} ${row.provider}/${row.model}`;
+        const agg = byCell.get(key) ?? { conform: 0, repaired: 0, samples: 0, pricedCost: 0, priced: 0 };
+        agg.samples += 1;
+        if (row.failureClass === null)
+            agg.conform += 1;
+        if (row.repairedBy !== null)
+            agg.repaired += 1;
+        if (!row.pricingMissing) {
+            agg.pricedCost += row.costEstimateUsd;
+            agg.priced += 1;
+        }
+        byCell.set(key, agg);
+    }
+    // A2: the pairwise fold rides the same pass — records attach to their
+    // (cell, model) row's stats; scope/RLS/contribution treatment identical.
+    const pairwiseByModel = aggregatePairwiseFromTelemetry(rows);
+    return [...byCell.entries()]
+        .sort(([a], [b]) => a.localeCompare(b))
+        .map(([key, agg]) => {
+        const [cellKey, modelKey] = key.split(" ");
+        const opponents = pairwiseByModel.get(cellKey)?.get(modelKey);
+        return {
+            cellKey,
+            modelKey,
+            stats: {
+                samples: agg.samples,
+                conform_rate: agg.samples === 0 ? 0 : agg.conform / agg.samples,
+                repair_rate: agg.samples === 0 ? 0 : agg.repaired / agg.samples,
+                mean_cost_usd: agg.priced === 0 ? null : agg.pricedCost / agg.priced,
+                scope: aggregationScope(opts),
+                ...(opponents !== undefined && opponents.size > 0
+                    ? {
+                        pairwise: Object.fromEntries([...opponents.entries()].sort(([a], [b]) => a.localeCompare(b))),
+                    }
+                    : {}),
+            },
+        };
+    });
+}
+/** Upsert cells into priors_cells (service role — the ONLY principal that
+ * can touch the table; RLS denies everyone else including anon). */
+async function writePriorsCells(cells, config) {
+    if (cells.length === 0)
+        return;
+    const fetchFn = config.fetchFn ?? fetch;
+    const base = config.supabaseUrl.replace(/\/$/, "");
+    const response = await fetchFn(`${base}/rest/v1/priors_cells?on_conflict=cell_key,model_key`, {
+        method: "POST",
+        headers: {
+            "content-type": "application/json",
+            apikey: config.serviceRoleKey,
+            authorization: `Bearer ${config.serviceRoleKey}`,
+            prefer: "resolution=merge-duplicates,return=minimal",
+        },
+        body: JSON.stringify(cells.map((c) => ({
+            cell_key: c.cellKey,
+            model_key: c.modelKey,
+            stats: c.stats,
+            updated_at: new Date().toISOString(),
+        }))),
+    });
+    if (!response.ok) {
+        throw new Error(`priors_cells upsert → ${response.status} ${await response.text().catch(() => "")}`);
+    }
+}

package/dist/query/grammar.d.ts

@@ -0,0 +1,35 @@
+/**
+ * The telemetry query grammar — Phase 4 WS5 ("your exhaust is your data").
+ * A hand-rolled recursive-descent parser over a DELIBERATELY tiny SQL-ish
+ * language:
+ *
+ *   SELECT item (, item)* FROM inferences
+ *     [WHERE cond (AND cond)*] [GROUP BY col (, col)*] [LIMIT n]
+ *   item := col | count(*) | sum(m)|avg(m)|min(m)|max(m)
+ *   col  := route|provider|model|ts|failure_class|served_variant|tag.<name>
+ *   m    := cost|tokens_in|tokens_out|tokens_cached|latency_ms|ttfb_ms
+ *   cond := col op literal      op := = != > < >= <=
+ *
+ * Security model = the allowlist: unknown tokens throw, string literals
+ * only ever travel as bind parameters, the output is a single read-only
+ * SELECT. There is no raw-SQL path — by construction, not by filtering.
+ * Row scoping: a token's scope is ANDed into WHERE as one more bind
+ * parameter; the caller cannot opt out of it.
+ */
+export declare class QueryGrammarError extends Error {
+    constructor(message: string);
+}
+/** What a query token is allowed to see. */
+export type QueryScope = {
+    readonly kind: "all";
+} | {
+    readonly kind: "tag";
+    readonly tag: string;
+    readonly value: string;
+};
+export interface CompiledQuery {
+    readonly sql: string;
+    readonly params: ReadonlyArray<string | number>;
+}
+export declare function compileQuery(input: string, scope: QueryScope): CompiledQuery;
+//# sourceMappingURL=grammar.d.ts.map

package/dist/query/grammar.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"grammar.d.ts","sourceRoot":"","sources":["../../src/query/grammar.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,qBAAa,iBAAkB,SAAQ,KAAK;gBAC9B,OAAO,EAAE,MAAM;CAI5B;AAED,4CAA4C;AAC5C,MAAM,MAAM,UAAU,GAClB;IAAE,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAA;CAAE,GACxB;IAAE,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC;IAAC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC;AAE3E,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC;CACjD;AAwHD,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,GAAG,aAAa,CAkJ5E"}

package/dist/query/grammar.js

@@ -0,0 +1,279 @@
+"use strict";
+/**
+ * The telemetry query grammar — Phase 4 WS5 ("your exhaust is your data").
+ * A hand-rolled recursive-descent parser over a DELIBERATELY tiny SQL-ish
+ * language:
+ *
+ *   SELECT item (, item)* FROM inferences
+ *     [WHERE cond (AND cond)*] [GROUP BY col (, col)*] [LIMIT n]
+ *   item := col | count(*) | sum(m)|avg(m)|min(m)|max(m)
+ *   col  := route|provider|model|ts|failure_class|served_variant|tag.<name>
+ *   m    := cost|tokens_in|tokens_out|tokens_cached|latency_ms|ttfb_ms
+ *   cond := col op literal      op := = != > < >= <=
+ *
+ * Security model = the allowlist: unknown tokens throw, string literals
+ * only ever travel as bind parameters, the output is a single read-only
+ * SELECT. There is no raw-SQL path — by construction, not by filtering.
+ * Row scoping: a token's scope is ANDed into WHERE as one more bind
+ * parameter; the caller cannot opt out of it.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.QueryGrammarError = void 0;
+exports.compileQuery = compileQuery;
+class QueryGrammarError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = "QueryGrammarError";
+    }
+}
+exports.QueryGrammarError = QueryGrammarError;
+const PLAIN_COLUMNS = {
+    route: "route",
+    provider: "provider",
+    model: "model",
+    ts: "ts",
+    failure_class: "failure_class",
+    served_variant: "served_variant",
+};
+const METRICS = {
+    cost: "cost_estimate_usd",
+    tokens_in: "tokens_in",
+    tokens_out: "tokens_out",
+    tokens_cached: "tokens_cached",
+    latency_ms: "latency_ms",
+    ttfb_ms: "ttfb_ms",
+};
+const FUNCTIONS = new Set(["sum", "avg", "min", "max", "count"]);
+const OPERATORS = new Set(["=", "!=", ">", "<", ">=", "<="]);
+const TAG_NAME = /^[a-zA-Z_][a-zA-Z0-9_]*$/;
+const DEFAULT_LIMIT = 1000;
+const MAX_LIMIT = 10000;
+function tokenize(input) {
+    const tokens = [];
+    let i = 0;
+    while (i < input.length) {
+        const ch = input[i];
+        if (/\s/.test(ch)) {
+            i += 1;
+            continue;
+        }
+        if (ch === "'") {
+            // Single-quoted string; '' escapes a quote. Anything else inside is data.
+            let value = "";
+            i += 1;
+            for (;;) {
+                if (i >= input.length)
+                    throw new QueryGrammarError("unterminated string literal");
+                if (input[i] === "'") {
+                    if (input[i + 1] === "'") {
+                        value += "'";
+                        i += 2;
+                        continue;
+                    }
+                    i += 1;
+                    break;
+                }
+                value += input[i];
+                i += 1;
+            }
+            tokens.push({ kind: "string", text: value });
+            continue;
+        }
+        if (/[0-9]/.test(ch) || (ch === "." && /[0-9]/.test(input[i + 1] ?? ""))) {
+            let num = "";
+            while (i < input.length && /[0-9.]/.test(input[i])) {
+                num += input[i];
+                i += 1;
+            }
+            if (!/^\d+(\.\d+)?$/.test(num))
+                throw new QueryGrammarError(`bad number: ${num}`);
+            tokens.push({ kind: "number", text: num });
+            continue;
+        }
+        if (/[a-zA-Z_]/.test(ch)) {
+            let word = "";
+            while (i < input.length && /[a-zA-Z0-9_.]/.test(input[i])) {
+                word += input[i];
+                i += 1;
+            }
+            tokens.push({ kind: "word", text: word });
+            continue;
+        }
+        if (ch === ">" || ch === "<" || ch === "!") {
+            const two = input.slice(i, i + 2);
+            if (two === ">=" || two === "<=" || two === "!=") {
+                tokens.push({ kind: "punct", text: two });
+                i += 2;
+                continue;
+            }
+            if (ch === "!")
+                throw new QueryGrammarError("unexpected '!'");
+            tokens.push({ kind: "punct", text: ch });
+            i += 1;
+            continue;
+        }
+        if (ch === "=" || ch === "," || ch === "(" || ch === ")" || ch === "*") {
+            tokens.push({ kind: "punct", text: ch });
+            i += 1;
+            continue;
+        }
+        throw new QueryGrammarError(`unexpected character: ${JSON.stringify(ch)}`);
+    }
+    return tokens;
+}
+/** Column reference → SQL expression. tag.<name> compiles to a
+ * json_extract over tags_json with a VALIDATED identifier (never
+ * interpolated user text beyond the identifier alphabet). */
+function columnSql(word) {
+    if (word.startsWith("tag.")) {
+        const tag = word.slice(4);
+        if (!TAG_NAME.test(tag))
+            throw new QueryGrammarError(`bad tag name: ${tag}`);
+        return `json_extract(tags_json, '$.${tag}')`;
+    }
+    const column = PLAIN_COLUMNS[word];
+    if (column === undefined) {
+        throw new QueryGrammarError(`unknown column "${word}" — allowed: ${Object.keys(PLAIN_COLUMNS).join(", ")}, tag.<name>`);
+    }
+    return column;
+}
+function compileQuery(input, scope) {
+    const tokens = tokenize(input);
+    let pos = 0;
+    const peek = () => tokens[pos];
+    function next() {
+        const token = tokens[pos];
+        if (token === undefined)
+            throw new QueryGrammarError("unexpected end of query");
+        pos += 1;
+        return token;
+    }
+    function expectWord(word) {
+        const token = next();
+        if (token.kind !== "word" || token.text.toLowerCase() !== word) {
+            throw new QueryGrammarError(`expected ${word.toUpperCase()}, got "${token.text}"`);
+        }
+    }
+    function expectPunct(text) {
+        const token = next();
+        if (token.kind !== "punct" || token.text !== text) {
+            throw new QueryGrammarError(`expected "${text}", got "${token.text}"`);
+        }
+    }
+    expectWord("select");
+    const selectSql = [];
+    for (;;) {
+        const token = next();
+        if (token.kind !== "word")
+            throw new QueryGrammarError(`bad select item: "${token.text}"`);
+        const lower = token.text.toLowerCase();
+        if (FUNCTIONS.has(lower) && peek()?.kind === "punct" && peek()?.text === "(") {
+            expectPunct("(");
+            const arg = next();
+            expectPunct(")");
+            if (lower === "count") {
+                if (arg.kind !== "punct" || arg.text !== "*") {
+                    throw new QueryGrammarError("count takes exactly count(*)");
+                }
+                selectSql.push(`COUNT(*) AS "count"`);
+            }
+            else {
+                if (arg.kind !== "word")
+                    throw new QueryGrammarError(`bad metric in ${lower}()`);
+                const metric = METRICS[arg.text.toLowerCase()];
+                if (metric === undefined) {
+                    throw new QueryGrammarError(`unknown metric "${arg.text}" — allowed: ${Object.keys(METRICS).join(", ")}`);
+                }
+                selectSql.push(`${lower.toUpperCase()}(${metric}) AS "${lower}_${arg.text.toLowerCase()}"`);
+            }
+        }
+        else {
+            // tag.client aliases to "client" — result keys read naturally.
+            const alias = token.text.startsWith("tag.") ? token.text.slice(4) : token.text;
+            selectSql.push(`${columnSql(token.text)} AS "${alias}"`);
+        }
+        if (peek()?.kind === "punct" && peek()?.text === ",") {
+            pos += 1;
+            continue;
+        }
+        break;
+    }
+    expectWord("from");
+    const table = next();
+    if (table.kind !== "word" || table.text.toLowerCase() !== "inferences") {
+        throw new QueryGrammarError(`only "inferences" is queryable, got "${table.text}"`);
+    }
+    const whereSql = [];
+    const params = [];
+    if (peek()?.kind === "word" && peek()?.text.toLowerCase() === "where") {
+        pos += 1;
+        for (;;) {
+            const column = next();
+            if (column.kind !== "word")
+                throw new QueryGrammarError(`bad condition column`);
+            const columnExpr = METRICS[column.text.toLowerCase()] !== undefined
+                ? METRICS[column.text.toLowerCase()]
+                : columnSql(column.text);
+            const op = next();
+            if (op.kind !== "punct" || !OPERATORS.has(op.text)) {
+                throw new QueryGrammarError(`bad operator "${op.text}" — allowed: ${[...OPERATORS].join(" ")}`);
+            }
+            const value = next();
+            if (value.kind === "string") {
+                params.push(value.text);
+            }
+            else if (value.kind === "number") {
+                params.push(Number(value.text));
+            }
+            else {
+                throw new QueryGrammarError(`condition value must be a 'string' or number, got "${value.text}"`);
+            }
+            whereSql.push(`${columnExpr} ${op.text} ?`);
+            if (peek()?.kind === "word" && peek()?.text.toLowerCase() === "and") {
+                pos += 1;
+                continue;
+            }
+            break;
+        }
+    }
+    // Row scoping — ANDed in unconditionally; the caller cannot remove it.
+    if (scope.kind === "tag") {
+        if (!TAG_NAME.test(scope.tag))
+            throw new QueryGrammarError(`bad scope tag: ${scope.tag}`);
+        whereSql.push(`json_extract(tags_json, '$.${scope.tag}') = ?`);
+        params.push(scope.value);
+    }
+    const groupSql = [];
+    if (peek()?.kind === "word" && peek()?.text.toLowerCase() === "group") {
+        pos += 1;
+        expectWord("by");
+        for (;;) {
+            const column = next();
+            if (column.kind !== "word")
+                throw new QueryGrammarError("bad GROUP BY column");
+            groupSql.push(columnSql(column.text));
+            if (peek()?.kind === "punct" && peek()?.text === ",") {
+                pos += 1;
+                continue;
+            }
+            break;
+        }
+    }
+    let limit = DEFAULT_LIMIT;
+    if (peek()?.kind === "word" && peek()?.text.toLowerCase() === "limit") {
+        pos += 1;
+        const n = next();
+        if (n.kind !== "number" || !/^\d+$/.test(n.text)) {
+            throw new QueryGrammarError("LIMIT takes a bare integer");
+        }
+        limit = Math.min(Number(n.text), MAX_LIMIT);
+    }
+    if (pos !== tokens.length) {
+        throw new QueryGrammarError(`unexpected trailing input: "${tokens[pos].text}"`);
+    }
+    const sql = `SELECT ${selectSql.join(", ")} FROM inferences` +
+        (whereSql.length > 0 ? ` WHERE ${whereSql.join(" AND ")}` : "") +
+        (groupSql.length > 0 ? ` GROUP BY ${groupSql.join(", ")}` : "") +
+        ` LIMIT ${limit}`;
+    return { sql, params };
+}

package/dist/query/run.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Query execution — Phase 4 WS5. Opens the telemetry file READONLY and runs
+ * one compiled (grammar-validated, parameterized) SELECT. The readonly
+ * handle is defense in depth under the grammar allowlist.
+ */
+import type { CompiledQuery } from "./grammar";
+export declare function runTelemetryQuery(dbPath: string, compiled: CompiledQuery): Array<Record<string, unknown>>;
+//# sourceMappingURL=run.d.ts.map

package/dist/query/run.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../src/query/run.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAE/C,wBAAgB,iBAAiB,CAC/B,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,aAAa,GACtB,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAOhC"}

package/dist/query/run.js

@@ -0,0 +1,21 @@
+"use strict";
+/**
+ * Query execution — Phase 4 WS5. Opens the telemetry file READONLY and runs
+ * one compiled (grammar-validated, parameterized) SELECT. The readonly
+ * handle is defense in depth under the grammar allowlist.
+ */
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runTelemetryQuery = runTelemetryQuery;
+const better_sqlite3_1 = __importDefault(require("better-sqlite3"));
+function runTelemetryQuery(dbPath, compiled) {
+    const db = new better_sqlite3_1.default(dbPath, { readonly: true, fileMustExist: true });
+    try {
+        return db.prepare(compiled.sql).all(...compiled.params);
+    }
+    finally {
+        db.close();
+    }
+}