@keel_flow/runtime 0.2.0

package/dist/spec-checker/bidirectional.js

@@ -0,0 +1,97 @@
+import { computeCost } from "../cost.js";
+import { BIDIRECTIONAL_SYSTEM, BIDIRECTIONAL_USER } from "./prompts.js";
+import { summarizeDiff } from "./claims.js";
+const summaryTool = {
+    name: "emit_diff_summary",
+    description: "Emit a 1-3 bullet summary of what the diff does.",
+    inputSchema: {
+        type: "object",
+        properties: {
+            summary: { type: "string" },
+        },
+        required: ["summary"],
+    },
+};
+function tokenize(s) {
+    return new Set(s
+        .toLowerCase()
+        .replace(/[^a-z0-9\s]/g, " ")
+        .split(/\s+/)
+        .filter((t) => t.length > 2));
+}
+export function jaccard(a, b) {
+    const ta = tokenize(a);
+    const tb = tokenize(b);
+    if (ta.size === 0 && tb.size === 0)
+        return 1;
+    let intersection = 0;
+    for (const t of ta)
+        if (tb.has(t))
+            intersection++;
+    const union = ta.size + tb.size - intersection;
+    if (union === 0)
+        return 0;
+    return intersection / union;
+}
+export function cosine(a, b) {
+    if (a.length !== b.length) {
+        throw new Error("cosine: vectors must have equal length");
+    }
+    let dot = 0;
+    let na = 0;
+    let nb = 0;
+    for (let i = 0; i < a.length; i++) {
+        const ai = a[i] ?? 0;
+        const bi = b[i] ?? 0;
+        dot += ai * bi;
+        na += ai * ai;
+        nb += bi * bi;
+    }
+    if (na === 0 || nb === 0)
+        return 0;
+    return dot / (Math.sqrt(na) * Math.sqrt(nb));
+}
+export async function bidirectionalCheck(provider, model, specSummary, diff, opts = {}) {
+    const threshold = opts.threshold ?? 0.55;
+    const diffSummary = summarizeDiff(diff);
+    const response = await provider.generate({
+        model,
+        system: BIDIRECTIONAL_SYSTEM,
+        messages: [
+            { role: "user", content: [{ type: "text", text: BIDIRECTIONAL_USER(diffSummary) }] },
+        ],
+        tools: [summaryTool],
+        toolChoice: { name: "emit_diff_summary" },
+        maxTokens: 512,
+    });
+    const toolUse = response.content.find((b) => b.type === "tool_use");
+    let summary = "";
+    if (toolUse && toolUse.type === "tool_use") {
+        const input = toolUse.input;
+        if (typeof input.summary === "string")
+            summary = input.summary;
+    }
+    let similarity;
+    if (opts.embed) {
+        const [specVec, sumVec] = await Promise.all([
+            opts.embed(specSummary),
+            opts.embed(summary),
+        ]);
+        similarity = cosine(specVec, sumVec);
+    }
+    else {
+        similarity = jaccard(specSummary, summary);
+    }
+    const costUsd = computeCost(provider.kind, model, response.usage.inputTokens, response.usage.outputTokens);
+    return {
+        summary,
+        similarity,
+        divergent: similarity < threshold,
+        stats: {
+            inputTokens: response.usage.inputTokens,
+            outputTokens: response.usage.outputTokens,
+            costUsd,
+        },
+    };
+}
+//# sourceMappingURL=bidirectional.js.map

package/dist/spec-checker/bidirectional.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"bidirectional.js","sourceRoot":"","sources":["../../src/spec-checker/bidirectional.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AACxE,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAoB5C,MAAM,WAAW,GAAmB;IAClC,IAAI,EAAE,mBAAmB;IACzB,WAAW,EAAE,kDAAkD;IAC/D,WAAW,EAAE;QACX,IAAI,EAAE,QAAQ;QACd,UAAU,EAAE;YACV,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;SAC5B;QACD,QAAQ,EAAE,CAAC,SAAS,CAAC;KACtB;CACF,CAAC;AAEF,SAAS,QAAQ,CAAC,CAAS;IACzB,OAAO,IAAI,GAAG,CACZ,CAAC;SACE,WAAW,EAAE;SACb,OAAO,CAAC,cAAc,EAAE,GAAG,CAAC;SAC5B,KAAK,CAAC,KAAK,CAAC;SACZ,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAC/B,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,CAAS,EAAE,CAAS;IAC1C,MAAM,EAAE,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;IACvB,MAAM,EAAE,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;IACvB,IAAI,EAAE,CAAC,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,IAAI,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAC7C,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,KAAK,MAAM,CAAC,IAAI,EAAE;QAAE,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;YAAE,YAAY,EAAE,CAAC;IAClD,MAAM,KAAK,GAAG,EAAE,CAAC,IAAI,GAAG,EAAE,CAAC,IAAI,GAAG,YAAY,CAAC;IAC/C,IAAI,KAAK,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAC1B,OAAO,YAAY,GAAG,KAAK,CAAC;AAC9B,CAAC;AAED,MAAM,UAAU,MAAM,CAAC,CAAW,EAAE,CAAW;IAC7C,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;IAC5D,CAAC;IACD,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,EAAE,GAAG,CAAC,CAAC;IACX,IAAI,EAAE,GAAG,CAAC,CAAC;IACX,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACrB,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACrB,GAAG,IAAI,EAAE,GAAG,EAAE,CAAC;QACf,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC;QACd,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC;IAChB,CAAC;IACD,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACnC,OAAO,GAAG,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;AAC/C,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,QAAuB,EACvB,KAAa,EACb,WAAmB,EACnB,IAAU,EACV,OAA0B,EAAE;IAE5B,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC;IACzC,MAAM,WAAW,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;IAExC,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC;QACvC,KAAK;QACL,MAAM,EAAE,oBAAoB;QAC5B,QAAQ,EAAE;YACR,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,kBAAkB,CAAC,WAAW,CAAC,EAAE,CAAC,EAAE;SACrF;QACD,KAAK,EAAE,CAAC,WAAW,CAAC;QACpB,UAAU,EAAE,EAAE,IAAI,EAAE,mBAAmB,EAAE;QACzC,SAAS,EAAE,GAAG;KACf,CAAC,CAAC;IAEH,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC;IACpE,IAAI,OAAO,GAAG,EAAE,CAAC;IACjB,IAAI,OAAO,IAAI,OAAO,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;QAC3C,MAAM,KAAK,GAAG,OAAO,CAAC,KAA8B,CAAC;QACrD,IAAI,OAAO,KAAK,CAAC,OAAO,KAAK,QAAQ;YAAE,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC;IACjE,CAAC;IAED,IAAI,UAAkB,CAAC;IACvB,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;QACf,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAC1C,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC;YACvB,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC;SACpB,CAAC,CAAC;QACH,UAAU,GAAG,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IACvC,CAAC;SAAM,CAAC;QACN,UAAU,GAAG,OAAO,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;IAC7C,CAAC;IAED,MAAM,OAAO,GAAG,WAAW,CACzB,QAAQ,CAAC,IAAI,EACb,KAAK,EACL,QAAQ,CAAC,KAAK,CAAC,WAAW,EAC1B,QAAQ,CAAC,KAAK,CAAC,YAAY,CAC5B,CAAC;IAEF,OAAO;QACL,OAAO;QACP,UAAU;QACV,SAAS,EAAE,UAAU,GAAG,SAAS;QACjC,KAAK,EAAE;YACL,WAAW,EAAE,QAAQ,CAAC,KAAK,CAAC,WAAW;YACvC,YAAY,EAAE,QAAQ,CAAC,KAAK,CAAC,YAAY;YACzC,OAAO;SACR;KACF,CAAC;AACJ,CAAC"}

package/dist/spec-checker/calibration.d.ts

@@ -0,0 +1,15 @@
+import type { AgentEvent } from "@keel_flow/schema";
+export interface SpecTypeStats {
+    n: number;
+    precisionApprox: number;
+}
+export interface CalibrationReport {
+    calibrated: boolean;
+    perSpecType?: Record<string, SpecTypeStats>;
+}
+export interface LoadCalibrationOpts {
+    readEvents: () => Promise<AgentEvent[]>;
+    minSamples?: number;
+}
+export declare function loadCalibration(opts: LoadCalibrationOpts): Promise<CalibrationReport>;
+//# sourceMappingURL=calibration.d.ts.map

package/dist/spec-checker/calibration.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"calibration.d.ts","sourceRoot":"","sources":["../../src/spec-checker/calibration.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAEpD,MAAM,WAAW,aAAa;IAC5B,CAAC,EAAE,MAAM,CAAC;IACV,eAAe,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,iBAAiB;IAChC,UAAU,EAAE,OAAO,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;CAC7C;AAED,MAAM,WAAW,mBAAmB;IAClC,UAAU,EAAE,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC;IACxC,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAYD,wBAAsB,eAAe,CACnC,IAAI,EAAE,mBAAmB,GACxB,OAAO,CAAC,iBAAiB,CAAC,CA4D5B"}

package/dist/spec-checker/calibration.js

@@ -0,0 +1,58 @@
+function specTypeKey(summary) {
+    return summary
+        .toLowerCase()
+        .replace(/[^a-z0-9\s]/g, " ")
+        .split(/\s+/)
+        .filter((t) => t.length > 0)
+        .slice(0, 3)
+        .join(" ");
+}
+export async function loadCalibration(opts) {
+    const minSamples = opts.minSamples ?? 50;
+    const events = await opts.readEvents();
+    const rubricEvents = events.filter((e) => e.kind === "spec-judge.rubric");
+    const claimEvents = events.filter((e) => e.kind === "spec-judge.claim");
+    if (rubricEvents.length === 0)
+        return { calibrated: false };
+    const summaryByEventId = new Map();
+    for (const r of rubricEvents) {
+        const payload = r.payload;
+        if (typeof payload.specSummary === "string") {
+            const key = typeof payload.eventId === "string" ? payload.eventId : r.id;
+            summaryByEventId.set(key, payload.specSummary);
+        }
+    }
+    const perGroup = new Map();
+    for (const c of claimEvents) {
+        const parentId = c.parentEventId;
+        if (!parentId)
+            continue;
+        const summary = summaryByEventId.get(parentId);
+        if (!summary)
+            continue;
+        const key = specTypeKey(summary);
+        const payload = c.payload;
+        const confidence = typeof payload.confidence === "number" ? payload.confidence : 0;
+        const verdict = typeof payload.verdict === "string" ? payload.verdict : "";
+        const entry = perGroup.get(key) ?? { n: 0, highConfHits: 0 };
+        entry.n += 1;
+        if (confidence >= 0.8 && (verdict === "pass" || verdict === "fail")) {
+            entry.highConfHits += 1;
+        }
+        perGroup.set(key, entry);
+    }
+    const perSpecType = {};
+    let totalN = 0;
+    for (const [k, v] of perGroup.entries()) {
+        totalN += v.n;
+        perSpecType[k] = {
+            n: v.n,
+            precisionApprox: v.n === 0 ? 0 : v.highConfHits / v.n,
+        };
+    }
+    return {
+        calibrated: totalN >= minSamples,
+        perSpecType,
+    };
+}
+//# sourceMappingURL=calibration.js.map

package/dist/spec-checker/calibration.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"calibration.js","sourceRoot":"","sources":["../../src/spec-checker/calibration.ts"],"names":[],"mappings":"AAiBA,SAAS,WAAW,CAAC,OAAe;IAClC,OAAO,OAAO;SACX,WAAW,EAAE;SACb,OAAO,CAAC,cAAc,EAAE,GAAG,CAAC;SAC5B,KAAK,CAAC,KAAK,CAAC;SACZ,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;SAC3B,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;SACX,IAAI,CAAC,GAAG,CAAC,CAAC;AACf,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,IAAyB;IAEzB,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC;IACzC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;IAEvC,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,mBAAmB,CAAC,CAAC;IAC1E,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,kBAAkB,CAAC,CAAC;IAExE,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC;IAE5D,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAAkB,CAAC;IACnD,KAAK,MAAM,CAAC,IAAI,YAAY,EAAE,CAAC;QAC7B,MAAM,OAAO,GAAG,CAAC,CAAC,OAAuD,CAAC;QAC1E,IAAI,OAAO,OAAO,CAAC,WAAW,KAAK,QAAQ,EAAE,CAAC;YAC5C,MAAM,GAAG,GACP,OAAO,OAAO,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAC/D,gBAAgB,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;QACjD,CAAC;IACH,CAAC;IAED,MAAM,QAAQ,GAAG,IAAI,GAAG,EAGrB,CAAC;IAEJ,KAAK,MAAM,CAAC,IAAI,WAAW,EAAE,CAAC;QAC5B,MAAM,QAAQ,GAAG,CAAC,CAAC,aAAa,CAAC;QACjC,IAAI,CAAC,QAAQ;YAAE,SAAS;QACxB,MAAM,OAAO,GAAG,gBAAgB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC/C,IAAI,CAAC,OAAO;YAAE,SAAS;QACvB,MAAM,GAAG,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;QACjC,MAAM,OAAO,GAAG,CAAC,CAAC,OAGjB,CAAC;QACF,MAAM,UAAU,GACd,OAAO,OAAO,CAAC,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;QAClE,MAAM,OAAO,GAAG,OAAO,OAAO,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;QAE3E,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC;QAC7D,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC;QACb,IAAI,UAAU,IAAI,GAAG,IAAI,CAAC,OAAO,KAAK,MAAM,IAAI,OAAO,KAAK,MAAM,CAAC,EAAE,CAAC;YACpE,KAAK,CAAC,YAAY,IAAI,CAAC,CAAC;QAC1B,CAAC;QACD,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IAC3B,CAAC;IAED,MAAM,WAAW,GAAkC,EAAE,CAAC;IACtD,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;QACxC,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC;QACd,WAAW,CAAC,CAAC,CAAC,GAAG;YACf,CAAC,EAAE,CAAC,CAAC,CAAC;YACN,eAAe,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC;SACtD,CAAC;IACJ,CAAC;IAED,OAAO;QACL,UAAU,EAAE,MAAM,IAAI,UAAU;QAChC,WAAW;KACZ,CAAC;AACJ,CAAC"}

package/dist/spec-checker/claims.d.ts

@@ -0,0 +1,26 @@
+import type { SpecClaimVerdict, SpecRubric } from "@keel_flow/schema";
+import type { Diff } from "@keel_flow/core";
+import type { ModelProvider } from "../provider.js";
+export interface ClaimJudgeStats {
+    inputTokens: number;
+    outputTokens: number;
+    costUsd: number;
+}
+export interface ClaimJudgeResult {
+    verdict: SpecClaimVerdict;
+    stats: ClaimJudgeStats;
+}
+export declare function summarizeDiff(diff: Diff): string;
+export interface JudgeClaimOpts {
+    swapped?: boolean;
+}
+export declare function judgeOneClaim(provider: ModelProvider, model: string, specSummary: string, claim: SpecRubric["claims"][number], diff: Diff, opts?: JudgeClaimOpts): Promise<ClaimJudgeResult>;
+export interface JudgeClaimsOpts {
+    concurrency?: number;
+}
+export interface JudgeClaimsResult {
+    verdicts: SpecClaimVerdict[];
+    stats: ClaimJudgeStats;
+}
+export declare function judgeClaims(provider: ModelProvider, model: string, specSummary: string, rubric: SpecRubric, diff: Diff, opts?: JudgeClaimsOpts, onResult?: (verdict: SpecClaimVerdict, stats: ClaimJudgeStats, swapped: boolean) => void): Promise<JudgeClaimsResult>;
+//# sourceMappingURL=claims.d.ts.map

package/dist/spec-checker/claims.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"claims.d.ts","sourceRoot":"","sources":["../../src/spec-checker/claims.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,gBAAgB,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AACtE,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,KAAK,EAAE,aAAa,EAAkB,MAAM,gBAAgB,CAAC;AAIpE,MAAM,WAAW,eAAe;IAC9B,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,gBAAgB,CAAC;IAC1B,KAAK,EAAE,eAAe,CAAC;CACxB;AAoBD,wBAAgB,aAAa,CAAC,IAAI,EAAE,IAAI,GAAG,MAAM,CA+BhD;AAED,MAAM,WAAW,cAAc;IAC7B,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,wBAAsB,aAAa,CACjC,QAAQ,EAAE,aAAa,EACvB,KAAK,EAAE,MAAM,EACb,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,EACnC,IAAI,EAAE,IAAI,EACV,IAAI,GAAE,cAAmB,GACxB,OAAO,CAAC,gBAAgB,CAAC,CAsC3B;AAED,MAAM,WAAW,eAAe;IAC9B,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,EAAE,gBAAgB,EAAE,CAAC;IAC7B,KAAK,EAAE,eAAe,CAAC;CACxB;AAED,wBAAsB,WAAW,CAC/B,QAAQ,EAAE,aAAa,EACvB,KAAK,EAAE,MAAM,EACb,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,UAAU,EAClB,IAAI,EAAE,IAAI,EACV,IAAI,GAAE,eAAoB,EAC1B,QAAQ,CAAC,EAAE,CACT,OAAO,EAAE,gBAAgB,EACzB,KAAK,EAAE,eAAe,EACtB,OAAO,EAAE,OAAO,KACb,IAAI,GACR,OAAO,CAAC,iBAAiB,CAAC,CA+B5B"}

package/dist/spec-checker/claims.js

@@ -0,0 +1,104 @@
+import { SpecClaimVerdictSchema } from "@keel_flow/schema";
+import { computeCost } from "../cost.js";
+import { CLAIM_SYSTEM, CLAIM_USER, CLAIM_USER_SWAPPED } from "./prompts.js";
+const claimTool = {
+    name: "emit_claim_verdict",
+    description: "Emit a single verdict for one claim.",
+    inputSchema: {
+        type: "object",
+        properties: {
+            claimId: { type: "string" },
+            verdict: { type: "string", enum: ["pass", "fail", "partial", "na"] },
+            evidence: { type: "string" },
+            confidence: { type: "number", minimum: 0, maximum: 1 },
+        },
+        required: ["claimId", "verdict", "confidence"],
+    },
+};
+const DIFF_FILE_MAX_CHARS = 4000;
+const DIFF_TOTAL_MAX_CHARS = 24000;
+export function summarizeDiff(diff) {
+    if (diff.files.length === 0)
+        return "(empty diff)";
+    const parts = [];
+    let totalChars = 0;
+    for (const f of diff.files) {
+        const header = `${f.status} ${f.path} (+${f.linesAdded}/-${f.linesRemoved})`;
+        let filePart = header;
+        if (f.addedText && f.addedText.length > 0) {
+            const budget = Math.min(DIFF_FILE_MAX_CHARS, DIFF_TOTAL_MAX_CHARS - totalChars - header.length - 20);
+            if (budget > 0) {
+                const excerpt = f.addedText.length <= budget
+                    ? f.addedText
+                    : f.addedText.slice(0, budget) + "\n…[truncated]";
+                filePart = `${header}\n${excerpt}`;
+            }
+        }
+        if (totalChars + filePart.length > DIFF_TOTAL_MAX_CHARS) {
+            parts.push(header + "\n…[truncated]");
+            break;
+        }
+        totalChars += filePart.length;
+        parts.push(filePart);
+    }
+    return parts.join("\n\n");
+}
+export async function judgeOneClaim(provider, model, specSummary, claim, diff, opts = {}) {
+    const diffSummary = summarizeDiff(diff);
+    const userPrompt = opts.swapped
+        ? CLAIM_USER_SWAPPED(specSummary, claim.text, claim.id, diffSummary)
+        : CLAIM_USER(specSummary, claim.text, claim.id, diffSummary);
+    const response = await provider.generate({
+        model,
+        system: CLAIM_SYSTEM,
+        messages: [{ role: "user", content: [{ type: "text", text: userPrompt }] }],
+        tools: [claimTool],
+        toolChoice: { name: "emit_claim_verdict" },
+        maxTokens: 1024,
+    });
+    const toolUse = response.content.find((b) => b.type === "tool_use");
+    if (!toolUse || toolUse.type !== "tool_use") {
+        throw new Error(`judgeOneClaim: no emit_claim_verdict tool_use returned for claim ${claim.id}`);
+    }
+    const parsed = SpecClaimVerdictSchema.parse(toolUse.input);
+    const costUsd = computeCost(provider.kind, model, response.usage.inputTokens, response.usage.outputTokens);
+    return {
+        verdict: { ...parsed, claimId: claim.id },
+        stats: {
+            inputTokens: response.usage.inputTokens,
+            outputTokens: response.usage.outputTokens,
+            costUsd,
+        },
+    };
+}
+export async function judgeClaims(provider, model, specSummary, rubric, diff, opts = {}, onResult) {
+    const concurrency = Math.max(1, opts.concurrency ?? 4);
+    const items = rubric.claims;
+    const verdicts = new Array(items.length);
+    const aggStats = {
+        inputTokens: 0,
+        outputTokens: 0,
+        costUsd: 0,
+    };
+    let cursor = 0;
+    async function worker() {
+        while (true) {
+            const idx = cursor++;
+            if (idx >= items.length)
+                return;
+            const claim = items[idx];
+            if (!claim)
+                return;
+            const r = await judgeOneClaim(provider, model, specSummary, claim, diff);
+            verdicts[idx] = r.verdict;
+            aggStats.inputTokens += r.stats.inputTokens;
+            aggStats.outputTokens += r.stats.outputTokens;
+            aggStats.costUsd += r.stats.costUsd;
+            if (onResult)
+                onResult(r.verdict, r.stats, false);
+        }
+    }
+    await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, () => worker()));
+    return { verdicts, stats: aggStats };
+}
+//# sourceMappingURL=claims.js.map

package/dist/spec-checker/claims.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"claims.js","sourceRoot":"","sources":["../../src/spec-checker/claims.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,sBAAsB,EAAE,MAAM,mBAAmB,CAAC;AAI3D,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AAa5E,MAAM,SAAS,GAAmB;IAChC,IAAI,EAAE,oBAAoB;IAC1B,WAAW,EAAE,sCAAsC;IACnD,WAAW,EAAE;QACX,IAAI,EAAE,QAAQ;QACd,UAAU,EAAE;YACV,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC3B,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,EAAE;YACpE,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC5B,UAAU,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE;SACvD;QACD,QAAQ,EAAE,CAAC,SAAS,EAAE,SAAS,EAAE,YAAY,CAAC;KAC/C;CACF,CAAC;AAEF,MAAM,mBAAmB,GAAG,IAAI,CAAC;AACjC,MAAM,oBAAoB,GAAG,KAAK,CAAC;AAEnC,MAAM,UAAU,aAAa,CAAC,IAAU;IACtC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,cAAc,CAAC;IAEnD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;QAC3B,MAAM,MAAM,GAAG,GAAG,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,UAAU,KAAK,CAAC,CAAC,YAAY,GAAG,CAAC;QAC7E,IAAI,QAAQ,GAAG,MAAM,CAAC;QAEtB,IAAI,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1C,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,mBAAmB,EAAE,oBAAoB,GAAG,UAAU,GAAG,MAAM,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC;YACrG,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;gBACf,MAAM,OAAO,GACX,CAAC,CAAC,SAAS,CAAC,MAAM,IAAI,MAAM;oBAC1B,CAAC,CAAC,CAAC,CAAC,SAAS;oBACb,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,GAAG,gBAAgB,CAAC;gBACtD,QAAQ,GAAG,GAAG,MAAM,KAAK,OAAO,EAAE,CAAC;YACrC,CAAC;QACH,CAAC;QAED,IAAI,UAAU,GAAG,QAAQ,CAAC,MAAM,GAAG,oBAAoB,EAAE,CAAC;YACxD,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,gBAAgB,CAAC,CAAC;YACtC,MAAM;QACR,CAAC;QAED,UAAU,IAAI,QAAQ,CAAC,MAAM,CAAC;QAC9B,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACvB,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAC5B,CAAC;AAMD,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,QAAuB,EACvB,KAAa,EACb,WAAmB,EACnB,KAAmC,EACnC,IAAU,EACV,OAAuB,EAAE;IAEzB,MAAM,WAAW,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;IACxC,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO;QAC7B,CAAC,CAAC,kBAAkB,CAAC,WAAW,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE,WAAW,CAAC;QACpE,CAAC,CAAC,UAAU,CAAC,WAAW,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC;IAE/D,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC;QACvC,KAAK;QACL,MAAM,EAAE,YAAY;QACpB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,EAAE,CAAC;QAC3E,KAAK,EAAE,CAAC,SAAS,CAAC;QAClB,UAAU,EAAE,EAAE,IAAI,EAAE,oBAAoB,EAAE;QAC1C,SAAS,EAAE,IAAI;KAChB,CAAC,CAAC;IAEH,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC;IACpE,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;QAC5C,MAAM,IAAI,KAAK,CACb,oEAAoE,KAAK,CAAC,EAAE,EAAE,CAC/E,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,sBAAsB,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAC3D,MAAM,OAAO,GAAG,WAAW,CACzB,QAAQ,CAAC,IAAI,EACb,KAAK,EACL,QAAQ,CAAC,KAAK,CAAC,WAAW,EAC1B,QAAQ,CAAC,KAAK,CAAC,YAAY,CAC5B,CAAC;IAEF,OAAO;QACL,OAAO,EAAE,EAAE,GAAG,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,EAAE;QACzC,KAAK,EAAE;YACL,WAAW,EAAE,QAAQ,CAAC,KAAK,CAAC,WAAW;YACvC,YAAY,EAAE,QAAQ,CAAC,KAAK,CAAC,YAAY;YACzC,OAAO;SACR;KACF,CAAC;AACJ,CAAC;AAWD,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,QAAuB,EACvB,KAAa,EACb,WAAmB,EACnB,MAAkB,EAClB,IAAU,EACV,OAAwB,EAAE,EAC1B,QAIS;IAET,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;IACvD,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC;IAC5B,MAAM,QAAQ,GAAuB,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAC7D,MAAM,QAAQ,GAAoB;QAChC,WAAW,EAAE,CAAC;QACd,YAAY,EAAE,CAAC;QACf,OAAO,EAAE,CAAC;KACX,CAAC;IAEF,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,UAAU,MAAM;QACnB,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC;YACrB,IAAI,GAAG,IAAI,KAAK,CAAC,MAAM;gBAAE,OAAO;YAChC,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC;YACzB,IAAI,CAAC,KAAK;gBAAE,OAAO;YACnB,MAAM,CAAC,GAAG,MAAM,aAAa,CAAC,QAAQ,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC;YACzE,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC;YAC1B,QAAQ,CAAC,WAAW,IAAI,CAAC,CAAC,KAAK,CAAC,WAAW,CAAC;YAC5C,QAAQ,CAAC,YAAY,IAAI,CAAC,CAAC,KAAK,CAAC,YAAY,CAAC;YAC9C,QAAQ,CAAC,OAAO,IAAI,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC;YACpC,IAAI,QAAQ;gBAAE,QAAQ,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QACpD,CAAC;IACH,CAAC;IAED,MAAM,OAAO,CAAC,GAAG,CACf,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,MAAM,EAAE,CAAC,CAC5E,CAAC;IAEF,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;AACvC,CAAC"}

package/dist/spec-checker/index.d.ts

@@ -0,0 +1,40 @@
+import type Anthropic from "@anthropic-ai/sdk";
+import type { Diff } from "@keel_flow/core";
+import type { ArchitectureMap, SpecCheckMode, SpecCheckResult, Violation } from "@keel_flow/schema";
+import type { TelemetryEmitter } from "@keel_flow/telemetry";
+import type { ModelProvider } from "../provider.js";
+export interface SpecCheckerOptions {
+    provider?: ModelProvider;
+    client?: Anthropic;
+    model?: string;
+    mode?: SpecCheckMode;
+    telemetry?: TelemetryEmitter;
+    embed?: (text: string) => Promise<number[]>;
+    bidirectionalThreshold?: number;
+    claimConcurrency?: number;
+}
+export interface SpecCheckArgs {
+    diff: Diff;
+    architecture: ArchitectureMap;
+    specSummary: string;
+}
+export interface SpecCheckerCallable extends SpecCheckResult {
+    passed: boolean;
+    violations: Violation[];
+}
+export type SpecChecker = (args: SpecCheckArgs) => Promise<SpecCheckerCallable>;
+export declare function toGateCheck(r: SpecCheckResult): {
+    passed: boolean;
+    violations: Violation[];
+    pending?: boolean;
+};
+export declare function createSpecChecker(opts?: SpecCheckerOptions): SpecChecker;
+export { decomposeSpec } from "./rubric.js";
+export { judgeClaims, judgeOneClaim, summarizeDiff } from "./claims.js";
+export { bidirectionalCheck, jaccard, cosine } from "./bidirectional.js";
+export { adversarialPass } from "./adversarial.js";
+export { runWithPositionSwap } from "./bias.js";
+export { aggregateVerdict } from "./aggregate.js";
+export { loadCalibration } from "./calibration.js";
+export type { CalibrationReport, LoadCalibrationOpts } from "./calibration.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/spec-checker/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/spec-checker/index.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,SAAS,MAAM,mBAAmB,CAAC;AAC/C,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,KAAK,EACV,eAAe,EACf,aAAa,EACb,eAAe,EAEf,SAAS,EACV,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AAE7D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AASpD,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,EAAE,aAAa,CAAC;IACzB,MAAM,CAAC,EAAE,SAAS,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,aAAa,CAAC;IACrB,SAAS,CAAC,EAAE,gBAAgB,CAAC;IAC7B,KAAK,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAC5C,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,IAAI,CAAC;IACX,YAAY,EAAE,eAAe,CAAC;IAC9B,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,mBAAoB,SAAQ,eAAe;IAC1D,MAAM,EAAE,OAAO,CAAC;IAChB,UAAU,EAAE,SAAS,EAAE,CAAC;CACzB;AAED,MAAM,MAAM,WAAW,GAAG,CACxB,IAAI,EAAE,aAAa,KAChB,OAAO,CAAC,mBAAmB,CAAC,CAAC;AA4ClC,wBAAgB,WAAW,CAAC,CAAC,EAAE,eAAe,GAAG;IAC/C,MAAM,EAAE,OAAO,CAAC;IAChB,UAAU,EAAE,SAAS,EAAE,CAAC;IACxB,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB,CA+BA;AAED,wBAAgB,iBAAiB,CAAC,IAAI,CAAC,EAAE,kBAAkB,GAAG,WAAW,CAkQxE;AAED,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,WAAW,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AACxE,OAAO,EAAE,kBAAkB,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AACzE,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,EAAE,mBAAmB,EAAE,MAAM,WAAW,CAAC;AAChD,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACnD,YAAY,EAAE,iBAAiB,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC"}