@sanity/ailf 5.0.0 → 6.1.0

package/dist/_vendor/ailf-core/services/diagnosis/cards/failure-mode-summary.js

@@ -0,0 +1,140 @@
+/**
+ * failure-mode-summary card — deterministic projection over Report.summary.failureModes.
+ *
+ * Pure computation, no LLM call. Identifies the dominant failure mode
+ * across all dimensions in the report's slim failure-mode summary.
+ *
+ * D-05: `.refine(buildFailureModeRefinement())` rejects cross-dimension
+ * (dimension, failureMode) pairs that the schema otherwise would accept —
+ * turning a "Zod-passes, semantically wrong" LLM output into a
+ * `parseFailed: true` degraded card. For this deterministic card, the
+ * refinement also defends against bad Report data.
+ *
+ * Schema is in the D0045 trust-boundary scan root; `satisfies` clause is
+ * mandatory (cards/ is a SCAN_ROOT in check-trust-boundary-satisfies.ts).
+ *
+ * @see .planning/phases/05-diagnosis-engine-cli-llm-cards/05-CONTEXT.md (D-05)
+ * @see packages/core/src/services/diagnosis/card-validators.ts
+ */
+import { z } from "zod";
+import { CANONICAL_DIMENSIONS, failureModesForDimension, } from "../../../grader/failure-modes/index.js";
+import { buildFailureModeRefinement } from "../card-validators.js";
+// ---------------------------------------------------------------------------
+// Body schema (D0045 trust boundary — satisfies required; D-05 refine)
+// ---------------------------------------------------------------------------
+export const FailureModeSummaryBodySchema = z
+    .object({
+    summary: z.string().min(1).max(800),
+    dimension: z.string().min(1),
+    failureMode: z.string().min(1),
+    count: z.number().int().nonnegative(),
+    sampleSize: z.number().int().nonnegative(),
+})
+    .refine(buildFailureModeRefinement(), {
+    message: "failureMode is not in the canonical taxonomy for this dimension",
+    path: ["failureMode"],
+});
+// ---------------------------------------------------------------------------
+// Private helper — find the dimension a failure mode belongs to
+// ---------------------------------------------------------------------------
+/**
+ * Per-family dimension preference order.
+ *
+ * Some failure modes (e.g., `missing-docs`) appear in multiple dimension
+ * families. When we know the report's eval mode, we should resolve the mode
+ * to a dimension in the matching family first, falling back to the linear
+ * scan only when the mode-preferred family doesn't carry the failure mode.
+ */
+const MODE_TO_PREFERRED_DIMENSIONS = {
+    literacy: ["task-completion", "code-correctness", "doc-coverage"],
+    "mcp-server": [
+        "mcp-behavior",
+        "input-validation",
+        "output-correctness",
+        "error-handling",
+        "security",
+    ],
+    "knowledge-probe": [
+        "knowledge-probe",
+        "factual-correctness",
+        "completeness",
+        "currency",
+    ],
+    "agent-harness": [
+        "agent-harness",
+        "process-quality",
+        "agent-output",
+        "tool-usage",
+    ],
+    custom: [],
+};
+/**
+ * Find a canonical dimension whose taxonomy includes `mode`. When a
+ * `preferredEvalMode` is supplied, prefer dimensions in the eval mode's
+ * family — e.g. a mode appearing in both literacy and MCP resolves to MCP on
+ * an MCP-only run. Falls back to the linear scan of CANONICAL_DIMENSIONS so
+ * cross-cutting modes (and modes from unknown eval modes) still resolve.
+ *
+ * Returns `undefined` if the mode is not in any dimension's taxonomy.
+ */
+function findDimensionForMode(mode, preferredEvalMode) {
+    if (preferredEvalMode) {
+        const preferred = MODE_TO_PREFERRED_DIMENSIONS[preferredEvalMode] ?? [];
+        for (const dim of preferred) {
+            if (failureModesForDimension(dim).includes(mode)) {
+                return dim;
+            }
+        }
+    }
+    for (const dim of CANONICAL_DIMENSIONS) {
+        if (failureModesForDimension(dim).includes(mode)) {
+            return dim;
+        }
+    }
+    return undefined;
+}
+// ---------------------------------------------------------------------------
+// Generator
+// ---------------------------------------------------------------------------
+export const generateFailureModeSummary = async (report) => {
+    const slimFm = report.summary.failureModes;
+    if (!slimFm ||
+        !slimFm.topTitles ||
+        slimFm.topTitles.length === 0 ||
+        slimFm.totalJudgments === 0) {
+        return {
+            status: "missing",
+            cardType: "failure-mode-summary",
+            reason: "report has no failure modes",
+        };
+    }
+    // Find the top entry — topTitles is already sorted by count descending
+    const topEntry = slimFm.topTitles.reduce((best, entry) => (entry.count > best.count ? entry : best), slimFm.topTitles[0]);
+    const failureMode = topEntry.category;
+    const dimension = findDimensionForMode(failureMode, report.provenance?.mode);
+    if (!dimension) {
+        return {
+            status: "missing",
+            cardType: "failure-mode-summary",
+            reason: `failure mode "${failureMode}" is not in the canonical taxonomy`,
+        };
+    }
+    const sampleSize = slimFm.totalJudgments;
+    const summary = `The most frequent failure mode is "${failureMode}" (${topEntry.count} of ${sampleSize} judgments in dimension "${dimension}").`;
+    const body = FailureModeSummaryBodySchema.parse({
+        summary,
+        dimension,
+        failureMode,
+        count: topEntry.count,
+        sampleSize,
+    });
+    return {
+        status: "ready",
+        cardType: "failure-mode-summary",
+        body,
+        meta: {
+            cardVersion: "failure-mode-summary@0.1.0",
+            generatedAt: new Date().toISOString(),
+        },
+    };
+};

package/dist/_vendor/ailf-core/services/diagnosis/cards/index.d.ts

@@ -0,0 +1,49 @@
+/**
+ * Card-generator barrel — exports all 8 generators + DIAGNOSIS_CARD_GENERATORS.
+ *
+ * This barrel lives in @sanity/ailf-core so the Plan-06 API route can import
+ * `DIAGNOSIS_CARD_GENERATORS` via `import { DIAGNOSIS_CARD_GENERATORS } from
+ * "@sanity/ailf-core"` without depending on `@sanity/ailf` (D-01 boundary).
+ *
+ * This is a CARD-GENERATOR barrel only — no vendor SDK classes, no adapter
+ * implementations. Cards speak the `LLMClient` port exclusively (D0051).
+ *
+ * TypeScript exhaustiveness: `CardType` is a 8-element literal union; the
+ * `Record<CardType, CardGenerator>` annotation causes a build error if any
+ * key is missing or extra.
+ *
+ * @see .planning/phases/05-diagnosis-engine-cli-llm-cards/05-CONTEXT.md (D-01)
+ * @see docs/decisions/D0051-llm-client-port.md
+ */
+import { generateAreaSummary } from "./area-summary.js";
+import { generateFailureModeSummary } from "./failure-mode-summary.js";
+import { generateNoIssues } from "./no-issues.js";
+import { generateTopRecommendations } from "./top-recommendations.js";
+import { generateWeakestArea } from "./weakest-area.js";
+import { generateLowConfidenceAttribution } from "./low-confidence-attribution.js";
+import { generateDocAttributionSpotlight } from "./doc-attribution-spotlight.js";
+import { generateRegressionVsBaseline } from "./regression-vs-baseline.js";
+import type { CardGenerator } from "../../diagnosis-runner.js";
+import type { CardType } from "../../../types/diagnosis.js";
+/**
+ * Canonical version of the card-registry surface — bumped whenever any card
+ * generator or body schema in this barrel changes. Used as the `cardVersion`
+ * fallback in version-resolver helpers (CLI `interpret`, API
+ * `versionsFromRecord`) so the four-version cache invalidation envelope stays
+ * in sync with the actual registry.
+ *
+ * Mirrors the pattern of `diagnosisVersion` exported from `diagnosis-runner.ts`.
+ */
+export declare const CARD_REGISTRY_VERSION = "0.1.0";
+/**
+ * The canonical card-generator registry for the diagnosis engine.
+ *
+ * `Readonly<Record<CardType, CardGenerator>>` — TypeScript exhaustiveness
+ * ensures all 8 `CardType` strings appear (no rogue keys, no missing keys).
+ * The composition root (`packages/eval/src/composition-root.ts`) passes this
+ * directly into `createDiagnosisRunner(deps)`.
+ *
+ * Also consumed by the Plan-06 API route, which imports via `@sanity/ailf-core`.
+ */
+export declare const DIAGNOSIS_CARD_GENERATORS: Readonly<Record<CardType, CardGenerator>>;
+export { generateAreaSummary, generateFailureModeSummary, generateNoIssues, generateTopRecommendations, generateWeakestArea, generateLowConfidenceAttribution, generateDocAttributionSpotlight, generateRegressionVsBaseline, };

package/dist/_vendor/ailf-core/services/diagnosis/cards/index.js

@@ -0,0 +1,65 @@
+/**
+ * Card-generator barrel — exports all 8 generators + DIAGNOSIS_CARD_GENERATORS.
+ *
+ * This barrel lives in @sanity/ailf-core so the Plan-06 API route can import
+ * `DIAGNOSIS_CARD_GENERATORS` via `import { DIAGNOSIS_CARD_GENERATORS } from
+ * "@sanity/ailf-core"` without depending on `@sanity/ailf` (D-01 boundary).
+ *
+ * This is a CARD-GENERATOR barrel only — no vendor SDK classes, no adapter
+ * implementations. Cards speak the `LLMClient` port exclusively (D0051).
+ *
+ * TypeScript exhaustiveness: `CardType` is a 8-element literal union; the
+ * `Record<CardType, CardGenerator>` annotation causes a build error if any
+ * key is missing or extra.
+ *
+ * @see .planning/phases/05-diagnosis-engine-cli-llm-cards/05-CONTEXT.md (D-01)
+ * @see docs/decisions/D0051-llm-client-port.md
+ */
+import { generateAreaSummary } from "./area-summary.js";
+import { generateFailureModeSummary } from "./failure-mode-summary.js";
+import { generateNoIssues } from "./no-issues.js";
+import { generateTopRecommendations } from "./top-recommendations.js";
+import { generateWeakestArea } from "./weakest-area.js";
+import { generateLowConfidenceAttribution } from "./low-confidence-attribution.js";
+import { generateDocAttributionSpotlight } from "./doc-attribution-spotlight.js";
+import { generateRegressionVsBaseline } from "./regression-vs-baseline.js";
+// ---------------------------------------------------------------------------
+// Card registry version (cache invalidation segment)
+// ---------------------------------------------------------------------------
+/**
+ * Canonical version of the card-registry surface — bumped whenever any card
+ * generator or body schema in this barrel changes. Used as the `cardVersion`
+ * fallback in version-resolver helpers (CLI `interpret`, API
+ * `versionsFromRecord`) so the four-version cache invalidation envelope stays
+ * in sync with the actual registry.
+ *
+ * Mirrors the pattern of `diagnosisVersion` exported from `diagnosis-runner.ts`.
+ */
+export const CARD_REGISTRY_VERSION = "0.1.0";
+// ---------------------------------------------------------------------------
+// DIAGNOSIS_CARD_GENERATORS — full 8-card registry literal
+// ---------------------------------------------------------------------------
+/**
+ * The canonical card-generator registry for the diagnosis engine.
+ *
+ * `Readonly<Record<CardType, CardGenerator>>` — TypeScript exhaustiveness
+ * ensures all 8 `CardType` strings appear (no rogue keys, no missing keys).
+ * The composition root (`packages/eval/src/composition-root.ts`) passes this
+ * directly into `createDiagnosisRunner(deps)`.
+ *
+ * Also consumed by the Plan-06 API route, which imports via `@sanity/ailf-core`.
+ */
+export const DIAGNOSIS_CARD_GENERATORS = {
+    "area-summary": generateAreaSummary,
+    "failure-mode-summary": generateFailureModeSummary,
+    "no-issues": generateNoIssues,
+    "top-recommendations": generateTopRecommendations,
+    "weakest-area": generateWeakestArea,
+    "low-confidence-attribution": generateLowConfidenceAttribution,
+    "doc-attribution-spotlight": generateDocAttributionSpotlight,
+    "regression-vs-baseline": generateRegressionVsBaseline,
+};
+// ---------------------------------------------------------------------------
+// Individual re-exports (for callers that want a single generator)
+// ---------------------------------------------------------------------------
+export { generateAreaSummary, generateFailureModeSummary, generateNoIssues, generateTopRecommendations, generateWeakestArea, generateLowConfidenceAttribution, generateDocAttributionSpotlight, generateRegressionVsBaseline, };

package/dist/_vendor/ailf-core/services/diagnosis/cards/low-confidence-attribution.d.ts

@@ -0,0 +1,27 @@
+/**
+ * low-confidence-attribution card — LLM-driven uncertain-attribution finder.
+ *
+ * Model: claude-sonnet-4-6 (routine per AI-SPEC §4 model routing)
+ * Version: low-confidence-attribution@0.1.0
+ *
+ * Landmine 11: this card reads `ctx.judgmentAttributions` (NOT Report.summary).
+ * Returns `status: "missing"` when attributions are undefined or empty BEFORE
+ * calling the LLM — this is a structural seam, not an error path.
+ *
+ * Schema is in the D0045 trust-boundary scan root; `satisfies` clause is
+ * mandatory.
+ *
+ * @see .planning/phases/05-diagnosis-engine-cli-llm-cards/05-RESEARCH.md (Landmine 11)
+ * @see docs/decisions/D0052-judgment-ref-granularity.md
+ */
+import { z } from "zod";
+import type { CardGenerator } from "../../diagnosis-runner.js";
+export declare const LowConfidenceAttributionBodySchema: z.ZodObject<{
+    summary: z.ZodString;
+    judgmentRefs: z.ZodArray<z.ZodObject<{
+        taskId: z.ZodString;
+        modelId: z.ZodString;
+        dimension: z.ZodString;
+    }, z.core.$strip>>;
+}, z.core.$strip>;
+export declare const generateLowConfidenceAttribution: CardGenerator;

package/dist/_vendor/ailf-core/services/diagnosis/cards/low-confidence-attribution.js

@@ -0,0 +1,93 @@
+/**
+ * low-confidence-attribution card — LLM-driven uncertain-attribution finder.
+ *
+ * Model: claude-sonnet-4-6 (routine per AI-SPEC §4 model routing)
+ * Version: low-confidence-attribution@0.1.0
+ *
+ * Landmine 11: this card reads `ctx.judgmentAttributions` (NOT Report.summary).
+ * Returns `status: "missing"` when attributions are undefined or empty BEFORE
+ * calling the LLM — this is a structural seam, not an error path.
+ *
+ * Schema is in the D0045 trust-boundary scan root; `satisfies` clause is
+ * mandatory.
+ *
+ * @see .planning/phases/05-diagnosis-engine-cli-llm-cards/05-RESEARCH.md (Landmine 11)
+ * @see docs/decisions/D0052-judgment-ref-granularity.md
+ */
+import { z } from "zod";
+import { modelId as mkModelId } from "../../../ports/llm-client.js";
+import { buildLowConfidenceAttributionPrompt } from "../prompt-builders.js";
+// ---------------------------------------------------------------------------
+// Body schema (D0045 trust boundary — satisfies required)
+// ---------------------------------------------------------------------------
+export const LowConfidenceAttributionBodySchema = z.object({
+    summary: z.string().min(1).max(800),
+    judgmentRefs: z
+        .array(z.object({
+        taskId: z.string().min(1),
+        modelId: z.string().min(1),
+        dimension: z.string().min(1),
+    }))
+        .min(1),
+});
+// ---------------------------------------------------------------------------
+// Generator
+// ---------------------------------------------------------------------------
+const CARD_MODEL = mkModelId("anthropic:claude-sonnet-4-6");
+export const generateLowConfidenceAttribution = async (report, ctx) => {
+    // C1: no LLM → missing
+    if (!ctx.llm) {
+        return {
+            status: "missing",
+            cardType: "low-confidence-attribution",
+            reason: "no LLMClient wired",
+        };
+    }
+    // L1: Landmine 11 — short-circuit BEFORE calling LLM when no attribution data
+    if (!ctx.judgmentAttributions || ctx.judgmentAttributions.length === 0) {
+        return {
+            status: "missing",
+            cardType: "low-confidence-attribution",
+            reason: "no attribution data for this run",
+        };
+    }
+    // Also short-circuit when every JudgmentAttribution has an empty
+    // `attributions` array. The card schema requires `judgmentRefs.min(1)`, so
+    // the LLM would have nothing to cite — degrade gracefully instead of
+    // building an empty-table prompt and failing parse downstream.
+    const hasAnyAttributions = ctx.judgmentAttributions.some((ja) => ja.attributions.length > 0);
+    if (!hasAnyAttributions) {
+        return {
+            status: "missing",
+            cardType: "low-confidence-attribution",
+            reason: "attribution entries exist but all are empty",
+        };
+    }
+    const prompt = buildLowConfidenceAttributionPrompt(report, ctx.judgmentAttributions);
+    // Destructure `cost` and `model` from the LLMClient return —
+    // already provided per llm-client.ts:139-144, previously discarded.
+    const { value, usage, cost, model } = await ctx.llm.completeStructured({
+        model: CARD_MODEL,
+        prompt: `${prompt.system}\n\n${prompt.user}`,
+        schema: LowConfidenceAttributionBodySchema,
+        temperature: 0.1,
+        maxTokens: 2000,
+        context: {
+            feature: "diagnosis",
+            runId: ctx.runId,
+            cardId: "low-confidence-attribution",
+        },
+    });
+    return {
+        status: "ready",
+        cardType: "low-confidence-attribution",
+        body: value,
+        meta: {
+            cardVersion: "low-confidence-attribution@0.1.0",
+            tokenUsage: { input: usage.promptTokens, output: usage.completionTokens },
+            generatedAt: new Date().toISOString(),
+            cost,
+            model,
+        },
+    };
+};

package/dist/_vendor/ailf-core/services/diagnosis/cards/no-issues.d.ts

@@ -0,0 +1,32 @@
+/**
+ * no-issues card — deterministic guard against sycophantic "all is well" reports.
+ *
+ * Only fires when ALL areas in the report scored at or above
+ * `NO_ISSUES_THRESHOLD`. The threshold is calibrated to keep the firing rate
+ * ≤30% per AI-SPEC §1b failure-mode #7 (sycophantic no-issues).
+ *
+ * Pure computation, no LLM call. Uses `report.summary.scores` — each
+ * `FeatureScore.totalScore` is the composite (0–100 scale).
+ *
+ * Schema is in the D0045 trust-boundary scan root; `satisfies` clause is
+ * mandatory (cards/ is a SCAN_ROOT in check-trust-boundary-satisfies.ts).
+ *
+ * @see .planning/phases/05-diagnosis-engine-cli-llm-cards/05-AI-SPEC.md §1b failure-mode #7
+ */
+import { z } from "zod";
+import type { CardGenerator } from "../../diagnosis-runner.js";
+/**
+ * Threshold above which a Report area is considered "no issues" per the
+ * team's action threshold (AI-SPEC §1b failure-mode #7). Tuned against
+ * the fixture set in no-issues.test.ts to keep firing rate ≤30%.
+ *
+ * At 85, only reports where every area scores ≥85 trigger this card.
+ * On the 10-report fixture spanning 0–99, only reports at [90,92],
+ * [95,97], [98,99] qualify — a 3/10 = 30% firing rate (right at the cap).
+ */
+export declare const NO_ISSUES_THRESHOLD = 85;
+export declare const NoIssuesBodySchema: z.ZodObject<{
+    summary: z.ZodString;
+    thresholdScore: z.ZodNumber;
+}, z.core.$strip>;
+export declare const generateNoIssues: CardGenerator;

package/dist/_vendor/ailf-core/services/diagnosis/cards/no-issues.js

@@ -0,0 +1,71 @@
+/**
+ * no-issues card — deterministic guard against sycophantic "all is well" reports.
+ *
+ * Only fires when ALL areas in the report scored at or above
+ * `NO_ISSUES_THRESHOLD`. The threshold is calibrated to keep the firing rate
+ * ≤30% per AI-SPEC §1b failure-mode #7 (sycophantic no-issues).
+ *
+ * Pure computation, no LLM call. Uses `report.summary.scores` — each
+ * `FeatureScore.totalScore` is the composite (0–100 scale).
+ *
+ * Schema is in the D0045 trust-boundary scan root; `satisfies` clause is
+ * mandatory (cards/ is a SCAN_ROOT in check-trust-boundary-satisfies.ts).
+ *
+ * @see .planning/phases/05-diagnosis-engine-cli-llm-cards/05-AI-SPEC.md §1b failure-mode #7
+ */
+import { z } from "zod";
+// ---------------------------------------------------------------------------
+// Threshold constant (calibration)
+// ---------------------------------------------------------------------------
+/**
+ * Threshold above which a Report area is considered "no issues" per the
+ * team's action threshold (AI-SPEC §1b failure-mode #7). Tuned against
+ * the fixture set in no-issues.test.ts to keep firing rate ≤30%.
+ *
+ * At 85, only reports where every area scores ≥85 trigger this card.
+ * On the 10-report fixture spanning 0–99, only reports at [90,92],
+ * [95,97], [98,99] qualify — a 3/10 = 30% firing rate (right at the cap).
+ */
+export const NO_ISSUES_THRESHOLD = 85;
+// ---------------------------------------------------------------------------
+// Body schema (D0045 trust boundary — satisfies required)
+// ---------------------------------------------------------------------------
+export const NoIssuesBodySchema = z.object({
+    summary: z.string().min(1).max(500),
+    thresholdScore: z.number(),
+});
+// ---------------------------------------------------------------------------
+// Generator
+// ---------------------------------------------------------------------------
+export const generateNoIssues = async (report) => {
+    const scores = report.summary.scores;
+    if (!scores || scores.length === 0) {
+        return {
+            status: "missing",
+            cardType: "no-issues",
+            reason: "report has no areas",
+        };
+    }
+    const allAboveThreshold = scores.every((s) => s.totalScore >= NO_ISSUES_THRESHOLD);
+    if (!allAboveThreshold) {
+        return {
+            status: "missing",
+            cardType: "no-issues",
+            reason: "at least one area scored below threshold",
+        };
+    }
+    const summary = `All ${scores.length} areas scored ≥${NO_ISSUES_THRESHOLD} — no action required.`;
+    const body = NoIssuesBodySchema.parse({
+        summary,
+        thresholdScore: NO_ISSUES_THRESHOLD,
+    });
+    return {
+        status: "ready",
+        cardType: "no-issues",
+        body,
+        meta: {
+            cardVersion: "no-issues@0.1.0",
+            generatedAt: new Date().toISOString(),
+        },
+    };
+};

package/dist/_vendor/ailf-core/services/diagnosis/cards/regression-vs-baseline.d.ts

@@ -0,0 +1,44 @@
+/**
+ * regression-vs-baseline card — LLM-driven run comparison card.
+ *
+ * Model: claude-opus-4-6 (high-stakes per AI-SPEC §4 model routing)
+ * Version: regression-vs-baseline@0.1.0
+ *
+ * DIAG-05: emits ONLY when `ctx.baseline` is set. When baseline is absent,
+ * returns `status: "missing", reason: "no --compare baseline supplied"`.
+ *
+ * Mitigations:
+ * - failure-mode #1: `buildRegressionVsBaselinePrompt` computes deltas in JS
+ *   BEFORE the LLM call; schema refine asserts sign-consistency (R3)
+ *
+ * Schema is in the D0045 trust-boundary scan root; `satisfies` clause is
+ * mandatory.
+ *
+ * @see .planning/phases/05-diagnosis-engine-cli-llm-cards/05-AI-SPEC.md §3 lines 603-664
+ */
+import { z } from "zod";
+import type { CardGenerator } from "../../diagnosis-runner.js";
+/**
+ * Module-level static shape only. Per-call adds:
+ * - sign-consistency refine (R3): Math.sign(pointsDelta) === directionSign(direction)
+ */
+export declare const RegressionVsBaselineBodySchema: z.ZodObject<{
+    summary: z.ZodString;
+    deltas: z.ZodArray<z.ZodObject<{
+        area: z.ZodString;
+        direction: z.ZodEnum<{
+            improved: "improved";
+            regressed: "regressed";
+            unchanged: "unchanged";
+        }>;
+        pointsDelta: z.ZodNumber;
+        drivers: z.ZodArray<z.ZodString>;
+    }, z.core.$strip>>;
+    overallTrend: z.ZodEnum<{
+        "net-improved": "net-improved";
+        "net-regressed": "net-regressed";
+        mixed: "mixed";
+        stable: "stable";
+    }>;
+}, z.core.$strip>;
+export declare const generateRegressionVsBaseline: CardGenerator;

package/dist/_vendor/ailf-core/services/diagnosis/cards/regression-vs-baseline.js

@@ -0,0 +1,130 @@
+/**
+ * regression-vs-baseline card — LLM-driven run comparison card.
+ *
+ * Model: claude-opus-4-6 (high-stakes per AI-SPEC §4 model routing)
+ * Version: regression-vs-baseline@0.1.0
+ *
+ * DIAG-05: emits ONLY when `ctx.baseline` is set. When baseline is absent,
+ * returns `status: "missing", reason: "no --compare baseline supplied"`.
+ *
+ * Mitigations:
+ * - failure-mode #1: `buildRegressionVsBaselinePrompt` computes deltas in JS
+ *   BEFORE the LLM call; schema refine asserts sign-consistency (R3)
+ *
+ * Schema is in the D0045 trust-boundary scan root; `satisfies` clause is
+ * mandatory.
+ *
+ * @see .planning/phases/05-diagnosis-engine-cli-llm-cards/05-AI-SPEC.md §3 lines 603-664
+ */
+import { z } from "zod";
+import { modelId as mkModelId } from "../../../ports/llm-client.js";
+import { buildRegressionVsBaselinePrompt } from "../prompt-builders.js";
+// ---------------------------------------------------------------------------
+// Helper: direction sign check
+// ---------------------------------------------------------------------------
+/**
+ * Map direction label to its expected sign. Returns the expected sign:
+ * +1 for improved, -1 for regressed, 0 for unchanged.
+ */
+function directionSign(direction) {
+    if (direction === "improved")
+        return 1;
+    if (direction === "regressed")
+        return -1;
+    return 0;
+}
+// ---------------------------------------------------------------------------
+// Body schema (D0045 trust boundary — satisfies required)
+// ---------------------------------------------------------------------------
+/**
+ * Module-level static shape only. Per-call adds:
+ * - sign-consistency refine (R3): Math.sign(pointsDelta) === directionSign(direction)
+ */
+export const RegressionVsBaselineBodySchema = z.object({
+    summary: z.string().min(1).max(800),
+    deltas: z
+        .array(z.object({
+        area: z.string().min(1),
+        direction: z.enum(["improved", "regressed", "unchanged"]),
+        pointsDelta: z.number(),
+        drivers: z.array(z.string()),
+    }))
+        .max(10),
+    overallTrend: z.enum(["net-improved", "net-regressed", "mixed", "stable"]),
+});
+// ---------------------------------------------------------------------------
+// Generator
+// ---------------------------------------------------------------------------
+const CARD_MODEL = mkModelId("anthropic:claude-opus-4-6");
+export const generateRegressionVsBaseline = async (report, ctx) => {
+    // C1: no LLM → missing
+    if (!ctx.llm) {
+        return {
+            status: "missing",
+            cardType: "regression-vs-baseline",
+            reason: "no LLMClient wired",
+        };
+    }
+    // R1: DIAG-05 no-auto-comparison — only emits when baseline is supplied
+    if (!ctx.baseline) {
+        return {
+            status: "missing",
+            cardType: "regression-vs-baseline",
+            reason: "no --compare baseline supplied",
+        };
+    }
+    // Compute deltas in JS BEFORE the LLM call (failure-mode #1 mitigation).
+    // The prompt embeds the precomputed numbers so the LLM cannot fabricate deltas.
+    const { system, user } = buildRegressionVsBaselinePrompt(report, ctx.baseline);
+    // Per-call schema: additive R3 sign-consistency refine
+    // (AI-SPEC §3 Pitfall 1 — schema local to generator, not module scope)
+    const PerCallSchema = z
+        .object({
+        summary: z.string().min(1).max(800),
+        deltas: z
+            .array(z.object({
+            area: z.string().min(1),
+            direction: z.enum(["improved", "regressed", "unchanged"]),
+            pointsDelta: z.number(),
+            drivers: z.array(z.string()),
+        }))
+            .max(10),
+        overallTrend: z.enum([
+            "net-improved",
+            "net-regressed",
+            "mixed",
+            "stable",
+        ]),
+    })
+        // R3: sign-consistency guardrail — LLM must not invert direction vs delta
+        .refine((body) => body.deltas.every((d) => Math.sign(d.pointsDelta) === directionSign(d.direction)), {
+        message: "direction label must match sign of pointsDelta (improved=positive, regressed=negative, unchanged=zero)",
+        path: ["deltas"],
+    });
+    // Destructure `cost` and `model` from the LLMClient return —
+    // already provided per llm-client.ts:139-144, previously discarded.
+    const { value, usage, cost, model } = await ctx.llm.completeStructured({
+        model: CARD_MODEL,
+        prompt: `${system}\n\n${user}`,
+        schema: PerCallSchema,
+        temperature: 0.1,
+        maxTokens: 2000,
+        context: {
+            feature: "diagnosis",
+            runId: ctx.runId,
+            cardId: "regression-vs-baseline",
+        },
+    });
+    return {
+        status: "ready",
+        cardType: "regression-vs-baseline",
+        body: value,
+        meta: {
+            cardVersion: "regression-vs-baseline@0.1.0",
+            tokenUsage: { input: usage.promptTokens, output: usage.completionTokens },
+            generatedAt: new Date().toISOString(),
+            cost,
+            model,
+        },
+    };
+};