@sanity/ailf 7.1.0 → 7.2.0

package/dist/pipeline/extract-grader-judgments-resilient.d.ts

@@ -0,0 +1,88 @@
+/**
+ * pipeline/extract-grader-judgments-resilient.ts
+ *
+ * Resilient grader-judgment extraction for the `calculate-scores` persist
+ * junction.
+ *
+ * Background: `calculateAndWriteScores` extracts grader judgments from the
+ * eval results file(s), then writes `grader-judgments.json` only when the
+ * array is non-empty (the `judgments.length > 0` guard). A runtime anomaly was
+ * observed where `extractGraderJudgments` returned 0 judgments for a results
+ * file that demonstrably contained classifiable llm-rubric components — the
+ * same file, read tens of milliseconds later by `extractStoredTestResults`,
+ * yielded the full set (entries with populated dimensions). The empty array
+ * silently skipped the write, so gap-analysis and compute-attribution skipped
+ * and the report shipped with a score but no tests.
+ *
+ * The committed code reads the file via a pure `readFileSync` with identical
+ * classification on both sides, so the divergence is not reproducible from the
+ * source + captured artifacts — it is a transient read anomaly at the live
+ * junction. This wrapper does not pretend to know the mechanism; it makes the
+ * junction observable and recovers from the transient:
+ *
+ *  1. **Instruments** — logs the resolved path(s), file size/mtime, parsed
+ *     result count, and judgment count on every run (never silent on 0), so a
+ *     future empty-judgments persist is diagnosable from the run log alone.
+ *  2. **Self-heals** — when extraction yields 0 judgments but a results file
+ *     exists, it re-extracts with bounded retries. A later read that yields
+ *     judgments proves the initial 0 was transient; the recovered judgments
+ *     are returned. If every attempt yields 0, severity is decided by an
+ *     independent classifiable-component count: a genuinely judgment-free run
+ *     (all api-errors / no llm-rubric) logs a warning, while 0 judgments
+ *     against present classifiable components logs an error (the downstream
+ *     gap-analysis fail-loud guard is the backstop).
+ *
+ * The extractor and fs helpers are injected so the wrapper is unit-testable
+ * without importing the ~3000-line scoring module (which would be circular)
+ * or touching the real filesystem.
+ */
+import type { GraderJudgment, GraderReliability, Logger } from "../_vendor/ailf-core/index.d.ts";
+/** Telemetry sink threaded into each extraction (shared reliability counters). */
+export interface ExtractionTelemetry {
+    reliability: GraderReliability;
+    runId?: string;
+}
+/** Cheap on-disk stat used for diagnostics and the retry gate. */
+export interface FileStat {
+    exists: boolean;
+    mtimeMs: number;
+    size: number;
+}
+/** Injectable seams — defaults wire the real fs; tests substitute fakes. */
+export interface ResilientExtractionDeps {
+    /**
+     * The real `extractGraderJudgments`. Injected (rather than imported) to
+     * avoid a circular dependency with `calculate-scores.ts`.
+     */
+    extract: (path: string, telemetry?: ExtractionTelemetry) => GraderJudgment[];
+    /** Parsed result-entry count for a path — diagnostics only. */
+    countResults?: (path: string) => number;
+    /**
+     * Count of classifiable llm-rubric components for a path. Used only to set
+     * the severity of a persistent-empty extraction: a file with classifiable
+     * components but 0 extracted judgments is an error; a file with none (all
+     * api-errors / no llm-rubric) is a benign empty.
+     */
+    countClassifiable?: (path: string) => number;
+    /** On-disk stat (existence + size + mtime). */
+    statFile?: (path: string) => FileStat;
+    /** Backoff between self-heal attempts. */
+    sleep?: (ms: number) => Promise<void>;
+}
+export interface ResilientExtractionOptions {
+    /** Total extraction attempts when the first yields 0 (default 3, min 1). */
+    maxAttempts?: number;
+    /** Delay before each retry, in ms (default 200). */
+    delayMs?: number;
+    deps: ResilientExtractionDeps;
+}
+/**
+ * Extract grader judgments across one or more results files, instrumented and
+ * self-healing. See the module header for the rationale.
+ *
+ * @param resultsPaths  One or more results files (e.g. baseline + agentic in
+ *   literacy full mode). Missing paths are skipped.
+ * @param telemetry  Shared reliability sink threaded into every extraction.
+ * @param log  Pipeline logger — the junction is logged here on every run.
+ */
+export declare function extractGraderJudgmentsResilient(resultsPaths: readonly string[], telemetry: ExtractionTelemetry | undefined, log: Logger, options: ResilientExtractionOptions): Promise<GraderJudgment[]>;

package/dist/pipeline/extract-grader-judgments-resilient.js

@@ -0,0 +1,122 @@
+/**
+ * pipeline/extract-grader-judgments-resilient.ts
+ *
+ * Resilient grader-judgment extraction for the `calculate-scores` persist
+ * junction.
+ *
+ * Background: `calculateAndWriteScores` extracts grader judgments from the
+ * eval results file(s), then writes `grader-judgments.json` only when the
+ * array is non-empty (the `judgments.length > 0` guard). A runtime anomaly was
+ * observed where `extractGraderJudgments` returned 0 judgments for a results
+ * file that demonstrably contained classifiable llm-rubric components — the
+ * same file, read tens of milliseconds later by `extractStoredTestResults`,
+ * yielded the full set (entries with populated dimensions). The empty array
+ * silently skipped the write, so gap-analysis and compute-attribution skipped
+ * and the report shipped with a score but no tests.
+ *
+ * The committed code reads the file via a pure `readFileSync` with identical
+ * classification on both sides, so the divergence is not reproducible from the
+ * source + captured artifacts — it is a transient read anomaly at the live
+ * junction. This wrapper does not pretend to know the mechanism; it makes the
+ * junction observable and recovers from the transient:
+ *
+ *  1. **Instruments** — logs the resolved path(s), file size/mtime, parsed
+ *     result count, and judgment count on every run (never silent on 0), so a
+ *     future empty-judgments persist is diagnosable from the run log alone.
+ *  2. **Self-heals** — when extraction yields 0 judgments but a results file
+ *     exists, it re-extracts with bounded retries. A later read that yields
+ *     judgments proves the initial 0 was transient; the recovered judgments
+ *     are returned. If every attempt yields 0, severity is decided by an
+ *     independent classifiable-component count: a genuinely judgment-free run
+ *     (all api-errors / no llm-rubric) logs a warning, while 0 judgments
+ *     against present classifiable components logs an error (the downstream
+ *     gap-analysis fail-loud guard is the backstop).
+ *
+ * The extractor and fs helpers are injected so the wrapper is unit-testable
+ * without importing the ~3000-line scoring module (which would be circular)
+ * or touching the real filesystem.
+ */
+import { existsSync, statSync } from "node:fs";
+const defaultStat = (path) => {
+    if (!existsSync(path))
+        return { exists: false, mtimeMs: 0, size: 0 };
+    const s = statSync(path);
+    return { exists: true, mtimeMs: s.mtimeMs, size: s.size };
+};
+const defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+/**
+ * Extract grader judgments across one or more results files, instrumented and
+ * self-healing. See the module header for the rationale.
+ *
+ * @param resultsPaths  One or more results files (e.g. baseline + agentic in
+ *   literacy full mode). Missing paths are skipped.
+ * @param telemetry  Shared reliability sink threaded into every extraction.
+ * @param log  Pipeline logger — the junction is logged here on every run.
+ */
+export async function extractGraderJudgmentsResilient(resultsPaths, telemetry, log, options) {
+    const { extract } = options.deps;
+    const statFile = options.deps.statFile ?? defaultStat;
+    const sleep = options.deps.sleep ?? defaultSleep;
+    const { countResults, countClassifiable } = options.deps;
+    const maxAttempts = Math.max(1, options.maxAttempts ?? 3);
+    const delayMs = options.delayMs ?? 200;
+    const present = resultsPaths.filter((p) => statFile(p).exists);
+    const extractAll = () => {
+        const all = [];
+        for (const p of present) {
+            all.push(...extract(p, telemetry));
+        }
+        return all;
+    };
+    const diag = (path) => {
+        const st = statFile(path);
+        return {
+            mtimeMs: st.mtimeMs,
+            path,
+            sizeBytes: st.size,
+            ...(countResults ? { resultCount: countResults(path) } : {}),
+        };
+    };
+    // Attempt 1 — always instrument the junction so 0 is never silent.
+    let judgments = extractAll();
+    for (const p of present) {
+        log.info("Grader judgments — persist junction read", diag(p));
+    }
+    log.info(`Grader judgments extracted: ${judgments.length} total`, {
+        judgmentCount: judgments.length,
+        paths: present,
+    });
+    if (judgments.length > 0)
+        return judgments;
+    // 0 judgments and no results file present → genuinely nothing to grade.
+    // A missing file cannot become non-empty within the retry window.
+    if (present.length === 0) {
+        log.info("No grader judgments — no results file present (nothing to grade)");
+        return judgments;
+    }
+    // Results file(s) exist but extraction yielded 0 judgments — a suspected
+    // transient read anomaly. Loud diagnostic, then bounded self-heal retries;
+    // the same file read tens of ms later has been observed to yield the full set.
+    log.warn("Grader extraction returned 0 judgments despite present results file(s) — suspected transient read anomaly; attempting self-heal", { paths: present.map(diag) });
+    for (let attempt = 2; attempt <= maxAttempts; attempt++) {
+        await sleep(delayMs);
+        judgments = extractAll();
+        log.warn(`Grader self-heal attempt ${attempt}/${maxAttempts}: ${judgments.length} judgment(s)`);
+        if (judgments.length > 0) {
+            log.warn(`Grader self-heal recovered ${judgments.length} grader judgment(s) on attempt ${attempt} — the initial empty extraction was a transient read anomaly`);
+            return judgments;
+        }
+    }
+    // Still empty after every attempt. Severity depends on whether the files
+    // actually contain classifiable components.
+    const classifiable = countClassifiable
+        ? present.reduce((n, p) => n + countClassifiable(p), 0)
+        : undefined;
+    if (classifiable === 0) {
+        log.warn(`No grader judgments after ${maxAttempts} attempt(s) — results contain no classifiable llm-rubric components (e.g. all api-errors); nothing to persist`);
+    }
+    else {
+        log.error(`Grader judgments empty after ${maxAttempts} attempt(s) but ${classifiable ?? "an unknown number of"} classifiable component(s) present in the results file(s) — persisting none; downstream gap-analysis/attribution will fail loud`, { paths: present.map(diag) });
+    }
+    return judgments;
+}

package/dist/report-store.d.ts

@@ -216,6 +216,7 @@ export interface SanityReportDoc {
     _type: string;
     comparison: null | Omit<ComparisonReport, "baseline" | "experiment">;
     completedAt: string;
+    degraded?: Report["degraded"];
     durationMs: number;
     provenance: Report["provenance"];
     reportId: ReportId;

package/dist/report-store.js

@@ -477,6 +477,7 @@ export function toSanityReportDoc(report) {
         _type: REPORT_TYPE,
         comparison,
         completedAt: report.completedAt,
+        ...(report.degraded ? { degraded: report.degraded } : {}),
         durationMs: report.durationMs,
         provenance: report.provenance,
         reportId: report.id,
@@ -526,6 +527,7 @@ export function toReport(doc) {
         artifactManifest,
         comparison: doc.comparison,
         completedAt: doc.completedAt,
+        degraded: doc.degraded,
         durationMs: doc.durationMs,
         id: doc.reportId,
         provenance: doc.provenance,

package/dist/sanity/queries.d.ts

@@ -69,7 +69,7 @@ export declare const ALL_ARTICLES_QUERY = "\n  *[_type == \"article\"\n    && !(
  *
  * Usage: client.fetch(ARTICLES_METADATA_BY_SLUGS_QUERY, { slugs: ["slug-a", "slug-b"] })
  */
-export declare const ARTICLES_METADATA_BY_SLUGS_QUERY = "\n  *[_type == \"article\"\n    && slug.current in $slugs\n    && !(_id in path(\"drafts.**\"))\n  ] {\n    \"slug\": slug.current,\n    _id,\n    _rev,\n    title\n  }\n";
+export declare const ARTICLES_METADATA_BY_SLUGS_QUERY = "\n  *[_type == \"article\"\n    && slug.current in $slugs\n    && !(_id in path(\"drafts.**\"))\n  ] {\n    \"slug\": slug.current,\n    \"sectionSlug\": primarySection->slug.current,\n    _id,\n    _rev,\n    title\n  }\n";
 /**
  * Fetch a single article by its slug — identical to ARTICLE_BY_SLUG_QUERY
  * but designed to be called with a perspective-enabled client.

package/dist/sanity/queries.js

@@ -203,6 +203,7 @@ export const ARTICLES_METADATA_BY_SLUGS_QUERY = `
     && !(_id in path("drafts.**"))
   ] {
     "slug": slug.current,
+    "sectionSlug": primarySection->slug.current,
     _id,
     _rev,
     title

package/dist/sources.js

@@ -37,6 +37,44 @@ const DEFAULT_SOURCE = {
     studioOrigin: "https://admin.sanity.io",
     urls: [],
 };
+/**
+ * Apply `SourceOverrides` + env-var fallbacks to `DEFAULT_SOURCE`.
+ *
+ * The DEFAULT_SOURCE early-return branches are taken when `config/sources`
+ * is missing or empty — the production state, since the named source
+ * definitions actually live in the `sanity-literacy` preset's `sourceDefs`
+ * (which `loadSource` doesn't consult). Returning `DEFAULT_SOURCE`
+ * verbatim drops every override the caller passed in, including
+ * `perspective` — observed live as production-source release evals
+ * fetching the published doc revision (W0295).
+ *
+ * The merge order mirrors the priority-1 (env-baseUrl) branch. The two
+ * paths diverge in three ways, all intentional: this branch (a) pins
+ * `baseUrl` / `llmsTxt` / `name` / `priorityDomain` to `DEFAULT_SOURCE`,
+ * (b) returns `documentIds: []` (the prior `DEFAULT_SOURCE` shape) where
+ * priority-1 would return `undefined` — both fall through the same
+ * `length > 0` consumer check, so behaviorally equivalent.
+ */
+function applyOverridesToDefault(overrides) {
+    const allowedOrigins = overrides?.allowedOrigins ?? parseAllowedOriginsEnv();
+    const headers = overrides?.headers ?? parseHeadersEnv();
+    return {
+        ...DEFAULT_SOURCE,
+        ...(allowedOrigins ? { allowedOrigins } : {}),
+        // oxlint-disable-next-line @typescript-eslint/prefer-nullish-coalescing -- empty string env var should fall back
+        dataset: overrides?.dataset ?? (process.env.SANITY_DATASET || "next"),
+        documentIds: overrides?.documentIds ?? parseDocumentIdsEnv() ?? [],
+        ...(headers ? { headers } : {}),
+        // oxlint-disable-next-line @typescript-eslint/prefer-nullish-coalescing -- empty string env var should fall back
+        perspective: overrides?.perspective ?? (process.env.SANITY_PERSPECTIVE || undefined),
+        // oxlint-disable-next-line @typescript-eslint/prefer-nullish-coalescing -- empty string env var should fall back
+        projectId: overrides?.projectId ?? (process.env.SANITY_PROJECT_ID || "3do82whm"),
+        // oxlint-disable-next-line @typescript-eslint/prefer-nullish-coalescing -- empty string env var should fall back
+        studioOrigin: overrides?.studioOrigin ??
+            (process.env.SANITY_STUDIO_ORIGIN || "https://admin.sanity.io"),
+        urls: overrides?.directUrls ?? parseDirectUrlsEnv(),
+    };
+}
 // ---------------------------------------------------------------------------
 // Validation
 // ---------------------------------------------------------------------------
@@ -117,12 +155,12 @@ export function loadSource(name, overrides, logger) {
             defaultBaseUrl: DEFAULT_SOURCE.baseUrl,
         });
         console.log("  No config/sources found, using built-in default (sanity.io production)");
-        return DEFAULT_SOURCE;
+        return applyOverridesToDefault(overrides);
     }
     if (!rawFile?.sources || Object.keys(rawFile.sources).length === 0) {
         log.debug("config/sources is empty, falling back to built-in default");
         console.log("  config/sources is empty, using built-in default");
-        return DEFAULT_SOURCE;
+        return applyOverridesToDefault(overrides);
     }
     // Resolve which source to use
     const sourceName = 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sanity/ailf",
-  "version": "7.1.0",
+  "version": "7.2.0",
   "private": false,
   "publishConfig": {
     "access": "public"