@sanity/ailf 0.1.32 → 0.1.34

package/dist/_vendor/ailf-core/ports/context.d.ts

@@ -177,7 +177,7 @@ export interface AppContext {
  */
 export interface ReportStorePort {
     /** Auto-compare against the most recent comparable baseline */
-    autoCompare(currentSummary: unknown, provenance: unknown, completedAt: unknown): Promise<unknown>;
+    autoCompare(currentSummary: unknown, provenance: unknown, completedAt: unknown, scopedAreas?: Set<string>): Promise<unknown>;
     /** Find a report by its eval fingerprint (for cache lookup) */
     findByFingerprint(fingerprint: string): Promise<null | unknown>;
     /** Find the most recent comparable baseline for auto-comparison */

package/dist/_vendor/ailf-core/types/index.d.ts

@@ -853,7 +853,7 @@ export interface AttributionReport {
     untrackedDocs: string[];
 }
 /** Classification of a feature area's score change */
-export type ChangeClass = "improved" | "regressed" | "unchanged";
+export type ChangeClass = "improved" | "not-evaluated" | "regressed" | "unchanged";
 /** Options for the compare function */
 export interface CompareOptions {
     /** Grader consistency data — if provided, used to compute empirical noise threshold */
@@ -927,6 +927,12 @@ export interface ComparisonReport {
     noiseThreshold: number;
     /** Whether the noise threshold was derived from empirical grader consistency data */
     noiseThresholdEmpirical: boolean;
+    /**
+     * Areas present in only one summary (not evaluated in both runs).
+     * These are excluded from improved/regressed/unchanged classification
+     * because comparing against a missing score is meaningless.
+     */
+    notEvaluated: string[];
     /** Areas that regressed beyond the noise threshold */
     regressed: string[];
     /** Areas within the noise threshold */

package/dist/adapters/api-client/build-request.js

@@ -101,6 +101,13 @@ export async function buildRemoteRequest(options) {
         raw.discoveryReport = true;
     if (config.noRemoteCache)
         raw.noRemoteCache = true;
+    // Caller git metadata — auto-detect from CI environment variables.
+    // When running via `ailf pipeline --remote` in a GitHub Actions workflow,
+    // the GITHUB_* env vars identify the *calling* repo (not the AILF core
+    // repo). This ensures report provenance attributes to the right repo.
+    const callerGit = detectCallerGit();
+    if (callerGit)
+        raw.callerGit = callerGit;
     // 4. Validate the assembled request
     const parsed = PipelineRequestSchema.parse(raw);
     return { request: parsed, taskCount: tasks.length };
@@ -180,3 +187,32 @@ function buildFilterOptions(config) {
         return undefined;
     return { areas, taskIds, tags };
 }
+/**
+ * Auto-detect caller git metadata from GitHub Actions environment variables.
+ *
+ * When the CLI runs in a calling repo's CI (via `npx @sanity/ailf pipeline
+ * --remote`), the GITHUB_* env vars reflect that repo — not the AILF core
+ * repo. We capture them here so the API can carry them through to report
+ * provenance.
+ *
+ * Returns undefined when not running in GitHub Actions.
+ */
+function detectCallerGit() {
+    const repo = process.env.GITHUB_REPOSITORY;
+    if (!repo)
+        return undefined;
+    const sha = process.env.GITHUB_SHA;
+    const ref = process.env.GITHUB_REF ?? "";
+    // For PRs, GITHUB_HEAD_REF is the source branch name (e.g., "fix/docs").
+    // For pushes, GITHUB_REF_NAME is the branch (e.g., "main").
+    const branch = process.env.GITHUB_HEAD_REF || process.env.GITHUB_REF_NAME;
+    // Extract PR number from GITHUB_REF (refs/pull/123/merge)
+    const prMatch = ref.match(/^refs\/pull\/(\d+)\//);
+    const prNumber = prMatch ? parseInt(prMatch[1], 10) : undefined;
+    return {
+        repo,
+        ...(sha ? { sha } : {}),
+        ...(branch ? { branch } : {}),
+        ...(prNumber ? { prNumber } : {}),
+    };
+}

package/dist/orchestration/steps/publish-report-step.js

@@ -82,8 +82,16 @@ export class PublishReportStep {
         const durationMs = Date.now() - this.pipelineStart;
         // Auto-compare against most recent comparable baseline.
         // Returns the comparison + baseline report ID for lineage tracking.
+        //
+        // When release auto-scope is active, the current experiment only covers
+        // a subset of areas. We pass the evaluated area set so autoCompare can
+        // scope the baseline to match — preventing mismatched areas from
+        // polluting the overall delta.
+        const evaluatedAreas = state.releaseAutoScope
+            ? new Set(summary.scores.map((s) => s.feature))
+            : undefined;
         const autoCompareResult = ctx.reportStore
-            ? (await ctx.reportStore.autoCompare(summary, provenance, now))
+            ? (await ctx.reportStore.autoCompare(summary, provenance, now, evaluatedAreas))
             : null;
         const comparison = autoCompareResult?.comparison ?? null;
         // Record which report we compared against in lineage

package/dist/pipeline/compare.js

@@ -51,11 +51,14 @@ export function compare(baseline, experiment, options) {
     // Identify mismatched areas
     const onlyInBaseline = [...baselineAreas].filter((a) => !experimentAreas.has(a));
     const onlyInExperiment = [...experimentAreas].filter((a) => !baselineAreas.has(a));
-    // Build per-area deltas
+    // Build per-area deltas.
+    // Areas present in only one summary get change: "not-evaluated" — comparing
+    // against a missing score is meaningless (it would produce false ±100 deltas).
+    const mismatchedSet = new Set([...onlyInBaseline, ...onlyInExperiment]);
     const areas = [...allAreas]
         .sort()
-        .map((area) => buildAreaDelta(area, findScore(baseline.scores, area), findScore(experiment.scores, area), threshold));
-    // Classify areas
+        .map((area) => buildAreaDelta(area, findScore(baseline.scores, area), findScore(experiment.scores, area), threshold, mismatchedSet.has(area)));
+    // Classify areas — mismatched areas are excluded from all three buckets
     const improved = areas
         .filter((a) => a.change === "improved")
         .map((a) => a.area);
@@ -65,6 +68,9 @@ export function compare(baseline, experiment, options) {
     const unchanged = areas
         .filter((a) => a.change === "unchanged")
         .map((a) => a.area);
+    const notEvaluated = areas
+        .filter((a) => a.change === "not-evaluated")
+        .map((a) => a.area);
     // Per-area deltas as a record
     const perArea = {};
     for (const a of areas) {
@@ -128,12 +134,13 @@ export function compare(baseline, experiment, options) {
         },
         noiseThreshold: threshold,
         noiseThresholdEmpirical: empirical,
+        notEvaluated,
         regressed,
         unchanged,
     };
 }
 /** Build an AreaDelta from baseline and experiment scores for a single area */
-function buildAreaDelta(area, baselineScore, experimentScore, threshold) {
+function buildAreaDelta(area, baselineScore, experimentScore, threshold, isMismatched = false) {
     const b = baselineScore;
     const e = experimentScore;
     const bTotal = b?.totalScore ?? 0;
@@ -174,7 +181,7 @@ function buildAreaDelta(area, baselineScore, experimentScore, threshold) {
         area,
         baseline: bTotal,
         ceilingDelta: eCeiling - bCeiling,
-        change: classifyChange(delta, threshold),
+        change: isMismatched ? "not-evaluated" : classifyChange(delta, threshold),
         delta,
         dimensions: {
             codeCorrectness: {

package/dist/pipeline/pr-comment.js

@@ -295,7 +295,7 @@ function generateComment(summary, options = {}) {
         if (hasActualDeltas) {
             lines.push("| Feature | Baseline | Current | Delta | Actual Δ | Ret. Gap Δ | Infra Δ |");
             lines.push("|---------|----------|---------|-------|----------|------------|---------|");
-            for (const a of report.areas) {
+            for (const a of report.areas.filter((a) => a.change !== "not-evaluated")) {
                 const icon = a.change === "improved"
                     ? "📈"
                     : a.change === "regressed"
@@ -313,7 +313,7 @@ function generateComment(summary, options = {}) {
         else {
             lines.push("| Feature | Baseline | Current | Delta | Task | Code | Docs |");
             lines.push("|---------|----------|---------|-------|------|------|------|");
-            for (const a of report.areas) {
+            for (const a of report.areas.filter((a) => a.change !== "not-evaluated")) {
                 const icon = a.change === "improved"
                     ? "📈"
                     : a.change === "regressed"
@@ -334,6 +334,9 @@ function generateComment(summary, options = {}) {
         if (report.unchanged.length > 0) {
             parts.push(`➡️ ${report.unchanged.length} unchanged`);
         }
+        if (report.notEvaluated?.length > 0) {
+            parts.push(`⏭️ ${report.notEvaluated.length} not evaluated`);
+        }
         if (parts.length > 0) {
             const isEmpirical = "noiseThresholdEmpirical" in report &&
                 report.noiseThresholdEmpirical === true;

package/dist/pipeline/release-report.js

@@ -36,6 +36,10 @@ export function buildReleaseImpactReport(classification, comparison, attribution
     const confirmedUnchanged = [];
     if (comparison) {
         for (const areaDelta of comparison.areas) {
+            // Skip areas that weren't evaluated in both runs — these are
+            // mismatched areas (e.g., auto-scoped release eval vs full baseline).
+            if (areaDelta.change === "not-evaluated")
+                continue;
             const regressed = areaDelta.delta < -threshold;
             // Find tasks and their attributed documents for this area
             const areaTasks = [];

package/dist/report-store.d.ts

@@ -51,9 +51,13 @@ export declare class ReportStore {
      * Returns the comparison plus the baseline report ID so the caller
      * can record `provenance.lineage.comparedAgainst`.
      *
+     * @param scopedAreas When provided, the baseline's scores are filtered to
+     *   only include these areas before comparison. This prevents mismatched
+     *   areas from polluting the overall delta (e.g., release auto-scope
+     *   evaluates only GROQ but the baseline has all areas).
      * @returns The comparison result with baseline ID, or null if no baseline found
      */
-    autoCompare(currentSummary: ScoreSummary, provenance: ReportProvenance, completedAt: ISOTimestamp): Promise<AutoCompareResult | null>;
+    autoCompare(currentSummary: ScoreSummary, provenance: ReportProvenance, completedAt: ISOTimestamp, scopedAreas?: Set<string>): Promise<AutoCompareResult | null>;
     /**
      * Find a report by its evaluation fingerprint (cross-environment cache lookup).
      *

package/dist/report-store.js

@@ -49,9 +49,13 @@ export class ReportStore {
      * Returns the comparison plus the baseline report ID so the caller
      * can record `provenance.lineage.comparedAgainst`.
      *
+     * @param scopedAreas When provided, the baseline's scores are filtered to
+     *   only include these areas before comparison. This prevents mismatched
+     *   areas from polluting the overall delta (e.g., release auto-scope
+     *   evaluates only GROQ but the baseline has all areas).
      * @returns The comparison result with baseline ID, or null if no baseline found
      */
-    async autoCompare(currentSummary, provenance, completedAt) {
+    async autoCompare(currentSummary, provenance, completedAt, scopedAreas) {
         // 1. Prefer explicit lineage source (deterministic re-run comparison)
         const rerunSourceId = provenance.lineage?.rerunOf;
         let baseline = null;
@@ -76,7 +80,30 @@ export class ReportStore {
             return null;
         }
         try {
-            const comparison = compare(baseline.summary, currentSummary);
+            // When auto-scope is active, filter the baseline to only include
+            // areas that were actually evaluated. This produces a fair
+            // comparison where the overall delta reflects only tested areas.
+            let baselineSummary = baseline.summary;
+            if (scopedAreas && scopedAreas.size > 0) {
+                const filteredScores = baselineSummary.scores.filter((s) => scopedAreas.has(s.feature));
+                if (filteredScores.length > 0 &&
+                    filteredScores.length < baselineSummary.scores.length) {
+                    const len = filteredScores.length;
+                    const avgScore = filteredScores.reduce((s, sc) => s + sc.totalScore, 0) / len;
+                    const avgDocLift = filteredScores.reduce((s, sc) => s + sc.docLift, 0) / len;
+                    baselineSummary = {
+                        ...baselineSummary,
+                        overall: {
+                            ...baselineSummary.overall,
+                            avgScore,
+                            avgDocLift,
+                        },
+                        scores: filteredScores,
+                    };
+                    console.log(`  🎯 Scoped baseline to ${filteredScores.length} of ${baseline.summary.scores.length} areas for comparison`);
+                }
+            }
+            const comparison = compare(baselineSummary, currentSummary);
             return { baselineReportId: baseline.id, comparison };
         }
         catch (error) {

package/dist/sinks/slack/format.js

@@ -110,6 +110,16 @@ export function formatRegressionAlert(report) {
             type: "section",
         });
     }
+    // Not-evaluated areas — informational mention
+    if (comparison.notEvaluated?.length > 0) {
+        blocks.push({
+            text: {
+                text: `⏭️ ${comparison.notEvaluated.length} area${comparison.notEvaluated.length === 1 ? "" : "s"} not evaluated: ${comparison.notEvaluated.join(", ")}`,
+                type: "mrkdwn",
+            },
+            type: "section",
+        });
+    }
     return {
         blocks,
         text: `📉 AI Literacy Score Regression: ${baselineScore} → ${experimentScore} (${formatDelta(delta)})`,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sanity/ailf",
-  "version": "0.1.32",
+  "version": "0.1.34",
   "private": false,
   "publishConfig": {
     "access": "restricted"