@sanity/ailf 0.1.33 → 0.2.0

package/dist/orchestration/steps/calculate-scores-step.js

@@ -63,8 +63,9 @@ export class CalculateScoresStep {
         catch {
             // Non-fatal — proceed without source metadata
         }
+        let belowCritical = [];
         try {
-            calculateAndWriteScores({
+            const result = calculateAndWriteScores({
                 allowedOrigins: ctx.config.allowedOrigins,
                 mode: ctx.config.mode,
                 resolvedSource,
@@ -75,25 +76,14 @@ export class CalculateScoresStep {
                 searchMode: ctx.config.searchMode,
                 source: ctx.config.source,
             });
+            belowCritical = result.belowCritical;
         }
         catch (err) {
-            const code = err !== null && typeof err === "object" && "status" in err
-                ? err.status
-                : undefined;
-            if (code !== undefined && code !== 1) {
-                return {
-                    durationMs: Date.now() - start,
-                    error: `calculate-scores failed with exit code ${code}`,
-                    status: "failed",
-                };
-            }
-            if (code === undefined) {
-                return {
-                    durationMs: Date.now() - start,
-                    error: `calculate-scores failed: ${err instanceof Error ? err.message : String(err)}`,
-                    status: "failed",
-                };
-            }
+            return {
+                durationMs: Date.now() - start,
+                error: `calculate-scores failed: ${err instanceof Error ? err.message : String(err)}`,
+                status: "failed",
+            };
         }
         // Postcondition: score summary exists and is valid
         const summaryIssues = checkScoreSummaryValid(ctx.config.rootDir);
@@ -105,10 +95,19 @@ export class CalculateScoresStep {
                 status: "failed",
             };
         }
+        // Propagate belowCritical into pipeline state for downstream consumers
+        // (e.g., orchestrator reporting, publish step metadata).
+        // This is informational — the pipeline continues to run subsequent steps.
+        if (belowCritical.length > 0) {
+            state.belowCritical = belowCritical;
+        }
+        const criticalSuffix = belowCritical.length > 0
+            ? ` (${belowCritical.length} area(s) below critical threshold: ${belowCritical.join(", ")})`
+            : "";
         return {
             durationMs: Date.now() - start,
             status: "success",
-            summary: "Scores calculated and summary written",
+            summary: `Scores calculated and summary written${criticalSuffix}`,
         };
     }
     cacheInputs(ctx) {

package/dist/orchestration/steps/publish-report-step.js

@@ -14,6 +14,7 @@ import { readFileSync } from "fs";
 import { resolve } from "path";
 import { checkScoreSummaryValid } from "../../pipeline/checks.js";
 import { buildProvenance, } from "../../pipeline/provenance.js";
+import { generateReportTitle } from "../../pipeline/report-title.js";
 import { generateReportId } from "../../report-store.js";
 import { withRetry } from "../../sinks/retry.js";
 export class PublishReportStep {
@@ -82,8 +83,16 @@ export class PublishReportStep {
         const durationMs = Date.now() - this.pipelineStart;
         // Auto-compare against most recent comparable baseline.
         // Returns the comparison + baseline report ID for lineage tracking.
+        //
+        // When release auto-scope is active, the current experiment only covers
+        // a subset of areas. We pass the evaluated area set so autoCompare can
+        // scope the baseline to match — preventing mismatched areas from
+        // polluting the overall delta.
+        const evaluatedAreas = state.releaseAutoScope
+            ? new Set(summary.scores.map((s) => s.feature))
+            : undefined;
         const autoCompareResult = ctx.reportStore
-            ? (await ctx.reportStore.autoCompare(summary, provenance, now))
+            ? (await ctx.reportStore.autoCompare(summary, provenance, now, evaluatedAreas))
             : null;
         const comparison = autoCompareResult?.comparison ?? null;
         // Record which report we compared against in lineage
@@ -93,6 +102,7 @@ export class PublishReportStep {
                 comparedAgainst: autoCompareResult.baselineReportId,
             };
         }
+        const title = generateReportTitle({ provenance });
         const report = {
             comparison: comparison ?? undefined,
             completedAt: now,
@@ -101,6 +111,7 @@ export class PublishReportStep {
             provenance,
             summary,
             tag: this.options.publishTag ?? ctx.config.publishTag,
+            title,
         };
         // Share reportId with downstream steps (CallbackStep + orchestrator job update)
         state.reportId = reportId;

package/dist/pipeline/calculate-scores.d.ts

@@ -99,4 +99,9 @@ export interface CalculateScoresOptions {
     /** Documentation source name */
     source?: string;
 }
-export declare function calculateAndWriteScores(options: CalculateScoresOptions): void;
+/** Result from calculateAndWriteScores — replaces process.exit() calls. */
+export interface CalculateScoresResult {
+    /** Feature areas that scored below the critical threshold (40). */
+    belowCritical: string[];
+}
+export declare function calculateAndWriteScores(options: CalculateScoresOptions): CalculateScoresResult;

package/dist/pipeline/calculate-scores.js

@@ -674,15 +674,10 @@ export function calculateAndWriteScores(options) {
     const resultsIssues = checkResultsExist(ROOT, baselineResultsPath);
     const resultsErrors = resultsIssues.filter((i) => i.severity === "error");
     if (resultsErrors.length > 0) {
-        console.error("❌ Results validation failed:");
-        for (const e of resultsErrors) {
-            console.error(`  ERROR: ${e.message}`);
-            if (e.path) {
-                console.error(`         at ${e.path}`);
-            }
-        }
-        console.error("\nRun 'pnpm eval' first to generate results, then 'pnpm calculate-scores'.");
-        process.exit(1);
+        const details = resultsErrors
+            .map((e) => (e.path ? `${e.message} (at ${e.path})` : e.message))
+            .join("; ");
+        throw new Error(`Results validation failed: ${details}. Run 'pnpm eval' first to generate results.`);
     }
     console.log(`Reading results from: ${baselineResultsPath}`);
     if (source) {
@@ -750,10 +745,7 @@ export function calculateAndWriteScores(options) {
         writeFileSync(join(outDir, "grader-judgments.json"), JSON.stringify(judgments, null, 2));
         console.log(`Grader judgments written to results/latest/grader-judgments.json (${judgments.length} judgments)`);
     }
-    // Exit with non-zero if any area below critical threshold
-    if (summary.belowCritical.length > 0) {
-        process.exit(1);
-    }
+    return { belowCritical: summary.belowCritical };
 }
 function printPerModelReport(perModel) {
     console.log("-".repeat(80));

package/dist/pipeline/compare.js

@@ -51,11 +51,14 @@ export function compare(baseline, experiment, options) {
     // Identify mismatched areas
     const onlyInBaseline = [...baselineAreas].filter((a) => !experimentAreas.has(a));
     const onlyInExperiment = [...experimentAreas].filter((a) => !baselineAreas.has(a));
-    // Build per-area deltas
+    // Build per-area deltas.
+    // Areas present in only one summary get change: "not-evaluated" — comparing
+    // against a missing score is meaningless (it would produce false ±100 deltas).
+    const mismatchedSet = new Set([...onlyInBaseline, ...onlyInExperiment]);
     const areas = [...allAreas]
         .sort()
-        .map((area) => buildAreaDelta(area, findScore(baseline.scores, area), findScore(experiment.scores, area), threshold));
-    // Classify areas
+        .map((area) => buildAreaDelta(area, findScore(baseline.scores, area), findScore(experiment.scores, area), threshold, mismatchedSet.has(area)));
+    // Classify areas — mismatched areas are excluded from all three buckets
     const improved = areas
         .filter((a) => a.change === "improved")
         .map((a) => a.area);
@@ -65,6 +68,9 @@ export function compare(baseline, experiment, options) {
     const unchanged = areas
         .filter((a) => a.change === "unchanged")
         .map((a) => a.area);
+    const notEvaluated = areas
+        .filter((a) => a.change === "not-evaluated")
+        .map((a) => a.area);
     // Per-area deltas as a record
     const perArea = {};
     for (const a of areas) {
@@ -128,12 +134,13 @@ export function compare(baseline, experiment, options) {
         },
         noiseThreshold: threshold,
         noiseThresholdEmpirical: empirical,
+        notEvaluated,
         regressed,
         unchanged,
     };
 }
 /** Build an AreaDelta from baseline and experiment scores for a single area */
-function buildAreaDelta(area, baselineScore, experimentScore, threshold) {
+function buildAreaDelta(area, baselineScore, experimentScore, threshold, isMismatched = false) {
     const b = baselineScore;
     const e = experimentScore;
     const bTotal = b?.totalScore ?? 0;
@@ -174,7 +181,7 @@ function buildAreaDelta(area, baselineScore, experimentScore, threshold) {
         area,
         baseline: bTotal,
         ceilingDelta: eCeiling - bCeiling,
-        change: classifyChange(delta, threshold),
+        change: isMismatched ? "not-evaluated" : classifyChange(delta, threshold),
         delta,
         dimensions: {
             codeCorrectness: {

package/dist/pipeline/generate-configs.js

@@ -264,15 +264,10 @@ export function generateConfigs(options) {
     const modelIssues = validateModelsYaml(rootDir);
     const modelErrors = modelIssues.filter((i) => i.severity === "error");
     if (modelErrors.length > 0) {
-        console.error("❌ config/models.yaml validation failed:");
-        for (const e of modelErrors) {
-            console.error(`  ERROR: ${e.message}`);
-            if (e.path) {
-                console.error(`         at ${e.path}`);
-            }
-        }
-        console.error("\nFix config/models.yaml before generating configs. Run 'pnpm validate' for details.");
-        process.exit(1);
+        const details = modelErrors
+            .map((e) => (e.path ? `${e.message} (at ${e.path})` : e.message))
+            .join("; ");
+        throw new Error(`config/models.yaml validation failed: ${details}. Run 'pnpm validate' for details.`);
     }
     console.log("Loading config/models.yaml...");
     const models = loadModels(rootDir);

package/dist/pipeline/mirror-repo-tasks.d.ts

@@ -24,6 +24,15 @@ export interface MirrorOptions {
     /** If true, log what would be done without writing */
     dryRun?: boolean;
 }
+/** Authorship info extracted from git context or GitHub Actions environment. */
+export interface GitAuthor {
+    /** Git commit author name (e.g., "Jordan Smith") */
+    gitName?: string;
+    /** Git commit author email (e.g., "jordan@example.com") */
+    gitEmail?: string;
+    /** GitHub username (from GITHUB_ACTOR or event payload) */
+    githubUsername?: string;
+}
 export interface GitContext {
     /** Full repo identifier (e.g., "sanity-io/visual-editing") */
     repo: string;
@@ -35,6 +44,8 @@ export interface GitContext {
     branch: string;
     /** HEAD commit SHA */
     commitSha: string;
+    /** Author of the current commit/trigger */
+    author: GitAuthor;
 }
 export interface MirrorResult {
     /** Total tasks processed */
@@ -84,3 +95,69 @@ export declare function mirrorDocId(owner: string, repo: string, taskId: string)
  * that's not mirrored.
  */
 export declare function computeTaskHash(task: TaskDefinition): string;
+/** @internal Exported for testing — not part of the public API. */
+export declare function buildMirrorDocument(task: TaskDefinition, opts: {
+    contentHash: string;
+    docId: string;
+    /** Existing author from the current mirror document (write-once preservation) */
+    existingAuthor?: GitAuthor;
+    git: GitContext;
+    slugToDocId: Map<string, string>;
+}): {
+    baseline?: {
+        rubric?: "full" | "abbreviated" | "none" | undefined;
+        enabled?: boolean | undefined;
+    } | undefined;
+    _id: string;
+    _type: string;
+    ownership: string;
+    status: "active" | "draft" | "paused" | "archived";
+    assert: Record<string, unknown>[];
+    canonicalDocs: ({
+        _key: string;
+        reason: string;
+    } | {
+        refType: string;
+        path: string;
+        _key: string;
+        reason: string;
+    } | {
+        doc?: {
+            _ref: string;
+            _type: string;
+        } | undefined;
+        docId?: string | undefined;
+        refType: string;
+        _key: string;
+        reason: string;
+    } | {
+        refType: string;
+        perspective: string;
+        _key: string;
+        reason: string;
+    })[];
+    description: string;
+    docCoverage: boolean;
+    featureArea: {
+        _ref: string;
+        _type: string;
+    };
+    id: {
+        _type: string;
+        current: string;
+    };
+    origin: {
+        branch: string;
+        commitSha: string;
+        contentHash: string;
+        lastSyncedAt: string;
+        path: string;
+        repo: string;
+        repoName: string;
+        repoOwner: string;
+        type: string;
+        author: GitAuthor;
+        lastEditor: GitAuthor;
+    };
+    taskPrompt: string;
+};

package/dist/pipeline/mirror-repo-tasks.js

@@ -13,7 +13,8 @@
  * @see docs/exec-plans/tasks-as-content/phase-5-content-lake-mirroring.md
  */
 import { createHash } from "crypto";
-import { isSlugRef, } from "../_vendor/ailf-core/index.js";
+import { readFileSync } from "fs";
+import { isIdRef, isPathRef, isPerspectiveRef, isSlugRef, } from "../_vendor/ailf-core/index.js";
 // ---------------------------------------------------------------------------
 // Public API
 // ---------------------------------------------------------------------------
@@ -56,22 +57,30 @@ export async function mirrorRepoTasks(options) {
     const areas = [...new Set(tasks.map((t) => t.featureArea))];
     const createdAreas = await ensureFeatureAreas(client, areas, dryRun);
     result.areasCreated = createdAreas;
-    // Fetch existing mirror document content hashes for change detection
+    // Fetch existing mirror document state for change detection + ownership check
     const mirrorIds = tasks.map((t) => mirrorDocId(git.owner, git.name, t.id));
-    const existingHashes = await fetchExistingHashes(client, mirrorIds);
+    const existingDocState = await fetchExistingDocState(client, mirrorIds);
     // Mirror each task
     for (const task of tasks) {
         try {
             const docId = mirrorDocId(git.owner, git.name, task.id);
+            const existing = existingDocState.get(docId);
+            // Skip graduated tasks — ownership was changed to "studio"
+            if (existing?.ownership === "studio") {
+                console.log(`  ℹ️  Skipping "${task.id}" — graduated to Studio ownership`);
+                result.skipped++;
+                continue;
+            }
             const contentHash = computeTaskHash(task);
             // Skip unchanged
-            if (existingHashes.get(docId) === contentHash) {
+            if (existing?.hash === contentHash) {
                 result.skipped++;
                 continue;
             }
             const doc = buildMirrorDocument(task, {
                 contentHash,
                 docId,
+                existingAuthor: existing?.existingAuthor,
                 git,
                 slugToDocId,
             });
@@ -106,12 +115,15 @@ export async function detectGitContext(repoTasksPath) {
     if (ghRepo) {
         const [owner, name] = ghRepo.split("/");
         const branch = ghHeadRef || ghRef.replace("refs/heads/", "").replace("refs/tags/", "");
+        // Extract author from GitHub Actions environment
+        const author = detectGitHubActionsAuthor();
         return {
             repo: ghRepo,
             owner: owner ?? "unknown",
             name: name ?? "unknown",
             branch: branch || "unknown",
             commitSha: ghSha || "unknown",
+            author,
         };
     }
     // Fallback: try git CLI
@@ -134,12 +146,15 @@ export async function detectGitContext(repoTasksPath) {
             remote.match(/([^/]+)\/([^/.]+?)(?:\.git)?$/);
         const owner = match?.[1] ?? "unknown";
         const name = match?.[2] ?? "unknown";
+        // Extract author from git log
+        const author = detectGitCliAuthor(repoTasksPath, execSync);
         return {
             repo: `${owner}/${name}`,
             owner,
             name,
             branch,
             commitSha,
+            author,
         };
     }
     catch {
@@ -149,10 +164,63 @@ export async function detectGitContext(repoTasksPath) {
             name: "unknown",
             branch: "unknown",
             commitSha: "unknown",
+            author: {},
         };
     }
 }
 // ---------------------------------------------------------------------------
+// Author detection helpers
+// ---------------------------------------------------------------------------
+/**
+ * Extract author info from GitHub Actions environment variables and
+ * the webhook event payload (GITHUB_EVENT_PATH).
+ */
+function detectGitHubActionsAuthor() {
+    const ghActor = process.env.GITHUB_ACTOR ?? undefined;
+    const author = { githubUsername: ghActor };
+    // Try to read richer author info from the event payload
+    const eventPath = process.env.GITHUB_EVENT_PATH;
+    if (eventPath) {
+        try {
+            const event = JSON.parse(readFileSync(eventPath, "utf-8"));
+            // Push event: head_commit.author has name + email
+            if (event.head_commit?.author) {
+                author.gitName = event.head_commit.author.name ?? undefined;
+                author.gitEmail = event.head_commit.author.email ?? undefined;
+            }
+            // PR event: pull_request.user.login is the PR author
+            if (event.pull_request?.user?.login) {
+                author.githubUsername = event.pull_request.user.login;
+            }
+        }
+        catch {
+            // Event payload parsing is best-effort — fall through with GITHUB_ACTOR only
+        }
+    }
+    return author;
+}
+/**
+ * Extract author info from git CLI (local fallback when not in GitHub Actions).
+ */
+function detectGitCliAuthor(cwd, execSyncFn) {
+    try {
+        const authorLine = execSyncFn('git log -1 --format="%an|%ae"', {
+            encoding: "utf-8",
+            cwd,
+        })
+            .toString()
+            .trim();
+        const [gitName, gitEmail] = authorLine.split("|");
+        return {
+            gitName: gitName || undefined,
+            gitEmail: gitEmail || undefined,
+        };
+    }
+    catch {
+        return {};
+    }
+}
+// ---------------------------------------------------------------------------
 // Document ID scheme
 // ---------------------------------------------------------------------------
 /**
@@ -247,42 +315,82 @@ async function ensureFeatureAreas(client, areas, dryRun) {
     await transaction.commit();
     return missing;
 }
-// ---------------------------------------------------------------------------
-// Fetch existing content hashes
-// ---------------------------------------------------------------------------
 /**
- * Fetch existing mirror documents' content hashes for change detection.
- * The hash is stored in origin.contentHash on the document.
+ * Fetch existing mirror documents' ownership and content hashes.
+ *
+ * The ownership field determines whether the mirror step is allowed to
+ * update the document:
+ * - `"repo"` or absent → safe to update (active mirror)
+ * - `"studio"` → skip (graduated to Studio ownership)
+ *
+ * The hash is stored in origin.contentHash for change detection.
  */
-async function fetchExistingHashes(client, docIds) {
+async function fetchExistingDocState(client, docIds) {
     if (docIds.length === 0)
         return new Map();
-    const query = `*[_id in $ids] { _id, "hash": origin.contentHash }`;
-    const results = await client.fetch(query, {
-        ids: docIds,
-    });
+    const query = `*[_id in $ids] {
+    _id,
+    "hash": origin.contentHash,
+    ownership,
+    "existingAuthor": origin.author
+  }`;
+    const results = await client.fetch(query, { ids: docIds });
     const map = new Map();
     for (const r of results) {
-        if (r.hash)
-            map.set(r._id, r.hash);
+        map.set(r._id, {
+            hash: r.hash ?? undefined,
+            ownership: r.ownership ?? undefined,
+            existingAuthor: r.existingAuthor ?? undefined,
+        });
     }
     return map;
 }
 // ---------------------------------------------------------------------------
 // Build mirror document
 // ---------------------------------------------------------------------------
-function buildMirrorDocument(task, opts) {
-    const { contentHash, docId, git, slugToDocId } = opts;
-    // Build canonical docs with resolved references.
-    // Only slug refs can be resolved to article references today.
-    // Other ref types (path, id, perspective) are stored with reason only.
+/** @internal Exported for testing — not part of the public API. */
+export function buildMirrorDocument(task, opts) {
+    const { contentHash, docId, existingAuthor, git, slugToDocId } = opts;
+    // Build canonical docs with resolved references and correct refType.
+    // Each ref type gets the appropriate resolution fields set on the
+    // mirror document so Studio can display them correctly.
     const canonicalDocs = task.canonicalDocs.map((ref, i) => {
-        const resolvedId = isSlugRef(ref) ? slugToDocId.get(ref.slug) : undefined;
-        return {
-            _key: `cd${i}`,
-            ...(resolvedId ? { doc: { _ref: resolvedId, _type: "reference" } } : {}),
-            reason: ref.reason ?? "",
-        };
+        const base = { _key: `cd${i}`, reason: ref.reason ?? "" };
+        if (isSlugRef(ref)) {
+            const resolvedId = slugToDocId.get(ref.slug);
+            // When a slug resolves to a document, store as "id" ref with
+            // the resolved article reference. When unresolved, store as
+            // "slug" so Studio knows the resolution strategy even if the
+            // article doesn't exist yet.
+            return {
+                ...base,
+                refType: resolvedId ? "id" : "slug",
+                ...(resolvedId
+                    ? { doc: { _ref: resolvedId, _type: "reference" } }
+                    : {}),
+            };
+        }
+        if (isPathRef(ref)) {
+            return { ...base, refType: "path", path: ref.path };
+        }
+        if (isIdRef(ref)) {
+            return {
+                ...base,
+                refType: "id",
+                ...(ref.id
+                    ? { doc: { _ref: ref.id, _type: "reference" }, docId: ref.id }
+                    : {}),
+            };
+        }
+        if (isPerspectiveRef(ref)) {
+            return {
+                ...base,
+                refType: "perspective",
+                perspective: ref.perspective,
+            };
+        }
+        // Unknown ref type — store with reason only
+        return base;
     });
     // Build assertions
     const assertArray = task.assertions.map((a, i) => {
@@ -315,6 +423,8 @@ function buildMirrorDocument(task, opts) {
     return {
         _id: docId,
         _type: "ailf.task",
+        ownership: "repo",
+        status: task.status ?? "active",
         assert: assertArray,
         canonicalDocs,
         description: task.description,
@@ -325,6 +435,7 @@ function buildMirrorDocument(task, opts) {
         },
         id: { _type: "slug", current: task.id },
         origin: {
+            // Existing provenance fields
             branch: git.branch,
             commitSha: git.commitSha,
             contentHash,
@@ -334,6 +445,9 @@ function buildMirrorDocument(task, opts) {
             repoName: git.name,
             repoOwner: git.owner,
             type: "repo",
+            // Authorship: author is write-once (preserve existing), lastEditor always updates
+            author: existingAuthor ?? git.author,
+            lastEditor: git.author,
         },
         taskPrompt: task.taskPrompt,
         ...(task.baseline

package/dist/pipeline/pr-comment.js

@@ -295,7 +295,7 @@ function generateComment(summary, options = {}) {
         if (hasActualDeltas) {
             lines.push("| Feature | Baseline | Current | Delta | Actual Δ | Ret. Gap Δ | Infra Δ |");
             lines.push("|---------|----------|---------|-------|----------|------------|---------|");
-            for (const a of report.areas) {
+            for (const a of report.areas.filter((a) => a.change !== "not-evaluated")) {
                 const icon = a.change === "improved"
                     ? "📈"
                     : a.change === "regressed"
@@ -313,7 +313,7 @@ function generateComment(summary, options = {}) {
         else {
             lines.push("| Feature | Baseline | Current | Delta | Task | Code | Docs |");
             lines.push("|---------|----------|---------|-------|------|------|------|");
-            for (const a of report.areas) {
+            for (const a of report.areas.filter((a) => a.change !== "not-evaluated")) {
                 const icon = a.change === "improved"
                     ? "📈"
                     : a.change === "regressed"
@@ -334,6 +334,9 @@ function generateComment(summary, options = {}) {
         if (report.unchanged.length > 0) {
             parts.push(`➡️ ${report.unchanged.length} unchanged`);
         }
+        if (report.notEvaluated?.length > 0) {
+            parts.push(`⏭️ ${report.notEvaluated.length} not evaluated`);
+        }
         if (parts.length > 0) {
             const isEmpirical = "noiseThresholdEmpirical" in report &&
                 report.noiseThresholdEmpirical === true;

package/dist/pipeline/release-report.js

@@ -36,6 +36,10 @@ export function buildReleaseImpactReport(classification, comparison, attribution
     const confirmedUnchanged = [];
     if (comparison) {
         for (const areaDelta of comparison.areas) {
+            // Skip areas that weren't evaluated in both runs — these are
+            // mismatched areas (e.g., auto-scoped release eval vs full baseline).
+            if (areaDelta.change === "not-evaluated")
+                continue;
             const regressed = areaDelta.delta < -threshold;
             // Find tasks and their attributed documents for this area
             const areaTasks = [];