@sanity/ailf 1.0.0 → 2.0.1

package/dist/artifact-capture/comparator.js

@@ -0,0 +1,493 @@
+/**
+ * CaptureComparator — compares two capture directories and produces a diff report.
+ *
+ * Reads manifest.json from both directories and computes:
+ * - Inventory diff (added/removed/common artifacts)
+ * - Content diff (structural or strict, for common artifacts)
+ * - Score comparison (from score-summary.json)
+ * - Timing comparison (from pipeline-context.json)
+ * - Metadata comparison (mode, variant, config keys)
+ * - Security scan (regex for leaked secrets)
+ *
+ * Implementation for the types defined in @sanity/ailf-core.
+ */
+import { existsSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+// ---------------------------------------------------------------------------
+// Defaults
+// ---------------------------------------------------------------------------
+const DEFAULT_OPTIONS = {
+    mode: "inventory",
+    scoreThresholds: { aggregate: 5, perTask: 10 },
+    timingThresholds: { multiplier: 2.0 },
+    jsonDiffDepth: 3,
+};
+const DEFAULT_EPHEMERAL_FIELDS = new Set([
+    "captureId",
+    "startedAt",
+    "completedAt",
+    "capturedAt",
+    "durationMs",
+]);
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+/**
+ * Compare two capture directories and produce a structured diff report.
+ *
+ * @param baselineDir - Path to the baseline capture directory (contains manifest.json)
+ * @param experimentDir - Path to the experiment capture directory
+ * @param opts - Comparison options (mode, thresholds, etc.)
+ */
+export function compareCaptures(baselineDir, experimentDir, opts) {
+    const options = { ...DEFAULT_OPTIONS, ...opts };
+    const ephemeral = new Set([
+        ...DEFAULT_EPHEMERAL_FIELDS,
+        ...(options.ephemeralFields ?? []),
+    ]);
+    const baselineManifest = readManifest(baselineDir);
+    const experimentManifest = readManifest(experimentDir);
+    // Inventory diff
+    const inventory = computeInventoryDiff(baselineManifest, experimentManifest);
+    // Security scan (always runs)
+    const security = scanForSecrets(baselineDir, experimentDir, baselineManifest, experimentManifest);
+    // Content diff (structural/strict only)
+    let content;
+    if (options.mode !== "inventory" && inventory.common.length > 0) {
+        content = computeContentDiffs(baselineDir, experimentDir, baselineManifest, experimentManifest, inventory.common, options.mode, options.jsonDiffDepth ?? 3, ephemeral);
+    }
+    // Score comparison
+    const scores = computeScoreDiff(baselineDir, experimentDir, baselineManifest, experimentManifest, options.scoreThresholds ?? DEFAULT_OPTIONS.scoreThresholds);
+    // Timing comparison
+    const timing = computeTimingDiff(baselineDir, experimentDir, baselineManifest, experimentManifest, options.timingThresholds ?? DEFAULT_OPTIONS.timingThresholds);
+    // Metadata comparison
+    const metadata = computeMetadataDiff(baselineDir, experimentDir, baselineManifest, experimentManifest, ephemeral);
+    // Determine equivalence
+    const violations = [];
+    if (inventory.added.length > 0)
+        violations.push(`${inventory.added.length} artifact(s) added`);
+    if (inventory.removed.length > 0)
+        violations.push(`${inventory.removed.length} artifact(s) removed`);
+    if (content && content.length > 0)
+        violations.push(`${content.length} artifact(s) changed`);
+    if (scores?.breaches.length)
+        violations.push(`${scores.breaches.length} score breach(es)`);
+    if (timing?.breaches.length)
+        violations.push(`${timing.breaches.length} timing breach(es)`);
+    if (security.leaksFound)
+        violations.push(`${security.violations.length} secret leak(s)`);
+    const equivalent = violations.length === 0;
+    return {
+        equivalent,
+        summary: equivalent
+            ? "Captures are equivalent."
+            : `Differences found: ${violations.join("; ")}.`,
+        mode: options.mode,
+        inventory,
+        ...(content ? { content } : {}),
+        ...(scores ? { scores } : {}),
+        ...(timing ? { timing } : {}),
+        ...(metadata ? { metadata } : {}),
+        security,
+    };
+}
+// ---------------------------------------------------------------------------
+// Manifest reading
+// ---------------------------------------------------------------------------
+function readManifest(dir) {
+    const manifestPath = join(dir, "manifest.json");
+    if (!existsSync(manifestPath)) {
+        throw new Error(`No manifest.json found in ${dir}`);
+    }
+    return JSON.parse(readFileSync(manifestPath, "utf-8"));
+}
+// ---------------------------------------------------------------------------
+// Inventory diff
+// ---------------------------------------------------------------------------
+function artifactKey(entry) {
+    return `${entry.step}/${entry.type}`;
+}
+function computeInventoryDiff(baseline, experiment) {
+    const baselineKeys = new Set(baseline.artifacts.map(artifactKey));
+    const experimentKeys = new Set(experiment.artifacts.map(artifactKey));
+    const added = [...experimentKeys].filter((k) => !baselineKeys.has(k));
+    const removed = [...baselineKeys].filter((k) => !experimentKeys.has(k));
+    const common = [...baselineKeys].filter((k) => experimentKeys.has(k));
+    return { added, removed, common };
+}
+// ---------------------------------------------------------------------------
+// Content diff
+// ---------------------------------------------------------------------------
+function computeContentDiffs(baselineDir, experimentDir, baselineManifest, experimentManifest, commonKeys, mode, depth, ephemeral) {
+    const diffs = [];
+    const baselineByKey = new Map(baselineManifest.artifacts.map((a) => [artifactKey(a), a]));
+    const experimentByKey = new Map(experimentManifest.artifacts.map((a) => [artifactKey(a), a]));
+    for (const key of commonKeys) {
+        const baseEntry = baselineByKey.get(key);
+        const expEntry = experimentByKey.get(key);
+        const basePath = join(baselineDir, baseEntry.path);
+        const expPath = join(experimentDir, expEntry.path);
+        if (!existsSync(basePath) || !existsSync(expPath))
+            continue;
+        const baseContent = readFileSync(basePath, "utf-8");
+        const expContent = readFileSync(expPath, "utf-8");
+        if (baseEntry.format === "json") {
+            try {
+                const baseData = JSON.parse(baseContent);
+                const expData = JSON.parse(expContent);
+                const stripped1 = stripEphemeral(baseData, ephemeral);
+                const stripped2 = stripEphemeral(expData, ephemeral);
+                if (mode === "strict") {
+                    if (JSON.stringify(stripped1) !== JSON.stringify(stripped2)) {
+                        const changes = diffJson(stripped1, stripped2, "", depth);
+                        if (changes.length > 0) {
+                            diffs.push({ artifactKey: key, format: "json", changes });
+                        }
+                    }
+                }
+                else {
+                    // structural — compare keys/types only
+                    const changes = diffJsonStructural(stripped1, stripped2, "", depth);
+                    if (changes.length > 0) {
+                        diffs.push({ artifactKey: key, format: "json", changes });
+                    }
+                }
+            }
+            catch {
+                // Can't parse JSON — fall through to text comparison
+                if (baseContent !== expContent) {
+                    diffs.push({
+                        artifactKey: key,
+                        format: "text",
+                        changes: computeLineDiff(baseContent, expContent),
+                    });
+                }
+            }
+        }
+        else {
+            // Text/markdown — line-level diff
+            if (baseContent !== expContent) {
+                diffs.push({
+                    artifactKey: key,
+                    format: baseEntry.format,
+                    changes: computeLineDiff(baseContent, expContent),
+                });
+            }
+        }
+    }
+    return diffs;
+}
+function computeLineDiff(a, b) {
+    const aLines = a.split("\n");
+    const bLines = b.split("\n");
+    const aFreq = new Map();
+    for (const l of aLines)
+        aFreq.set(l, (aFreq.get(l) ?? 0) + 1);
+    const bFreq = new Map();
+    for (const l of bLines)
+        bFreq.set(l, (bFreq.get(l) ?? 0) + 1);
+    let addedLines = 0;
+    let removedLines = 0;
+    for (const [line, count] of bFreq) {
+        addedLines += Math.max(0, count - (aFreq.get(line) ?? 0));
+    }
+    for (const [line, count] of aFreq) {
+        removedLines += Math.max(0, count - (bFreq.get(line) ?? 0));
+    }
+    return { addedLines, removedLines };
+}
+// ---------------------------------------------------------------------------
+// JSON diffing
+// ---------------------------------------------------------------------------
+function stripEphemeral(data, ephemeral) {
+    if (typeof data !== "object" || data === null)
+        return data;
+    if (Array.isArray(data))
+        return data.map((item) => stripEphemeral(item, ephemeral));
+    const result = {};
+    for (const [key, value] of Object.entries(data)) {
+        if (ephemeral.has(key))
+            continue;
+        result[key] = stripEphemeral(value, ephemeral);
+    }
+    return result;
+}
+/** Strict diff — compares values at each key path. */
+function diffJson(a, b, path, depth) {
+    if (depth <= 0)
+        return [];
+    if (a === b)
+        return [];
+    const entries = [];
+    if (typeof a !== "object" ||
+        typeof b !== "object" ||
+        a === null ||
+        b === null) {
+        return [{ path: path || "(root)", baseline: a, experiment: b }];
+    }
+    if (Array.isArray(a) && Array.isArray(b)) {
+        if (JSON.stringify(a) !== JSON.stringify(b)) {
+            entries.push({ path: path || "(root)", baseline: a, experiment: b });
+        }
+        return entries;
+    }
+    const aObj = a;
+    const bObj = b;
+    const allKeys = new Set([...Object.keys(aObj), ...Object.keys(bObj)]);
+    for (const key of allKeys) {
+        const subPath = path ? `${path}.${key}` : key;
+        if (!(key in aObj)) {
+            entries.push({ path: subPath, experiment: bObj[key] });
+        }
+        else if (!(key in bObj)) {
+            entries.push({ path: subPath, baseline: aObj[key] });
+        }
+        else {
+            entries.push(...diffJson(aObj[key], bObj[key], subPath, depth - 1));
+        }
+    }
+    return entries;
+}
+/** Structural diff — only checks that keys exist and types match. */
+function diffJsonStructural(a, b, path, depth) {
+    if (depth <= 0)
+        return [];
+    const typeA = typeof a;
+    const typeB = typeof b;
+    if (typeA !== typeB) {
+        return [{ path: path || "(root)", baseline: typeA, experiment: typeB }];
+    }
+    if (typeA !== "object" || a === null || b === null)
+        return [];
+    if (Array.isArray(a) !== Array.isArray(b)) {
+        return [
+            {
+                path: path || "(root)",
+                baseline: Array.isArray(a) ? "array" : "object",
+                experiment: Array.isArray(b) ? "array" : "object",
+            },
+        ];
+    }
+    if (Array.isArray(a))
+        return []; // Arrays: structural match if both are arrays
+    const entries = [];
+    const aObj = a;
+    const bObj = b;
+    const allKeys = new Set([...Object.keys(aObj), ...Object.keys(bObj)]);
+    for (const key of allKeys) {
+        const subPath = path ? `${path}.${key}` : key;
+        if (!(key in aObj)) {
+            entries.push({ path: subPath, experiment: typeof bObj[key] });
+        }
+        else if (!(key in bObj)) {
+            entries.push({ path: subPath, baseline: typeof aObj[key] });
+        }
+        else {
+            entries.push(...diffJsonStructural(aObj[key], bObj[key], subPath, depth - 1));
+        }
+    }
+    return entries;
+}
+// ---------------------------------------------------------------------------
+// Score comparison
+// ---------------------------------------------------------------------------
+function computeScoreDiff(baselineDir, experimentDir, baselineManifest, experimentManifest, thresholds) {
+    const baseScore = findAndReadArtifact(baselineDir, baselineManifest, "calculate-scores", "score-summary");
+    const expScore = findAndReadArtifact(experimentDir, experimentManifest, "calculate-scores", "score-summary");
+    if (!baseScore || !expScore)
+        return undefined;
+    const baselineMean = baseScore.aggregate ?? 0;
+    const currentMean = expScore.aggregate ?? 0;
+    const delta = currentMean - baselineMean;
+    // Per-task comparison
+    const baseScores = baseScore.scores ?? [];
+    const expScores = expScore.scores ?? [];
+    const expByTask = new Map(expScores.map((s) => [s.task ?? "", s.score ?? 0]));
+    const perTask = [];
+    const breaches = [];
+    for (const base of baseScores) {
+        const task = base.task ?? "";
+        const baseVal = base.score ?? 0;
+        const expVal = expByTask.get(task);
+        if (expVal !== undefined) {
+            const taskDelta = expVal - baseVal;
+            perTask.push({
+                task,
+                baseline: baseVal,
+                current: expVal,
+                delta: taskDelta,
+            });
+            if (taskDelta < -thresholds.perTask) {
+                breaches.push(`${task}: dropped ${Math.abs(taskDelta)} points`);
+            }
+        }
+    }
+    if (delta < -thresholds.aggregate) {
+        breaches.push(`Aggregate score dropped ${Math.abs(delta).toFixed(1)} points (threshold: ${thresholds.aggregate})`);
+    }
+    return { baselineMean, currentMean, delta, perTask, breaches };
+}
+// ---------------------------------------------------------------------------
+// Timing comparison
+// ---------------------------------------------------------------------------
+function computeTimingDiff(baselineDir, experimentDir, baselineManifest, experimentManifest, thresholds) {
+    const baseCtx = findAndReadArtifact(baselineDir, baselineManifest, "pipeline", "pipeline-context");
+    const expCtx = findAndReadArtifact(experimentDir, experimentManifest, "pipeline", "pipeline-context");
+    if (!baseCtx || !expCtx)
+        return undefined;
+    const baseSteps = (baseCtx.steps ?? []).filter((s) => s.durationMs !== undefined);
+    const expSteps = (expCtx.steps ?? []).filter((s) => s.durationMs !== undefined);
+    const expByName = new Map(expSteps.map((s) => [s.name, s.durationMs]));
+    const perStep = [];
+    const breaches = [];
+    let totalBaseline = 0;
+    let totalExperiment = 0;
+    for (const base of baseSteps) {
+        const expMs = expByName.get(base.name);
+        if (expMs !== undefined) {
+            const baseMs = base.durationMs;
+            const ratio = baseMs > 0 ? expMs / baseMs : expMs > 0 ? Infinity : 1;
+            perStep.push({
+                step: base.name,
+                baselineMs: baseMs,
+                currentMs: expMs,
+                ratio,
+            });
+            const stepThreshold = thresholds.perStep?.[base.name] ?? thresholds.multiplier;
+            if (ratio > stepThreshold) {
+                breaches.push(`${base.name}: ${ratio.toFixed(1)}x slower (threshold: ${stepThreshold}x)`);
+            }
+            totalBaseline += baseMs;
+            totalExperiment += expMs;
+        }
+    }
+    return {
+        totalDeltaMs: totalExperiment - totalBaseline,
+        perStep,
+        breaches,
+    };
+}
+// ---------------------------------------------------------------------------
+// Metadata comparison
+// ---------------------------------------------------------------------------
+function computeMetadataDiff(baselineDir, experimentDir, baselineManifest, experimentManifest, ephemeral) {
+    const modeMatch = baselineManifest.pipeline.mode === experimentManifest.pipeline.mode;
+    const variantMatch = baselineManifest.pipeline.variant === experimentManifest.pipeline.variant;
+    // Compare config from pipeline-context
+    const baseCtx = findAndReadArtifact(baselineDir, baselineManifest, "pipeline", "pipeline-context");
+    const expCtx = findAndReadArtifact(experimentDir, experimentManifest, "pipeline", "pipeline-context");
+    let configDiffs = [];
+    if (baseCtx && expCtx) {
+        const baseConfig = baseCtx.config ?? {};
+        const expConfig = expCtx.config ?? {};
+        configDiffs = diffJsonStructural(stripEphemeral(baseConfig, ephemeral), stripEphemeral(expConfig, ephemeral), "config", 3);
+    }
+    return { modeMatch, variantMatch, configDiffs };
+}
+// ---------------------------------------------------------------------------
+// Security scan
+// ---------------------------------------------------------------------------
+/** Patterns that indicate potential secret values (not just key names). */
+const SECRET_VALUE_PATTERNS = [
+    /^sk-[a-zA-Z0-9_-]{20,}/, // OpenAI-style keys
+    /^sk[A-Z][a-zA-Z0-9]{20,}/, // Sanity-style tokens (e.g., skJ3rMwt…)
+    /^xoxb-/, // Slack bot tokens
+    /^ghp_/, // GitHub personal tokens
+    /^ghs_/, // GitHub server-to-server tokens
+    /^Bearer\s+\S{20,}/, // Authorization header values
+];
+/**
+ * Keys that should never appear with non-empty string values in captured artifacts.
+ *
+ * Uses case-insensitive matching without word boundaries to handle camelCase
+ * (e.g., "apiToken", "secretKey"). Intentionally broader than the orchestrator's
+ * sanitization pattern (/token|secret|key/i) — this also catches "password",
+ * "credential", and "authorization" in artifacts that bypass orchestrator
+ * sanitization (e.g., in-memory captures from PublishReportStep or CallbackStep).
+ */
+const SECRET_KEY_PATTERN = /(?:api[_-]?token|auth(?:orization|[_-]?token)|access[_-]?token)|secret|apiKey|api_key|password|credential|^set-cookie$|^cookie$/i;
+function scanForSecrets(baselineDir, experimentDir, baselineManifest, experimentManifest) {
+    const violations = [];
+    for (const [dir, manifest] of [
+        [experimentDir, experimentManifest],
+        [baselineDir, baselineManifest],
+    ]) {
+        for (const artifact of manifest.artifacts) {
+            const filePath = join(dir, artifact.path);
+            if (!existsSync(filePath))
+                continue;
+            const content = readFileSync(filePath, "utf-8");
+            if (artifact.format === "json") {
+                try {
+                    const data = JSON.parse(content);
+                    scanJsonForSecrets(data, artifact.path, "", violations);
+                }
+                catch {
+                    // Non-parseable — skip
+                }
+            }
+            else {
+                // Text/markdown: scan for secret-looking strings
+                for (const pattern of SECRET_VALUE_PATTERNS) {
+                    if (pattern.test(content)) {
+                        violations.push({
+                            file: artifact.path,
+                            detail: `Content matches secret pattern: ${pattern.source}`,
+                        });
+                    }
+                }
+            }
+        }
+    }
+    return { leaksFound: violations.length > 0, violations };
+}
+function scanJsonForSecrets(data, file, path, violations, depth = 0) {
+    if (depth > 10)
+        return;
+    if (typeof data === "string") {
+        for (const pattern of SECRET_VALUE_PATTERNS) {
+            if (pattern.test(data)) {
+                violations.push({
+                    file,
+                    detail: `Value at "${path}" matches secret pattern: ${pattern.source}`,
+                });
+            }
+        }
+    }
+    else if (Array.isArray(data)) {
+        for (let i = 0; i < data.length; i++) {
+            scanJsonForSecrets(data[i], file, `${path}[${i}]`, violations, depth + 1);
+        }
+    }
+    else if (typeof data === "object" && data !== null) {
+        for (const [key, value] of Object.entries(data)) {
+            // Check if a key name looks like it holds a secret AND has a string value
+            if (SECRET_KEY_PATTERN.test(key) &&
+                typeof value === "string" &&
+                value.length > 0) {
+                violations.push({
+                    file,
+                    detail: `Key "${path ? path + "." : ""}${key}" may contain a secret value`,
+                });
+            }
+            scanJsonForSecrets(value, file, path ? `${path}.${key}` : key, violations, depth + 1);
+        }
+    }
+}
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+function findAndReadArtifact(dir, manifest, step, type) {
+    const entry = manifest.artifacts.find((a) => a.step === step && a.type === type);
+    if (!entry)
+        return undefined;
+    const filePath = join(dir, entry.path);
+    if (!existsSync(filePath))
+        return undefined;
+    try {
+        return JSON.parse(readFileSync(filePath, "utf-8"));
+    }
+    catch {
+        return undefined;
+    }
+}

package/dist/artifact-capture/filesystem-collector.d.ts

@@ -0,0 +1,42 @@
+/**
+ * FilesystemArtifactCollector — writes captured artifacts to a local directory.
+ *
+ * Accumulates artifact entries in memory during pipeline execution.
+ * On flush(), creates a structured directory with one subdirectory per
+ * step, writes all artifacts, and generates a manifest.json.
+ *
+ * Design principles:
+ * - capture() and captureFile() are synchronous (no I/O during step execution)
+ * - flush() does all I/O at pipeline end
+ * - Failures in capture/captureFile are swallowed (P5: non-blocking)
+ */
+import type { ArtifactCollector, CaptureFlushResult } from "../_vendor/ailf-core/index.d.ts";
+export interface FilesystemCollectorOptions {
+    /** Base directory for capture output (e.g., results/captures/) */
+    captureDir: string;
+    /** Pipeline mode (for directory naming) */
+    mode: string;
+    /** Whether to compress on flush (Phase 5 — currently ignored) */
+    compress: boolean;
+    /** Whether mode-specific extras are enabled */
+    extras: boolean;
+    /** Pipeline metadata for the manifest */
+    pipeline?: {
+        variant?: string;
+        source?: string;
+        areas?: string[];
+    };
+}
+export declare class FilesystemArtifactCollector implements ArtifactCollector {
+    readonly enabled = true;
+    readonly extrasEnabled: boolean;
+    private readonly entries;
+    private readonly captureId;
+    private readonly outputDir;
+    private readonly startedAt;
+    private readonly options;
+    constructor(options: FilesystemCollectorOptions);
+    capture(step: string, type: string, data: unknown, meta?: Record<string, unknown>): void;
+    captureFile(step: string, type: string, filePath: string, meta?: Record<string, unknown>): void;
+    flush(): Promise<CaptureFlushResult>;
+}