@sanity/ailf 2.8.0 → 3.0.0

package/dist/artifact-capture/comparator.js

@@ -1,493 +0,0 @@
-/**
- * CaptureComparator — compares two capture directories and produces a diff report.
- *
- * Reads manifest.json from both directories and computes:
- * - Inventory diff (added/removed/common artifacts)
- * - Content diff (structural or strict, for common artifacts)
- * - Score comparison (from score-summary.json)
- * - Timing comparison (from pipeline-context.json)
- * - Metadata comparison (mode, variant, config keys)
- * - Security scan (regex for leaked secrets)
- *
- * Implementation for the types defined in @sanity/ailf-core.
- */
-import { existsSync, readFileSync } from "node:fs";
-import { join } from "node:path";
-// ---------------------------------------------------------------------------
-// Defaults
-// ---------------------------------------------------------------------------
-const DEFAULT_OPTIONS = {
-    mode: "inventory",
-    scoreThresholds: { aggregate: 5, perTask: 10 },
-    timingThresholds: { multiplier: 2.0 },
-    jsonDiffDepth: 3,
-};
-const DEFAULT_EPHEMERAL_FIELDS = new Set([
-    "captureId",
-    "startedAt",
-    "completedAt",
-    "capturedAt",
-    "durationMs",
-]);
-// ---------------------------------------------------------------------------
-// Public API
-// ---------------------------------------------------------------------------
-/**
- * Compare two capture directories and produce a structured diff report.
- *
- * @param baselineDir - Path to the baseline capture directory (contains manifest.json)
- * @param experimentDir - Path to the experiment capture directory
- * @param opts - Comparison options (mode, thresholds, etc.)
- */
-export function compareCaptures(baselineDir, experimentDir, opts) {
-    const options = { ...DEFAULT_OPTIONS, ...opts };
-    const ephemeral = new Set([
-        ...DEFAULT_EPHEMERAL_FIELDS,
-        ...(options.ephemeralFields ?? []),
-    ]);
-    const baselineManifest = readManifest(baselineDir);
-    const experimentManifest = readManifest(experimentDir);
-    // Inventory diff
-    const inventory = computeInventoryDiff(baselineManifest, experimentManifest);
-    // Security scan (always runs)
-    const security = scanForSecrets(baselineDir, experimentDir, baselineManifest, experimentManifest);
-    // Content diff (structural/strict only)
-    let content;
-    if (options.mode !== "inventory" && inventory.common.length > 0) {
-        content = computeContentDiffs(baselineDir, experimentDir, baselineManifest, experimentManifest, inventory.common, options.mode, options.jsonDiffDepth ?? 3, ephemeral);
-    }
-    // Score comparison
-    const scores = computeScoreDiff(baselineDir, experimentDir, baselineManifest, experimentManifest, options.scoreThresholds ?? DEFAULT_OPTIONS.scoreThresholds);
-    // Timing comparison
-    const timing = computeTimingDiff(baselineDir, experimentDir, baselineManifest, experimentManifest, options.timingThresholds ?? DEFAULT_OPTIONS.timingThresholds);
-    // Metadata comparison
-    const metadata = computeMetadataDiff(baselineDir, experimentDir, baselineManifest, experimentManifest, ephemeral);
-    // Determine equivalence
-    const violations = [];
-    if (inventory.added.length > 0)
-        violations.push(`${inventory.added.length} artifact(s) added`);
-    if (inventory.removed.length > 0)
-        violations.push(`${inventory.removed.length} artifact(s) removed`);
-    if (content && content.length > 0)
-        violations.push(`${content.length} artifact(s) changed`);
-    if (scores?.breaches.length)
-        violations.push(`${scores.breaches.length} score breach(es)`);
-    if (timing?.breaches.length)
-        violations.push(`${timing.breaches.length} timing breach(es)`);
-    if (security.leaksFound)
-        violations.push(`${security.violations.length} secret leak(s)`);
-    const equivalent = violations.length === 0;
-    return {
-        equivalent,
-        summary: equivalent
-            ? "Captures are equivalent."
-            : `Differences found: ${violations.join("; ")}.`,
-        mode: options.mode,
-        inventory,
-        ...(content ? { content } : {}),
-        ...(scores ? { scores } : {}),
-        ...(timing ? { timing } : {}),
-        ...(metadata ? { metadata } : {}),
-        security,
-    };
-}
-// ---------------------------------------------------------------------------
-// Manifest reading
-// ---------------------------------------------------------------------------
-function readManifest(dir) {
-    const manifestPath = join(dir, "manifest.json");
-    if (!existsSync(manifestPath)) {
-        throw new Error(`No manifest.json found in ${dir}`);
-    }
-    return JSON.parse(readFileSync(manifestPath, "utf-8"));
-}
-// ---------------------------------------------------------------------------
-// Inventory diff
-// ---------------------------------------------------------------------------
-function artifactKey(entry) {
-    return `${entry.step}/${entry.type}`;
-}
-function computeInventoryDiff(baseline, experiment) {
-    const baselineKeys = new Set(baseline.artifacts.map(artifactKey));
-    const experimentKeys = new Set(experiment.artifacts.map(artifactKey));
-    const added = [...experimentKeys].filter((k) => !baselineKeys.has(k));
-    const removed = [...baselineKeys].filter((k) => !experimentKeys.has(k));
-    const common = [...baselineKeys].filter((k) => experimentKeys.has(k));
-    return { added, removed, common };
-}
-// ---------------------------------------------------------------------------
-// Content diff
-// ---------------------------------------------------------------------------
-function computeContentDiffs(baselineDir, experimentDir, baselineManifest, experimentManifest, commonKeys, mode, depth, ephemeral) {
-    const diffs = [];
-    const baselineByKey = new Map(baselineManifest.artifacts.map((a) => [artifactKey(a), a]));
-    const experimentByKey = new Map(experimentManifest.artifacts.map((a) => [artifactKey(a), a]));
-    for (const key of commonKeys) {
-        const baseEntry = baselineByKey.get(key);
-        const expEntry = experimentByKey.get(key);
-        const basePath = join(baselineDir, baseEntry.path);
-        const expPath = join(experimentDir, expEntry.path);
-        if (!existsSync(basePath) || !existsSync(expPath))
-            continue;
-        const baseContent = readFileSync(basePath, "utf-8");
-        const expContent = readFileSync(expPath, "utf-8");
-        if (baseEntry.format === "json") {
-            try {
-                const baseData = JSON.parse(baseContent);
-                const expData = JSON.parse(expContent);
-                const stripped1 = stripEphemeral(baseData, ephemeral);
-                const stripped2 = stripEphemeral(expData, ephemeral);
-                if (mode === "strict") {
-                    if (JSON.stringify(stripped1) !== JSON.stringify(stripped2)) {
-                        const changes = diffJson(stripped1, stripped2, "", depth);
-                        if (changes.length > 0) {
-                            diffs.push({ artifactKey: key, format: "json", changes });
-                        }
-                    }
-                }
-                else {
-                    // structural — compare keys/types only
-                    const changes = diffJsonStructural(stripped1, stripped2, "", depth);
-                    if (changes.length > 0) {
-                        diffs.push({ artifactKey: key, format: "json", changes });
-                    }
-                }
-            }
-            catch {
-                // Can't parse JSON — fall through to text comparison
-                if (baseContent !== expContent) {
-                    diffs.push({
-                        artifactKey: key,
-                        format: "text",
-                        changes: computeLineDiff(baseContent, expContent),
-                    });
-                }
-            }
-        }
-        else {
-            // Text/markdown — line-level diff
-            if (baseContent !== expContent) {
-                diffs.push({
-                    artifactKey: key,
-                    format: baseEntry.format,
-                    changes: computeLineDiff(baseContent, expContent),
-                });
-            }
-        }
-    }
-    return diffs;
-}
-function computeLineDiff(a, b) {
-    const aLines = a.split("\n");
-    const bLines = b.split("\n");
-    const aFreq = new Map();
-    for (const l of aLines)
-        aFreq.set(l, (aFreq.get(l) ?? 0) + 1);
-    const bFreq = new Map();
-    for (const l of bLines)
-        bFreq.set(l, (bFreq.get(l) ?? 0) + 1);
-    let addedLines = 0;
-    let removedLines = 0;
-    for (const [line, count] of bFreq) {
-        addedLines += Math.max(0, count - (aFreq.get(line) ?? 0));
-    }
-    for (const [line, count] of aFreq) {
-        removedLines += Math.max(0, count - (bFreq.get(line) ?? 0));
-    }
-    return { addedLines, removedLines };
-}
-// ---------------------------------------------------------------------------
-// JSON diffing
-// ---------------------------------------------------------------------------
-function stripEphemeral(data, ephemeral) {
-    if (typeof data !== "object" || data === null)
-        return data;
-    if (Array.isArray(data))
-        return data.map((item) => stripEphemeral(item, ephemeral));
-    const result = {};
-    for (const [key, value] of Object.entries(data)) {
-        if (ephemeral.has(key))
-            continue;
-        result[key] = stripEphemeral(value, ephemeral);
-    }
-    return result;
-}
-/** Strict diff — compares values at each key path. */
-function diffJson(a, b, path, depth) {
-    if (depth <= 0)
-        return [];
-    if (a === b)
-        return [];
-    const entries = [];
-    if (typeof a !== "object" ||
-        typeof b !== "object" ||
-        a === null ||
-        b === null) {
-        return [{ path: path || "(root)", baseline: a, experiment: b }];
-    }
-    if (Array.isArray(a) && Array.isArray(b)) {
-        if (JSON.stringify(a) !== JSON.stringify(b)) {
-            entries.push({ path: path || "(root)", baseline: a, experiment: b });
-        }
-        return entries;
-    }
-    const aObj = a;
-    const bObj = b;
-    const allKeys = new Set([...Object.keys(aObj), ...Object.keys(bObj)]);
-    for (const key of allKeys) {
-        const subPath = path ? `${path}.${key}` : key;
-        if (!(key in aObj)) {
-            entries.push({ path: subPath, experiment: bObj[key] });
-        }
-        else if (!(key in bObj)) {
-            entries.push({ path: subPath, baseline: aObj[key] });
-        }
-        else {
-            entries.push(...diffJson(aObj[key], bObj[key], subPath, depth - 1));
-        }
-    }
-    return entries;
-}
-/** Structural diff — only checks that keys exist and types match. */
-function diffJsonStructural(a, b, path, depth) {
-    if (depth <= 0)
-        return [];
-    const typeA = typeof a;
-    const typeB = typeof b;
-    if (typeA !== typeB) {
-        return [{ path: path || "(root)", baseline: typeA, experiment: typeB }];
-    }
-    if (typeA !== "object" || a === null || b === null)
-        return [];
-    if (Array.isArray(a) !== Array.isArray(b)) {
-        return [
-            {
-                path: path || "(root)",
-                baseline: Array.isArray(a) ? "array" : "object",
-                experiment: Array.isArray(b) ? "array" : "object",
-            },
-        ];
-    }
-    if (Array.isArray(a))
-        return []; // Arrays: structural match if both are arrays
-    const entries = [];
-    const aObj = a;
-    const bObj = b;
-    const allKeys = new Set([...Object.keys(aObj), ...Object.keys(bObj)]);
-    for (const key of allKeys) {
-        const subPath = path ? `${path}.${key}` : key;
-        if (!(key in aObj)) {
-            entries.push({ path: subPath, experiment: typeof bObj[key] });
-        }
-        else if (!(key in bObj)) {
-            entries.push({ path: subPath, baseline: typeof aObj[key] });
-        }
-        else {
-            entries.push(...diffJsonStructural(aObj[key], bObj[key], subPath, depth - 1));
-        }
-    }
-    return entries;
-}
-// ---------------------------------------------------------------------------
-// Score comparison
-// ---------------------------------------------------------------------------
-function computeScoreDiff(baselineDir, experimentDir, baselineManifest, experimentManifest, thresholds) {
-    const baseScore = findAndReadArtifact(baselineDir, baselineManifest, "calculate-scores", "score-summary");
-    const expScore = findAndReadArtifact(experimentDir, experimentManifest, "calculate-scores", "score-summary");
-    if (!baseScore || !expScore)
-        return undefined;
-    const baselineMean = baseScore.aggregate ?? 0;
-    const currentMean = expScore.aggregate ?? 0;
-    const delta = currentMean - baselineMean;
-    // Per-task comparison
-    const baseScores = baseScore.scores ?? [];
-    const expScores = expScore.scores ?? [];
-    const expByTask = new Map(expScores.map((s) => [s.task ?? "", s.score ?? 0]));
-    const perTask = [];
-    const breaches = [];
-    for (const base of baseScores) {
-        const task = base.task ?? "";
-        const baseVal = base.score ?? 0;
-        const expVal = expByTask.get(task);
-        if (expVal !== undefined) {
-            const taskDelta = expVal - baseVal;
-            perTask.push({
-                task,
-                baseline: baseVal,
-                current: expVal,
-                delta: taskDelta,
-            });
-            if (taskDelta < -thresholds.perTask) {
-                breaches.push(`${task}: dropped ${Math.abs(taskDelta)} points`);
-            }
-        }
-    }
-    if (delta < -thresholds.aggregate) {
-        breaches.push(`Aggregate score dropped ${Math.abs(delta).toFixed(1)} points (threshold: ${thresholds.aggregate})`);
-    }
-    return { baselineMean, currentMean, delta, perTask, breaches };
-}
-// ---------------------------------------------------------------------------
-// Timing comparison
-// ---------------------------------------------------------------------------
-function computeTimingDiff(baselineDir, experimentDir, baselineManifest, experimentManifest, thresholds) {
-    const baseCtx = findAndReadArtifact(baselineDir, baselineManifest, "pipeline", "pipeline-context");
-    const expCtx = findAndReadArtifact(experimentDir, experimentManifest, "pipeline", "pipeline-context");
-    if (!baseCtx || !expCtx)
-        return undefined;
-    const baseSteps = (baseCtx.steps ?? []).filter((s) => s.durationMs !== undefined);
-    const expSteps = (expCtx.steps ?? []).filter((s) => s.durationMs !== undefined);
-    const expByName = new Map(expSteps.map((s) => [s.name, s.durationMs]));
-    const perStep = [];
-    const breaches = [];
-    let totalBaseline = 0;
-    let totalExperiment = 0;
-    for (const base of baseSteps) {
-        const expMs = expByName.get(base.name);
-        if (expMs !== undefined) {
-            const baseMs = base.durationMs;
-            const ratio = baseMs > 0 ? expMs / baseMs : expMs > 0 ? Infinity : 1;
-            perStep.push({
-                step: base.name,
-                baselineMs: baseMs,
-                currentMs: expMs,
-                ratio,
-            });
-            const stepThreshold = thresholds.perStep?.[base.name] ?? thresholds.multiplier;
-            if (ratio > stepThreshold) {
-                breaches.push(`${base.name}: ${ratio.toFixed(1)}x slower (threshold: ${stepThreshold}x)`);
-            }
-            totalBaseline += baseMs;
-            totalExperiment += expMs;
-        }
-    }
-    return {
-        totalDeltaMs: totalExperiment - totalBaseline,
-        perStep,
-        breaches,
-    };
-}
-// ---------------------------------------------------------------------------
-// Metadata comparison
-// ---------------------------------------------------------------------------
-function computeMetadataDiff(baselineDir, experimentDir, baselineManifest, experimentManifest, ephemeral) {
-    const modeMatch = baselineManifest.pipeline.mode === experimentManifest.pipeline.mode;
-    const variantMatch = baselineManifest.pipeline.variant === experimentManifest.pipeline.variant;
-    // Compare config from pipeline-context
-    const baseCtx = findAndReadArtifact(baselineDir, baselineManifest, "pipeline", "pipeline-context");
-    const expCtx = findAndReadArtifact(experimentDir, experimentManifest, "pipeline", "pipeline-context");
-    let configDiffs = [];
-    if (baseCtx && expCtx) {
-        const baseConfig = baseCtx.config ?? {};
-        const expConfig = expCtx.config ?? {};
-        configDiffs = diffJsonStructural(stripEphemeral(baseConfig, ephemeral), stripEphemeral(expConfig, ephemeral), "config", 3);
-    }
-    return { modeMatch, variantMatch, configDiffs };
-}
-// ---------------------------------------------------------------------------
-// Security scan
-// ---------------------------------------------------------------------------
-/** Patterns that indicate potential secret values (not just key names). */
-const SECRET_VALUE_PATTERNS = [
-    /^sk-[a-zA-Z0-9_-]{20,}/, // OpenAI-style keys
-    /^sk[A-Z][a-zA-Z0-9]{20,}/, // Sanity-style tokens (e.g., skJ3rMwt…)
-    /^xoxb-/, // Slack bot tokens
-    /^ghp_/, // GitHub personal tokens
-    /^ghs_/, // GitHub server-to-server tokens
-    /^Bearer\s+\S{20,}/, // Authorization header values
-];
-/**
- * Keys that should never appear with non-empty string values in captured artifacts.
- *
- * Uses case-insensitive matching without word boundaries to handle camelCase
- * (e.g., "apiToken", "secretKey"). Intentionally broader than the orchestrator's
- * sanitization pattern (/token|secret|key/i) — this also catches "password",
- * "credential", and "authorization" in artifacts that bypass orchestrator
- * sanitization (e.g., in-memory captures from PublishReportStep or CallbackStep).
- */
-const SECRET_KEY_PATTERN = /(?:api[_-]?token|auth(?:orization|[_-]?token)|access[_-]?token)|secret|apiKey|api_key|password|credential|^set-cookie$|^cookie$/i;
-function scanForSecrets(baselineDir, experimentDir, baselineManifest, experimentManifest) {
-    const violations = [];
-    for (const [dir, manifest] of [
-        [experimentDir, experimentManifest],
-        [baselineDir, baselineManifest],
-    ]) {
-        for (const artifact of manifest.artifacts) {
-            const filePath = join(dir, artifact.path);
-            if (!existsSync(filePath))
-                continue;
-            const content = readFileSync(filePath, "utf-8");
-            if (artifact.format === "json") {
-                try {
-                    const data = JSON.parse(content);
-                    scanJsonForSecrets(data, artifact.path, "", violations);
-                }
-                catch {
-                    // Non-parseable — skip
-                }
-            }
-            else {
-                // Text/markdown: scan for secret-looking strings
-                for (const pattern of SECRET_VALUE_PATTERNS) {
-                    if (pattern.test(content)) {
-                        violations.push({
-                            file: artifact.path,
-                            detail: `Content matches secret pattern: ${pattern.source}`,
-                        });
-                    }
-                }
-            }
-        }
-    }
-    return { leaksFound: violations.length > 0, violations };
-}
-function scanJsonForSecrets(data, file, path, violations, depth = 0) {
-    if (depth > 10)
-        return;
-    if (typeof data === "string") {
-        for (const pattern of SECRET_VALUE_PATTERNS) {
-            if (pattern.test(data)) {
-                violations.push({
-                    file,
-                    detail: `Value at "${path}" matches secret pattern: ${pattern.source}`,
-                });
-            }
-        }
-    }
-    else if (Array.isArray(data)) {
-        for (let i = 0; i < data.length; i++) {
-            scanJsonForSecrets(data[i], file, `${path}[${i}]`, violations, depth + 1);
-        }
-    }
-    else if (typeof data === "object" && data !== null) {
-        for (const [key, value] of Object.entries(data)) {
-            // Check if a key name looks like it holds a secret AND has a string value
-            if (SECRET_KEY_PATTERN.test(key) &&
-                typeof value === "string" &&
-                value.length > 0) {
-                violations.push({
-                    file,
-                    detail: `Key "${path ? path + "." : ""}${key}" may contain a secret value`,
-                });
-            }
-            scanJsonForSecrets(value, file, path ? `${path}.${key}` : key, violations, depth + 1);
-        }
-    }
-}
-// ---------------------------------------------------------------------------
-// Helpers
-// ---------------------------------------------------------------------------
-function findAndReadArtifact(dir, manifest, step, type) {
-    const entry = manifest.artifacts.find((a) => a.step === step && a.type === type);
-    if (!entry)
-        return undefined;
-    const filePath = join(dir, entry.path);
-    if (!existsSync(filePath))
-        return undefined;
-    try {
-        return JSON.parse(readFileSync(filePath, "utf-8"));
-    }
-    catch {
-        return undefined;
-    }
-}

package/dist/artifact-capture/filesystem-collector.d.ts

@@ -1,42 +0,0 @@
-/**
- * FilesystemArtifactCollector — writes captured artifacts to a local directory.
- *
- * Accumulates artifact entries in memory during pipeline execution.
- * On flush(), creates a structured directory with one subdirectory per
- * step, writes all artifacts, and generates a manifest.json.
- *
- * Design principles:
- * - capture() and captureFile() are synchronous (no I/O during step execution)
- * - flush() does all I/O at pipeline end
- * - Failures in capture/captureFile are swallowed (P5: non-blocking)
- */
-import type { ArtifactCollector, CaptureFlushResult } from "../_vendor/ailf-core/index.d.ts";
-export interface FilesystemCollectorOptions {
-    /** Base directory for capture output (e.g., results/captures/) */
-    captureDir: string;
-    /** Pipeline mode (for directory naming) */
-    mode: string;
-    /** Whether to compress on flush (Phase 5 — currently ignored) */
-    compress: boolean;
-    /** Whether mode-specific extras are enabled */
-    extras: boolean;
-    /** Pipeline metadata for the manifest */
-    pipeline?: {
-        variant?: string;
-        source?: string;
-        areas?: string[];
-    };
-}
-export declare class FilesystemArtifactCollector implements ArtifactCollector {
-    readonly enabled = true;
-    readonly extrasEnabled: boolean;
-    private readonly entries;
-    private readonly captureId;
-    private readonly outputDir;
-    private readonly startedAt;
-    private readonly options;
-    constructor(options: FilesystemCollectorOptions);
-    capture(step: string, type: string, data: unknown, meta?: Record<string, unknown>): void;
-    captureFile(step: string, type: string, filePath: string, meta?: Record<string, unknown>): void;
-    flush(): Promise<CaptureFlushResult>;
-}