@sanity/ailf 2.9.0 → 3.0.0

package/dist/commands/capture-compare.js

@@ -1,253 +0,0 @@
-/**
- * capture compare — compare two pipeline capture directories.
- *
- * Reads manifest.json from both captures, runs compareCaptures(),
- * and prints a human-readable table or JSON diff report.
- *
- * Supports both raw directories and .tar.gz archives.
- *
- * Exit codes:
- *   0 — captures are equivalent
- *   1 — differences found
- *   2 — error (missing files, invalid manifest, etc.)
- */
-import { execFileSync } from "node:child_process";
-import { existsSync, mkdtempSync, readdirSync, rmSync, writeFileSync, } from "node:fs";
-import { tmpdir } from "node:os";
-import { join, resolve } from "node:path";
-import { Command, Option } from "commander";
-import { compareCaptures } from "../artifact-capture/comparator.js";
-// ---------------------------------------------------------------------------
-// Command factory
-// ---------------------------------------------------------------------------
-export function createCaptureCompareCommand() {
-    return new Command("compare")
-        .description("Compare two pipeline capture directories")
-        .argument("<baseline>", "Path to baseline capture (directory or .tar.gz)")
-        .argument("<experiment>", "Path to experiment capture (directory or .tar.gz)")
-        .addOption(new Option("-m, --mode <mode>", "Comparison mode")
-        .choices(["inventory", "structural", "strict"])
-        .default("inventory"))
-        .option("-f, --format <fmt>", "Output format: table or json", "table")
-        .option("-o, --output <path>", "Write JSON report to file")
-        .option("--score-threshold <n>", "Aggregate score regression threshold (points)", parseFloat, 5)
-        .option("--task-threshold <n>", "Per-task score regression threshold (points)", parseFloat, 10)
-        .option("--timing-threshold <n>", "Step timing multiplier threshold", parseFloat, 2)
-        .option("--json-depth <n>", "JSON structural diff depth", parseInt, 3)
-        .action(async (baselinePath, experimentPath, opts) => {
-        const cleanups = [];
-        try {
-            const baseline = resolveCapturePath(resolve(baselinePath), cleanups);
-            const experiment = resolveCapturePath(resolve(experimentPath), cleanups);
-            console.log("");
-            console.log("  ailf capture compare");
-            console.log("  " + "─".repeat(40));
-            console.log("");
-            console.log(`  Baseline:   ${baselinePath}`);
-            console.log(`  Experiment: ${experimentPath}`);
-            console.log(`  Mode:       ${opts.mode}`);
-            console.log("");
-            const report = compareCaptures(baseline, experiment, {
-                mode: opts.mode,
-                scoreThresholds: {
-                    aggregate: opts.scoreThreshold,
-                    perTask: opts.taskThreshold,
-                },
-                timingThresholds: { multiplier: opts.timingThreshold },
-                jsonDiffDepth: opts.jsonDepth,
-            });
-            if (opts.format === "json") {
-                const json = JSON.stringify(report, null, 2);
-                if (opts.output) {
-                    writeFileSync(opts.output, json, "utf-8");
-                    console.log(`  Report written to ${opts.output}`);
-                }
-                else {
-                    console.log(json);
-                }
-            }
-            else {
-                printTableReport(report);
-                if (opts.output) {
-                    const json = JSON.stringify(report, null, 2);
-                    writeFileSync(opts.output, json, "utf-8");
-                    console.log(`  Report also written to ${opts.output}`);
-                }
-            }
-            process.exitCode = report.equivalent ? 0 : 1;
-        }
-        catch (err) {
-            console.error(`  Error: ${err instanceof Error ? err.message : String(err)}`);
-            process.exitCode = 2;
-        }
-        finally {
-            for (const cleanup of cleanups) {
-                try {
-                    cleanup();
-                }
-                catch {
-                    // Best-effort cleanup
-                }
-            }
-        }
-    });
-}
-// ---------------------------------------------------------------------------
-// Path resolution (handles tar.gz, subdirectories, raw dirs)
-// ---------------------------------------------------------------------------
-function resolveCapturePath(inputPath, cleanups) {
-    if (!existsSync(inputPath)) {
-        throw new Error(`Path does not exist: ${inputPath}`);
-    }
-    if (inputPath.endsWith(".tar.gz")) {
-        const tempDir = mkdtempSync(join(tmpdir(), "ailf-capture-cmp-"));
-        cleanups.push(() => rmSync(tempDir, { recursive: true, force: true }));
-        execFileSync("tar", ["-xzf", inputPath, "-C", tempDir]);
-        return findManifestDir(tempDir);
-    }
-    return findManifestDir(inputPath);
-}
-/**
- * Find the directory containing manifest.json.
- *
- * Handles two cases:
- * 1. Path IS the capture dir (contains manifest.json directly)
- * 2. Path is the parent captures/ dir (contains a single timestamped subdir)
- */
-function findManifestDir(dir) {
-    if (existsSync(join(dir, "manifest.json")))
-        return dir;
-    // Look one level down for a capture subdirectory
-    const entries = readdirSync(dir).filter((e) => !e.startsWith(".") && !e.endsWith(".tar.gz"));
-    for (const entry of entries) {
-        const sub = join(dir, entry);
-        if (existsSync(join(sub, "manifest.json")))
-            return sub;
-    }
-    throw new Error(`No manifest.json found in ${dir} or its subdirectories. ` +
-        `Is this a valid capture directory?`);
-}
-// ---------------------------------------------------------------------------
-// Table formatting
-// ---------------------------------------------------------------------------
-function printTableReport(report) {
-    // Inventory
-    console.log("  Inventory");
-    console.log("  " + "─".repeat(40));
-    console.log(`  Common:    ${report.inventory.common.length} artifact(s)`);
-    if (report.inventory.added.length > 0) {
-        console.log(`  Added:     ${report.inventory.added.length}  (${report.inventory.added.join(", ")})`);
-    }
-    else {
-        console.log("  Added:     0");
-    }
-    if (report.inventory.removed.length > 0) {
-        console.log(`  Removed:   ${report.inventory.removed.length}  (${report.inventory.removed.join(", ")})`);
-    }
-    else {
-        console.log("  Removed:   0");
-    }
-    console.log("");
-    // Content diff preview (structural/strict modes)
-    if (report.content && report.content.length > 0) {
-        console.log("  Content Changes");
-        console.log("  " + "─".repeat(40));
-        for (const diff of report.content.slice(0, 10)) {
-            printContentDiff(diff);
-        }
-        if (report.content.length > 10) {
-            console.log(`  ... and ${report.content.length - 10} more changed artifact(s)`);
-        }
-        console.log("");
-    }
-    // Scores
-    if (report.scores) {
-        console.log("  Scores");
-        console.log("  " + "─".repeat(40));
-        const { baselineMean, currentMean, delta } = report.scores;
-        const icon = delta > 0 ? "+" : delta < 0 ? "" : " ";
-        console.log(`  Aggregate: ${baselineMean.toFixed(1)} -> ${currentMean.toFixed(1)}  (${icon}${delta.toFixed(1)})`);
-        if (report.scores.breaches.length > 0) {
-            console.log(`  Breaches:  ${report.scores.breaches.length}`);
-            for (const b of report.scores.breaches) {
-                console.log(`    - ${b}`);
-            }
-        }
-        else {
-            console.log("  Breaches:  none");
-        }
-        console.log("");
-    }
-    // Timing
-    if (report.timing) {
-        console.log("  Timing");
-        console.log("  " + "─".repeat(40));
-        const { totalDeltaMs } = report.timing;
-        const sign = totalDeltaMs >= 0 ? "+" : "";
-        console.log(`  Total delta: ${sign}${totalDeltaMs}ms`);
-        if (report.timing.breaches.length > 0) {
-            console.log(`  Breaches:    ${report.timing.breaches.length}`);
-            for (const b of report.timing.breaches) {
-                console.log(`    - ${b}`);
-            }
-        }
-        else {
-            console.log("  Breaches:    none");
-        }
-        console.log("");
-    }
-    // Security
-    console.log("  Security");
-    console.log("  " + "─".repeat(40));
-    if (report.security.leaksFound) {
-        console.log(`  Leaks:     ${report.security.violations.length} finding(s)`);
-        for (const v of report.security.violations.slice(0, 5)) {
-            console.log(`    - ${v.file}: ${v.detail}`);
-        }
-        if (report.security.violations.length > 5) {
-            console.log(`    ... and ${report.security.violations.length - 5} more`);
-        }
-    }
-    else {
-        console.log("  Leaks:     none");
-    }
-    console.log("");
-    // Result
-    if (report.equivalent) {
-        console.log("  Result: EQUIVALENT");
-    }
-    else {
-        console.log(`  Result: DIFFERENCES FOUND`);
-        console.log(`  ${report.summary}`);
-    }
-    console.log("");
-}
-/**
- * Print up to 3 changed key paths for a content diff.
- */
-function printContentDiff(diff) {
-    console.log(`  ${diff.artifactKey} (${diff.format})`);
-    if (Array.isArray(diff.changes)) {
-        // JSON diff — show up to 3 changed paths
-        const jsonChanges = diff.changes;
-        for (const change of jsonChanges.slice(0, 3)) {
-            if (change.baseline === undefined) {
-                console.log(`    + ${change.path} (added)`);
-            }
-            else if (change.experiment === undefined) {
-                console.log(`    - ${change.path} (removed)`);
-            }
-            else {
-                console.log(`    ~ ${change.path} (changed)`);
-            }
-        }
-        if (jsonChanges.length > 3) {
-            console.log(`    ... ${jsonChanges.length - 3} more change(s)`);
-        }
-    }
-    else {
-        // Text/markdown diff
-        const { addedLines, removedLines } = diff.changes;
-        console.log(`    +${addedLines} / -${removedLines} lines`);
-    }
-}

package/dist/commands/capture-list.d.ts

@@ -1,12 +0,0 @@
-/**
- * capture list — list pipeline captures in a directory.
- *
- * Scans a capture directory for subdirectories containing manifest.json,
- * reads each manifest, and prints a summary table sorted by date.
- *
- * Usage:
- *   ailf capture list                          # default: .ailf/results/captures/
- *   ailf capture list ./my-captures            # custom directory
- */
-import { Command } from "commander";
-export declare function createCaptureListCommand(): Command;

package/dist/commands/capture-list.js

@@ -1,150 +0,0 @@
-/**
- * capture list — list pipeline captures in a directory.
- *
- * Scans a capture directory for subdirectories containing manifest.json,
- * reads each manifest, and prints a summary table sorted by date.
- *
- * Usage:
- *   ailf capture list                          # default: .ailf/results/captures/
- *   ailf capture list ./my-captures            # custom directory
- */
-import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
-import { join, resolve } from "node:path";
-import { Command } from "commander";
-import { resolveOutputDir } from "./shared/resolve-output-dir.js";
-// ---------------------------------------------------------------------------
-// Command factory
-// ---------------------------------------------------------------------------
-export function createCaptureListCommand() {
-    return new Command("list")
-        .description("List pipeline captures in a directory")
-        .argument("[dir]", "Captures directory (default: .ailf/results/captures/)")
-        .option("-f, --format <fmt>", "Output format: table or json", "table")
-        .action(async (dir, opts) => {
-        const captureDir = dir
-            ? resolve(dir)
-            : resolve(resolveOutputDir(), "..", "captures");
-        if (!existsSync(captureDir)) {
-            console.error(`  No captures directory found at ${captureDir}`);
-            console.error("  Run 'ailf pipeline --capture' to create captures.");
-            process.exitCode = 1;
-            return;
-        }
-        const captures = discoverCaptures(captureDir);
-        if (captures.length === 0) {
-            console.log(`  No captures found in ${captureDir}`);
-            return;
-        }
-        // Sort by startedAt descending (newest first)
-        captures.sort((a, b) => new Date(b.startedAt).getTime() - new Date(a.startedAt).getTime());
-        if (opts.format === "json") {
-            console.log(JSON.stringify(captures, null, 2));
-            return;
-        }
-        console.log("");
-        console.log("  ailf capture list");
-        console.log("  " + "─".repeat(60));
-        console.log("");
-        console.log("  " +
-            "Date".padEnd(22) +
-            "Mode".padEnd(18) +
-            "Artifacts".padEnd(12) +
-            "Size".padEnd(10) +
-            "Path");
-        console.log("  " + "─".repeat(60));
-        for (const c of captures) {
-            const date = new Date(c.startedAt).toLocaleString("en-US", {
-                month: "short",
-                day: "2-digit",
-                hour: "2-digit",
-                minute: "2-digit",
-            });
-            const mode = c.mode.padEnd(18);
-            const artifacts = String(c.artifactCount).padEnd(12);
-            const size = formatBytes(c.totalBytes).padEnd(10);
-            console.log(`  ${date.padEnd(22)}${mode}${artifacts}${size}${c.name}`);
-        }
-        console.log("");
-        console.log(`  ${captures.length} capture(s) found in ${captureDir}`);
-        console.log("");
-    });
-}
-function discoverCaptures(captureDir) {
-    const entries = [];
-    for (const name of readdirSync(captureDir)) {
-        if (name.startsWith("."))
-            continue;
-        const fullPath = join(captureDir, name);
-        // Raw directory with manifest.json
-        const manifestPath = join(fullPath, "manifest.json");
-        if (existsSync(manifestPath)) {
-            try {
-                const manifest = JSON.parse(readFileSync(manifestPath, "utf-8"));
-                const totalBytes = manifest.artifacts.reduce((sum, a) => sum + a.bytes, 0);
-                entries.push({
-                    name,
-                    path: fullPath,
-                    mode: manifest.pipeline.mode,
-                    startedAt: manifest.startedAt,
-                    artifactCount: manifest.artifacts.length,
-                    totalBytes,
-                    compressed: false,
-                });
-            }
-            catch {
-                // Skip unparseable manifests
-            }
-            continue;
-        }
-        // .tar.gz archive — read size but don't extract
-        if (name.endsWith(".tar.gz")) {
-            try {
-                const stat = statSync(fullPath);
-                entries.push({
-                    name,
-                    path: fullPath,
-                    mode: extractModeFromName(name),
-                    startedAt: extractTimestampFromName(name),
-                    artifactCount: -1, // Unknown without extracting
-                    totalBytes: stat.size,
-                    compressed: true,
-                });
-            }
-            catch {
-                // Skip
-            }
-        }
-    }
-    return entries;
-}
-/** Known modes — used to correctly extract mode from hyphenated filenames. */
-const KNOWN_MODES = [
-    "literacy",
-    "mcp-server",
-    "agent-harness",
-    "knowledge-probe",
-];
-function extractModeFromName(name) {
-    for (const mode of KNOWN_MODES) {
-        if (name.startsWith(mode + "-"))
-            return mode;
-    }
-    return name.split("-")[0] ?? "unknown";
-}
-function extractTimestampFromName(name) {
-    // Pattern: {mode}-YYYYMMDD-HHmmss-{id}.tar.gz
-    const match = name.match(/(\d{4})(\d{2})(\d{2})-(\d{2})(\d{2})(\d{2})/);
-    if (!match)
-        return "unknown";
-    const [, y, m, d, h, min, s] = match;
-    return `${y}-${m}-${d}T${h}:${min}:${s}Z`;
-}
-function formatBytes(bytes) {
-    if (bytes < 0)
-        return "?";
-    if (bytes < 1024)
-        return `${bytes}B`;
-    if (bytes < 1024 * 1024)
-        return `${(bytes / 1024).toFixed(1)}KB`;
-    return `${(bytes / 1024 / 1024).toFixed(1)}MB`;
-}

package/dist/commands/capture.d.ts

@@ -1,9 +0,0 @@
-/**
- * capture command — manage and compare pipeline artifact captures.
- *
- * Parent command for capture-related subcommands:
- *   ailf capture compare <baseline> <experiment>
- *   ailf capture list [dir]
- */
-import { Command } from "commander";
-export declare function createCaptureCommand(): Command;

package/dist/commands/capture.js

@@ -1,16 +0,0 @@
-/**
- * capture command — manage and compare pipeline artifact captures.
- *
- * Parent command for capture-related subcommands:
- *   ailf capture compare <baseline> <experiment>
- *   ailf capture list [dir]
- */
-import { Command } from "commander";
-import { createCaptureCompareCommand } from "./capture-compare.js";
-import { createCaptureListCommand } from "./capture-list.js";
-export function createCaptureCommand() {
-    const cmd = new Command("capture").description("Manage and compare pipeline artifact captures");
-    cmd.addCommand(createCaptureCompareCommand());
-    cmd.addCommand(createCaptureListCommand());
-    return cmd;
-}