@sanity/ailf 2.0.1 → 2.1.0

package/dist/pipeline/generate-configs.d.ts

@@ -1,92 +0,0 @@
-/**
- * pipeline/generate-configs.ts
- *
- * @deprecated This is the LEGACY compilation path. New code should use the
- * config compiler pipeline instead:
- *
- *   import { compileLiteracyTasks } from "./compiler/literacy-bridge.js"
- *   import { buildTaskGraph, compileToPromptfoo } from "./compiler/index.js"
- *
- * This file is retained behind the `--legacy-compiler` CLI flag as an
- * emergency fallback during the migration period. It will be removed once
- * the new compiler has been validated in production.
- *
- * @see packages/eval/src/pipeline/compiler/ — the new compiler pipeline
- * @see docs/archive/exec-plans/architecture-overhaul/phase-7-migrate-literacy.md
- *
- * ---
- *
- * Reads config/models.yaml (the central model registry) and generates all
- * promptfoo config files with the correct provider entries.
- *
- * This keeps model definitions in one place — add a model to config/models.yaml
- * and run `pnpm generate-configs` to propagate it to all eval modes.
- *
- * Generated configs:
- *   - promptfooconfig.yaml           (baseline: with-docs vs without-docs)
- *   - promptfooconfig.observed.yaml  (instrumented HTTP recording)
- *   - promptfooconfig.agentic.yaml   (agentic tool-calling: naive vs optimized)
- *
- * All functions accept rootDir as a parameter — no module-level constants.
- * No process.argv parsing. No env var fallbacks. Callers provide typed options.
- *
- * @see config/models.yaml — the central model registry
- * @see docs/archive/exec-plans/eliminate-lib-layer.md
- */
-import { type LiteracyTaskDefinition, type Logger } from "../_vendor/ailf-core/index.d.ts";
-import type { FilterOptions } from "./types.js";
-import { type ResolvedSourceConfig } from "../sources.js";
-export { extractModelName, extractProvider, mergeConfig, modelMatchesMode, } from "../_vendor/ailf-core/index.d.ts";
-/** Auto-discover all task YAML files in the tasks/ directory. */
-export declare function discoverTaskFiles(rootDir: string): string[];
-interface LoadedPrompts {
-    agentic: {
-        id: string;
-        label: string;
-        raw: string;
-    };
-    withDocs: {
-        id: string;
-        label: string;
-        raw: string;
-    };
-    withoutDocs: {
-        id: string;
-        label: string;
-        raw: string;
-    };
-}
-/**
- * Load prompt templates. Uses handler-owned literacy templates as defaults,
- * with config/prompts.ts as an override layer for user customization.
- */
-export declare function loadPrompts(rootDir: string): LoadedPrompts;
-/** Options for the generateConfigs function. */
-export interface GenerateConfigsOptions {
-    /** Allowed origins for agentic mode (controls source isolation assertion) */
-    allowedOrigins?: string[];
-    /** Filter to specific feature areas or task IDs */
-    filter?: FilterOptions;
-    /** Logger instance (defaults to ConsoleLogger) */
-    logger?: Logger;
-    /** Pre-resolved source config (skips loadSource() call) */
-    resolvedSource?: ResolvedSourceConfig;
-    /** Root directory of the eval package (required) */
-    rootDir: string;
-    /** Search mode for agentic mode */
-    searchMode?: string;
-    /** Documentation source name (e.g., "branch", "local") */
-    source?: string;
-    /** Pre-loaded literacy task definitions from a TaskSource adapter.
-     *  When provided, expandTaskDefinitions() is used instead of
-     *  loadAndExpandTasks() (which reads from tasks/*.yaml files). */
-    tasks?: LiteracyTaskDefinition[];
-}
-/**
- * Generate Promptfoo config files from models.yaml + task definitions.
- *
- * All parameters are passed via the typed options object — no process.argv
- * parsing or env var fallbacks. Callers (command handlers, orchestration
- * steps) are responsible for resolving options from their own context.
- */
-export declare function generateConfigs(options: GenerateConfigsOptions): void;

package/dist/pipeline/generate-configs.js

@@ -1,445 +0,0 @@
-/**
- * pipeline/generate-configs.ts
- *
- * @deprecated This is the LEGACY compilation path. New code should use the
- * config compiler pipeline instead:
- *
- *   import { compileLiteracyTasks } from "./compiler/literacy-bridge.js"
- *   import { buildTaskGraph, compileToPromptfoo } from "./compiler/index.js"
- *
- * This file is retained behind the `--legacy-compiler` CLI flag as an
- * emergency fallback during the migration period. It will be removed once
- * the new compiler has been validated in production.
- *
- * @see packages/eval/src/pipeline/compiler/ — the new compiler pipeline
- * @see docs/archive/exec-plans/architecture-overhaul/phase-7-migrate-literacy.md
- *
- * ---
- *
- * Reads config/models.yaml (the central model registry) and generates all
- * promptfoo config files with the correct provider entries.
- *
- * This keeps model definitions in one place — add a model to config/models.yaml
- * and run `pnpm generate-configs` to propagate it to all eval modes.
- *
- * Generated configs:
- *   - promptfooconfig.yaml           (baseline: with-docs vs without-docs)
- *   - promptfooconfig.observed.yaml  (instrumented HTTP recording)
- *   - promptfooconfig.agentic.yaml   (agentic tool-calling: naive vs optimized)
- *
- * All functions accept rootDir as a parameter — no module-level constants.
- * No process.argv parsing. No env var fallbacks. Callers provide typed options.
- *
- * @see config/models.yaml — the central model registry
- * @see docs/archive/exec-plans/eliminate-lib-layer.md
- */
-import { extractModelName, extractProvider, mergeConfig, } from "../_vendor/ailf-core/index.js";
-import { existsSync, readdirSync, writeFileSync } from "fs";
-import { resolve } from "path";
-import { dump } from "js-yaml";
-import { ConsoleLogger } from "../adapters/loggers/index.js";
-import { loadConfigFile } from "./compiler/config-loader.js";
-import { modelMatchesLiteracyVariant } from "./compiler/mode-bases/literacy.js";
-import { LITERACY_PROMPT_TEMPLATES } from "./compiler/mode-handlers/literacy/index.js";
-import { expandTaskDefinitions, loadAndExpandTasks } from "./expand-tasks.js";
-import { validateModelsYaml } from "./validate.js";
-import { LiteracyVariant } from "./normalize-mode.js";
-import { loadSource } from "../sources.js";
-// Re-export pure functions from core for backward compatibility.
-// Tests and other modules that previously imported from lib/generate-configs
-// can import from pipeline/generate-configs instead.
-export { extractModelName, extractProvider, mergeConfig, modelMatchesMode, } from "../_vendor/ailf-core/index.js";
-// ---------------------------------------------------------------------------
-// Helpers
-// ---------------------------------------------------------------------------
-/** Auto-discover all task YAML files in the tasks/ directory. */
-export function discoverTaskFiles(rootDir) {
-    const tasksDir = resolve(rootDir, "tasks");
-    if (!existsSync(tasksDir)) {
-        return []; // tasks may come from Content Lake instead
-    }
-    return readdirSync(tasksDir)
-        .filter((f) => (f.endsWith(".yaml") || f.endsWith(".yml")) && !f.startsWith("."))
-        .sort()
-        .map((f) => `file://tasks/${f}`);
-}
-/**
- * Load prompt templates. Uses handler-owned literacy templates as defaults,
- * with config/prompts.ts as an override layer for user customization.
- */
-export function loadPrompts(rootDir) {
-    const toPrompt = (entry) => ({
-        id: entry.id,
-        label: entry.label,
-        raw: entry.template,
-    });
-    // Load user overrides from config/prompts (may be empty after Wave 4)
-    let overrides = {};
-    try {
-        const loaded = loadConfigFile("prompts", rootDir).data;
-        // config/prompts.ts may export a Record (legacy) or an empty array (post-Wave 4)
-        if (loaded && !Array.isArray(loaded)) {
-            overrides = loaded;
-        }
-    }
-    catch {
-        // No config/prompts file — use handler defaults only
-    }
-    // Handler-owned templates are the canonical source; overrides take precedence
-    return {
-        agentic: overrides[LiteracyVariant.AGENTIC]
-            ? toPrompt(overrides[LiteracyVariant.AGENTIC])
-            : toPrompt(LITERACY_PROMPT_TEMPLATES[LiteracyVariant.AGENTIC]),
-        withDocs: overrides["with-docs"]
-            ? toPrompt(overrides["with-docs"])
-            : toPrompt(LITERACY_PROMPT_TEMPLATES["with-docs"]),
-        withoutDocs: overrides["without-docs"]
-            ? toPrompt(overrides["without-docs"])
-            : toPrompt(LITERACY_PROMPT_TEMPLATES["without-docs"]),
-    };
-}
-function loadModels(rootDir) {
-    return loadConfigFile("models", rootDir).data;
-}
-// ---------------------------------------------------------------------------
-// Shared components
-// ---------------------------------------------------------------------------
-const URL_EXTRACTION_ASSERT = {
-    type: "javascript",
-    value: `const urlPattern = /https?:\\/\\/[^\\s\\)\\"\\'\\\`>]+/g;
-const urls = [...new Set((output.match(urlPattern) || []))];
-const sanityUrls = urls.filter(u => u.includes('sanity.io'));
-return {
-  pass: true,
-  score: 1,
-  reason: JSON.stringify({
-    sanityUrls,
-    otherUrls: urls.filter(u => !u.includes('sanity.io')),
-    totalUrlCount: urls.length,
-    sanityUrlCount: sanityUrls.length
-  })
-};`,
-    weight: 0,
-};
-/**
- * Source isolation assertion — advisory (weight: 0).
- * Verifies that the agentic provider only fetched docs from allowed origins.
- * Only injected when origin sandboxing is active.
- */
-const SOURCE_ISOLATION_ASSERT = {
-    metadata: { dimension: "source-isolation" },
-    type: "javascript",
-    value: "file://dist/assertions/source-isolation.js",
-    weight: 0,
-};
-// ---------------------------------------------------------------------------
-// Config generators
-// ---------------------------------------------------------------------------
-function generateAgenticConfig(models, tests, prompts, source, searchMode, allowedOrigins) {
-    const naiveModels = models.models.filter((m) => modelMatchesLiteracyVariant(m, "agentic-naive"));
-    const optimizedModels = models.models.filter((m) => modelMatchesLiteracyVariant(m, "agentic-optimized"));
-    const providers = [];
-    // Build doc source config to inject into providers
-    const resolvedSearchMode = searchMode ?? "open";
-    const sourceConfig = source
-        ? {
-            ...(source.allowedOrigins?.length
-                ? { allowedOrigins: source.allowedOrigins }
-                : {}),
-            docBaseUrl: source.baseUrl,
-            ...(source.headers && Object.keys(source.headers).length > 0
-                ? { customHeaders: source.headers }
-                : {}),
-            llmsTxtUrl: source.llmsTxt,
-            ...(source.priorityDomain
-                ? { priorityDomain: source.priorityDomain }
-                : {}),
-            // Tool access control: search mode for web_search behavior
-            ...(resolvedSearchMode !== "open"
-                ? { searchMode: resolvedSearchMode }
-                : {}),
-        }
-        : {};
-    for (const model of naiveModels) {
-        const modelName = extractModelName(model.id);
-        const provider = extractProvider(model.id);
-        providers.push({
-            config: {
-                ...mergeConfig(models.defaults, model.config, {
-                    agentMode: "naive",
-                    maxToolRounds: models.defaults.maxToolRounds ?? 5,
-                    model: modelName,
-                    provider,
-                }),
-                ...(model.timeoutMs ? { timeoutMs: model.timeoutMs } : {}),
-                ...sourceConfig,
-                observe: true,
-                observerOptions: models.defaults.observerOptions ?? {},
-            },
-            id: "file://dist/agent-observer/agentic-provider.js",
-            label: `${model.label} (Naive Agent)`,
-        });
-    }
-    for (const model of optimizedModels) {
-        const modelName = extractModelName(model.id);
-        const provider = extractProvider(model.id);
-        providers.push({
-            config: {
-                ...mergeConfig(models.defaults, model.config, {
-                    agentMode: "optimized",
-                    maxToolRounds: models.defaults.maxToolRounds ?? 5,
-                    model: modelName,
-                    provider,
-                }),
-                ...(model.timeoutMs ? { timeoutMs: model.timeoutMs } : {}),
-                ...sourceConfig,
-                observe: true,
-                observerOptions: models.defaults.observerOptions ?? {},
-            },
-            id: "file://dist/agent-observer/agentic-provider.js",
-            label: `${model.label} (Optimized Agent)`,
-        });
-    }
-    // Inject source isolation assertion when origin sandboxing is active
-    const hasOriginSandbox = Boolean(allowedOrigins?.length);
-    const agenticAssertions = hasOriginSandbox ? [SOURCE_ISOLATION_ASSERT] : [];
-    return {
-        commandLineOptions: { table: false },
-        defaultTest: {
-            ...(agenticAssertions.length > 0 ? { assert: agenticAssertions } : {}),
-            options: {
-                provider: models.grader.id,
-                rubricProvider: models.grader.id,
-            },
-        },
-        description: "Sanity AI Literacy Evaluation — Agentic (naive vs optimized)",
-        ...(models.maxConcurrency
-            ? { evaluateOptions: { maxConcurrency: models.maxConcurrency } }
-            : {}),
-        outputPath: "results/latest/eval-results-agentic.json",
-        prompts: [prompts.agentic],
-        providers,
-        tests,
-    };
-}
-function generateBaselineConfig(models, tests, prompts) {
-    const baselineModels = models.models.filter((m) => modelMatchesLiteracyVariant(m, "baseline"));
-    const providers = baselineModels.map((model) => ({
-        config: {
-            ...mergeConfig(models.defaults, model.config),
-            ...(model.timeoutMs ? { timeoutMs: model.timeoutMs } : {}),
-        },
-        id: model.id,
-        label: model.label,
-    }));
-    return {
-        commandLineOptions: { table: false },
-        defaultTest: {
-            assert: [URL_EXTRACTION_ASSERT],
-            options: {
-                provider: models.grader.id,
-                rubricProvider: models.grader.id,
-            },
-        },
-        description: "Sanity AI Literacy Evaluation — Baseline",
-        ...(models.maxConcurrency
-            ? { evaluateOptions: { maxConcurrency: models.maxConcurrency } }
-            : {}),
-        outputPath: "results/latest/eval-results.json",
-        prompts: [prompts.withDocs, prompts.withoutDocs],
-        providers,
-        tests,
-    };
-}
-function generateObservedConfig(models, tests, prompts) {
-    const observedModels = models.models.filter((m) => modelMatchesLiteracyVariant(m, "observed"));
-    const providers = observedModels.map((model) => {
-        const modelName = extractModelName(model.id);
-        return {
-            config: {
-                ...mergeConfig(models.defaults, model.config),
-                ...(model.timeoutMs ? { timeoutMs: model.timeoutMs } : {}),
-                modelName,
-                observe: true,
-                recordOptions: models.defaults.observerOptions ?? {},
-            },
-            id: "file://dist/agent-observer/provider.js",
-            label: `${model.label} (Observed)`,
-        };
-    });
-    return {
-        commandLineOptions: { table: false },
-        defaultTest: {
-            options: {
-                provider: models.grader.id,
-                rubricProvider: models.grader.id,
-            },
-        },
-        description: "Sanity AI Literacy Evaluation — Observed",
-        ...(models.maxConcurrency
-            ? { evaluateOptions: { maxConcurrency: models.maxConcurrency } }
-            : {}),
-        outputPath: "results/latest/eval-results-observed.json",
-        prompts: [prompts.withDocs, prompts.withoutDocs],
-        providers,
-        tests,
-    };
-}
-// ---------------------------------------------------------------------------
-// Main entry point
-// ---------------------------------------------------------------------------
-/**
- * Generate Promptfoo config files from models.yaml + task definitions.
- *
- * All parameters are passed via the typed options object — no process.argv
- * parsing or env var fallbacks. Callers (command handlers, orchestration
- * steps) are responsible for resolving options from their own context.
- */
-export function generateConfigs(options) {
-    const { rootDir } = options;
-    const log = options.logger ?? new ConsoleLogger();
-    // Validate config/models.yaml before generating configs
-    const modelIssues = validateModelsYaml(rootDir);
-    const modelErrors = modelIssues.filter((i) => i.severity === "error");
-    if (modelErrors.length > 0) {
-        const details = modelErrors
-            .map((e) => (e.path ? `${e.message} (at ${e.path})` : e.message))
-            .join("; ");
-        throw new Error(`config/models.yaml validation failed: ${details}. Run 'pnpm validate' for details.`);
-    }
-    log.info("Loading config/models.yaml...");
-    const models = loadModels(rootDir);
-    const activeModels = models.models.filter((m) => m.id && m.label);
-    log.debug("Models loaded from config/models.yaml", {
-        totalModels: models.models.length,
-        activeModels: activeModels.length,
-        modelIds: activeModels.map((m) => m.id),
-        graderId: models.grader.id,
-        maxConcurrency: models.maxConcurrency,
-    });
-    log.info(`  Found ${activeModels.length} active model(s):`);
-    for (const m of activeModels) {
-        // oxlint-disable-next-line @typescript-eslint/prefer-nullish-coalescing -- empty array join → "all"
-        const modes = m.modes?.join(", ") || "all";
-        log.info(`    - ${m.label} (${m.id}) → [${modes}]`);
-    }
-    // oxlint-disable-next-line @typescript-eslint/prefer-nullish-coalescing -- empty label falls through to id
-    log.info(`  Grader: ${models.grader.label || models.grader.id}`);
-    // Build filter from options
-    const filter = options.filter?.areas || options.filter?.taskIds
-        ? options.filter
-        : undefined;
-    // Expand tasks — use GeneralizedTaskDefinition[] from TaskSource when provided,
-    // otherwise fall back to loading from tasks/*.yaml files.
-    let entries;
-    let agenticEntries;
-    if (options.tasks) {
-        // TaskSource path — tasks already loaded and filtered by the adapter
-        log.debug("Expanding tasks from TaskSource adapter", {
-            taskCount: options.tasks.length,
-            taskIds: options.tasks.map((t) => t.id),
-        });
-        const baselineResult = expandTaskDefinitions(options.tasks, rootDir, LiteracyVariant.STANDARD);
-        entries = baselineResult.entries;
-        log.info(`  Expanded ${baselineResult.stats.totalTasks} task(s) → ${baselineResult.stats.expandedTotal} test entries (from TaskSource)`);
-        const agenticResult = expandTaskDefinitions(options.tasks, rootDir, LiteracyVariant.AGENTIC);
-        agenticEntries = agenticResult.entries;
-        log.info(`  Agentic: ${agenticResult.stats.expandedTotal} entries (gold only, no baseline)`);
-    }
-    else {
-        // Legacy path — read from tasks/*.yaml files
-        const { entries: baselineEntries, stats } = loadAndExpandTasks(rootDir, filter, LiteracyVariant.STANDARD, log);
-        entries = baselineEntries;
-        log.info(`  Expanded ${stats.singleDefinitions} task(s) → ${stats.expandedTotal} test entries`);
-        if (stats.legacyEntries > 0) {
-            log.info(`  ⚠ ${stats.legacyEntries} legacy (paired) entries passed through unchanged`);
-        }
-        if (filter) {
-            const parts = [];
-            if (filter.areas) {
-                parts.push(`areas: ${filter.areas.join(", ")}`);
-            }
-            if (filter.taskIds) {
-                parts.push(`tasks: ${filter.taskIds.join(", ")}`);
-            }
-            log.info(`  Scoped to: ${parts.join("; ")}`);
-        }
-        const { entries: agenticFromYaml, stats: agenticStats } = loadAndExpandTasks(rootDir, filter, LiteracyVariant.AGENTIC, log);
-        agenticEntries = agenticFromYaml;
-        log.info(`  Agentic: ${agenticStats.expandedTotal} entries (gold only, no baseline)`);
-    }
-    // Write expanded tasks to generated files for Promptfoo to consume
-    const expandedPath = resolve(rootDir, "tasks", ".expanded.yaml");
-    const expandedYaml = dump(entries, {
-        forceQuotes: false,
-        lineWidth: 120,
-        noRefs: true,
-        quotingType: "'",
-    });
-    writeFileSync(expandedPath, `# .expanded.yaml\n#\n# AUTO-GENERATED — do not edit directly.\n# Source: tasks/*.yaml (single-definition format)\n# Run: pnpm generate-configs\n\n${expandedYaml}`, "utf-8");
-    log.info(`  ✓ tasks/.expanded.yaml (${entries.length} entries)`);
-    const agenticExpandedPath = resolve(rootDir, "tasks", ".expanded.agentic.yaml");
-    const agenticExpandedYaml = dump(agenticEntries, {
-        forceQuotes: false,
-        lineWidth: 120,
-        noRefs: true,
-        quotingType: "'",
-    });
-    writeFileSync(agenticExpandedPath, `# .expanded.agentic.yaml\n#\n# AUTO-GENERATED — do not edit directly.\n# Gold entries only (no baseline) for agentic evaluation mode.\n# Source: tasks/*.yaml (single-definition format)\n# Run: pnpm generate-configs\n\n${agenticExpandedYaml}`, "utf-8");
-    log.info(`  ✓ tasks/.expanded.agentic.yaml (${agenticEntries.length} entries)`);
-    const taskFiles = ["file://tasks/.expanded.yaml"];
-    const agenticTaskFiles = ["file://tasks/.expanded.agentic.yaml"];
-    // Load prompt templates
-    const prompts = loadPrompts(rootDir);
-    log.debug("Prompt templates loaded", {
-        keys: Object.keys(prompts),
-        withDocsId: prompts.withDocs.id,
-        withoutDocsId: prompts.withoutDocs.id,
-        agenticId: prompts.agentic.id,
-    });
-    log.info(`  Loaded prompts: ${Object.keys(prompts).join(", ")}`);
-    // Load optional documentation source configuration
-    // Pre-resolved source wins over name-based lookup
-    let source = options.resolvedSource;
-    const sourceName = options.source;
-    if (!source && sourceName) {
-        log.info(`\nLoading source: ${sourceName}`);
-        try {
-            source = loadSource(sourceName);
-        }
-        catch (err) {
-            const msg = err instanceof Error ? err.message : String(err);
-            log.warn(`\n⚠ Failed to load source "${sourceName}": ${msg}`);
-        }
-    }
-    if (source) {
-        log.info(`  Base URL: ${source.baseUrl}`);
-        log.info(`  Dataset: ${source.dataset}`);
-        if (source.allowedOrigins?.length) {
-            log.info(`  Allowed origins: ${source.allowedOrigins.join(", ")}`);
-        }
-    }
-    log.info("\nGenerating configs...");
-    writeConfig(rootDir, "promptfooconfig.yaml", generateBaselineConfig(models, taskFiles, prompts), `# promptfooconfig.yaml\n#\n# AUTO-GENERATED from config/models.yaml — do not edit directly.\n# Run: pnpm generate-configs\n`, log);
-    writeConfig(rootDir, "promptfooconfig.observed.yaml", generateObservedConfig(models, taskFiles, prompts), `# promptfooconfig.observed.yaml\n#\n# AUTO-GENERATED from config/models.yaml — do not edit directly.\n# Run: pnpm generate-configs\n`, log);
-    writeConfig(rootDir, "promptfooconfig.agentic.yaml", generateAgenticConfig(models, agenticTaskFiles, prompts, source, options.searchMode, options.allowedOrigins), `# promptfooconfig.agentic.yaml\n#\n# AUTO-GENERATED from config/models.yaml — do not edit directly.\n# Run: pnpm generate-configs\n`, log);
-    log.info("\nDone! Configs are ready.");
-    if (source) {
-        log.info(`  (using doc source: ${sourceName})`);
-    }
-}
-// ---------------------------------------------------------------------------
-// File writing
-// ---------------------------------------------------------------------------
-function writeConfig(rootDir, filename, config, header, log) {
-    const yamlStr = dump(config, {
-        forceQuotes: false,
-        lineWidth: 120,
-        noRefs: true,
-        quotingType: "'",
-    });
-    const content = `${header}\n${yamlStr}`;
-    const outPath = resolve(rootDir, filename);
-    writeFileSync(outPath, content, "utf-8");
-    log.info(`  ✓ ${filename}`);
-}

package/dist/pipeline/steps/calculate-scores-step.d.ts

@@ -1,11 +0,0 @@
-/**
- * Pipeline step: Calculate AI Literacy Scores from eval results.
- *
- * Preconditions: eval-results.json exists and is valid
- * Postconditions: score-summary.json exists and is valid
- *
- * Cache key: eval results JSON file(s)
- * Cache outputs: results/latest/score-summary.json
- */
-import type { EvalMode, StepResult } from "../types.js";
-export declare function runCalculateScores(source?: string, mode?: EvalMode, noCache?: boolean): StepResult;

package/dist/pipeline/steps/calculate-scores-step.js

@@ -1,89 +0,0 @@
-/**
- * Pipeline step: Calculate AI Literacy Scores from eval results.
- *
- * Preconditions: eval-results.json exists and is valid
- * Postconditions: score-summary.json exists and is valid
- *
- * Cache key: eval results JSON file(s)
- * Cache outputs: results/latest/score-summary.json
- */
-import { execSync } from "child_process";
-import { dirname, resolve } from "path";
-import { fileURLToPath } from "url";
-import { getStepInputPaths, hashFiles, lookupCache, recordCache, } from "../cache.js";
-import { checkResultsExist, checkScoreSummaryValid } from "../checks.js";
-import { RESULTS_FILES } from "./eval-step.js";
-const __dirname = dirname(fileURLToPath(import.meta.url));
-const ROOT = resolve(__dirname, "..", "..", "..");
-export function runCalculateScores(source, mode = "baseline", noCache = false) {
-    const start = Date.now();
-    // For full mode, use the baseline results file as the primary input
-    // (calculate-scores reads all available results files internally)
-    const primaryMode = mode === "full" ? "baseline" : mode;
-    const resultsFile = RESULTS_FILES[primaryMode];
-    const resultsIssues = checkResultsExist(ROOT, resultsFile);
-    const resultsErrors = resultsIssues.filter((i) => i.severity === "error");
-    if (resultsErrors.length > 0) {
-        return {
-            durationMs: Date.now() - start,
-            error: `Results missing: ${resultsErrors.map((e) => e.message).join("; ")}`,
-            status: "failed",
-        };
-    }
-    // Cache check
-    if (!noCache) {
-        const cacheResult = lookupCache(ROOT, "calculate-scores");
-        if (cacheResult.hit) {
-            return {
-                durationMs: Date.now() - start,
-                status: "success",
-                summary: `Skipped (cached) — ${cacheResult.entry.summary}`,
-            };
-        }
-    }
-    // Execute — note: calculate-scores exits 1 when areas are below critical,
-    // which is expected behavior, not an error
-    try {
-        const sourceArg = source ? ` --source ${source}` : "";
-        const resultsArg = primaryMode !== "baseline" ? ` ${resultsFile}` : "";
-        execSync(`tsx src/lib/calculate-scores.ts${resultsArg}${sourceArg}`, {
-            cwd: ROOT,
-            env: process.env,
-            stdio: "inherit",
-        });
-    }
-    catch (err) {
-        const code = err !== null && typeof err === "object" && "status" in err
-            ? err.status
-            : 1;
-        // Exit code 1 means "areas below critical" — that's expected
-        if (code !== 1) {
-            return {
-                durationMs: Date.now() - start,
-                error: `calculate-scores failed with exit code ${code}`,
-                status: "failed",
-            };
-        }
-    }
-    // Postcondition: score summary exists and is valid
-    const summaryIssues = checkScoreSummaryValid(ROOT);
-    const summaryErrors = summaryIssues.filter((i) => i.severity === "error");
-    if (summaryErrors.length > 0) {
-        return {
-            durationMs: Date.now() - start,
-            error: `Postcondition failed: ${summaryErrors.map((e) => e.message).join("; ")}`,
-            status: "failed",
-        };
-    }
-    const durationMs = Date.now() - start;
-    const summary = "Scores calculated and summary written";
-    // Record cache
-    if (!noCache) {
-        const inputPaths = getStepInputPaths(ROOT, "calculate-scores");
-        const inputHash = hashFiles(inputPaths);
-        recordCache(ROOT, "calculate-scores", inputHash, summary, durationMs, [
-            "results/latest/score-summary.json",
-        ]);
-    }
-    return { durationMs, status: "success", summary };
-}

package/dist/pipeline/steps/compare-step.d.ts

@@ -1,18 +0,0 @@
-/**
- * Pipeline step: Compare current scores against a baseline.
- *
- * Preconditions: score-summary.json exists
- * Postconditions: comparison-report.json written to results/latest/
- *
- * This step is optional — it only runs when --compare is passed
- * (or a baseline exists and auto-compare is enabled).
- */
-import type { CompareOptions, StepResult } from "../types.js";
-/**
- * Run comparison against a baseline.
- *
- * @param rootDir Package root directory
- * @param baselinePath Explicit baseline file path (optional — uses latest if omitted)
- * @param options Compare options (noise threshold, etc.)
- */
-export declare function runCompare(rootDir: string, baselinePath?: string, options?: CompareOptions): StepResult;