create-principles-disciple 1.89.0 → 1.91.0

package/plugin/dist/core/model-training-registry.js

@@ -1,474 +0,0 @@
-/**
- * Model Training Registry — Training Run, Checkpoint, and Eval Summary Lineage
- * =============================================================================
- *
- * PURPOSE: Establish strict auditable lineage from training run → checkpoint → eval
- * so that "deployable" is a controlled state, not a free-text field.
- *
- * ARCHITECTURE:
- *   - Registry file: {stateDir}/.state/nocturnal/training-registry.json
- *   - Three record types in one store: TrainingRun, Checkpoint, EvalSummary
- *   - File locking on all write operations
- *   - Family alignment enforced at every transition
- *
- * LINEAGE CHAIN (enforced):
- *   TrainingRun → Checkpoint → EvalSummary
- *   DatasetFingerprint → TrainingRun → Checkpoint → EvalSummary
- *
- * DEPLOYABILITY RULE:
- *   A Checkpoint can only be marked deployable if:
- *     1. It has an attached EvalSummary
- *     2. The EvalSummary has a verdict of 'pass' or 'compare_only' (not 'fail')
- *     3. The EvalSummary's targetModelFamily matches the Checkpoint's targetModelFamily
- *     4. The Checkpoint's trainRun is in 'completed' status
- *
- * DESIGN CONSTRAINTS:
- *   - No real training invocation (Phase 4 only)
- *   - No checkpoint deploy routing (Phase 5)
- *   - No automatic promotion
- *   - Registry is append-only for runs and checkpoints
- *   - EvalSummary attachment is the only mutable operation on a Checkpoint
- */
-import * as fs from 'fs';
-import * as path from 'path';
-import * as crypto from 'crypto';
-import { withLock } from '../utils/file-lock.js';
-import { atomicWriteFileSync } from '../utils/io.js';
-// ---------------------------------------------------------------------------
-// Constants
-// ---------------------------------------------------------------------------
-const REGISTRY_FILE = '.state/nocturnal/training-registry.json';
-// ---------------------------------------------------------------------------
-// Registry Path
-// ---------------------------------------------------------------------------
-function getRegistryPath(stateDir) {
-    return path.join(stateDir, REGISTRY_FILE);
-}
-/**
- * Ensure the registry directory exists.
- */
-function ensureRegistryDir(stateDir) {
-    const registryPath = getRegistryPath(stateDir);
-    const dir = path.dirname(registryPath);
-    if (!fs.existsSync(dir)) {
-        fs.mkdirSync(dir, { recursive: true });
-    }
-}
-// ---------------------------------------------------------------------------
-// File Operations
-// ---------------------------------------------------------------------------
-/**
- * Read the registry from disk. Returns empty registry if missing.
- */
-function readRegistry(stateDir) {
-    const registryPath = getRegistryPath(stateDir);
-    if (!fs.existsSync(registryPath)) {
-        return { trainingRuns: [], checkpoints: [], evalSummaries: [] };
-    }
-    try {
-        const content = fs.readFileSync(registryPath, 'utf-8');
-        return JSON.parse(content);
-    }
-    catch (err) {
-        console.warn(`[model-training-registry] Registry corrupted at ${registryPath}, recovering with empty state: ${String(err)}`);
-        return { trainingRuns: [], checkpoints: [], evalSummaries: [] };
-    }
-}
-/**
- * Write the registry to disk atomically.
- * Caller must hold the registry lock.
- */
-function writeRegistry(stateDir, registry) {
-    ensureRegistryDir(stateDir);
-    const registryPath = getRegistryPath(stateDir);
-    atomicWriteFileSync(registryPath, JSON.stringify(registry, null, 2));
-}
-/**
- * Execute a read-modify-write under an exclusive file lock.
- */
-function withRegistryLock(stateDir, fn) {
-    const registryPath = getRegistryPath(stateDir);
-    return withLock(registryPath, () => {
-        const registry = readRegistry(stateDir);
-        return fn(registry);
-    });
-}
-// ---------------------------------------------------------------------------
-// Training Run Operations
-// ---------------------------------------------------------------------------
-/**
- * Valid training run status transitions.
- * pending → running → completed | failed
- * (no backward transitions)
- */
-const VALID_RUN_TRANSITIONS = {
-    pending: ['running'],
-    running: ['completed', 'failed'],
-    completed: [], // terminal
-    failed: [], // terminal
-};
-/**
- * Register a new training run.
- *
- * @param stateDir - Workspace state directory
- * @param params - Run parameters
- * @returns The registered TrainingRun
- */
-export function registerTrainingRun(stateDir, params) {
-    return withRegistryLock(stateDir, (registry) => {
-        const now = new Date().toISOString();
-        const trainRunId = crypto.randomUUID();
-        const run = {
-            trainRunId,
-            experimentId: params.experimentId,
-            targetModelFamily: params.targetModelFamily,
-            datasetFingerprint: params.datasetFingerprint,
-            exportId: params.exportId,
-            sampleCount: params.sampleCount,
-            configFingerprint: params.configFingerprint,
-            createdAt: now,
-            status: 'pending',
-            checkpointIds: [],
-        };
-        registry.trainingRuns.push(run);
-        writeRegistry(stateDir, registry);
-        return run;
-    });
-}
-/**
- * Update a training run's status.
- *
- * @throws Error if run not found or transition is invalid
- */
-export function updateTrainingRunStatus(stateDir, trainRunId, newStatus, failureReason) {
-    return withRegistryLock(stateDir, (registry) => {
-        const idx = registry.trainingRuns.findIndex((r) => r.trainRunId === trainRunId);
-        if (idx === -1) {
-            throw new Error(`Training run not found: ${trainRunId}`);
-        }
-        const run = registry.trainingRuns[idx];
-        const allowed = VALID_RUN_TRANSITIONS[run.status];
-        if (!allowed.includes(newStatus)) {
-            throw new Error(`Invalid status transition for training run ${trainRunId}: ${run.status} → ${newStatus}. ` +
-                `Allowed transitions from ${run.status}: ${allowed.join(', ') || 'none'}`);
-        }
-        registry.trainingRuns[idx] = {
-            ...run,
-            status: newStatus,
-            completedAt: newStatus === 'completed' || newStatus === 'failed'
-                ? new Date().toISOString()
-                : undefined,
-            failureReason: newStatus === 'failed' ? failureReason : undefined,
-        };
-        writeRegistry(stateDir, registry);
-        return registry.trainingRuns[idx];
-    });
-}
-/**
- * Complete a training run (convenience wrapper).
- */
-export function completeTrainingRun(stateDir, trainRunId) {
-    return updateTrainingRunStatus(stateDir, trainRunId, 'completed');
-}
-/**
- * Fail a training run (convenience wrapper).
- */
-export function failTrainingRun(stateDir, trainRunId, reason) {
-    return updateTrainingRunStatus(stateDir, trainRunId, 'failed', reason);
-}
-/**
- * Start a training run (convenience wrapper).
- */
-export function startTrainingRun(stateDir, trainRunId) {
-    return updateTrainingRunStatus(stateDir, trainRunId, 'running');
-}
-/**
- * Get a training run by ID.
- */
-export function getTrainingRun(stateDir, trainRunId) {
-    const registry = readRegistry(stateDir);
-    return registry.trainingRuns.find((r) => r.trainRunId === trainRunId) ?? null;
-}
-/**
- * List all training runs, optionally filtered by status or family.
- */
-export function listTrainingRuns(stateDir, filter) {
-    const registry = readRegistry(stateDir);
-    let runs = registry.trainingRuns;
-    if (filter?.status) {
-        runs = runs.filter((r) => r.status === filter.status);
-    }
-    if (filter?.targetModelFamily) {
-        runs = runs.filter((r) => r.targetModelFamily === filter.targetModelFamily);
-    }
-    return runs.sort((a, b) => new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime());
-}
-// ---------------------------------------------------------------------------
-// Checkpoint Operations
-// ---------------------------------------------------------------------------
-/**
- * Register a checkpoint produced by a training run.
- *
- * @throws Error if the training run is not found
- * @throws Error if the targetModelFamily does not match the run's family
- */
-export function registerCheckpoint(stateDir, params) {
-    return withRegistryLock(stateDir, (registry) => {
-        // Verify the training run exists
-        const run = registry.trainingRuns.find((r) => r.trainRunId === params.trainRunId);
-        if (!run) {
-            throw new Error(`Training run not found: ${params.trainRunId}`);
-        }
-        // Verify family alignment
-        if (run.targetModelFamily !== params.targetModelFamily) {
-            throw new Error(`Target model family mismatch: checkpoint family "${params.targetModelFamily}" ` +
-                `does not match training run family "${run.targetModelFamily}"`);
-        }
-        const now = new Date().toISOString();
-        const checkpointId = crypto.randomUUID();
-        const checkpoint = {
-            checkpointId,
-            trainRunId: params.trainRunId,
-            targetModelFamily: params.targetModelFamily,
-            artifactPath: params.artifactPath,
-            createdAt: now,
-            deployable: false, // Always starts as false
-        };
-        registry.checkpoints.push(checkpoint);
-        // Update the training run's checkpoint IDs
-        const runIdx = registry.trainingRuns.findIndex((r) => r.trainRunId === params.trainRunId);
-        registry.trainingRuns[runIdx] = {
-            ...run,
-            checkpointIds: [...run.checkpointIds, checkpointId],
-        };
-        writeRegistry(stateDir, registry);
-        return checkpoint;
-    });
-}
-/**
- * Get a checkpoint by ID.
- */
-export function getCheckpoint(stateDir, checkpointId) {
-    const registry = readRegistry(stateDir);
-    return registry.checkpoints.find((c) => c.checkpointId === checkpointId) ?? null;
-}
-/**
- * List all checkpoints, optionally filtered.
- */
-export function listCheckpoints(stateDir, filter) {
-    const registry = readRegistry(stateDir);
-    let { checkpoints } = registry;
-    if (filter?.trainRunId) {
-        checkpoints = checkpoints.filter((c) => c.trainRunId === filter.trainRunId);
-    }
-    if (filter?.targetModelFamily) {
-        checkpoints = checkpoints.filter((c) => c.targetModelFamily === filter.targetModelFamily);
-    }
-    if (filter?.deployable !== undefined) {
-        checkpoints = checkpoints.filter((c) => c.deployable === filter.deployable);
-    }
-    return checkpoints.sort((a, b) => new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime());
-}
-/**
- * List all deployable checkpoints for a target model family.
- */
-export function listDeployableCheckpoints(stateDir, targetModelFamily) {
-    return listCheckpoints(stateDir, {
-        targetModelFamily,
-        deployable: true,
-    });
-}
-// ---------------------------------------------------------------------------
-// Eval Summary Operations
-// ---------------------------------------------------------------------------
-/**
- * Attach an eval summary to a checkpoint.
- *
- * @param stateDir - Workspace state directory
- * @param checkpointId - The checkpoint to attach to
- * @param summary - The eval summary to attach
- *
- * @throws Error if checkpoint not found
- * @throws Error if targetModelFamily mismatch between summary and checkpoint
- */
-export function attachEvalSummary(stateDir, checkpointId, summary) {
-    return withRegistryLock(stateDir, (registry) => {
-        const checkpointIdx = registry.checkpoints.findIndex((c) => c.checkpointId === checkpointId);
-        if (checkpointIdx === -1) {
-            throw new Error(`Checkpoint not found: ${checkpointId}`);
-        }
-        const checkpoint = registry.checkpoints[checkpointIdx];
-        // FAMILY ALIGNMENT — enforced fail-closed:
-        // An eval for a gpt-4 checkpoint cannot be attached to a claude-3 checkpoint
-        // (and vice versa), even if the eval verdict is 'pass'.
-        if (summary.targetModelFamily !== checkpoint.targetModelFamily) {
-            throw new Error(`Family mismatch: eval targets "${summary.targetModelFamily}" ` +
-                `but checkpoint "${checkpointId}" is "${checkpoint.targetModelFamily}". ` +
-                `EvalSummary.targetModelFamily must match the checkpoint's targetModelFamily.`);
-        }
-        const evalSummary = {
-            ...summary,
-            createdAt: new Date().toISOString(),
-        };
-        registry.evalSummaries.push(evalSummary);
-        // Update the checkpoint's lastEvalSummaryRef
-        registry.checkpoints[checkpointIdx] = {
-            ...checkpoint,
-            lastEvalSummaryRef: evalSummary.evalId,
-        };
-        writeRegistry(stateDir, registry);
-        return evalSummary;
-    });
-}
-/**
- * Get an eval summary by ID.
- */
-export function getEvalSummary(stateDir, evalId) {
-    const registry = readRegistry(stateDir);
-    return registry.evalSummaries.find((e) => e.evalId === evalId) ?? null;
-}
-/**
- * List eval summaries, optionally filtered.
- */
-export function listEvalSummaries(stateDir, filter) {
-    const registry = readRegistry(stateDir);
-    let evals = registry.evalSummaries;
-    if (filter?.checkpointId) {
-        evals = evals.filter((e) => e.checkpointId === filter.checkpointId);
-    }
-    if (filter?.benchmarkId) {
-        evals = evals.filter((e) => e.benchmarkId === filter.benchmarkId);
-    }
-    if (filter?.verdict) {
-        evals = evals.filter((e) => e.verdict === filter.verdict);
-    }
-    if (filter?.targetModelFamily) {
-        evals = evals.filter((e) => e.targetModelFamily === filter.targetModelFamily);
-    }
-    return evals.sort((a, b) => new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime());
-}
-// ---------------------------------------------------------------------------
-// Deployability — Core Gating Logic
-// ---------------------------------------------------------------------------
-/**
- * MARK the deployability status of a checkpoint.
- *
- * DEPLOYABILITY RULE (fail-closed):
- *   A checkpoint can only be marked deployable if ALL of:
- *     1. It has an attached EvalSummary (lastEvalSummaryRef is set)
- *     2. The attached EvalSummary has verdict 'pass' or 'compare_only' (not 'fail')
- *     3. The EvalSummary's targetModelFamily matches the Checkpoint's targetModelFamily
- *        NOTE: This is enforced at attachEvalSummary() time (see attachEvalSummary).
- *        If a mismatched-family eval is attached, attachEvalSummary throws before
- *        the registry is modified, so no eval with wrong family can ever reach here.
- *     4. The parent TrainingRun is in 'completed' status
- *
- * @param stateDir - Workspace state directory
- * @param checkpointId - The checkpoint to mark
- * @param deployable - true to mark as deployable; false to revoke
- *
- * @throws Error if checkpoint not found
- * @throws Error if preconditions for deployable=true are not met
- */
-export function markCheckpointDeployable(stateDir, checkpointId, deployable) {
-    return withRegistryLock(stateDir, (registry) => {
-        const idx = registry.checkpoints.findIndex((c) => c.checkpointId === checkpointId);
-        if (idx === -1) {
-            throw new Error(`Checkpoint not found: ${checkpointId}`);
-        }
-        const checkpoint = registry.checkpoints[idx];
-        if (deployable) {
-            // FAIL-CLOSED: Verify all preconditions
-            // 1. Must have an attached eval summary
-            if (!checkpoint.lastEvalSummaryRef) {
-                throw new Error(`Cannot mark checkpoint ${checkpointId} as deployable: ` +
-                    `no eval summary attached. Attach an EvalSummary first.`);
-            }
-            // 2. Find the eval summary
-            const evalSummary = registry.evalSummaries.find((e) => e.evalId === checkpoint.lastEvalSummaryRef);
-            if (!evalSummary) {
-                throw new Error(`Cannot mark checkpoint ${checkpointId} as deployable: ` +
-                    `eval summary "${checkpoint.lastEvalSummaryRef}" not found`);
-            }
-            // 3. Verdict must be 'pass' or 'compare_only' (not 'fail')
-            if (evalSummary.verdict === 'fail') {
-                throw new Error(`Cannot mark checkpoint ${checkpointId} as deployable: ` +
-                    `eval verdict is '${evalSummary.verdict}' (evalId: ${evalSummary.evalId}). ` +
-                    `Only 'pass' or 'compare_only' verdicts allow deployment.`);
-            }
-            // 4. Parent training run must be completed
-            const run = registry.trainingRuns.find((r) => r.trainRunId === checkpoint.trainRunId);
-            if (!run) {
-                throw new Error(`Cannot mark checkpoint ${checkpointId} as deployable: ` +
-                    `parent training run "${checkpoint.trainRunId}" not found`);
-            }
-            if (run.status !== 'completed') {
-                throw new Error(`Cannot mark checkpoint ${checkpointId} as deployable: ` +
-                    `parent training run is in '${run.status}' status (must be 'completed')`);
-            }
-        }
-        // Apply the update (both marking deployable and revoking deployability)
-        registry.checkpoints[idx] = {
-            ...checkpoint,
-            deployable,
-            // If revoking deployability, also clear the eval ref
-            lastEvalSummaryRef: deployable ? checkpoint.lastEvalSummaryRef : undefined,
-        };
-        writeRegistry(stateDir, registry);
-        return registry.checkpoints[idx];
-    });
-}
-/**
- * Convenience: check if a checkpoint is deployable.
- */
-export function isCheckpointDeployable(stateDir, checkpointId) {
-    const checkpoint = getCheckpoint(stateDir, checkpointId);
-    return checkpoint?.deployable ?? false;
-}
-// ---------------------------------------------------------------------------
-// Registry-Level Queries
-// ---------------------------------------------------------------------------
-/**
- * Get the full lineage chain for a checkpoint.
- * Returns: { run, checkpoint, eval? } or null if not found.
- */
-export function getCheckpointLineage(stateDir, checkpointId) {
-    const registry = readRegistry(stateDir);
-    const checkpoint = registry.checkpoints.find((c) => c.checkpointId === checkpointId);
-    if (!checkpoint)
-        return null;
-    const run = registry.trainingRuns.find((r) => r.trainRunId === checkpoint.trainRunId);
-    if (!run)
-        return null;
-    const eval_ = checkpoint.lastEvalSummaryRef
-        ? registry.evalSummaries.find((e) => e.evalId === checkpoint.lastEvalSummaryRef) ?? null
-        : null;
-    return { run, checkpoint, eval: eval_ ?? null };
-}
-/**
- * Get the complete registry (for debugging/admin purposes).
- */
-export function getFullRegistry(stateDir) {
-    return readRegistry(stateDir);
-}
-/**
- * Compute stats for the training registry.
- */
-export function getTrainingRegistryStats(stateDir) {
-    const registry = readRegistry(stateDir);
-    const runs = registry.trainingRuns;
-    const { checkpoints } = registry;
-    const evals = registry.evalSummaries;
-    return {
-        totalRuns: runs.length,
-        completedRuns: runs.filter((r) => r.status === 'completed').length,
-        failedRuns: runs.filter((r) => r.status === 'failed').length,
-        pendingRuns: runs.filter((r) => r.status === 'pending').length,
-        runningRuns: runs.filter((r) => r.status === 'running').length,
-        totalCheckpoints: checkpoints.length,
-        deployableCheckpoints: checkpoints.filter((c) => c.deployable).length,
-        totalEvals: evals.length,
-        passingEvals: evals.filter((e) => e.verdict === 'pass' || e.verdict === 'compare_only').length,
-        failingEvals: evals.filter((e) => e.verdict === 'fail').length,
-    };
-}