@simulatte/doppler 0.1.5 → 0.1.6

package/src/training/operator-artifacts.d.ts

@@ -0,0 +1,62 @@
+import type { LoadedTrainingWorkload, TrainingWorkloadPack } from './workloads.js';
+
+export interface TrainingRunLayout {
+  runRoot: string;
+  logs: string;
+  checkpoints: string;
+  eval: string;
+  scoreboard: string;
+  exports: string;
+  compare: string;
+  qualityGate: string;
+}
+
+export declare function normalizeTrainingTimestamp(value?: string | Date | null): string;
+
+export declare function createTrainingRunLayout(options: {
+  kind: string;
+  workloadId: string;
+  timestamp?: string | Date | null;
+}): Promise<TrainingRunLayout>;
+
+export declare function writeJsonArtifact(
+  filePath: string,
+  payload: Record<string, unknown>
+): Promise<{ path: string; sha256: string; relativePath: string }>;
+
+export declare function writeNdjsonRow(
+  filePath: string,
+  row: Record<string, unknown>
+): Promise<{ path: string; sha256: string; relativePath: string }>;
+
+export declare function writeWorkloadLock(
+  layout: TrainingRunLayout,
+  loadedWorkload: LoadedTrainingWorkload
+): Promise<{ path: string; sha256: string; relativePath: string }>;
+
+export declare function writeRunContract(
+  layout: TrainingRunLayout,
+  payload: Record<string, unknown>
+): Promise<{ path: string; sha256: string; relativePath: string }>;
+
+export declare function buildArtifactBase(options: {
+  artifactType: string;
+  reportId: string;
+  workload: TrainingWorkloadPack;
+  workloadPath: string;
+  workloadSha256: string;
+  datasetPath: string | null;
+  datasetHash: string | null;
+  baseModelId: string | null;
+  teacherModelId?: string | null;
+  studentModelId?: string | null;
+  stage?: string | null;
+  checkpointStep?: number | null;
+  parentArtifacts?: Array<Record<string, unknown>>;
+  runtime?: string;
+  surface?: string;
+  claimBoundary: string;
+  configHash: string;
+}): Record<string, unknown>;
+
+export declare function hashArtifactPayload(payload: Record<string, unknown>): string;

package/src/training/operator-artifacts.js

@@ -0,0 +1,140 @@
+import { mkdir, readFile, writeFile } from 'node:fs/promises';
+import { dirname, join, relative, resolve } from 'node:path';
+
+import { sha256Hex } from '../utils/sha256.js';
+import { serializeTrainingWorkloadLock } from './workloads.js';
+
+function stableSortObject(value) {
+  if (Array.isArray(value)) {
+    return value.map((entry) => stableSortObject(entry));
+  }
+  if (!value || typeof value !== 'object') {
+    return value;
+  }
+  const sorted = {};
+  for (const key of Object.keys(value).sort()) {
+    sorted[key] = stableSortObject(value[key]);
+  }
+  return sorted;
+}
+
+function stableJson(value) {
+  return JSON.stringify(stableSortObject(value));
+}
+
+export function normalizeTrainingTimestamp(value = null) {
+  const date = value instanceof Date
+    ? value
+    : (typeof value === 'string' && value.trim() ? new Date(value) : new Date());
+  return date.toISOString().replace(/[:]/g, '-');
+}
+
+export async function createTrainingRunLayout({ kind, workloadId, timestamp = null }) {
+  const normalizedKind = String(kind || '').trim();
+  const normalizedWorkloadId = String(workloadId || '').trim();
+  if (!normalizedKind || !normalizedWorkloadId) {
+    throw new Error('createTrainingRunLayout requires kind and workloadId.');
+  }
+  const ts = normalizeTrainingTimestamp(timestamp);
+  const runRoot = resolve('reports', 'training', normalizedKind, normalizedWorkloadId, ts);
+  const directories = {
+    runRoot,
+    logs: join(runRoot, 'logs'),
+    checkpoints: join(runRoot, 'checkpoints'),
+    eval: join(runRoot, 'eval'),
+    scoreboard: join(runRoot, 'scoreboard'),
+    exports: join(runRoot, 'exports'),
+    compare: join(runRoot, 'compare'),
+    qualityGate: join(runRoot, 'quality-gate'),
+  };
+  await Promise.all(Object.values(directories).map((dirPath) => mkdir(dirPath, { recursive: true })));
+  return directories;
+}
+
+export async function writeJsonArtifact(filePath, payload) {
+  const absolutePath = resolve(String(filePath));
+  const json = `${JSON.stringify(payload, null, 2)}\n`;
+  await mkdir(dirname(absolutePath), { recursive: true });
+  await writeFile(absolutePath, json, 'utf8');
+  return {
+    path: absolutePath,
+    sha256: sha256Hex(json),
+    relativePath: relative(process.cwd(), absolutePath),
+  };
+}
+
+export async function writeNdjsonRow(filePath, row) {
+  const absolutePath = resolve(String(filePath));
+  let existing = '';
+  try {
+    existing = await readFile(absolutePath, 'utf8');
+  } catch (error) {
+    if (error?.code !== 'ENOENT') {
+      throw error;
+    }
+  }
+  const serialized = `${existing}${JSON.stringify(row)}\n`;
+  await mkdir(dirname(absolutePath), { recursive: true });
+  await writeFile(absolutePath, serialized, 'utf8');
+  return {
+    path: absolutePath,
+    sha256: sha256Hex(serialized),
+    relativePath: relative(process.cwd(), absolutePath),
+  };
+}
+
+export async function writeWorkloadLock(layout, loadedWorkload) {
+  const filePath = join(layout.runRoot, 'workload.lock.json');
+  const payload = JSON.parse(serializeTrainingWorkloadLock(loadedWorkload));
+  return writeJsonArtifact(filePath, payload);
+}
+
+export async function writeRunContract(layout, payload) {
+  return writeJsonArtifact(join(layout.runRoot, 'run_contract.json'), payload);
+}
+
+export function buildArtifactBase({
+  artifactType,
+  reportId,
+  workload,
+  workloadPath,
+  workloadSha256,
+  datasetPath,
+  datasetHash,
+  baseModelId,
+  teacherModelId = null,
+  studentModelId = null,
+  stage = null,
+  checkpointStep = null,
+  parentArtifacts = [],
+  runtime = 'node',
+  surface = 'node',
+  claimBoundary,
+  configHash,
+}) {
+  return {
+    artifactType,
+    schemaVersion: 1,
+    reportId,
+    workloadId: workload.id,
+    workloadPath,
+    workloadSha256,
+    configHash,
+    datasetPath: datasetPath || null,
+    datasetHash: datasetHash || null,
+    baseModelId: baseModelId || null,
+    teacherModelId: teacherModelId || null,
+    studentModelId: studentModelId || null,
+    stage,
+    checkpointStep,
+    parentArtifacts,
+    generatedAt: new Date().toISOString(),
+    runtime,
+    surface,
+    claimBoundary,
+  };
+}
+
+export function hashArtifactPayload(payload) {
+  return sha256Hex(stableJson(payload));
+}

package/src/training/operator-command.d.ts

@@ -0,0 +1,5 @@
+import type { ToolingCommandRequest } from '../tooling/command-api.js';
+
+export declare function runTrainingOperatorCommand(
+  request: ToolingCommandRequest
+): Promise<Record<string, unknown>>;

package/src/training/operator-command.js

@@ -0,0 +1,453 @@
+import { readFile, readdir } from 'node:fs/promises';
+import { join, resolve } from 'node:path';
+
+import { loadTrainingWorkloadPack } from './workloads.js';
+import {
+  buildFrozenSubset,
+  createDistillationRunArtifacts,
+  evaluateDistillationCheckpoint,
+  runDistillationStageA,
+  runDistillationStageB,
+  watchDistillationCheckpoints,
+} from './distillation/index.js';
+import {
+  compareLoraRun,
+  evaluateLoraCheckpoint,
+  exportLoraCheckpoint,
+  qualityGateLoraRun,
+  runLoraPipeline,
+  watchLoraCheckpoints,
+} from './lora-pipeline.js';
+import { writeDistillCompareReport, writeDistillQualityGateReport } from './distillation/artifacts.js';
+
+async function listJsonFiles(rootDir) {
+  const results = [];
+  const entries = await readdir(rootDir, { withFileTypes: true });
+  for (const entry of entries) {
+    const absolutePath = join(rootDir, entry.name);
+    if (entry.isDirectory()) {
+      results.push(...await listJsonFiles(absolutePath));
+      continue;
+    }
+    if (entry.isFile() && entry.name.endsWith('.json')) {
+      results.push(absolutePath);
+    }
+  }
+  return results.sort((left, right) => left.localeCompare(right));
+}
+
+async function readJson(filePath) {
+  const raw = await readFile(filePath, 'utf8');
+  return JSON.parse(raw);
+}
+
+async function loadWorkloadFromRunRoot(runRoot) {
+  const lockPath = join(resolve(String(runRoot)), 'workload.lock.json');
+  const payload = await readJson(lockPath);
+  return {
+    absolutePath: payload.workloadPath,
+    path: payload.workloadPath,
+    raw: JSON.stringify(payload.workload),
+    workloadSha256: payload.workloadSha256,
+    workload: payload.workload,
+  };
+}
+
+function resolveStageEntry(workload, action) {
+  const stagePlan = Array.isArray(workload.pipeline?.stagePlan) ? workload.pipeline.stagePlan : [];
+  if (action === 'stage-a') {
+    return stagePlan.find((entry) => String(entry.trainingStage || entry.id || '').includes('stage_a')) || stagePlan[0];
+  }
+  if (action === 'stage-b') {
+    return stagePlan.find((entry) => String(entry.trainingStage || entry.id || '').includes('stage_b'))
+      || stagePlan[stagePlan.length - 1];
+  }
+  return null;
+}
+
+async function buildDistillSubsetIfNeeded(loadedWorkload, layout, subsetManifestPath = null) {
+  if (subsetManifestPath) {
+    const manifest = await readJson(subsetManifestPath);
+    return {
+      subsetManifestPath: resolve(String(subsetManifestPath)),
+      subsetJsonlPath: manifest.output?.subsetJsonlPath || loadedWorkload.workload.datasetPath,
+      manifest,
+    };
+  }
+  if (!loadedWorkload.workload.pipeline.subsetSpec) {
+    return null;
+  }
+  const subset = await buildFrozenSubset({
+    datasetPath: loadedWorkload.workload.datasetPath,
+    outputDir: join(layout.exports, 'subset'),
+    strictPairContract: loadedWorkload.workload.pipeline.strictPairContract === true,
+    sourceLangs: loadedWorkload.workload.pipeline.sourceLangs,
+    targetLangs: loadedWorkload.workload.pipeline.targetLangs,
+    pairAllowlist: loadedWorkload.workload.pipeline.pairAllowlist,
+    subsetSpec: loadedWorkload.workload.pipeline.subsetSpec,
+  });
+  return {
+    subsetManifestPath: subset.manifestPath,
+    subsetJsonlPath: subset.subsetJsonlPath,
+    manifest: subset.manifest,
+  };
+}
+
+async function compareDistillRun(runRoot) {
+  const evalDir = join(runRoot, 'eval');
+  const files = await listJsonFiles(evalDir);
+  const reports = [];
+  for (const filePath of files) {
+    const report = await readJson(filePath);
+    if (report?.artifactType === 'training_eval_report') {
+      reports.push(report);
+    }
+  }
+  const sorted = reports
+    .slice()
+    .sort((left, right) => Number(right?.primaryScore ?? Number.NEGATIVE_INFINITY) - Number(left?.primaryScore ?? Number.NEGATIVE_INFINITY));
+  const payload = {
+    artifactType: 'training_compare_report',
+    schemaVersion: 1,
+    generatedAt: new Date().toISOString(),
+    runRoot,
+    count: sorted.length,
+    best: sorted[0] || null,
+    reports: sorted.map((report) => ({
+      stage: report.stage || null,
+      checkpointId: report.checkpointId || null,
+      evalDatasetId: report.evalDatasetId || null,
+      primaryMetric: report.primaryMetric || null,
+      primaryScore: report.primaryScore ?? null,
+      bleu: report.bleu ?? null,
+      chrf: report.chrf ?? null,
+      reportPath: report.reportPath || null,
+    })),
+  };
+  const artifact = await writeDistillCompareReport({
+    compare: join(runRoot, 'compare'),
+  }, payload);
+  return {
+    ...payload,
+    comparePath: artifact.path,
+  };
+}
+
+async function qualityGateDistillRun(runRoot, loadedWorkload) {
+  const checks = [];
+  const required = [
+    join(runRoot, 'run_contract.json'),
+    join(runRoot, 'workload.lock.json'),
+  ];
+  for (const filePath of required) {
+    try {
+      await readFile(filePath, 'utf8');
+      checks.push({ path: filePath, ok: true });
+    } catch (error) {
+      checks.push({ path: filePath, ok: false, error: error?.message || String(error) });
+    }
+  }
+  const stageManifests = await listJsonFiles(join(runRoot, 'checkpoints'));
+  const expectedStageCount = Array.isArray(loadedWorkload.workload.pipeline?.stagePlan)
+    ? loadedWorkload.workload.pipeline.stagePlan.length
+    : 0;
+  const actualStageCount = stageManifests.filter((filePath) => filePath.endsWith('distill_stage_manifest.json')).length;
+  checks.push({
+    path: join(runRoot, 'checkpoints'),
+    ok: actualStageCount >= expectedStageCount,
+    expectedStageCount,
+    actualStageCount,
+  });
+  const passed = checks.every((entry) => entry.ok === true);
+  const payload = {
+    artifactType: 'training_quality_gate',
+    schemaVersion: 1,
+    generatedAt: new Date().toISOString(),
+    runRoot,
+    passed,
+    checks,
+  };
+  const artifact = await writeDistillQualityGateReport({
+    qualityGate: join(runRoot, 'quality-gate'),
+  }, payload);
+  return {
+    ...payload,
+    reportPath: artifact.path,
+  };
+}
+
+async function runDistillCommand(request) {
+  const action = String(request.action || '').trim();
+  const loadedWorkload = request.workloadPath
+    ? await loadTrainingWorkloadPack(request.workloadPath)
+    : await loadWorkloadFromRunRoot(request.runRoot);
+  const runArtifacts = await createDistillationRunArtifacts({
+    loadedWorkload,
+    runRoot: request.runRoot || null,
+    timestamp: request.timestamp || null,
+  });
+  if (action === 'subsets') {
+    const subset = await buildDistillSubsetIfNeeded(loadedWorkload, runArtifacts.layout, request.subsetManifest || null);
+    if (!subset) {
+      throw new Error(`Distill workload "${loadedWorkload.workload.id}" has no subsetSpec and no subsetManifest was provided.`);
+    }
+    return {
+      ok: true,
+      kind: 'distill',
+      action,
+      workloadId: loadedWorkload.workload.id,
+      runRoot: runArtifacts.layout.runRoot,
+      subset,
+    };
+  }
+  if (action === 'run' || action === 'stage-a' || action === 'stage-b') {
+    const subset = await buildDistillSubsetIfNeeded(loadedWorkload, runArtifacts.layout, request.subsetManifest || null);
+    const datasetPath = subset?.subsetJsonlPath || loadedWorkload.workload.datasetPath;
+    const stageResults = [];
+    if (action === 'run') {
+      for (const stageEntry of loadedWorkload.workload.pipeline.stagePlan) {
+        const stageResult = String(stageEntry.trainingStage || stageEntry.id || '').includes('stage_b')
+          ? await runDistillationStageB({
+            loadedWorkload,
+            stageEntry,
+            layout: runArtifacts.layout,
+            datasetPath,
+            priorStageResult: stageResults[stageResults.length - 1] || null,
+            legacyArtifactDir: join(runArtifacts.layout.runRoot, 'legacy-stage-artifacts'),
+            timestamp: request.timestamp || null,
+          })
+          : await runDistillationStageA({
+            loadedWorkload,
+            stageEntry,
+            layout: runArtifacts.layout,
+            datasetPath,
+            legacyArtifactDir: join(runArtifacts.layout.runRoot, 'legacy-stage-artifacts'),
+            timestamp: request.timestamp || null,
+          });
+        stageResults.push(stageResult);
+      }
+    } else {
+      const stageEntry = resolveStageEntry(loadedWorkload.workload, action);
+      if (!stageEntry) {
+        throw new Error(`Unable to resolve stage entry for action "${action}".`);
+      }
+      const stageResult = action === 'stage-b'
+        ? await runDistillationStageB({
+          loadedWorkload,
+          stageEntry,
+          layout: runArtifacts.layout,
+          datasetPath,
+          stageAArtifact: request.stageArtifact || null,
+          legacyArtifactDir: join(runArtifacts.layout.runRoot, 'legacy-stage-artifacts'),
+          timestamp: request.timestamp || null,
+        })
+        : await runDistillationStageA({
+          loadedWorkload,
+          stageEntry,
+          layout: runArtifacts.layout,
+          datasetPath,
+          legacyArtifactDir: join(runArtifacts.layout.runRoot, 'legacy-stage-artifacts'),
+          timestamp: request.timestamp || null,
+        });
+      stageResults.push(stageResult);
+    }
+    const compare = await compareDistillRun(runArtifacts.layout.runRoot);
+    const qualityGate = await qualityGateDistillRun(runArtifacts.layout.runRoot, loadedWorkload);
+    return {
+      ok: true,
+      kind: 'distill',
+      action,
+      workloadId: loadedWorkload.workload.id,
+      runRoot: runArtifacts.layout.runRoot,
+      subsetManifest: subset?.subsetManifestPath || null,
+      stageResults,
+      compare,
+      qualityGate,
+    };
+  }
+  if (action === 'eval') {
+    if (request.checkpointPath) {
+      return {
+        ok: true,
+        kind: 'distill',
+        action,
+        reports: await evaluateDistillationCheckpoint({
+          loadedWorkload,
+          checkpointPath: request.checkpointPath,
+          checkpointId: request.checkpointId || null,
+          checkpointStep: request.checkpointStep ?? null,
+          stageId: request.stageId || null,
+          evalDatasetId: request.evalDatasetId || null,
+          layout: runArtifacts.layout,
+        }),
+      };
+    }
+    const markerFiles = (await listJsonFiles(join(runArtifacts.layout.runRoot, 'checkpoints')))
+      .filter((filePath) => filePath.endsWith('checkpoint.complete.json'));
+    const reports = [];
+    for (const markerPath of markerFiles) {
+      const marker = await readJson(markerPath);
+      reports.push(...await evaluateDistillationCheckpoint({
+        loadedWorkload,
+        checkpointPath: marker.checkpointPath,
+        checkpointId: marker.checkpointId || null,
+        checkpointStep: marker.checkpointStep ?? null,
+        stageId: marker.stage || null,
+        evalDatasetId: request.evalDatasetId || null,
+        layout: runArtifacts.layout,
+        stageAArtifact: marker.stageArtifact || null,
+        stageAArtifactHash: marker.stageArtifactHash || null,
+      }));
+    }
+    return {
+      ok: true,
+      kind: 'distill',
+      action,
+      reports,
+    };
+  }
+  if (action === 'watch') {
+    return {
+      ok: true,
+      kind: 'distill',
+      action,
+      ...(await watchDistillationCheckpoints({
+        loadedWorkload,
+        layout: runArtifacts.layout,
+        pollIntervalMs: request.pollIntervalMs || null,
+        stopWhenIdle: request.stopWhenIdle === true,
+      })),
+    };
+  }
+  if (action === 'compare') {
+    return {
+      ok: true,
+      kind: 'distill',
+      action,
+      ...(await compareDistillRun(runArtifacts.layout.runRoot)),
+    };
+  }
+  if (action === 'quality-gate') {
+    return {
+      ok: true,
+      kind: 'distill',
+      action,
+      ...(await qualityGateDistillRun(runArtifacts.layout.runRoot, loadedWorkload)),
+    };
+  }
+  throw new Error(`Unsupported distill action "${action}".`);
+}
+
+async function runLoraCommand(request) {
+  const action = String(request.action || '').trim();
+  const loadedWorkload = request.workloadPath
+    ? await loadTrainingWorkloadPack(request.workloadPath)
+    : await loadWorkloadFromRunRoot(request.runRoot);
+  if (action === 'run') {
+    return runLoraPipeline({
+      loadedWorkload,
+      runRoot: request.runRoot || null,
+      timestamp: request.timestamp || null,
+    });
+  }
+  if (action === 'eval') {
+    const checkpointPath = request.checkpointPath
+      || (await selectLoraCheckpointPath(request.runRoot)).checkpointPath;
+    return {
+      ok: true,
+      kind: 'lora',
+      action,
+      reports: await evaluateLoraCheckpoint({
+        loadedWorkload,
+        checkpointPath,
+        checkpointId: request.checkpointId || null,
+        checkpointStep: request.checkpointStep ?? null,
+        layout: request.runRoot
+          ? { eval: join(resolve(String(request.runRoot)), 'eval') }
+          : null,
+      }),
+    };
+  }
+  if (action === 'watch') {
+    return {
+      ok: true,
+      kind: 'lora',
+      action,
+      ...(await watchLoraCheckpoints({
+        loadedWorkload,
+        runRoot: resolve(String(request.runRoot)),
+        pollIntervalMs: request.pollIntervalMs || null,
+        stopWhenIdle: request.stopWhenIdle === true,
+      })),
+    };
+  }
+  if (action === 'export') {
+    const checkpointSelection = request.checkpointPath
+      ? { checkpointPath: request.checkpointPath, checkpointId: request.checkpointId || null }
+      : await selectLoraCheckpointPath(request.runRoot);
+    return {
+      ok: true,
+      kind: 'lora',
+      action,
+      ...(await exportLoraCheckpoint({
+        loadedWorkload,
+        checkpointPath: checkpointSelection.checkpointPath,
+        checkpointId: checkpointSelection.checkpointId || null,
+        layout: request.runRoot
+          ? { exports: join(resolve(String(request.runRoot)), 'exports') }
+          : null,
+      })),
+    };
+  }
+  if (action === 'compare') {
+    return {
+      ok: true,
+      kind: 'lora',
+      action,
+      ...(await compareLoraRun({
+        runRoot: resolve(String(request.runRoot)),
+      })),
+    };
+  }
+  if (action === 'quality-gate') {
+    return {
+      ok: true,
+      kind: 'lora',
+      action,
+      ...(await qualityGateLoraRun({
+        runRoot: resolve(String(request.runRoot)),
+      })),
+    };
+  }
+  if (action === 'activate') {
+    throw new Error(
+      'lora activate is not supported in the Node operator runner. The active-model adapter surface currently lives in the browser provider.'
+    );
+  }
+  throw new Error(`Unsupported lora action "${action}".`);
+}
+
+async function selectLoraCheckpointPath(runRoot) {
+  const checkpointsDir = join(resolve(String(runRoot)), 'checkpoints');
+  const markers = (await listJsonFiles(checkpointsDir))
+    .filter((filePath) => filePath.endsWith('checkpoint.complete.json'));
+  const latest = markers[markers.length - 1];
+  if (!latest) {
+    throw new Error(`No finalized LoRA checkpoints found in ${checkpointsDir}.`);
+  }
+  const marker = await readJson(latest);
+  return {
+    checkpointPath: marker.checkpointPath,
+    checkpointId: marker.checkpointId || null,
+  };
+}
+
+export async function runTrainingOperatorCommand(request) {
+  if (request.command === 'distill') {
+    return runDistillCommand(request);
+  }
+  if (request.command === 'lora') {
+    return runLoraCommand(request);
+  }
+  throw new Error(`Unsupported training operator command "${request.command}".`);
+}

package/src/training/operator-eval.d.ts

@@ -0,0 +1,48 @@
+export interface BleuResult {
+  score: number;
+  brevityPenalty: number;
+  precisions: number[];
+  hypothesisLength: number;
+  referenceLength: number;
+}
+
+export interface ChrfResult {
+  score: number;
+  precision: number;
+  recall: number;
+}
+
+export declare function computeBleuScore(
+  hypotheses: string[],
+  references: string[],
+  options?: { maxOrder?: number }
+): BleuResult;
+
+export declare function computeChrfScore(
+  hypotheses: string[],
+  references: string[],
+  options?: { maxOrder?: number; beta?: number }
+): ChrfResult;
+
+export declare function computeExactMatch(
+  hypotheses: string[],
+  references: string[]
+): { score: number; matches: number; total: number };
+
+export declare function computeAccuracy(
+  labels: string[],
+  predictions: string[]
+): { score: number; matches: number; total: number };
+
+export declare function computeEvalMetrics(
+  evalKind: string,
+  hypotheses: string[],
+  references: string[],
+  options?: Record<string, unknown>
+): Record<string, unknown>;
+
+export declare function loadEvalDataset(datasetPath: string): Promise<{
+  absolutePath: string;
+  rows: unknown[];
+  raw: string;
+}>;