@simulatte/doppler 0.1.5 → 0.1.6

package/src/rules/tooling/command-runtime.rules.json

@@ -27,6 +27,24 @@
         "intent": "verify"
       }
     },
+    {
+      "match": {
+        "command": "lora"
+      },
+      "value": {
+        "suite": null,
+        "intent": null
+      }
+    },
+    {
+      "match": {
+        "command": "distill"
+      },
+      "value": {
+        "suite": null,
+        "intent": null
+      }
+    },
     {
       "match": {},
       "value": {

package/src/tooling/command-api.d.ts

@@ -1,10 +1,12 @@
 import type { ConverterConfigSchema } from '../config/schema/converter.schema.js';
 
-export type ToolingCommand = 'convert' | 'debug' | 'bench' | 'verify';
+export type ToolingCommand = 'convert' | 'debug' | 'bench' | 'verify' | 'lora' | 'distill';
 export type ToolingSurface = 'browser' | 'node';
 export type ToolingSuite = 'kernels' | 'inference' | 'training' | 'bench' | 'debug' | 'diffusion' | 'energy';
 export type ToolingIntent = 'verify' | 'investigate' | 'calibrate' | null;
 export type ToolingTrainingStage = 'stage1_joint' | 'stage2_base' | 'stage_a' | 'stage_b';
+export type ToolingDistillAction = 'run' | 'stage-a' | 'stage-b' | 'eval' | 'watch' | 'compare' | 'quality-gate' | 'subsets';
+export type ToolingLoraAction = 'run' | 'eval' | 'watch' | 'export' | 'compare' | 'quality-gate' | 'activate';
 
 export interface ToolingConvertExecutionPayload {
   workers?: number | null;
@@ -25,6 +27,7 @@ export interface ToolingConvertPayload {
 
 export interface ToolingCommandRequestInput {
   command: ToolingCommand;
+  action?: ToolingDistillAction | ToolingLoraAction;
   suite?: ToolingSuite;
   modelId?: string;
   trainingTests?: string[];
@@ -65,6 +68,17 @@ export interface ToolingCommandRequestInput {
   inputDir?: string;
   outputDir?: string;
   convertPayload?: ToolingConvertPayload;
+  workloadPath?: string;
+  runRoot?: string;
+  checkpointPath?: string;
+  checkpointId?: string;
+  checkpointStep?: number;
+  stageId?: string;
+  stageArtifact?: string;
+  subsetManifest?: string;
+  evalDatasetId?: string;
+  pollIntervalMs?: number;
+  stopWhenIdle?: boolean;
   captureOutput?: boolean;
   keepPipeline?: boolean;
   report?: Record<string, unknown> | null;
@@ -76,6 +90,7 @@ export interface ToolingCommandRequest {
   command: ToolingCommand;
   suite: ToolingSuite | null;
   intent: ToolingIntent;
+  action: ToolingDistillAction | ToolingLoraAction | null;
   modelId: string | null;
   trainingTests: string[] | null;
   trainingStage: ToolingTrainingStage | null;
@@ -115,6 +130,17 @@ export interface ToolingCommandRequest {
   inputDir: string | null;
   outputDir: string | null;
   convertPayload: ToolingConvertPayload | null;
+  workloadPath: string | null;
+  runRoot: string | null;
+  checkpointPath: string | null;
+  checkpointId: string | null;
+  checkpointStep: number | null;
+  stageId: string | null;
+  stageArtifact: string | null;
+  subsetManifest: string | null;
+  evalDatasetId: string | null;
+  pollIntervalMs: number | null;
+  stopWhenIdle: boolean | null;
   captureOutput: boolean;
   keepPipeline: boolean;
   report: Record<string, unknown> | null;

package/src/tooling/command-api.js

@@ -1,12 +1,14 @@
 import { isPlainObject } from '../utils/plain-object.js';
 import { selectRuleValue } from '../rules/rule-registry.js';
 
-const TOOLING_COMMAND_SET = ['convert', 'debug', 'bench', 'verify'];
+const TOOLING_COMMAND_SET = ['convert', 'debug', 'bench', 'verify', 'lora', 'distill'];
 const TOOLING_SURFACE_SET = ['browser', 'node'];
 const TOOLING_SUITE_SET = ['kernels', 'inference', 'training', 'bench', 'debug', 'diffusion', 'energy'];
 const TOOLING_INTENT_SET = ['verify', 'investigate', 'calibrate'];
 const VERIFY_SUITES = ['kernels', 'inference', 'training', 'diffusion', 'energy'];
 const TRAINING_STAGE_SET = ['stage1_joint', 'stage2_base', 'stage_a', 'stage_b'];
+const DISTILL_ACTION_SET = ['run', 'stage-a', 'stage-b', 'eval', 'watch', 'compare', 'quality-gate', 'subsets'];
+const LORA_ACTION_SET = ['run', 'eval', 'watch', 'export', 'compare', 'quality-gate', 'activate'];
 const TRAINING_COMMAND_SCHEMA_VERSION = 1;
 
 export const TOOLING_COMMANDS = Object.freeze([...TOOLING_COMMAND_SET]);
@@ -82,6 +84,15 @@ function asOptionalForceResumeReason(value, label) {
   return reason;
 }
 
+function asOptionalAction(value, label, allowed) {
+  const action = asOptionalString(value, label);
+  if (!action) return null;
+  if (!allowed.includes(action)) {
+    throw new Error(`tooling command: ${label} must be one of ${allowed.join(', ')}.`);
+  }
+  return action;
+}
+
 function assertCommand(value) {
   const command = asOptionalString(value, 'command');
   if (!command) {
@@ -246,6 +257,7 @@ function normalizeConvert(raw) {
     command: 'convert',
     suite: null,
     intent: null,
+    action: null,
     modelId: null,
     trainingTests: null,
     trainingStage: null,
@@ -285,6 +297,113 @@ function normalizeConvert(raw) {
     inputDir,
     outputDir,
     convertPayload: payload,
+    workloadPath: null,
+    runRoot: null,
+    checkpointPath: null,
+    checkpointId: null,
+    checkpointStep: null,
+    stageId: null,
+    stageArtifact: null,
+    subsetManifest: null,
+    evalDatasetId: null,
+    pollIntervalMs: null,
+    stopWhenIdle: null,
+    captureOutput: false,
+    keepPipeline: false,
+    report: asOptionalObject(raw.report, 'report'),
+    timestamp: raw.timestamp ?? null,
+    searchParams: raw.searchParams ?? null,
+  };
+}
+
+function normalizeTrainingOperatorCommand(raw, command) {
+  const allowedActions = command === 'distill' ? DISTILL_ACTION_SET : LORA_ACTION_SET;
+  const action = asOptionalAction(raw.action, 'action', allowedActions);
+  if (!action) {
+    throw new Error(`tooling command: ${command} requires action.`);
+  }
+  const workloadPath = asOptionalString(raw.workloadPath, 'workloadPath');
+  const runRoot = asOptionalString(raw.runRoot, 'runRoot');
+  const checkpointPath = asOptionalString(raw.checkpointPath, 'checkpointPath');
+  const checkpointId = asOptionalString(raw.checkpointId, 'checkpointId');
+  const checkpointStep = asOptionalPositiveInteger(raw.checkpointStep, 'checkpointStep');
+  const stageId = asOptionalString(raw.stageId, 'stageId');
+  const stageArtifact = asOptionalString(raw.stageArtifact, 'stageArtifact');
+  const subsetManifest = asOptionalString(raw.subsetManifest, 'subsetManifest');
+  const evalDatasetId = asOptionalString(raw.evalDatasetId, 'evalDatasetId');
+  const pollIntervalMs = asOptionalPositiveInteger(raw.pollIntervalMs, 'pollIntervalMs');
+  const stopWhenIdle = asOptionalBoolean(raw.stopWhenIdle, 'stopWhenIdle');
+  if (!workloadPath && !runRoot) {
+    throw new Error(`tooling command: ${command} requires workloadPath or runRoot.`);
+  }
+  if ((action === 'eval' || action === 'export') && !checkpointPath && !runRoot) {
+    throw new Error(`tooling command: ${command} ${action} requires checkpointPath or runRoot.`);
+  }
+  if (action === 'watch' && !runRoot) {
+    throw new Error(`tooling command: ${command} watch requires runRoot.`);
+  }
+  if ((action === 'compare' || action === 'quality-gate') && !runRoot) {
+    throw new Error(`tooling command: ${command} ${action} requires runRoot.`);
+  }
+  if (command === 'distill' && action === 'stage-b' && !stageArtifact && !runRoot) {
+    throw new Error('tooling command: distill stage-b requires stageArtifact or runRoot.');
+  }
+
+  return {
+    command,
+    suite: null,
+    intent: null,
+    action,
+    modelId: null,
+    trainingTests: null,
+    trainingStage: null,
+    trainingConfig: null,
+    stage1Artifact: null,
+    stage1ArtifactHash: null,
+    ulArtifactDir: null,
+    stageAArtifact: null,
+    stageAArtifactHash: null,
+    distillArtifactDir: null,
+    teacherModelId: null,
+    studentModelId: null,
+    distillDatasetId: null,
+    distillDatasetPath: null,
+    distillLanguagePair: null,
+    distillSourceLangs: null,
+    distillTargetLangs: null,
+    distillPairAllowlist: null,
+    strictPairContract: null,
+    distillShardIndex: null,
+    distillShardCount: null,
+    resumeFrom: null,
+    forceResume: null,
+    forceResumeReason: null,
+    forceResumeSource: null,
+    checkpointOperator: null,
+    trainingSchemaVersion: null,
+    trainingBenchSteps: null,
+    checkpointEvery: null,
+    workloadType: 'training',
+    modelUrl: null,
+    cacheMode: asOptionalCacheMode(raw.cacheMode, 'cacheMode'),
+    loadMode: asOptionalLoadMode(raw.loadMode, 'loadMode'),
+    runtimePreset: asOptionalString(raw.runtimePreset, 'runtimePreset'),
+    runtimeConfigUrl: asOptionalString(raw.runtimeConfigUrl, 'runtimeConfigUrl'),
+    runtimeConfig: asOptionalObject(raw.runtimeConfig, 'runtimeConfig'),
+    inputDir: null,
+    outputDir: null,
+    convertPayload: null,
+    workloadPath,
+    runRoot,
+    checkpointPath,
+    checkpointId,
+    checkpointStep,
+    stageId,
+    stageArtifact,
+    subsetManifest,
+    evalDatasetId,
+    pollIntervalMs,
+    stopWhenIdle,
     captureOutput: false,
     keepPipeline: false,
     report: asOptionalObject(raw.report, 'report'),
@@ -428,6 +547,7 @@ function normalizeSuiteCommand(raw, command) {
     command,
     suite,
     intent: runtimeContract.intent,
+    action: null,
     modelId,
     trainingTests,
     trainingStage,
@@ -469,6 +589,17 @@ function normalizeSuiteCommand(raw, command) {
     inputDir: null,
     outputDir: null,
     convertPayload: null,
+    workloadPath: null,
+    runRoot: null,
+    checkpointPath: null,
+    checkpointId: null,
+    checkpointStep: null,
+    stageId: null,
+    stageArtifact: null,
+    subsetManifest: null,
+    evalDatasetId: null,
+    pollIntervalMs: null,
+    stopWhenIdle: null,
     captureOutput: asOptionalBoolean(raw.captureOutput, 'captureOutput') ?? false,
     keepPipeline: asOptionalBoolean(raw.keepPipeline, 'keepPipeline') ?? false,
     report: asOptionalObject(raw.report, 'report'),
@@ -485,6 +616,9 @@ export function normalizeToolingCommandRequest(input) {
   if (command === 'convert') {
     return normalizeConvert(input);
   }
+  if (command === 'lora' || command === 'distill') {
+    return normalizeTrainingOperatorCommand(input, command);
+  }
   return normalizeSuiteCommand(input, command);
 }
 
@@ -514,8 +648,13 @@ export function ensureCommandSupportedOnSurface(commandRequest, surface) {
     throw new Error(`tooling command: unsupported surface "${surface}".`);
   }
 
-  // All commands are contractually available on both surfaces.
-  // Surface-specific capability checks happen in the runners.
+  if (
+    normalizedSurface === 'browser'
+    && (request.command === 'lora' || request.command === 'distill')
+  ) {
+    throw new Error(`tooling command: ${request.command} is currently Node-only and must fail closed on browser.`);
+  }
+
   return {
     request,
     surface: normalizedSurface,

package/src/tooling/node-browser-command-runner.d.ts

@@ -7,6 +7,10 @@ import type { BrowserCommandRunResult } from './browser-command-runner.js';
 
 export interface NodeBrowserCommandRunOptions {
   staticRootDir?: string;
+  staticMounts?: Array<{
+    urlPrefix: string;
+    rootDir: string;
+  }>;
   baseUrl?: string;
   host?: string;
   port?: number;

package/src/tooling/node-browser-command-runner.js

@@ -67,8 +67,61 @@ function resolveStaticPath(rootDir, requestPath) {
   return candidate;
 }
 
-async function resolveFileForRequest(rootDir, requestPath) {
-  const resolved = resolveStaticPath(rootDir, requestPath);
+function normalizeStaticMounts(mounts = []) {
+  if (!Array.isArray(mounts)) {
+    throw new Error('browser command: staticMounts must be an array.');
+  }
+
+  return mounts.map((mount, index) => {
+    if (!mount || typeof mount !== 'object' || Array.isArray(mount)) {
+      throw new Error(`browser command: staticMounts[${index}] must be an object.`);
+    }
+    const urlPrefix = String(mount.urlPrefix || '').trim();
+    const rootDir = String(mount.rootDir || '').trim();
+    if (!urlPrefix.startsWith('/')) {
+      throw new Error(`browser command: staticMounts[${index}].urlPrefix must start with "/".`);
+    }
+    if (!rootDir) {
+      throw new Error(`browser command: staticMounts[${index}].rootDir is required.`);
+    }
+    return {
+      urlPrefix: urlPrefix.replace(/\/+$/u, '') || '/',
+      rootDir: path.resolve(rootDir),
+    };
+  });
+}
+
+function findStaticRootForRequest(rootDir, mounts, requestPath) {
+  const normalizedPath = String(requestPath || '/');
+  let bestMount = null;
+
+  for (const mount of mounts) {
+    const prefix = mount.urlPrefix;
+    if (normalizedPath !== prefix && !normalizedPath.startsWith(`${prefix}/`)) {
+      continue;
+    }
+    if (!bestMount || prefix.length > bestMount.urlPrefix.length) {
+      bestMount = mount;
+    }
+  }
+
+  if (!bestMount) {
+    return {
+      effectiveRootDir: rootDir,
+      effectivePath: normalizedPath,
+    };
+  }
+
+  const relativePath = normalizedPath.slice(bestMount.urlPrefix.length) || '/';
+  return {
+    effectiveRootDir: bestMount.rootDir,
+    effectivePath: relativePath.startsWith('/') ? relativePath : `/${relativePath}`,
+  };
+}
+
+async function resolveFileForRequest(rootDir, mounts, requestPath) {
+  const { effectiveRootDir, effectivePath } = findStaticRootForRequest(rootDir, mounts, requestPath);
+  const resolved = resolveStaticPath(effectiveRootDir, effectivePath);
   if (!resolved) return null;
 
   let stats;
@@ -99,6 +152,7 @@ async function createStaticFileServer(options = {}) {
   const rootDir = path.resolve(
     options.rootDir || fileURLToPath(new URL('../../', import.meta.url))
   );
+  const staticMounts = normalizeStaticMounts(options.staticMounts || []);
   const host = String(options.host || DEFAULT_HOST);
   const port = Number.isFinite(options.port) ? Math.max(0, Math.floor(options.port)) : 0;
 
@@ -120,7 +174,7 @@ async function createStaticFileServer(options = {}) {
       return;
     }
 
-    const resolved = await resolveFileForRequest(rootDir, pathname);
+    const resolved = await resolveFileForRequest(rootDir, staticMounts, pathname);
     if (!resolved) {
       res.statusCode = 404;
       res.end('File not found');
@@ -545,6 +599,7 @@ export async function runBrowserCommandInNode(commandRequest, options = {}) {
     ? null
     : await createStaticFileServer({
       rootDir: options.staticRootDir,
+      staticMounts: options.staticMounts,
       host: options.host,
       port: serverPort,
     }).catch((error) => {

package/src/tooling/node-command-runner.js

@@ -18,6 +18,7 @@ import {
   getActiveKernelPathSource,
   setActiveKernelPath,
 } from '../config/kernel-path-loader.js';
+import { runTrainingOperatorCommand } from '../training/operator-command.js';
 
 function asOptionalPlainObject(value, label) {
   if (value == null) return null;
@@ -90,6 +91,20 @@ export async function runNodeCommand(commandRequest, options = {}) {
       });
     }
 
+    if (request.command === 'lora' || request.command === 'distill') {
+      const gpuOptionalActions = new Set(['compare', 'quality-gate', 'subsets']);
+      installNodeFileFetchShim();
+      if (!gpuOptionalActions.has(request.action)) {
+        await assertNodeWebGPUSupport();
+      }
+      const result = await runTrainingOperatorCommand(request);
+      return createToolingSuccessEnvelope({
+        surface: 'node',
+        request,
+        result,
+      });
+    }
+
     await assertNodeWebGPUSupport();
     const modules = await loadRuntimeModules();
     const runtimeBridge = {

package/src/tooling/node-webgpu.js

@@ -2,17 +2,6 @@ import { existsSync, readFileSync, statSync } from 'node:fs';
 import { dirname, isAbsolute, resolve } from 'node:path';
 import { fileURLToPath, pathToFileURL } from 'node:url';
 
-const DOPPLER_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..', '..');
-
-const DEFAULT_LOCAL_DOE_PROVIDER_PATH = resolve(
-  dirname(fileURLToPath(import.meta.url)),
-  '..',
-  '..',
-  '..',
-  'fawn',
-  'nursery',
-  'webgpu-doe',
-);
 const DEFAULT_DOE_PROVIDER_CREATE_ARGS = 'enable-dawn-features=allow_unsafe_apis';
 
 function hasNavigatorGpu() {
@@ -59,20 +48,7 @@ function resolveCandidateModuleSpecifier(candidate) {
 }
 
 function resolveDefaultWebgpuModuleSpecifiers() {
-  const specifiers = [];
-  const localCandidates = [
-    resolve(process.cwd(), '..', 'fawn', 'nursery', 'webgpu-doe'),
-    DEFAULT_LOCAL_DOE_PROVIDER_PATH,
-  ];
-  for (const localCandidate of localCandidates) {
-    const resolvedPath = resolveNodeModuleFilePath(localCandidate);
-    if (resolvedPath) {
-      specifiers.push(pathToFileURL(resolvedPath).href);
-    }
-  }
-  specifiers.push('@simulatte/webgpu-doe');
-  specifiers.push('webgpu');
-  return [...new Set(specifiers)];
+  return ['@simulatte/webgpu', 'webgpu'];
 }
 
 function resolveWebgpuModuleSpecifiers() {
@@ -89,80 +65,26 @@ function resolveWebgpuModuleSpecifiers() {
   };
 }
 
-function resolveWorkspaceWebgpuProviderPath() {
-  const candidates = [
-    resolve(process.cwd(), 'node_modules', 'webgpu'),
-    resolve(DOPPLER_ROOT, 'node_modules', 'webgpu'),
-  ];
-  for (const candidate of candidates) {
-    const resolvedPath = resolveNodeModuleFilePath(candidate);
-    if (resolvedPath) {
-      return resolvedPath;
-    }
-  }
-  return null;
-}
-
 function isDoeWebgpuSpecifier(specifier) {
-  if (specifier === '@simulatte/webgpu-doe') {
-    return true;
-  }
-  if (typeof specifier !== 'string') {
-    return false;
-  }
-  if (specifier.includes('/webgpu-doe/')) {
+  if (specifier === '@simulatte/webgpu') {
     return true;
   }
-  return specifier.includes('webgpu-doe') && specifier.startsWith('file://');
-}
-
-function resolveDoeProviderOverride(specifier) {
-  const explicitProvider = process.env.FAWN_WEBGPU_NODE_PROVIDER_MODULE;
-  if (typeof explicitProvider === 'string' && explicitProvider.trim().length > 0) {
-    return null;
-  }
-  if (!isDoeWebgpuSpecifier(specifier)) {
-    return null;
-  }
-  return resolveWorkspaceWebgpuProviderPath();
+  return typeof specifier === 'string'
+    && specifier.startsWith('file://')
+    && specifier.includes('@simulatte/webgpu');
 }
 
 async function importWithProviderOverride(specifier) {
-  const providerOverride = resolveDoeProviderOverride(specifier);
   const shouldApplyCreateArgsDefault = isDoeWebgpuSpecifier(specifier)
     && !(typeof process.env.FAWN_WEBGPU_CREATE_ARGS === 'string' && process.env.FAWN_WEBGPU_CREATE_ARGS.trim().length > 0);
-  if (!providerOverride) {
-    if (!shouldApplyCreateArgsDefault) {
-      return import(specifier);
-    }
-    process.env.FAWN_WEBGPU_CREATE_ARGS = DEFAULT_DOE_PROVIDER_CREATE_ARGS;
-    try {
-      return await import(specifier);
-    } finally {
-      delete process.env.FAWN_WEBGPU_CREATE_ARGS;
-    }
-  }
-  const hadProvider = Object.prototype.hasOwnProperty.call(process.env, 'FAWN_WEBGPU_NODE_PROVIDER_MODULE');
-  const previousProvider = process.env.FAWN_WEBGPU_NODE_PROVIDER_MODULE;
-  const hadCreateArgs = Object.prototype.hasOwnProperty.call(process.env, 'FAWN_WEBGPU_CREATE_ARGS');
-  const previousCreateArgs = process.env.FAWN_WEBGPU_CREATE_ARGS;
-  process.env.FAWN_WEBGPU_NODE_PROVIDER_MODULE = providerOverride;
-  if (!hadCreateArgs) {
-    process.env.FAWN_WEBGPU_CREATE_ARGS = DEFAULT_DOE_PROVIDER_CREATE_ARGS;
+  if (!shouldApplyCreateArgsDefault) {
+    return import(specifier);
   }
+  process.env.FAWN_WEBGPU_CREATE_ARGS = DEFAULT_DOE_PROVIDER_CREATE_ARGS;
   try {
     return await import(specifier);
   } finally {
-    if (hadProvider) {
-      process.env.FAWN_WEBGPU_NODE_PROVIDER_MODULE = previousProvider;
-    } else {
-      delete process.env.FAWN_WEBGPU_NODE_PROVIDER_MODULE;
-    }
-    if (hadCreateArgs) {
-      process.env.FAWN_WEBGPU_CREATE_ARGS = previousCreateArgs;
-    } else {
-      delete process.env.FAWN_WEBGPU_CREATE_ARGS;
-    }
+    delete process.env.FAWN_WEBGPU_CREATE_ARGS;
   }
 }
 

package/src/training/checkpoint-watch.d.ts

@@ -0,0 +1,7 @@
+export declare function watchFinalizedCheckpoints(options: {
+  checkpointsDir: string;
+  manifestPath: string;
+  pollIntervalMs?: number | null;
+  stopWhenIdle?: boolean;
+  onCheckpoint: (markerPath: string) => Promise<void> | void;
+}): Promise<{ ok: true; processedCount: number; manifestPath: string }>;

package/src/training/checkpoint-watch.js

@@ -0,0 +1,106 @@
+import { readdir, readFile } from 'node:fs/promises';
+import { join, resolve } from 'node:path';
+
+import { writeJsonArtifact } from './operator-artifacts.js';
+
+async function listCheckpointMarkers(checkpointsDir) {
+  const absoluteDir = resolve(String(checkpointsDir));
+  const entries = await readdir(absoluteDir, { withFileTypes: true });
+  const markers = [];
+  for (const entry of entries) {
+    if (!entry.isDirectory()) {
+      continue;
+    }
+    const entryPath = join(absoluteDir, entry.name);
+    const markerPath = join(entryPath, 'checkpoint.complete.json');
+    try {
+      await readFile(markerPath, 'utf8');
+      markers.push(markerPath);
+      continue;
+    } catch (error) {
+      if (error?.code !== 'ENOENT') {
+        throw error;
+      }
+    }
+    markers.push(...await listCheckpointMarkers(entryPath));
+  }
+  return markers.sort((left, right) => left.localeCompare(right));
+}
+
+async function ensureDirectoryExists(directoryPath) {
+  try {
+    const entries = await readdir(directoryPath, { withFileTypes: true });
+    return Array.isArray(entries);
+  } catch (error) {
+    if (error?.code === 'ENOENT') {
+      return false;
+    }
+    throw error;
+  }
+}
+
+async function readProcessedManifest(manifestPath) {
+  try {
+    const raw = await readFile(manifestPath, 'utf8');
+    const parsed = JSON.parse(raw);
+    const processed = Array.isArray(parsed?.processedCheckpointMarkers)
+      ? parsed.processedCheckpointMarkers.filter((entry) => typeof entry === 'string')
+      : [];
+    return new Set(processed);
+  } catch (error) {
+    if (error?.code === 'ENOENT') {
+      return new Set();
+    }
+    throw error;
+  }
+}
+
+export async function watchFinalizedCheckpoints(options) {
+  const checkpointsDir = resolve(String(options.checkpointsDir));
+  const manifestPath = resolve(String(options.manifestPath));
+  const pollIntervalMs = Number.isFinite(options.pollIntervalMs)
+    ? Math.max(100, Math.floor(options.pollIntervalMs))
+    : 2000;
+  const stopWhenIdle = options.stopWhenIdle === true;
+  const onCheckpoint = typeof options.onCheckpoint === 'function'
+    ? options.onCheckpoint
+    : null;
+  if (!onCheckpoint) {
+    throw new Error('watchFinalizedCheckpoints requires onCheckpoint(markerPath).');
+  }
+
+  const processed = await readProcessedManifest(manifestPath);
+  let idlePolls = 0;
+  for (;;) {
+    const checkpointsExist = await ensureDirectoryExists(checkpointsDir);
+    const markers = checkpointsExist
+      ? await listCheckpointMarkers(checkpointsDir)
+      : [];
+    let sawNewMarker = false;
+    for (const markerPath of markers) {
+      if (processed.has(markerPath)) continue;
+      sawNewMarker = true;
+      await onCheckpoint(markerPath);
+      processed.add(markerPath);
+      await writeJsonArtifact(manifestPath, {
+        artifactType: 'training_checkpoint_watch_manifest',
+        schemaVersion: 1,
+        generatedAt: new Date().toISOString(),
+        processedCheckpointMarkers: [...processed].sort((left, right) => left.localeCompare(right)),
+      });
+    }
+    if (!sawNewMarker) {
+      idlePolls += 1;
+      if (stopWhenIdle && idlePolls > 0) {
+        return {
+          ok: true,
+          processedCount: processed.size,
+          manifestPath,
+        };
+      }
+    } else {
+      idlePolls = 0;
+    }
+    await new Promise((resolvePromise) => setTimeout(resolvePromise, pollIntervalMs));
+  }
+}

package/src/training/checkpoint.d.ts

@@ -23,7 +23,12 @@ export declare function saveCheckpoint(
   key: string,
   data: unknown,
   options?: CheckpointStoreOptions
-): Promise<void>;
+): Promise<{
+  key: string;
+  path: string | null;
+  metadata: Record<string, unknown>;
+  data: unknown;
+}>;
 
 export declare function loadCheckpoint(
   key: string,