@tuned-tensor/local 0.2.6 → 0.2.8

package/dist/index.js

@@ -1,36 +1,55 @@
 #!/usr/bin/env node
-import { realpathSync } from "node:fs";
-import { readFile } from "node:fs/promises";
-import { resolve } from "node:path";
+import { readFileSync, realpathSync } from "node:fs";
+import { createHash } from "node:crypto";
+import { mkdir, open, readFile, stat } from "node:fs/promises";
+import { spawn } from "node:child_process";
+import { basename, dirname, join, resolve } from "node:path";
 import { cwd } from "node:process";
 import { fileURLToPath } from "node:url";
 import { compareRuns } from "./compare.js";
+import { assertArtifactManifest, claimRunArtifactDirectory, defaultArtifactPrefix, resolveRunArtifacts } from "./artifacts.js";
 import { fineTuneRunRequestSchema, localBehaviorSpecFileSchema, localRunnerConfigSchema, specSnapshotSchema } from "./contracts.js";
 import { buildSystemMessage } from "./dataset.js";
+import { assertEvaluationScoringReady } from "./evaluation.js";
 import { runLocalLabelingJob } from "./labeling.js";
-import { loadLocalRunnerConfig, runLocalFineTuneStage, } from "./orchestrator.js";
+import { loadLocalRunnerConfig, fingerprintLocalBaseModel, runLocalFineTuneStage, } from "./orchestrator.js";
 import { runDoctor } from "./doctor.js";
-import { resolveTrainingModel } from "./model-registry.js";
-import { createLocalStore } from "./store.js";
+import { assertUsableModelArtifact, resolveTrainingModel } from "./model-registry.js";
+import { buildLocalModelServerLaunch, serveLocalModel } from "./model-server.js";
+import { prefetchBaseModel } from "./prefetch.js";
+import { createLocalStore, isTerminalRunState } from "./store.js";
 import { serveLocalDashboard } from "./server.js";
-import { DEFAULT_LOCAL_SPEC_PATH, initLocalSpecFile, loadLocalRunInput, runRequestFromLocalSpec, } from "./local-project.js";
+import { DEFAULT_LOCAL_SPEC_PATH, assertLocalRunInputReady, initLocalRunnerConfigFile, initLocalSpecFile, loadLocalRunInput, runRequestFromLocalSpec, } from "./local-project.js";
 import { sanitizeLogLine } from "./run-reporter.js";
 export * from "./compare.js";
 export * from "./contracts.js";
 export * from "./dataset.js";
 export * from "./labeling.js";
 export * from "./labeling-sanitize.js";
+export * from "./model-server.js";
 export * from "./orchestrator.js";
 export * from "./local-project.js";
 export * from "./openrouter.js";
+export * from "./prefetch.js";
 export * from "./run-reporter.js";
 export * from "./server.js";
 export * from "./store.js";
+function packageVersion() {
+    try {
+        const packageJson = JSON.parse(readFileSync(fileURLToPath(new URL("../package.json", import.meta.url)), "utf8"));
+        return typeof packageJson.version === "string" ? packageJson.version : "unknown";
+    }
+    catch {
+        return "unknown";
+    }
+}
+export const TT_LOCAL_VERSION = packageVersion();
 export function getLocalRunnerInfo() {
     return {
         name: "tuned-tensor-local",
         status: "local-runner-preview",
         description: "Local fine-tuning runner for single-GPU uv/Python hosts.",
+        version: TT_LOCAL_VERSION,
     };
 }
 /**
@@ -79,72 +98,343 @@ async function loadDotEnvFromCwd() {
 }
 function readOption(argv, name) {
     const index = argv.indexOf(name);
-    if (index === -1)
-        return undefined;
-    return argv[index + 1];
+    if (index !== -1)
+        return argv[index + 1];
+    const inline = argv.find((value) => value.startsWith(`${name}=`));
+    return inline?.slice(name.length + 1);
 }
 function hasFlag(argv, name) {
     return argv.includes(name);
 }
 function printHelp() {
-    console.log(`tt-local
+    console.log(`Usage: tt-local <command> [options]
 
 Commands:
-  info
-  init [--name "My Local Model"] [--model Qwen/Qwen3.5-2B] [--output tunedtensor.json] [--force]
-  doctor [--config local-runner.json]
+  info                              Show package and runner information
+  init [--name "My Local Model"] [--model Qwen/Qwen3.5-2B] [--output tunedtensor.json] [--profile spark] [--force]
+  doctor [tunedtensor.json|request.json] [--config local-runner.json]
   validate [tunedtensor.json|request.json] [--config local-runner.json]
   run [tunedtensor.json|request.json] [--config local-runner.json] [--stage all|prepare|baseline|train|candidate|score|report]
       [--run-id uuid] [--parent-model local-model-id] [--parent-model-artifact path-or-file-uri]
-      [--model-artifact path-or-file-uri] [--force] [--dry-run] [--verbose] [--quiet]
+      [--model-artifact path-or-file-uri] [--force] [--dry-run] [--detach] [--verbose] [--quiet]
   label <input.jsonl|input.csv> [--input-column col] [--spec tunedtensor.json] [--system-prompt "..."]
         [--model teacher-model] [--output labeled.jsonl] [--config local-runner.json] [--dry-run]
   serve [--config local-runner.json] [--host 127.0.0.1] [--port 8787]
   runs list|get|events|watch|report|compare|cancel|reconcile [args] [--config local-runner.json]
-  models list|get [args] [--config local-runner.json]
+  models list|get|verify|prefetch|verify-base|serve [args] [--config local-runner.json]
   specs list|get|import [args] [--config local-runner.json]
   store rebuild-index [--config local-runner.json]
 
+Global options:
+  -h, --help                       Show help
+  -V, --version                    Show the installed version
+
 The run command writes local artifacts under config.artifactRoot, defaulting to
 .tt-local/artifacts. The file-backed local store defaults to
 ~/.tuned-tensor-local unless config.storeRoot or TT_LOCAL_HOME is set.`);
 }
-const optionNamesWithValues = new Set([
-    "--config",
-    "--host",
-    "--port",
-    "--name",
-    "--model",
-    "--output",
-    "--id",
-    "--user-id",
-    "--run-number",
-    "--run-id",
-    "--stage",
-    "--model-artifact",
-    "--parent-model",
-    "--parent-model-artifact",
-    "--input-column",
-    "--system-prompt",
-    "--spec",
-    "--format",
-]);
-function readPositionals(argv, startIndex = 3) {
+const CONFIG_OPTION = { name: "--config", value: "path", description: "Local runner config JSON path" };
+const USER_ID_OPTION = { name: "--user-id", value: "id", description: "Override the local user ID" };
+const RUN_NUMBER_OPTION = { name: "--run-number", value: "number", description: "Override the local run number" };
+const VERBOSE_OPTION = { name: "--verbose", description: "Stream subprocess output" };
+const QUIET_OPTION = { name: "--quiet", description: "Suppress progress output on stderr" };
+const COMMAND_DEFINITIONS = {
+    info: {
+        usage: "tt-local info",
+        description: "Show the installed TT Local version and runner status.",
+        options: [],
+        maxPositionals: 0,
+    },
+    init: {
+        usage: "tt-local init [options]",
+        description: "Create a local tunedtensor.json behavior spec.",
+        options: [
+            { name: "--name", value: "name", description: "Behavior spec name" },
+            { name: "--model", value: "model", description: "Base model ID" },
+            { name: "--output", value: "path", description: "Output spec path" },
+            { name: "--profile", value: "profile", description: "Write a durable runner config (spark)" },
+            { name: "--config-output", value: "path", description: "Profile config output path" },
+            { name: "--force", description: "Overwrite an existing output file" },
+        ],
+        maxPositionals: 0,
+    },
+    doctor: {
+        usage: "tt-local doctor [tunedtensor.json|request.json] [--config path]",
+        description: "Check the host and optional run input before starting work.",
+        options: [CONFIG_OPTION],
+        maxPositionals: 1,
+    },
+    validate: {
+        usage: "tt-local validate [tunedtensor.json|request.json] [options]",
+        description: "Validate a behavior spec or run request without executing it.",
+        options: [CONFIG_OPTION, USER_ID_OPTION, RUN_NUMBER_OPTION],
+        maxPositionals: 1,
+    },
+    run: {
+        usage: "tt-local run [tunedtensor.json|request.json] [options]",
+        description: "Run the local fine-tuning workflow.",
+        options: [
+            CONFIG_OPTION,
+            USER_ID_OPTION,
+            RUN_NUMBER_OPTION,
+            { name: "--run-id", value: "uuid", description: "Resume or override a run ID" },
+            { name: "--stage", value: "stage", description: "Run through a specific workflow stage" },
+            { name: "--model-artifact", value: "uri", description: "Use an existing model artifact" },
+            { name: "--parent-model", value: "model-id", description: "Continue training a stored local model" },
+            { name: "--parent-model-artifact", value: "uri", description: "Continue training a model artifact" },
+            { name: "--force", description: "Recompute the selected stage" },
+            { name: "--dry-run", description: "Write representative artifacts without GPU work" },
+            { name: "--detach", description: "Run in the background and return the run ID immediately" },
+            VERBOSE_OPTION,
+            QUIET_OPTION,
+        ],
+        maxPositionals: 1,
+    },
+    label: {
+        usage: "tt-local label <input.jsonl|input.csv> [options]",
+        description: "Label local rows with a teacher model.",
+        options: [
+            CONFIG_OPTION,
+            { name: "--input-column", value: "column", description: "CSV input column" },
+            { name: "--spec", value: "path", description: "Behavior spec path" },
+            { name: "--system-prompt", value: "text", description: "Explicit teacher system prompt" },
+            { name: "--model", value: "model", description: "Teacher model ID" },
+            { name: "--output", value: "path", description: "Output JSONL path" },
+            { name: "--format", value: "jsonl|csv", description: "Force the source format" },
+            { name: "--dry-run", description: "Parse and sanitize without teacher calls" },
+            VERBOSE_OPTION,
+            QUIET_OPTION,
+        ],
+        minPositionals: 1,
+        maxPositionals: 1,
+        missingPositionalsMessage: "label requires <input.jsonl|input.csv>",
+    },
+    serve: {
+        usage: "tt-local serve [--config path] [--host host] [--port port]",
+        description: "Serve the local runs dashboard.",
+        options: [
+            CONFIG_OPTION,
+            { name: "--host", value: "host", description: "Dashboard bind host" },
+            { name: "--port", value: "port", description: "Dashboard bind port" },
+        ],
+        maxPositionals: 0,
+    },
+};
+const COMMAND_GROUPS = {
+    runs: {
+        description: "Inspect and manage locally stored runs.",
+        defaultSubcommand: "list",
+        subcommands: {
+            list: { usage: "tt-local runs list [--config path]", description: "List local runs.", options: [CONFIG_OPTION], maxPositionals: 0 },
+            get: { usage: "tt-local runs get <run-id> [--config path]", description: "Get a local run.", options: [CONFIG_OPTION], minPositionals: 1, maxPositionals: 1, missingPositionalsMessage: "runs get requires <run-id>" },
+            events: { usage: "tt-local runs events <run-id> [--config path]", description: "List run events.", options: [CONFIG_OPTION], minPositionals: 1, maxPositionals: 1, missingPositionalsMessage: "runs events requires <run-id>" },
+            watch: { usage: "tt-local runs watch <run-id> [--config path]", description: "Watch a run until it finishes.", options: [CONFIG_OPTION], minPositionals: 1, maxPositionals: 1, missingPositionalsMessage: "runs watch requires <run-id>" },
+            report: { usage: "tt-local runs report <run-id> [--config path]", description: "Get a run comparison report.", options: [CONFIG_OPTION], minPositionals: 1, maxPositionals: 1, missingPositionalsMessage: "runs report requires <run-id>" },
+            compare: { usage: "tt-local runs compare <run-id-a> <run-id-b> [--config path]", description: "Compare two run reports.", options: [CONFIG_OPTION], minPositionals: 2, maxPositionals: 2, missingPositionalsMessage: "runs compare requires <run-id-a> <run-id-b>" },
+            cancel: { usage: "tt-local runs cancel <run-id> [--config path]", description: "Request cancellation of a run.", options: [CONFIG_OPTION], minPositionals: 1, maxPositionals: 1, missingPositionalsMessage: "runs cancel requires <run-id>" },
+            reconcile: { usage: "tt-local runs reconcile [--config path]", description: "Rebuild local store indexes.", options: [CONFIG_OPTION], maxPositionals: 0 },
+        },
+    },
+    models: {
+        description: "Inspect, verify, prefetch, or serve local models.",
+        defaultSubcommand: "list",
+        subcommands: {
+            list: { usage: "tt-local models list [--config path]", description: "List local models.", options: [CONFIG_OPTION], maxPositionals: 0 },
+            get: { usage: "tt-local models get <model-id> [--config path]", description: "Get a local model.", options: [CONFIG_OPTION], minPositionals: 1, maxPositionals: 1, missingPositionalsMessage: "models get requires <model-id>" },
+            verify: {
+                usage: "tt-local models verify <model-id-or-artifact-path> [--config path]",
+                description: "Verify a stored model or manifested artifact path.",
+                options: [CONFIG_OPTION],
+                minPositionals: 1,
+                maxPositionals: 1,
+                missingPositionalsMessage: "models verify requires <model-id-or-artifact-path>",
+            },
+            prefetch: {
+                usage: "tt-local models prefetch [tunedtensor.json|request.json] [options]",
+                description: "Download the configured base model before a run.",
+                options: [CONFIG_OPTION, USER_ID_OPTION, RUN_NUMBER_OPTION, VERBOSE_OPTION, QUIET_OPTION],
+                maxPositionals: 1,
+            },
+            "verify-base": {
+                usage: "tt-local models verify-base [tunedtensor.json|request.json] [options]",
+                description: "Verify that the configured base-model snapshot is complete and locally available.",
+                options: [CONFIG_OPTION, USER_ID_OPTION, RUN_NUMBER_OPTION, VERBOSE_OPTION, QUIET_OPTION],
+                maxPositionals: 1,
+            },
+            serve: {
+                usage: "tt-local models serve <model-id> [options]",
+                description: "Serve a verified adapter through an OpenAI-compatible local API.",
+                options: [
+                    CONFIG_OPTION,
+                    { name: "--host", value: "host", description: "Bind host (localhost by default)" },
+                    { name: "--port", value: "port", description: "Bind port" },
+                    { name: "--device", value: "device", description: "auto, cpu, cuda, or mps" },
+                    { name: "--max-tokens", value: "count", description: "Default response token limit" },
+                    { name: "--temperature", value: "number", description: "Default sampling temperature" },
+                    { name: "--top-p", value: "number", description: "Default nucleus sampling threshold" },
+                    { name: "--max-concurrent-requests", value: "count", description: "Concurrent generation limit" },
+                    { name: "--spec", value: "path", description: "Behavior spec whose instructions are enforced" },
+                    { name: "--no-spec-prompt", description: "Do not enforce the stored behavior-spec prompt" },
+                    { name: "--allow-remote", description: "Allow a non-loopback bind" },
+                    { name: "--api-key-env", value: "name", description: "Environment variable containing a bearer token" },
+                    { name: "--print-command", description: "Validate and print the launch plan without starting" },
+                ],
+                minPositionals: 1,
+                maxPositionals: 1,
+                missingPositionalsMessage: "models serve requires <model-id>",
+            },
+        },
+    },
+    specs: {
+        description: "Inspect and import local behavior specs.",
+        defaultSubcommand: "list",
+        subcommands: {
+            list: { usage: "tt-local specs list [--config path]", description: "List local behavior specs.", options: [CONFIG_OPTION], maxPositionals: 0 },
+            get: { usage: "tt-local specs get <spec-id> [--config path]", description: "Get a local behavior spec.", options: [CONFIG_OPTION], minPositionals: 1, maxPositionals: 1, missingPositionalsMessage: "specs get requires <spec-id>" },
+            import: {
+                usage: "tt-local specs import <spec-or-request.json> [options]",
+                description: "Import a behavior spec into the local store.",
+                options: [CONFIG_OPTION, USER_ID_OPTION, RUN_NUMBER_OPTION, { name: "--id", value: "id", description: "ID for a raw spec snapshot" }],
+                minPositionals: 1,
+                maxPositionals: 1,
+                missingPositionalsMessage: "specs import requires <spec-or-request.json>",
+            },
+        },
+    },
+    store: {
+        description: "Maintain the local file-backed store.",
+        subcommands: {
+            "rebuild-index": { usage: "tt-local store rebuild-index [--config path]", description: "Rebuild store indexes from durable records.", options: [CONFIG_OPTION], maxPositionals: 0 },
+        },
+    },
+};
+function hasHelpFlag(argv) {
+    return argv.includes("--help") || argv.includes("-h");
+}
+function printCommandHelp(definition) {
+    console.log(`Usage: ${definition.usage}\n\n${definition.description}`);
+    if (definition.options.length > 0) {
+        console.log("\nOptions:");
+        for (const option of definition.options) {
+            const label = option.value ? `${option.name} <${option.value}>` : option.name;
+            console.log(`  ${label.padEnd(34)} ${option.description}`);
+        }
+    }
+    console.log("  -h, --help                        Show help");
+}
+function printGroupHelp(command, group) {
+    console.log(`Usage: tt-local ${command} <command> [options]\n\n${group.description}\n\nCommands:`);
+    for (const [name, definition] of Object.entries(group.subcommands)) {
+        console.log(`  ${name.padEnd(16)} ${definition.description}`);
+    }
+    console.log("\nRun `tt-local " + command + " <command> --help` for command-specific help.");
+}
+function parseCommandArguments(tokens, definition) {
+    const options = new Map(definition.options.map((option) => [option.name, option]));
+    const seen = new Set();
     const positionals = [];
-    for (let index = startIndex; index < argv.length; index += 1) {
-        const arg = argv[index];
-        if (!arg)
-            continue;
-        if (optionNamesWithValues.has(arg)) {
-            index += 1;
+    let optionsEnded = false;
+    for (let index = 0; index < tokens.length; index += 1) {
+        const token = tokens[index];
+        if (!optionsEnded && token === "--") {
+            optionsEnded = true;
             continue;
         }
-        if (arg.startsWith("--"))
+        if (!optionsEnded && token.startsWith("-")) {
+            const equalsIndex = token.indexOf("=");
+            const name = equalsIndex === -1 ? token : token.slice(0, equalsIndex);
+            const inlineValue = equalsIndex === -1 ? undefined : token.slice(equalsIndex + 1);
+            const option = options.get(name);
+            if (!option)
+                throw new Error(`Unknown option: ${name}`);
+            if (seen.has(name))
+                throw new Error(`Option ${name} may only be specified once.`);
+            seen.add(name);
+            if (option.value) {
+                if (inlineValue !== undefined) {
+                    if (!inlineValue)
+                        throw new Error(`Option ${name} requires a value.`);
+                    continue;
+                }
+                const value = tokens[index + 1];
+                if (value === undefined || value.startsWith("-")) {
+                    throw new Error(`Option ${name} requires a value.`);
+                }
+                index += 1;
+            }
+            else if (inlineValue !== undefined) {
+                throw new Error(`Option ${name} does not accept a value.`);
+            }
             continue;
-        positionals.push(arg);
+        }
+        positionals.push(token);
+    }
+    if (positionals.length < (definition.minPositionals ?? 0)) {
+        throw new Error(definition.missingPositionalsMessage ?? `Missing required argument. Usage: ${definition.usage}`);
+    }
+    if (definition.maxPositionals !== undefined && positionals.length > definition.maxPositionals) {
+        throw new Error(`Too many arguments. Usage: ${definition.usage}`);
     }
     return positionals;
 }
+function parseCli(argv) {
+    const command = argv[2] ?? "info";
+    if (command === "--help" || command === "-h") {
+        return { command: "info", positionals: [], help: "top" };
+    }
+    if (command === "--version" || command === "-V") {
+        if (argv.length > 3)
+            throw new Error(`${command} does not accept arguments.`);
+        return { command, positionals: [], help: "command" };
+    }
+    if (command.startsWith("-"))
+        throw new Error(`Unknown option: ${command}`);
+    const definition = COMMAND_DEFINITIONS[command];
+    if (definition) {
+        if (hasHelpFlag(argv.slice(3))) {
+            return { command, positionals: [], help: "command", definition };
+        }
+        return {
+            command,
+            positionals: parseCommandArguments(argv.slice(3), definition),
+            help: "top",
+            definition,
+        };
+    }
+    const group = COMMAND_GROUPS[command];
+    if (!group)
+        throw new Error(`Unknown command: ${command}`);
+    if (argv[3] === "--help" || argv[3] === "-h") {
+        return { command, positionals: [], help: "group" };
+    }
+    const candidate = argv[3];
+    let subcommand;
+    let tokenStart;
+    if (candidate && !candidate.startsWith("-")) {
+        subcommand = candidate;
+        tokenStart = 4;
+    }
+    else if (group.defaultSubcommand) {
+        subcommand = group.defaultSubcommand;
+        tokenStart = 3;
+    }
+    else {
+        throw new Error(`${command} requires a subcommand. Run 'tt-local ${command} --help'.`);
+    }
+    const subcommandDefinition = group.subcommands[subcommand];
+    if (!subcommandDefinition)
+        throw new Error(`Unknown ${command} command: ${subcommand}`);
+    if (hasHelpFlag(argv.slice(tokenStart))) {
+        return { command, subcommand, positionals: [], help: "command", definition: subcommandDefinition };
+    }
+    return {
+        command,
+        subcommand,
+        positionals: parseCommandArguments(argv.slice(tokenStart), subcommandDefinition),
+        help: "top",
+        definition: subcommandDefinition,
+    };
+}
 function readNumberOption(argv, name) {
     const value = readOption(argv, name);
     return value ? Number(value) : undefined;
@@ -233,10 +523,45 @@ async function parentModelSelectionFromArgv(argv, config) {
         return undefined;
     const store = createLocalStore(config.storeRoot);
     const record = await store.getModel(parentModel);
+    const verified = await verifyStoredModel(record, { requireServable: true });
+    const contract = verified.contract;
+    const baseModelRevision = typeof contract.base_model_revision === "string"
+        ? contract.base_model_revision
+        : undefined;
+    const baseModelArtifactUri = typeof contract.base_model_artifact_uri === "string"
+        ? contract.base_model_artifact_uri
+        : undefined;
+    const baseModelFingerprint = typeof contract.base_model_fingerprint === "string"
+        ? contract.base_model_fingerprint
+        : undefined;
+    if (baseModelArtifactUri && baseModelFingerprint) {
+        if (!config.paths.baseModel) {
+            throw new Error(`Parent model ${record.id} was trained from a local base snapshot. Configure paths.baseModel to the same `
+                + "snapshot before continuing it.");
+        }
+        const configuredFingerprint = await fingerprintLocalBaseModel(config.paths.baseModel);
+        if (configuredFingerprint !== baseModelFingerprint) {
+            throw new Error(`Configured paths.baseModel content does not match parent model ${record.id}.`);
+        }
+    }
+    else if (!baseModelRevision) {
+        throw new Error(`Parent model ${record.id} does not record an immutable base-model revision or local snapshot fingerprint. `
+            + "Use --parent-model-artifact only if you intentionally accept responsibility for that unpinned artifact.");
+    }
+    if (baseModelRevision && config.paths.baseModel) {
+        const snapshotRevision = resolve(config.paths.baseModel).match(/[\\/]snapshots[\\/]([^\\/]+)(?:[\\/]|$)/)?.[1];
+        if (!baseModelFingerprint && snapshotRevision !== baseModelRevision) {
+            throw new Error(`Parent model ${record.id} requires base revision ${baseModelRevision}; configured paths.baseModel does not `
+                + "identify that snapshot revision.");
+        }
+    }
     return {
         artifactUri: record.artifact_uri,
         modelId: record.id,
         baseModel: record.base_model,
+        baseModelRevision,
+        baseModelArtifactUri,
+        baseModelFingerprint,
     };
 }
 function withParentModelArtifact(request, parentModel) {
@@ -246,10 +571,16 @@ function withParentModelArtifact(request, parentModel) {
         throw new Error(`Parent model ${parentModel.modelId ?? ""} uses base model ${parentModel.baseModel}, `
             + `but this run uses ${request.spec_snapshot.base_model}. Continue from a model with the same base model.`);
     }
+    const requestedRevision = request.hyperparameters.base_model_revision;
+    if (parentModel.baseModelRevision && requestedRevision && parentModel.baseModelRevision !== requestedRevision) {
+        throw new Error(`Parent model ${parentModel.modelId ?? ""} uses base revision ${parentModel.baseModelRevision}, `
+            + `but this run requests ${requestedRevision}.`);
+    }
     return fineTuneRunRequestSchema.parse({
         ...request,
         hyperparameters: {
             ...request.hyperparameters,
+            ...(parentModel.baseModelRevision ? { base_model_revision: parentModel.baseModelRevision } : {}),
             parent_model_artifact: parentModel.artifactUri,
         },
     });
@@ -257,26 +588,265 @@ function withParentModelArtifact(request, parentModel) {
 async function sleep(ms) {
     await new Promise((resolveSleep) => setTimeout(resolveSleep, ms));
 }
-function isTerminalStatus(status) {
-    return status === "completed" || status === "failed" || status === "cancelled";
+function detachedRunArgv(argv, runId) {
+    const tokens = argv.slice(3);
+    const output = [];
+    for (let index = 0; index < tokens.length; index += 1) {
+        const token = tokens[index];
+        if (token === "--detach")
+            continue;
+        if (token === "--run-id") {
+            index += 1;
+            continue;
+        }
+        if (token.startsWith("--run-id="))
+            continue;
+        output.push(token);
+    }
+    return ["run", ...output, "--run-id", runId];
+}
+async function launchDetachedRun(args) {
+    if (!process.argv[1])
+        throw new Error("Cannot determine the tt-local CLI entrypoint for --detach.");
+    const prefix = args.request.artifacts?.prefix ?? defaultArtifactPrefix({
+        userId: args.request.user_id,
+        behaviorSpecId: args.request.behavior_spec_id,
+        runId: args.request.run_id,
+    });
+    const artifacts = resolveRunArtifacts({ artifactRoot: args.config.artifactRoot, prefix });
+    await claimRunArtifactDirectory({
+        artifacts,
+        runId: args.request.run_id,
+        userId: args.request.user_id,
+        behaviorSpecId: args.request.behavior_spec_id,
+    });
+    const store = createLocalStore(args.config.storeRoot);
+    const existing = await store.getRun(args.request.run_id).catch(() => null);
+    if (existing) {
+        await store.syncRunRequest(args.request);
+    }
+    else {
+        await store.startRun({ request: args.request, artifactDir: artifacts.runDir });
+    }
+    const logPath = resolve(artifacts.runDir, "detached.log");
+    await mkdir(dirname(logPath), { recursive: true });
+    const log = await open(logPath, "a");
+    try {
+        const child = spawn(process.execPath, [
+            ...process.execArgv,
+            process.argv[1],
+            ...detachedRunArgv(args.argv, args.request.run_id),
+        ], {
+            detached: true,
+            stdio: ["ignore", log.fd, log.fd],
+            env: { ...process.env, TT_LOCAL_DETACHED: "1" },
+        });
+        await new Promise((resolveSpawn, reject) => {
+            child.once("spawn", resolveSpawn);
+            child.once("error", reject);
+        });
+        if (!child.pid)
+            throw new Error("Detached tt-local process did not report a PID.");
+        child.unref();
+        return { pid: child.pid, logPath, artifactDir: artifacts.runDir };
+    }
+    catch (error) {
+        await store.failRun(args.request.run_id, `Failed to launch detached workflow: ${error instanceof Error ? error.message : String(error)}`).catch(() => undefined);
+        throw error;
+    }
+    finally {
+        await log.close();
+    }
+}
+async function verifyStoredModel(model, options = {}) {
+    const manifestPath = join(model.artifact_dir, "artifact-manifest.json");
+    const integrity = await assertArtifactManifest(manifestPath, {
+        requiredPaths: ["stage-metadata.json", "training-report.json"],
+        scopeToRequired: true,
+        verifyModel: true,
+    });
+    const manifest = JSON.parse(await readFile(manifestPath, "utf8"));
+    if (!manifest.model) {
+        throw new Error(`Artifact manifest does not contain a model contract: ${manifestPath}`);
+    }
+    const customNonServable = manifest.model.framework !== "transformers-peft"
+        && manifest.model.servable === false;
+    const artifact = await assertUsableModelArtifact(model.artifact_uri, {
+        allowUnrecognizedPayload: customNonServable && !options.requireServable,
+    });
+    if (typeof manifest.model.base_model_artifact_uri === "string"
+        && typeof manifest.model.base_model_fingerprint === "string") {
+        const actualBaseFingerprint = await fingerprintLocalBaseModel(manifest.model.base_model_artifact_uri);
+        if (actualBaseFingerprint !== manifest.model.base_model_fingerprint) {
+            throw new Error("Recorded local base-model content no longer matches the model artifact contract.");
+        }
+    }
+    if (typeof manifest.model.artifact_root !== "string"
+        || resolve(manifest.model.artifact_root) !== resolve(artifact.path)) {
+        throw new Error("Stored model record does not match the artifact covered by its manifest.");
+    }
+    if (manifest.model.base_model !== model.base_model) {
+        throw new Error("Stored model base model does not match its artifact manifest.");
+    }
+    if (options.requireServable && manifest.model.servable !== true) {
+        throw new Error(`Model ${model.id} is valid but its artifact contract does not mark it as locally servable.`);
+    }
+    return {
+        manifest_path: manifestPath,
+        integrity,
+        artifact,
+        contract: manifest.model,
+    };
+}
+async function verifyModelArtifactPath(input) {
+    const inputPath = resolve(input);
+    let manifestPath;
+    let artifactUri = inputPath;
+    if (basename(inputPath) === "artifact-manifest.json") {
+        manifestPath = inputPath;
+        const raw = JSON.parse(await readFile(inputPath, "utf8"));
+        if (typeof raw.model?.artifact_root !== "string") {
+            throw new Error(`Artifact manifest does not contain a model contract: ${inputPath}`);
+        }
+        artifactUri = raw.model.artifact_root;
+    }
+    else {
+        const metadata = await stat(inputPath).catch(() => null);
+        if (!metadata)
+            throw new Error(`Model not found and artifact path does not exist: ${input}`);
+        let current = metadata.isDirectory() ? inputPath : dirname(inputPath);
+        for (let depth = 0; depth < 8; depth += 1) {
+            const candidate = join(current, "artifact-manifest.json");
+            const raw = await readFile(candidate, "utf8").catch(() => null);
+            if (raw) {
+                const parsed = JSON.parse(raw);
+                if (typeof parsed.model?.artifact_root === "string"
+                    && resolve(parsed.model.artifact_root) === resolve(inputPath)) {
+                    manifestPath = candidate;
+                    break;
+                }
+            }
+            const parent = dirname(current);
+            if (parent === current)
+                break;
+            current = parent;
+        }
+    }
+    if (!manifestPath) {
+        throw new Error(`No artifact manifest covering model path ${inputPath} was found in its parent run directory.`);
+    }
+    const integrity = await assertArtifactManifest(manifestPath, {
+        requiredPaths: ["stage-metadata.json", "training-report.json"],
+        scopeToRequired: true,
+        verifyModel: true,
+    });
+    const manifest = JSON.parse(await readFile(manifestPath, "utf8"));
+    if (!manifest.model)
+        throw new Error(`Artifact manifest does not contain a model contract: ${manifestPath}`);
+    const customNonServable = manifest.model.framework !== "transformers-peft"
+        && manifest.model.servable === false;
+    const artifact = await assertUsableModelArtifact(artifactUri, {
+        allowUnrecognizedPayload: customNonServable,
+    });
+    if (typeof manifest.model.base_model_artifact_uri === "string"
+        && typeof manifest.model.base_model_fingerprint === "string") {
+        const actualBaseFingerprint = await fingerprintLocalBaseModel(manifest.model.base_model_artifact_uri);
+        if (actualBaseFingerprint !== manifest.model.base_model_fingerprint) {
+            throw new Error("Recorded local base-model content no longer matches the model artifact contract.");
+        }
+    }
+    if (typeof manifest.model.artifact_root !== "string"
+        || resolve(manifest.model.artifact_root) !== resolve(artifact.path)) {
+        throw new Error("Model path does not match the artifact covered by its manifest.");
+    }
+    return { manifest_path: manifestPath, integrity, artifact, contract: manifest.model };
+}
+async function modelSystemPrompt(args) {
+    const specPath = readOption(args.argv, "--spec");
+    if (specPath && hasFlag(args.argv, "--no-spec-prompt")) {
+        throw new Error("Use only one of --spec or --no-spec-prompt.");
+    }
+    if (hasFlag(args.argv, "--no-spec-prompt"))
+        return undefined;
+    let spec;
+    if (specPath) {
+        const input = JSON.parse(await readFile(resolve(specPath), "utf8"));
+        const local = localBehaviorSpecFileSchema.safeParse(input);
+        const request = fineTuneRunRequestSchema.safeParse(input);
+        const snapshot = specSnapshotSchema.safeParse(input);
+        const parsedSpec = local.success ? local.data : request.success ? request.data.spec_snapshot : snapshot.success ? snapshot.data : null;
+        if (!parsedSpec)
+            throw new Error(`--spec must contain a TT Local behavior spec or run request: ${resolve(specPath)}`);
+        spec = parsedSpec;
+    }
+    else {
+        const runRequestPath = join(args.store.paths.runsDir, args.model.run_id, "request.json");
+        let persistedRequest = null;
+        try {
+            persistedRequest = fineTuneRunRequestSchema.parse(JSON.parse(await readFile(runRequestPath, "utf8")));
+        }
+        catch (error) {
+            if (error.code !== "ENOENT")
+                throw error;
+        }
+        spec = persistedRequest?.spec_snapshot
+            ?? (await args.store.getSpec(args.model.behavior_spec_id)).spec;
+    }
+    if (spec.base_model !== args.model.base_model) {
+        throw new Error(`Behavior spec base model ${spec.base_model} does not match stored model base ${args.model.base_model}.`);
+    }
+    const prompt = buildSystemMessage(spec);
+    const metadata = JSON.parse(await readFile(join(args.model.artifact_dir, "stage-metadata.json"), "utf8"));
+    const promptHash = createHash("sha256").update(prompt).digest("hex");
+    if (metadata.system_prompt_sha256 !== promptHash) {
+        throw new Error("Behavior spec instructions do not match the prompt fingerprint used for this trained model. "
+            + "Pass the original --spec, or use --no-spec-prompt only when this change is intentional.");
+    }
+    return prompt;
+}
+function modelServeDevice(argv) {
+    const value = readOption(argv, "--device");
+    if (value === undefined || value === "auto" || value === "cpu" || value === "cuda" || value === "mps") {
+        return value;
+    }
+    throw new Error(`--device must be one of auto, cpu, cuda, mps; got: ${value}`);
 }
 async function main(argv) {
-    const command = argv[2] ?? "info";
+    const cli = parseCli(argv);
+    const command = cli.command;
+    if (command === "--version" || command === "-V") {
+        console.log(TT_LOCAL_VERSION);
+        return;
+    }
+    if (cli.help === "top" && (argv[2] === "--help" || argv[2] === "-h")) {
+        printHelp();
+        return;
+    }
+    if (cli.help === "group") {
+        printGroupHelp(command, COMMAND_GROUPS[command]);
+        return;
+    }
+    if (cli.help === "command" && cli.definition) {
+        printCommandHelp(cli.definition);
+        return;
+    }
     const loadedEnv = await loadDotEnvFromCwd();
     if (loadedEnv.length > 0 && !hasFlag(argv, "--quiet")) {
         process.stderr.write(`[tt-local] loaded ${loadedEnv.join(", ")} from .env\n`);
     }
-    if (command === "info" || command === "--help" || command === "-h") {
+    if (command === "info") {
         const info = getLocalRunnerInfo();
         console.log(`${info.name}: ${info.description}`);
+        console.log(`Version: ${info.version}`);
         console.log(`Status: ${info.status}`);
-        if (command !== "info")
-            printHelp();
         return;
     }
     if (command === "doctor") {
         const config = await configFromArgv(argv);
-        const checks = await runDoctor(config);
+        const request = cli.positionals[0]
+            ? (await loadLocalRunInput(resolve(cli.positionals[0]))).request
+            : undefined;
+        const checks = await runDoctor(config, request);
         const ok = checks.every((check) => check.ok);
         printJson({ ok, checks });
         if (!ok)
@@ -285,35 +855,52 @@ async function main(argv) {
     }
     if (command === "init") {
         const outputPath = resolve(readOption(argv, "--output") ?? DEFAULT_LOCAL_SPEC_PATH);
+        const profile = readOption(argv, "--profile");
+        if (profile !== undefined && profile !== "spark") {
+            throw new Error(`--profile must be spark, got: ${profile}`);
+        }
         const spec = await initLocalSpecFile({
             outputPath,
             name: readOption(argv, "--name") ?? "Local Tuned Tensor Spec",
             baseModel: readOption(argv, "--model") ?? "Qwen/Qwen3.5-2B",
             force: hasFlag(argv, "--force"),
         });
+        const configPath = profile
+            ? resolve(readOption(argv, "--config-output") ?? resolve(dirname(outputPath), "local-runner.json"))
+            : undefined;
+        if (profile && configPath) {
+            await initLocalRunnerConfigFile({
+                outputPath: configPath,
+                profile,
+                force: hasFlag(argv, "--force"),
+            });
+        }
         printJson({
             ok: true,
             path: outputPath,
             id: spec.id,
             name: spec.name,
             base_model: spec.base_model,
+            config_path: configPath ?? null,
         });
         return;
     }
     if (command === "validate") {
-        const inputPath = resolve(readPositionals(argv)[0] ?? DEFAULT_LOCAL_SPEC_PATH);
+        const inputPath = resolve(cli.positionals[0] ?? DEFAULT_LOCAL_SPEC_PATH);
         const input = await loadLocalRunInput(inputPath, {
             userId: readOption(argv, "--user-id"),
             runNumber: readNumberOption(argv, "--run-number"),
         });
         const request = input.request;
         const config = await configFromArgv(argv);
+        assertLocalRunInputReady(request);
         assertSupportedValidateShape(request, config);
+        assertEvaluationScoringReady(config);
         printJson({
             ok: true,
             input_kind: input.kind,
             input_path: input.path,
-            run_id: request.run_id,
+            run_id: input.kind === "request" ? request.run_id : null,
             behavior_spec_id: request.behavior_spec_id,
             training_method: request.training_method,
             base_model: request.spec_snapshot.base_model,
@@ -326,7 +913,7 @@ async function main(argv) {
         return;
     }
     if (command === "run") {
-        const inputPath = resolve(readPositionals(argv)[0] ?? DEFAULT_LOCAL_SPEC_PATH);
+        const inputPath = resolve(cli.positionals[0] ?? DEFAULT_LOCAL_SPEC_PATH);
         const configInput = await configFromArgv(argv);
         const config = localRunnerConfigSchema.parse({
             ...configInput,
@@ -347,6 +934,28 @@ async function main(argv) {
             ? fineTuneRunRequestSchema.parse({ ...input.request, run_id: runId })
             : input.request;
         const request = withParentModelArtifact(baseRequest, await parentModelSelectionFromArgv(argv, config));
+        assertLocalRunInputReady(request);
+        assertSupportedValidateShape(request, config);
+        assertEvaluationScoringReady(config);
+        if (hasFlag(argv, "--detach")) {
+            const detached = await launchDetachedRun({ argv, request, config });
+            const configPath = readOption(argv, "--config");
+            const configSuffix = configPath ? ` --config ${JSON.stringify(resolve(configPath))}` : "";
+            printJson({
+                status: "queued",
+                detached: true,
+                run_id: request.run_id,
+                behavior_spec_id: request.behavior_spec_id,
+                pid: detached.pid,
+                log_path: detached.logPath,
+                artifact_dir: detached.artifactDir,
+                next: {
+                    watch: `tt-local runs watch ${request.run_id}${configSuffix}`,
+                    cancel: `tt-local runs cancel ${request.run_id}${configSuffix}`,
+                },
+            });
+            return;
+        }
         const stage = readRunStage(argv);
         const result = await runLocalFineTuneStage({
             request,
@@ -379,7 +988,7 @@ async function main(argv) {
         }
         else {
             printJson({
-                status: "completed",
+                status: "stage_completed",
                 stage: result.stage,
                 input_kind: input.kind,
                 run_id: result.request.run_id,
@@ -392,7 +1001,7 @@ async function main(argv) {
         return;
     }
     if (command === "label") {
-        const sourcePath = readPositionals(argv)[0];
+        const sourcePath = cli.positionals[0];
         if (!sourcePath)
             throw new Error("label requires <input.jsonl|input.csv>");
         const configInput = await configFromArgv(argv);
@@ -450,32 +1059,32 @@ async function main(argv) {
         return;
     }
     if (command === "runs") {
-        const subcommand = argv[3] ?? "list";
+        const subcommand = cli.subcommand;
         const config = await configFromArgv(argv);
         const store = createLocalStore(config.storeRoot);
         if (subcommand === "list")
             return printJson(await store.listRuns());
         if (subcommand === "get") {
-            const id = argv[4];
+            const id = cli.positionals[0];
             if (!id)
                 throw new Error("runs get requires <run-id>");
             return printJson(await store.getRun(id));
         }
         if (subcommand === "events") {
-            const id = argv[4];
+            const id = cli.positionals[0];
             if (!id)
                 throw new Error("runs events requires <run-id>");
             return printJson(await store.getRunEvents(id));
         }
         if (subcommand === "report") {
-            const id = argv[4];
+            const id = cli.positionals[0];
             if (!id)
                 throw new Error("runs report requires <run-id>");
             return printJson(await store.getRunReport(id));
         }
         if (subcommand === "compare") {
-            const idA = argv[4];
-            const idB = argv[5];
+            const idA = cli.positionals[0];
+            const idB = cli.positionals[1];
             if (!idA || !idB)
                 throw new Error("runs compare requires <run-id-a> <run-id-b>");
             const [reportA, reportB] = await Promise.all([
@@ -485,14 +1094,15 @@ async function main(argv) {
             return printJson(compareRuns(reportA, reportB));
         }
         if (subcommand === "cancel") {
-            const id = argv[4];
+            const id = cli.positionals[0];
             if (!id)
                 throw new Error("runs cancel requires <run-id>");
             await store.cancelRun(id);
-            return printJson({ ok: true, run_id: id });
+            const run = await store.getRun(id);
+            return printJson({ ok: true, run_id: run.id, status: run.status, current_stage: run.current_stage });
         }
         if (subcommand === "watch") {
-            const id = argv[4];
+            const id = cli.positionals[0];
             if (!id)
                 throw new Error("runs watch requires <run-id>");
             const printed = new Set();
@@ -505,7 +1115,7 @@ async function main(argv) {
                     console.log(`${event.occurred_at} ${event.stage} ${event.status}: ${event.message}`);
                 }
                 const run = await store.getRun(id);
-                if (isTerminalStatus(run.status))
+                if (isTerminalRunState(run))
                     return;
                 await sleep(1000);
             }
@@ -517,33 +1127,116 @@ async function main(argv) {
         throw new Error(`Unknown runs command: ${subcommand}`);
     }
     if (command === "models") {
-        const subcommand = argv[3] ?? "list";
+        const subcommand = cli.subcommand;
         const config = await configFromArgv(argv);
         const store = createLocalStore(config.storeRoot);
         if (subcommand === "list")
             return printJson(await store.listModels());
         if (subcommand === "get") {
-            const id = argv[4];
+            const id = cli.positionals[0];
             if (!id)
                 throw new Error("models get requires <model-id>");
             return printJson(await store.getModel(id));
         }
+        if (subcommand === "verify") {
+            const id = cli.positionals[0];
+            if (!id)
+                throw new Error("models verify requires <model-id-or-artifact-path>");
+            if (await stat(resolve(id)).then(() => true, () => false)) {
+                return printJson({ ok: true, model: null, ...await verifyModelArtifactPath(id) });
+            }
+            const model = await store.getModel(id);
+            const verified = await verifyStoredModel(model);
+            return printJson({ ok: true, model, ...verified });
+        }
+        if (subcommand === "prefetch" || subcommand === "verify-base") {
+            const inputPath = resolve(cli.positionals[0] ?? DEFAULT_LOCAL_SPEC_PATH);
+            const input = await loadLocalRunInput(inputPath, {
+                userId: readOption(argv, "--user-id"),
+                runNumber: readNumberOption(argv, "--run-number"),
+            });
+            if (!hasFlag(argv, "--quiet")) {
+                for (const warning of input.warnings) {
+                    process.stderr.write(`[tt-local] warning: ${warning}\n`);
+                }
+            }
+            const report = await prefetchBaseModel({
+                request: input.request,
+                config,
+                localOnly: subcommand === "verify-base",
+                reporter: createConsoleReporter({
+                    verbose: hasFlag(argv, "--verbose"),
+                    quiet: hasFlag(argv, "--quiet"),
+                }),
+            });
+            return printJson({
+                ...report,
+                input_kind: input.kind,
+                input_path: input.path,
+            });
+        }
+        if (subcommand === "serve") {
+            const id = cli.positionals[0];
+            if (!id)
+                throw new Error("models serve requires <model-id>");
+            const model = await store.getModel(id);
+            const verified = await verifyStoredModel(model, { requireServable: true });
+            const launch = buildLocalModelServerLaunch({
+                model,
+                config,
+                options: {
+                    host: readOption(argv, "--host"),
+                    port: readNumberOption(argv, "--port"),
+                    device: modelServeDevice(argv),
+                    maxTokens: readNumberOption(argv, "--max-tokens"),
+                    temperature: readNumberOption(argv, "--temperature"),
+                    topP: readNumberOption(argv, "--top-p"),
+                    maxConcurrentRequests: readNumberOption(argv, "--max-concurrent-requests"),
+                    systemPrompt: await modelSystemPrompt({ argv, model, store }),
+                    allowRemote: hasFlag(argv, "--allow-remote"),
+                    apiKeyEnv: readOption(argv, "--api-key-env"),
+                    baseModelRevision: (() => {
+                        const contract = verified.contract;
+                        return typeof contract.base_model_revision === "string" ? contract.base_model_revision : undefined;
+                    })(),
+                    baseModelArtifactUri: (() => {
+                        const contract = verified.contract;
+                        return typeof contract.base_model_artifact_uri === "string" ? contract.base_model_artifact_uri : undefined;
+                    })(),
+                },
+            });
+            if (hasFlag(argv, "--print-command")) {
+                return printJson({
+                    ok: true,
+                    model_id: model.id,
+                    url: launch.url,
+                    command: launch.displayCommand,
+                    artifact_path: launch.artifactPath,
+                    manifest_path: verified.manifest_path,
+                    integrity: verified.integrity,
+                });
+            }
+            process.stderr.write(`[tt-local] verified ${verified.integrity.checked} artifact file(s)\n`);
+            process.stderr.write(`[tt-local] model API: ${launch.url}\n`);
+            await serveLocalModel(launch);
+            return;
+        }
         throw new Error(`Unknown models command: ${subcommand}`);
     }
     if (command === "specs") {
-        const subcommand = argv[3] ?? "list";
+        const subcommand = cli.subcommand;
         const config = await configFromArgv(argv);
         const store = createLocalStore(config.storeRoot);
         if (subcommand === "list")
             return printJson(await store.listSpecs());
         if (subcommand === "get") {
-            const id = argv[4];
+            const id = cli.positionals[0];
             if (!id)
                 throw new Error("specs get requires <spec-id>");
             return printJson(await store.getSpec(id));
         }
         if (subcommand === "import") {
-            const path = argv[4];
+            const path = cli.positionals[0];
             if (!path)
                 throw new Error("specs import requires <spec-or-request.json>");
             const input = JSON.parse(await readFile(resolve(path), "utf8"));
@@ -566,7 +1259,7 @@ async function main(argv) {
         throw new Error(`Unknown specs command: ${subcommand}`);
     }
     if (command === "store") {
-        const subcommand = argv[3];
+        const subcommand = cli.subcommand;
         const config = await configFromArgv(argv);
         const store = createLocalStore(config.storeRoot);
         if (subcommand === "rebuild-index") {