@elizaos/plugin-training 2.0.3-beta.6 → 2.0.3-beta.7

package/dist/services/training-vast-service.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/services/training-vast-service.ts"],"sourcesContent":["/**\n * Vast.ai training-job orchestration service.\n *\n * Owns:\n *   - Loading the model registry from `<trainingRoot>/scripts/dump_registry_json.py`\n *     (Python subprocess) and caching it in-memory with manual refresh.\n *   - Spawning `bash <trainingRoot>/scripts/train_vast.sh provision-and-train ...`\n *     for new jobs and recording state transitions in the JSONL job store.\n *   - Spawning `python <trainingRoot>/scripts/eval_checkpoint.py ...` for ad-hoc\n *     evals against an existing job's run directory.\n *   - Reading the inference-stats JSONL the inference side appends to\n *     (`~/.eliza/inference-stats.jsonl` by default).\n *\n * `trainingRoot` is the eliza-1 training package, resolved by default to\n * `eliza/packages/training/` (relative to this file's compiled location).\n *\n * Subprocess invariants:\n *   - Every shell-out uses `spawn` with an arg array. User-supplied values\n *     (registry_key, label) are validated against an explicit whitelist\n *     before they ever reach `spawn`. We never interpolate user input into\n *     shell strings.\n *   - The training root is locked at construction time and is the cwd for\n *     every spawn. Resolved once via the absolute path of this file.\n */\n\nimport { spawn } from \"node:child_process\";\nimport { existsSync, promises as fs } from \"node:fs\";\nimport { join, resolve } from \"node:path\";\nimport { fileURLToPath } from \"node:url\";\nimport { logger } from \"@elizaos/core\";\nimport {\n  aggregateInferenceStats,\n  emptyInferenceStatsAggregate,\n  type InferenceStatRow,\n  type InferenceStatsAggregate,\n  parseStatRow,\n} from \"./vast-inference-stats.js\";\nimport {\n  type InferenceEndpointRecord,\n  inferenceStatsPath,\n  readInferenceEndpoints,\n  readJobLogTail,\n  type VastJobRecord,\n  VastJobStore,\n  writeInferenceEndpoints,\n} from \"./vast-job-store.js\";\nimport {\n  runCapture,\n  runDetachedToLog,\n  VastServiceError,\n} from \"./vast-subprocess.js\";\n\nexport { VastServiceError } from \"./vast-subprocess.js\";\n\nexport interface VastRegistryEntry {\n  eliza_short_name: string;\n  eliza_repo_id: string;\n  gguf_repo_id: string;\n  base_hf_id: string;\n  tier: string;\n  inference_max_context: number;\n}\n\nexport type VastRegistry = Record<string, VastRegistryEntry>;\n\nexport interface CreateJobInput {\n  registry_key: string;\n  epochs: number;\n  run_name?: string;\n}\n\nexport interface EvalCheckpointInput {\n  registry_key: string;\n  checkpoint_dir: string;\n  val_jsonl?: string;\n  max_examples?: number;\n}\n\nexport type { InferenceStatRow, InferenceStatsAggregate };\n\nexport interface RegistryListing {\n  short_name: string;\n  entry: VastRegistryEntry;\n}\n\nexport interface CheckpointInfo {\n  name: string;\n  path: string;\n  step: number | null;\n  evaluated: boolean;\n  eval_summary: Record<string, unknown> | null;\n}\n\n/**\n * Running cost snapshot for one Vast.ai job.\n *\n * Shape matches the JSON emitted by `scripts/lib/vast_budget.py snapshot\n * --json`. Mapped 1:1 to the React panel field-set so we don't need a\n * separate DTO.\n */\nexport interface VastJobBudget {\n  job_id: string;\n  instance_id: number | null;\n  pipeline: string;\n  run_name: string;\n  gpu_name: string;\n  num_gpus: number;\n  gpu_sku: string;\n  state: string;\n  uptime_seconds: number;\n  uptime_pretty: string;\n  dph_total: number;\n  total_so_far_usd: number;\n  soft_cap_usd: number | null;\n  hard_cap_usd: number | null;\n  over_soft: boolean;\n  over_hard: boolean;\n  fetched_at: number;\n}\n\nexport interface VastTrainingServiceOptions {\n  /** Override the training package root used to locate `scripts/...`. */\n  trainingRoot?: string;\n  /** Override the python launcher used for the registry dump + eval. */\n  pythonLauncher?: { command: string; preArgs: string[] };\n  /** Optional injected store for test isolation. */\n  store?: VastJobStore;\n  /** Override the spawn factory for tests. */\n  spawnImpl?: typeof spawn;\n}\n\nconst DEFAULT_PYTHON: { command: string; preArgs: string[] } = {\n  command: \"uv\",\n  preArgs: [\"run\", \"--quiet\", \"python\"],\n};\n\nconst HERE = fileURLToPath(import.meta.url);\nconst TRAINING_ROOT_DEFAULT = resolve(\n  HERE,\n  \"..\",\n  \"..\",\n  \"..\",\n  \"..\",\n  \"..\",\n  \"packages\",\n  \"training\",\n);\n\nconst RUN_NAME_PATTERN = /^[A-Za-z0-9._-]{1,64}$/;\nconst LABEL_PATTERN = /^[A-Za-z0-9._-]{1,64}$/;\n\n/**\n * Strict subset of the registry-key shape we accept from clients. The actual\n * authoritative whitelist is the registry dump itself; this is just a cheap\n * pre-filter that keeps obviously hostile values from ever reaching spawn.\n */\nconst REGISTRY_KEY_PATTERN = /^[A-Za-z0-9._-]{1,64}$/;\n\nexport class VastTrainingService {\n  private readonly store: VastJobStore;\n  private readonly trainingRoot: string;\n  private readonly python: { command: string; preArgs: string[] };\n  private readonly spawnImpl: typeof spawn;\n  private registryCache: VastRegistry | null = null;\n  private registryLoadedAt: string | null = null;\n\n  constructor(options: VastTrainingServiceOptions = {}) {\n    this.store = options.store ?? new VastJobStore();\n    this.trainingRoot = options.trainingRoot ?? TRAINING_ROOT_DEFAULT;\n    this.python = options.pythonLauncher ?? DEFAULT_PYTHON;\n    this.spawnImpl = options.spawnImpl ?? spawn;\n  }\n\n  // ── Registry ──────────────────────────────────────────────────────────\n\n  async getRegistry(refresh = false): Promise<VastRegistry> {\n    if (this.registryCache && !refresh) return this.registryCache;\n    const dumpScript = join(\n      this.trainingRoot,\n      \"scripts\",\n      \"dump_registry_json.py\",\n    );\n    if (!existsSync(dumpScript)) {\n      throw new VastServiceError(\n        `Registry dump script not found at ${dumpScript}`,\n        503,\n      );\n    }\n    const stdout = await this.runCapture(\n      this.python.command,\n      [...this.python.preArgs, dumpScript],\n      { cwd: this.trainingRoot },\n    );\n    let parsed: unknown;\n    try {\n      parsed = JSON.parse(stdout);\n    } catch (err) {\n      throw new VastServiceError(\n        `Registry dump produced non-JSON output: ${err instanceof Error ? err.message : String(err)}`,\n        500,\n      );\n    }\n    if (!parsed || typeof parsed !== \"object\" || Array.isArray(parsed)) {\n      throw new VastServiceError(\n        \"Registry dump returned unexpected shape\",\n        500,\n      );\n    }\n    const out: VastRegistry = {};\n    for (const [key, value] of Object.entries(\n      parsed as Record<string, unknown>,\n    )) {\n      const entry = narrowRegistryEntry(value);\n      if (entry) out[key] = entry;\n    }\n    this.registryCache = out;\n    this.registryLoadedAt = new Date().toISOString();\n    return out;\n  }\n\n  async listRegistry(refresh = false): Promise<{\n    loaded_at: string | null;\n    entries: RegistryListing[];\n  }> {\n    const registry = await this.getRegistry(refresh);\n    const entries = Object.entries(registry).map(([short_name, entry]) => ({\n      short_name,\n      entry,\n    }));\n    return { loaded_at: this.registryLoadedAt, entries };\n  }\n\n  async ensureRegistryKey(key: string): Promise<VastRegistryEntry> {\n    if (!REGISTRY_KEY_PATTERN.test(key)) {\n      throw new VastServiceError(\n        \"registry_key must match [A-Za-z0-9._-]{1,64}\",\n        400,\n      );\n    }\n    const registry = await this.getRegistry(false);\n    const entry = registry[key];\n    if (!entry) {\n      throw new VastServiceError(\n        `Unknown registry_key '${key}'. Refresh /api/training/vast/models to update the cache.`,\n        400,\n      );\n    }\n    return entry;\n  }\n\n  // ── Jobs ──────────────────────────────────────────────────────────────\n\n  async listJobs(): Promise<VastJobRecord[]> {\n    return this.store.list();\n  }\n\n  async getJob(jobId: string): Promise<VastJobRecord | null> {\n    return this.store.get(jobId);\n  }\n\n  async createJob(input: CreateJobInput): Promise<VastJobRecord> {\n    await this.ensureRegistryKey(input.registry_key);\n    if (\n      !Number.isInteger(input.epochs) ||\n      input.epochs < 1 ||\n      input.epochs > 64\n    ) {\n      throw new VastServiceError(\n        \"epochs must be an integer between 1 and 64\",\n        400,\n      );\n    }\n    if (\n      input.run_name !== undefined &&\n      !RUN_NAME_PATTERN.test(input.run_name)\n    ) {\n      throw new VastServiceError(\n        \"run_name must match [A-Za-z0-9._-]{1,64}\",\n        400,\n      );\n    }\n    const job_id = makeJobId();\n    const run_name = input.run_name ?? defaultRunName(input.registry_key);\n    const now = new Date().toISOString();\n    const record: VastJobRecord = {\n      job_id,\n      run_name,\n      registry_key: input.registry_key,\n      status: \"queued\",\n      epochs: input.epochs,\n      created_at: now,\n      updated_at: now,\n      started_at: null,\n      ended_at: null,\n      vast_instance_id: null,\n      exit_code: null,\n      error: null,\n    };\n    await this.store.insert(record);\n    void this.dispatchJob(record);\n    return record;\n  }\n\n  async cancelJob(jobId: string): Promise<VastJobRecord> {\n    const job = await this.store.get(jobId);\n    if (!job) throw new VastServiceError(\"Job not found\", 404);\n    if (job.status === \"completed\" || job.status === \"failed\") {\n      throw new VastServiceError(\n        `Cannot cancel job in terminal state '${job.status}'`,\n        409,\n      );\n    }\n    if (job.status === \"cancelled\") return job;\n    return this.store.update(jobId, {\n      status: \"cancelled\",\n      ended_at: new Date().toISOString(),\n    });\n  }\n\n  async readJobLog(jobId: string, tailLines: number): Promise<string[]> {\n    const job = await this.store.get(jobId);\n    if (!job) throw new VastServiceError(\"Job not found\", 404);\n    return readJobLogTail(jobId, tailLines);\n  }\n\n  async runEval(\n    jobId: string,\n    input: Partial<EvalCheckpointInput>,\n  ): Promise<{\n    job_id: string;\n    exit_code: number;\n    out_path: string;\n    summary: Record<string, unknown> | null;\n  }> {\n    const job = await this.store.get(jobId);\n    if (!job) throw new VastServiceError(\"Job not found\", 404);\n    const evalScript = join(this.trainingRoot, \"scripts\", \"eval_checkpoint.py\");\n    if (!existsSync(evalScript)) {\n      throw new VastServiceError(\n        `eval_checkpoint.py not found at ${evalScript} — CheckpointSyncAgent has not landed it yet`,\n        503,\n      );\n    }\n    const checkpointDir =\n      input.checkpoint_dir ??\n      join(this.trainingRoot, \"checkpoints\", job.run_name, \"final\");\n    if (!isSafeCheckpointPath(checkpointDir, this.trainingRoot)) {\n      throw new VastServiceError(\n        \"checkpoint_dir must resolve under the training root\",\n        400,\n      );\n    }\n    const valJsonl = input.val_jsonl ?? \"data/smoke/val.jsonl\";\n    if (!isSafeRelativePath(valJsonl)) {\n      throw new VastServiceError(\n        \"val_jsonl must be a relative path without traversal\",\n        400,\n      );\n    }\n    const maxExamples = input.max_examples ?? 50;\n    if (\n      !Number.isInteger(maxExamples) ||\n      maxExamples < 1 ||\n      maxExamples > 5000\n    ) {\n      throw new VastServiceError(\n        \"max_examples must be an integer between 1 and 5000\",\n        400,\n      );\n    }\n    const outPath = join(\n      this.trainingRoot,\n      \"checkpoints\",\n      job.run_name,\n      `eval-${Date.now()}.json`,\n    );\n    const args = [\n      ...this.python.preArgs,\n      evalScript,\n      \"--checkpoint\",\n      checkpointDir,\n      \"--registry-key\",\n      job.registry_key,\n      \"--val-jsonl\",\n      valJsonl,\n      \"--max-examples\",\n      String(maxExamples),\n      \"--out\",\n      outPath,\n    ];\n    const exitCode = await this.runDetachedToLog(\n      jobId,\n      this.python.command,\n      args,\n      this.trainingRoot,\n    );\n    let summary: Record<string, unknown> | null = null;\n    if (existsSync(outPath)) {\n      try {\n        const raw = await fs.readFile(outPath, \"utf8\");\n        const parsed: unknown = JSON.parse(raw);\n        if (parsed && typeof parsed === \"object\" && !Array.isArray(parsed)) {\n          summary = parsed as Record<string, unknown>;\n        }\n      } catch (err) {\n        logger.warn(\n          `[VastTrainingService] failed to read eval output ${outPath}: ${err instanceof Error ? err.message : String(err)}`,\n        );\n      }\n    }\n    return { job_id: jobId, exit_code: exitCode, out_path: outPath, summary };\n  }\n\n  // ── Checkpoints ───────────────────────────────────────────────────────\n\n  async listCheckpointsForRegistryKey(\n    registryKey: string,\n  ): Promise<CheckpointInfo[]> {\n    await this.ensureRegistryKey(registryKey);\n    const checkpointsRoot = join(this.trainingRoot, \"checkpoints\");\n    if (!existsSync(checkpointsRoot)) return [];\n    const out: CheckpointInfo[] = [];\n    const runDirs = await fs.readdir(checkpointsRoot, { withFileTypes: true });\n    for (const runDir of runDirs) {\n      if (!runDir.isDirectory()) continue;\n      if (!runDir.name.startsWith(registryKey.replace(/\\./g, \"-\"))) continue;\n      const runPath = join(checkpointsRoot, runDir.name);\n      const status = join(runPath, \"STATUS.md\");\n      if (existsSync(status)) {\n        try {\n          const body = await fs.readFile(status, \"utf8\");\n          if (body.includes(\"FAILED RUN\")) continue;\n        } catch {\n          continue;\n        }\n      }\n      const ckptEntries = await fs.readdir(runPath, { withFileTypes: true });\n      for (const ckpt of ckptEntries) {\n        if (!ckpt.isDirectory()) continue;\n        if (!/^checkpoint-\\d+$|^final$/.test(ckpt.name)) continue;\n        const ckptPath = join(runPath, ckpt.name);\n        const stepMatch = /^checkpoint-(\\d+)$/.exec(ckpt.name);\n        const evalDoneFile = join(ckptPath, \"_eval.json\");\n        let evalSummary: Record<string, unknown> | null = null;\n        const evaluated = existsSync(evalDoneFile);\n        if (evaluated) {\n          try {\n            const raw = await fs.readFile(evalDoneFile, \"utf8\");\n            const parsed: unknown = JSON.parse(raw);\n            if (\n              parsed &&\n              typeof parsed === \"object\" &&\n              !Array.isArray(parsed)\n            ) {\n              evalSummary = parsed as Record<string, unknown>;\n            }\n          } catch {\n            evalSummary = null;\n          }\n        }\n        out.push({\n          name: `${runDir.name}/${ckpt.name}`,\n          path: ckptPath,\n          step: stepMatch ? Number(stepMatch[1]) : null,\n          evaluated,\n          eval_summary: evalSummary,\n        });\n      }\n    }\n    out.sort((a, b) => (b.step ?? -1) - (a.step ?? -1));\n    return out;\n  }\n\n  // ── Inference endpoints ───────────────────────────────────────────────\n\n  async listInferenceEndpoints(): Promise<InferenceEndpointRecord[]> {\n    return readInferenceEndpoints();\n  }\n\n  async createInferenceEndpoint(input: {\n    label: string;\n    base_url: string;\n    registry_key: string;\n  }): Promise<InferenceEndpointRecord> {\n    if (!LABEL_PATTERN.test(input.label)) {\n      throw new VastServiceError(\"label must match [A-Za-z0-9._-]{1,64}\", 400);\n    }\n    let url: URL;\n    try {\n      url = new URL(input.base_url);\n    } catch {\n      throw new VastServiceError(\"base_url must be a valid URL\", 400);\n    }\n    if (url.protocol !== \"http:\" && url.protocol !== \"https:\") {\n      throw new VastServiceError(\"base_url must use http:// or https://\", 400);\n    }\n    await this.ensureRegistryKey(input.registry_key);\n    const endpoints = await readInferenceEndpoints();\n    if (endpoints.some((e) => e.label === input.label)) {\n      throw new VastServiceError(\n        `Inference endpoint with label '${input.label}' already exists`,\n        409,\n      );\n    }\n    const record: InferenceEndpointRecord = {\n      id: `ep_${makeShortId()}`,\n      label: input.label,\n      base_url: input.base_url,\n      registry_key: input.registry_key,\n      created_at: new Date().toISOString(),\n    };\n    endpoints.push(record);\n    await writeInferenceEndpoints(endpoints);\n    return record;\n  }\n\n  async deleteInferenceEndpoint(id: string): Promise<boolean> {\n    const endpoints = await readInferenceEndpoints();\n    const next = endpoints.filter((e) => e.id !== id);\n    if (next.length === endpoints.length) return false;\n    await writeInferenceEndpoints(next);\n    return true;\n  }\n\n  /**\n   * Read a per-job budget snapshot via `scripts.lib.vast_budget snapshot\n   * --json`. Returns `null` when the job has no provisioned instance\n   * yet (the UI shows a \"not provisioned\" state in that case).\n   *\n   * The python module is the single source of truth — it both renders\n   * the watcher's status line and answers this endpoint, so the UI and\n   * the watcher never disagree about the budget state.\n   */\n  async getJobBudget(jobId: string): Promise<VastJobBudget | null> {\n    const job = await this.store.get(jobId);\n    if (!job) throw new VastServiceError(\"Job not found\", 404);\n    const instanceIdRaw = job.vast_instance_id;\n    if (!instanceIdRaw) return null;\n    const instanceId = Number(instanceIdRaw);\n    if (!Number.isFinite(instanceId) || instanceId <= 0) return null;\n    const dumpScript = \"-m\";\n    let stdout: string;\n    try {\n      stdout = await this.runCapture(\n        this.python.command,\n        [\n          ...this.python.preArgs,\n          dumpScript,\n          \"scripts.lib.vast_budget\",\n          \"snapshot\",\n          String(instanceId),\n          \"--pipeline\",\n          job.registry_key,\n          \"--run-name\",\n          job.run_name,\n          \"--json\",\n        ],\n        { cwd: this.trainingRoot },\n      );\n    } catch (err) {\n      // Treat any backend error (vastai unreachable, instance destroyed)\n      // as \"snapshot unavailable\" — the UI shows a stale-data marker\n      // rather than failing the whole panel.\n      logger.warn(\n        `[VastTrainingService] budget snapshot failed for ${jobId}: ${err instanceof Error ? err.message : String(err)}`,\n      );\n      return null;\n    }\n    const trimmed = stdout.trim();\n    if (!trimmed) return null;\n    let parsed: unknown;\n    try {\n      parsed = JSON.parse(trimmed);\n    } catch {\n      return null;\n    }\n    return narrowBudgetSnapshot(parsed, jobId);\n  }\n\n  async getInferenceStats(\n    label: string | null,\n    lastMinutes: number,\n  ): Promise<InferenceStatsAggregate> {\n    if (\n      !Number.isFinite(lastMinutes) ||\n      lastMinutes <= 0 ||\n      lastMinutes > 24 * 60\n    ) {\n      throw new VastServiceError(\n        \"last_minutes must be a positive number ≤ 1440\",\n        400,\n      );\n    }\n    if (label !== null && !LABEL_PATTERN.test(label)) {\n      throw new VastServiceError(\"label must match [A-Za-z0-9._-]{1,64}\", 400);\n    }\n    const path = inferenceStatsPath();\n    if (!existsSync(path)) {\n      return emptyInferenceStatsAggregate(label, lastMinutes);\n    }\n    const cutoffMs = Date.now() - lastMinutes * 60_000;\n    const raw = await fs.readFile(path, \"utf8\");\n    const samples: InferenceStatRow[] = [];\n    for (const line of raw.split(\"\\n\")) {\n      const trimmed = line.trim();\n      if (!trimmed) continue;\n      const row = parseStatRow(trimmed);\n      if (!row) continue;\n      if (label !== null && row.label !== label) continue;\n      const ts = Date.parse(row.ts);\n      if (!Number.isFinite(ts) || ts < cutoffMs) continue;\n      samples.push(row);\n    }\n    return aggregateInferenceStats(samples, label, lastMinutes);\n  }\n\n  // ── Internals ─────────────────────────────────────────────────────────\n\n  private async dispatchJob(record: VastJobRecord): Promise<void> {\n    const trainScript = join(this.trainingRoot, \"scripts\", \"train_vast.sh\");\n    if (!existsSync(trainScript)) {\n      logger.error(\n        `[VastTrainingService] train_vast.sh not found at ${trainScript}`,\n      );\n      await this.store.update(record.job_id, {\n        status: \"failed\",\n        ended_at: new Date().toISOString(),\n        error: \"train_vast.sh not found\",\n      });\n      return;\n    }\n    await this.store.update(record.job_id, {\n      status: \"provisioning\",\n      started_at: new Date().toISOString(),\n    });\n    const args = [\n      trainScript,\n      \"provision-and-train\",\n      \"--registry-key\",\n      record.registry_key,\n      \"--epochs\",\n      String(record.epochs),\n    ];\n    try {\n      const exitCode = await this.runDetachedToLog(\n        record.job_id,\n        \"bash\",\n        args,\n        this.trainingRoot,\n        {\n          RUN_NAME: record.run_name,\n        },\n      );\n      const instanceId = await this.discoverInstanceIdForRun(record);\n      if (exitCode === 0) {\n        await this.store.update(record.job_id, {\n          status: \"completed\",\n          ended_at: new Date().toISOString(),\n          exit_code: exitCode,\n          vast_instance_id: instanceId,\n        });\n      } else {\n        await this.store.update(record.job_id, {\n          status: \"failed\",\n          ended_at: new Date().toISOString(),\n          exit_code: exitCode,\n          vast_instance_id: instanceId,\n          error: `train_vast.sh exited with code ${exitCode}`,\n        });\n      }\n    } catch (err) {\n      logger.error(\n        `[VastTrainingService] dispatch failure for ${record.job_id}: ${err instanceof Error ? (err.stack ?? err.message) : String(err)}`,\n      );\n      await this.store.update(record.job_id, {\n        status: \"failed\",\n        ended_at: new Date().toISOString(),\n        error: err instanceof Error ? err.message : String(err),\n      });\n    }\n  }\n\n  private async discoverInstanceIdForRun(\n    record: VastJobRecord,\n  ): Promise<string | null> {\n    const stateFile = join(\n      this.trainingRoot,\n      \".eliza\",\n      \"vast-state\",\n      `${record.run_name}.json`,\n    );\n    if (existsSync(stateFile)) {\n      try {\n        const raw = await fs.readFile(stateFile, \"utf8\");\n        const parsed: unknown = JSON.parse(raw);\n        if (parsed && typeof parsed === \"object\") {\n          const obj = parsed as Record<string, unknown>;\n          const id = obj.instance_id ?? obj.vast_instance_id;\n          if (typeof id === \"string\" && id.trim()) return id.trim();\n          if (typeof id === \"number\") return String(id);\n        }\n      } catch (err) {\n        logger.warn(\n          `[VastTrainingService] could not parse vast-state file ${stateFile}: ${err instanceof Error ? err.message : String(err)}`,\n        );\n      }\n    }\n    const logTail = await readJobLogTail(record.job_id, 200);\n    for (const line of logTail) {\n      const match = /ELIZA_VAST_INSTANCE_ID=([A-Za-z0-9_-]+)/.exec(line);\n      if (match) return match[1];\n    }\n    return null;\n  }\n\n  private runCapture(\n    command: string,\n    args: string[],\n    options: { cwd: string },\n  ): Promise<string> {\n    return runCapture(this.spawnImpl, command, args, options);\n  }\n\n  private runDetachedToLog(\n    jobId: string,\n    command: string,\n    args: string[],\n    cwd: string,\n    extraEnv: NodeJS.ProcessEnv = {},\n  ): Promise<number> {\n    return runDetachedToLog(\n      this.spawnImpl,\n      jobId,\n      command,\n      args,\n      cwd,\n      extraEnv,\n    );\n  }\n}\n\n// ── Helpers ────────────────────────────────────────────────────────────\n\nfunction defaultRunName(registryKey: string): string {\n  return `${registryKey.replace(/\\./g, \"-\")}-apollo`;\n}\n\nfunction makeJobId(): string {\n  return `vjob_${makeShortId()}`;\n}\n\nfunction makeShortId(): string {\n  // 12 base36 chars from crypto-quality randomness.\n  const buf = new Uint8Array(8);\n  if (typeof globalThis.crypto.getRandomValues === \"function\") {\n    globalThis.crypto.getRandomValues(buf);\n  } else {\n    for (let i = 0; i < buf.length; i += 1)\n      buf[i] = Math.floor(Math.random() * 256);\n  }\n  let n = 0n;\n  for (const byte of buf) n = (n << 8n) | BigInt(byte);\n  return n.toString(36).padStart(12, \"0\").slice(0, 12);\n}\n\nfunction narrowRegistryEntry(value: unknown): VastRegistryEntry | null {\n  if (!value || typeof value !== \"object\") return null;\n  const obj = value as Record<string, unknown>;\n  if (\n    typeof obj.eliza_short_name === \"string\" &&\n    typeof obj.eliza_repo_id === \"string\" &&\n    typeof obj.gguf_repo_id === \"string\" &&\n    typeof obj.base_hf_id === \"string\" &&\n    typeof obj.tier === \"string\" &&\n    typeof obj.inference_max_context === \"number\"\n  ) {\n    return {\n      eliza_short_name: obj.eliza_short_name,\n      eliza_repo_id: obj.eliza_repo_id,\n      gguf_repo_id: obj.gguf_repo_id,\n      base_hf_id: obj.base_hf_id,\n      tier: obj.tier,\n      inference_max_context: obj.inference_max_context,\n    };\n  }\n  return null;\n}\n\nfunction narrowBudgetSnapshot(\n  raw: unknown,\n  jobId: string,\n): VastJobBudget | null {\n  if (!raw || typeof raw !== \"object\" || Array.isArray(raw)) return null;\n  const obj = raw as Record<string, unknown>;\n  const num = (k: string): number | null => {\n    const v = obj[k];\n    if (typeof v !== \"number\" || !Number.isFinite(v)) return null;\n    return v;\n  };\n  const str = (k: string): string | null => {\n    const v = obj[k];\n    return typeof v === \"string\" ? v : null;\n  };\n  const bool = (k: string): boolean | null => {\n    const v = obj[k];\n    return typeof v === \"boolean\" ? v : null;\n  };\n  const instanceId = num(\"instance_id\");\n  const pipeline = str(\"pipeline\");\n  const runName = str(\"run_name\");\n  const gpuName = str(\"gpu_name\");\n  const gpuSku = str(\"gpu_sku\");\n  const state = str(\"state\");\n  const uptime = num(\"uptime_seconds\");\n  const uptimePretty = str(\"uptime_pretty\");\n  const dph = num(\"dph_total\");\n  const total = num(\"total_so_far_usd\");\n  const fetchedAt = num(\"fetched_at\");\n  const numGpus = num(\"num_gpus\");\n  const overSoft = bool(\"over_soft\");\n  const overHard = bool(\"over_hard\");\n  if (\n    instanceId === null ||\n    !pipeline ||\n    runName === null ||\n    !gpuName ||\n    !gpuSku ||\n    !state ||\n    uptime === null ||\n    !uptimePretty ||\n    dph === null ||\n    total === null ||\n    fetchedAt === null ||\n    numGpus === null ||\n    overSoft === null ||\n    overHard === null\n  ) {\n    return null;\n  }\n  // Caps may legitimately be null when ELIZA_VAST_MAX_USD is unset.\n  const softRaw = obj.soft_cap_usd;\n  const hardRaw = obj.hard_cap_usd;\n  const soft =\n    typeof softRaw === \"number\" && Number.isFinite(softRaw) ? softRaw : null;\n  const hard =\n    typeof hardRaw === \"number\" && Number.isFinite(hardRaw) ? hardRaw : null;\n  return {\n    job_id: jobId,\n    instance_id: instanceId,\n    pipeline,\n    run_name: runName,\n    gpu_name: gpuName,\n    num_gpus: numGpus,\n    gpu_sku: gpuSku,\n    state,\n    uptime_seconds: uptime,\n    uptime_pretty: uptimePretty,\n    dph_total: dph,\n    total_so_far_usd: total,\n    soft_cap_usd: soft,\n    hard_cap_usd: hard,\n    over_soft: overSoft,\n    over_hard: overHard,\n    fetched_at: fetchedAt,\n  };\n}\n\nfunction isSafeRelativePath(p: string): boolean {\n  if (typeof p !== \"string\" || !p.trim()) return false;\n  if (p.startsWith(\"/\") || p.includes(\"..\")) return false;\n  return true;\n}\n\nfunction isSafeCheckpointPath(\n  checkpointDir: string,\n  trainingRoot: string,\n): boolean {\n  if (typeof checkpointDir !== \"string\" || !checkpointDir.trim()) return false;\n  const root = resolve(trainingRoot);\n  const resolved = resolve(checkpointDir);\n  return resolved === root || resolved.startsWith(`${root}/`);\n}\n"],"mappings":"AAyBA,SAAS,aAAa;AACtB,SAAS,YAAY,YAAY,UAAU;AAC3C,SAAS,MAAM,eAAe;AAC9B,SAAS,qBAAqB;AAC9B,SAAS,cAAc;AACvB;AAAA,EACE;AAAA,EACA;AAAA,EAGA;AAAA,OACK;AACP;AAAA,EAEE;AAAA,EACA;AAAA,EACA;AAAA,EAEA;AAAA,EACA;AAAA,OACK;AACP;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,OACK;AAEP,SAAS,oBAAAA,yBAAwB;AA+EjC,MAAM,iBAAyD;AAAA,EAC7D,SAAS;AAAA,EACT,SAAS,CAAC,OAAO,WAAW,QAAQ;AACtC;AAEA,MAAM,OAAO,cAAc,YAAY,GAAG;AAC1C,MAAM,wBAAwB;AAAA,EAC5B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAEA,MAAM,mBAAmB;AACzB,MAAM,gBAAgB;AAOtB,MAAM,uBAAuB;AAEtB,MAAM,oBAAoB;AAAA,EACd;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACT,gBAAqC;AAAA,EACrC,mBAAkC;AAAA,EAE1C,YAAY,UAAsC,CAAC,GAAG;AACpD,SAAK,QAAQ,QAAQ,SAAS,IAAI,aAAa;AAC/C,SAAK,eAAe,QAAQ,gBAAgB;AAC5C,SAAK,SAAS,QAAQ,kBAAkB;AACxC,SAAK,YAAY,QAAQ,aAAa;AAAA,EACxC;AAAA;AAAA,EAIA,MAAM,YAAY,UAAU,OAA8B;AACxD,QAAI,KAAK,iBAAiB,CAAC,QAAS,QAAO,KAAK;AAChD,UAAM,aAAa;AAAA,MACjB,KAAK;AAAA,MACL;AAAA,MACA;AAAA,IACF;AACA,QAAI,CAAC,WAAW,UAAU,GAAG;AAC3B,YAAM,IAAI;AAAA,QACR,qCAAqC,UAAU;AAAA,QAC/C;AAAA,MACF;AAAA,IACF;AACA,UAAM,SAAS,MAAM,KAAK;AAAA,MACxB,KAAK,OAAO;AAAA,MACZ,CAAC,GAAG,KAAK,OAAO,SAAS,UAAU;AAAA,MACnC,EAAE,KAAK,KAAK,aAAa;AAAA,IAC3B;AACA,QAAI;AACJ,QAAI;AACF,eAAS,KAAK,MAAM,MAAM;AAAA,IAC5B,SAAS,KAAK;AACZ,YAAM,IAAI;AAAA,QACR,2CAA2C,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,QAC3F;AAAA,MACF;AAAA,IACF;AACA,QAAI,CAAC,UAAU,OAAO,WAAW,YAAY,MAAM,QAAQ,MAAM,GAAG;AAClE,YAAM,IAAI;AAAA,QACR;AAAA,QACA;AAAA,MACF;AAAA,IACF;AACA,UAAM,MAAoB,CAAC;AAC3B,eAAW,CAAC,KAAK,KAAK,KAAK,OAAO;AAAA,MAChC;AAAA,IACF,GAAG;AACD,YAAM,QAAQ,oBAAoB,KAAK;AACvC,UAAI,MAAO,KAAI,GAAG,IAAI;AAAA,IACxB;AACA,SAAK,gBAAgB;AACrB,SAAK,oBAAmB,oBAAI,KAAK,GAAE,YAAY;AAC/C,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,aAAa,UAAU,OAG1B;AACD,UAAM,WAAW,MAAM,KAAK,YAAY,OAAO;AAC/C,UAAM,UAAU,OAAO,QAAQ,QAAQ,EAAE,IAAI,CAAC,CAAC,YAAY,KAAK,OAAO;AAAA,MACrE;AAAA,MACA;AAAA,IACF,EAAE;AACF,WAAO,EAAE,WAAW,KAAK,kBAAkB,QAAQ;AAAA,EACrD;AAAA,EAEA,MAAM,kBAAkB,KAAyC;AAC/D,QAAI,CAAC,qBAAqB,KAAK,GAAG,GAAG;AACnC,YAAM,IAAI;AAAA,QACR;AAAA,QACA;AAAA,MACF;AAAA,IACF;AACA,UAAM,WAAW,MAAM,KAAK,YAAY,KAAK;AAC7C,UAAM,QAAQ,SAAS,GAAG;AAC1B,QAAI,CAAC,OAAO;AACV,YAAM,IAAI;AAAA,QACR,yBAAyB,GAAG;AAAA,QAC5B;AAAA,MACF;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA;AAAA,EAIA,MAAM,WAAqC;AACzC,WAAO,KAAK,MAAM,KAAK;AAAA,EACzB;AAAA,EAEA,MAAM,OAAO,OAA8C;AACzD,WAAO,KAAK,MAAM,IAAI,KAAK;AAAA,EAC7B;AAAA,EAEA,MAAM,UAAU,OAA+C;AAC7D,UAAM,KAAK,kBAAkB,MAAM,YAAY;AAC/C,QACE,CAAC,OAAO,UAAU,MAAM,MAAM,KAC9B,MAAM,SAAS,KACf,MAAM,SAAS,IACf;AACA,YAAM,IAAI;AAAA,QACR;AAAA,QACA;AAAA,MACF;AAAA,IACF;AACA,QACE,MAAM,aAAa,UACnB,CAAC,iBAAiB,KAAK,MAAM,QAAQ,GACrC;AACA,YAAM,IAAI;AAAA,QACR;AAAA,QACA;AAAA,MACF;AAAA,IACF;AACA,UAAM,SAAS,UAAU;AACzB,UAAM,WAAW,MAAM,YAAY,eAAe,MAAM,YAAY;AACpE,UAAM,OAAM,oBAAI,KAAK,GAAE,YAAY;AACnC,UAAM,SAAwB;AAAA,MAC5B;AAAA,MACA;AAAA,MACA,cAAc,MAAM;AAAA,MACpB,QAAQ;AAAA,MACR,QAAQ,MAAM;AAAA,MACd,YAAY;AAAA,MACZ,YAAY;AAAA,MACZ,YAAY;AAAA,MACZ,UAAU;AAAA,MACV,kBAAkB;AAAA,MAClB,WAAW;AAAA,MACX,OAAO;AAAA,IACT;AACA,UAAM,KAAK,MAAM,OAAO,MAAM;AAC9B,SAAK,KAAK,YAAY,MAAM;AAC5B,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,UAAU,OAAuC;AACrD,UAAM,MAAM,MAAM,KAAK,MAAM,IAAI,KAAK;AACtC,QAAI,CAAC,IAAK,OAAM,IAAI,iBAAiB,iBAAiB,GAAG;AACzD,QAAI,IAAI,WAAW,eAAe,IAAI,WAAW,UAAU;AACzD,YAAM,IAAI;AAAA,QACR,wCAAwC,IAAI,MAAM;AAAA,QAClD;AAAA,MACF;AAAA,IACF;AACA,QAAI,IAAI,WAAW,YAAa,QAAO;AACvC,WAAO,KAAK,MAAM,OAAO,OAAO;AAAA,MAC9B,QAAQ;AAAA,MACR,WAAU,oBAAI,KAAK,GAAE,YAAY;AAAA,IACnC,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,WAAW,OAAe,WAAsC;AACpE,UAAM,MAAM,MAAM,KAAK,MAAM,IAAI,KAAK;AACtC,QAAI,CAAC,IAAK,OAAM,IAAI,iBAAiB,iBAAiB,GAAG;AACzD,WAAO,eAAe,OAAO,SAAS;AAAA,EACxC;AAAA,EAEA,MAAM,QACJ,OACA,OAMC;AACD,UAAM,MAAM,MAAM,KAAK,MAAM,IAAI,KAAK;AACtC,QAAI,CAAC,IAAK,OAAM,IAAI,iBAAiB,iBAAiB,GAAG;AACzD,UAAM,aAAa,KAAK,KAAK,cAAc,WAAW,oBAAoB;AAC1E,QAAI,CAAC,WAAW,UAAU,GAAG;AAC3B,YAAM,IAAI;AAAA,QACR,mCAAmC,UAAU;AAAA,QAC7C;AAAA,MACF;AAAA,IACF;AACA,UAAM,gBACJ,MAAM,kBACN,KAAK,KAAK,cAAc,eAAe,IAAI,UAAU,OAAO;AAC9D,QAAI,CAAC,qBAAqB,eAAe,KAAK,YAAY,GAAG;AAC3D,YAAM,IAAI;AAAA,QACR;AAAA,QACA;AAAA,MACF;AAAA,IACF;AACA,UAAM,WAAW,MAAM,aAAa;AACpC,QAAI,CAAC,mBAAmB,QAAQ,GAAG;AACjC,YAAM,IAAI;AAAA,QACR;AAAA,QACA;AAAA,MACF;AAAA,IACF;AACA,UAAM,cAAc,MAAM,gBAAgB;AAC1C,QACE,CAAC,OAAO,UAAU,WAAW,KAC7B,cAAc,KACd,cAAc,KACd;AACA,YAAM,IAAI;AAAA,QACR;AAAA,QACA;AAAA,MACF;AAAA,IACF;AACA,UAAM,UAAU;AAAA,MACd,KAAK;AAAA,MACL;AAAA,MACA,IAAI;AAAA,MACJ,QAAQ,KAAK,IAAI,CAAC;AAAA,IACpB;AACA,UAAM,OAAO;AAAA,MACX,GAAG,KAAK,OAAO;AAAA,MACf;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,IAAI;AAAA,MACJ;AAAA,MACA;AAAA,MACA;AAAA,MACA,OAAO,WAAW;AAAA,MAClB;AAAA,MACA;AAAA,IACF;AACA,UAAM,WAAW,MAAM,KAAK;AAAA,MAC1B;AAAA,MACA,KAAK,OAAO;AAAA,MACZ;AAAA,MACA,KAAK;AAAA,IACP;AACA,QAAI,UAA0C;AAC9C,QAAI,WAAW,OAAO,GAAG;AACvB,UAAI;AACF,cAAM,MAAM,MAAM,GAAG,SAAS,SAAS,MAAM;AAC7C,cAAM,SAAkB,KAAK,MAAM,GAAG;AACtC,YAAI,UAAU,OAAO,WAAW,YAAY,CAAC,MAAM,QAAQ,MAAM,GAAG;AAClE,oBAAU;AAAA,QACZ;AAAA,MACF,SAAS,KAAK;AACZ,eAAO;AAAA,UACL,oDAAoD,OAAO,KAAK,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,QAClH;AAAA,MACF;AAAA,IACF;AACA,WAAO,EAAE,QAAQ,OAAO,WAAW,UAAU,UAAU,SAAS,QAAQ;AAAA,EAC1E;AAAA;AAAA,EAIA,MAAM,8BACJ,aAC2B;AAC3B,UAAM,KAAK,kBAAkB,WAAW;AACxC,UAAM,kBAAkB,KAAK,KAAK,cAAc,aAAa;AAC7D,QAAI,CAAC,WAAW,eAAe,EAAG,QAAO,CAAC;AAC1C,UAAM,MAAwB,CAAC;AAC/B,UAAM,UAAU,MAAM,GAAG,QAAQ,iBAAiB,EAAE,eAAe,KAAK,CAAC;AACzE,eAAW,UAAU,SAAS;AAC5B,UAAI,CAAC,OAAO,YAAY,EAAG;AAC3B,UAAI,CAAC,OAAO,KAAK,WAAW,YAAY,QAAQ,OAAO,GAAG,CAAC,EAAG;AAC9D,YAAM,UAAU,KAAK,iBAAiB,OAAO,IAAI;AACjD,YAAM,SAAS,KAAK,SAAS,WAAW;AACxC,UAAI,WAAW,MAAM,GAAG;AACtB,YAAI;AACF,gBAAM,OAAO,MAAM,GAAG,SAAS,QAAQ,MAAM;AAC7C,cAAI,KAAK,SAAS,YAAY,EAAG;AAAA,QACnC,QAAQ;AACN;AAAA,QACF;AAAA,MACF;AACA,YAAM,cAAc,MAAM,GAAG,QAAQ,SAAS,EAAE,eAAe,KAAK,CAAC;AACrE,iBAAW,QAAQ,aAAa;AAC9B,YAAI,CAAC,KAAK,YAAY,EAAG;AACzB,YAAI,CAAC,2BAA2B,KAAK,KAAK,IAAI,EAAG;AACjD,cAAM,WAAW,KAAK,SAAS,KAAK,IAAI;AACxC,cAAM,YAAY,qBAAqB,KAAK,KAAK,IAAI;AACrD,cAAM,eAAe,KAAK,UAAU,YAAY;AAChD,YAAI,cAA8C;AAClD,cAAM,YAAY,WAAW,YAAY;AACzC,YAAI,WAAW;AACb,cAAI;AACF,kBAAM,MAAM,MAAM,GAAG,SAAS,cAAc,MAAM;AAClD,kBAAM,SAAkB,KAAK,MAAM,GAAG;AACtC,gBACE,UACA,OAAO,WAAW,YAClB,CAAC,MAAM,QAAQ,MAAM,GACrB;AACA,4BAAc;AAAA,YAChB;AAAA,UACF,QAAQ;AACN,0BAAc;AAAA,UAChB;AAAA,QACF;AACA,YAAI,KAAK;AAAA,UACP,MAAM,GAAG,OAAO,IAAI,IAAI,KAAK,IAAI;AAAA,UACjC,MAAM;AAAA,UACN,MAAM,YAAY,OAAO,UAAU,CAAC,CAAC,IAAI;AAAA,UACzC;AAAA,UACA,cAAc;AAAA,QAChB,CAAC;AAAA,MACH;AAAA,IACF;AACA,QAAI,KAAK,CAAC,GAAG,OAAO,EAAE,QAAQ,OAAO,EAAE,QAAQ,GAAG;AAClD,WAAO;AAAA,EACT;AAAA;AAAA,EAIA,MAAM,yBAA6D;AACjE,WAAO,uBAAuB;AAAA,EAChC;AAAA,EAEA,MAAM,wBAAwB,OAIO;AACnC,QAAI,CAAC,cAAc,KAAK,MAAM,KAAK,GAAG;AACpC,YAAM,IAAI,iBAAiB,yCAAyC,GAAG;AAAA,IACzE;AACA,QAAI;AACJ,QAAI;AACF,YAAM,IAAI,IAAI,MAAM,QAAQ;AAAA,IAC9B,QAAQ;AACN,YAAM,IAAI,iBAAiB,gCAAgC,GAAG;AAAA,IAChE;AACA,QAAI,IAAI,aAAa,WAAW,IAAI,aAAa,UAAU;AACzD,YAAM,IAAI,iBAAiB,yCAAyC,GAAG;AAAA,IACzE;AACA,UAAM,KAAK,kBAAkB,MAAM,YAAY;AAC/C,UAAM,YAAY,MAAM,uBAAuB;AAC/C,QAAI,UAAU,KAAK,CAAC,MAAM,EAAE,UAAU,MAAM,KAAK,GAAG;AAClD,YAAM,IAAI;AAAA,QACR,kCAAkC,MAAM,KAAK;AAAA,QAC7C;AAAA,MACF;AAAA,IACF;AACA,UAAM,SAAkC;AAAA,MACtC,IAAI,MAAM,YAAY,CAAC;AAAA,MACvB,OAAO,MAAM;AAAA,MACb,UAAU,MAAM;AAAA,MAChB,cAAc,MAAM;AAAA,MACpB,aAAY,oBAAI,KAAK,GAAE,YAAY;AAAA,IACrC;AACA,cAAU,KAAK,MAAM;AACrB,UAAM,wBAAwB,SAAS;AACvC,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,wBAAwB,IAA8B;AAC1D,UAAM,YAAY,MAAM,uBAAuB;AAC/C,UAAM,OAAO,UAAU,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE;AAChD,QAAI,KAAK,WAAW,UAAU,OAAQ,QAAO;AAC7C,UAAM,wBAAwB,IAAI;AAClC,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAWA,MAAM,aAAa,OAA8C;AAC/D,UAAM,MAAM,MAAM,KAAK,MAAM,IAAI,KAAK;AACtC,QAAI,CAAC,IAAK,OAAM,IAAI,iBAAiB,iBAAiB,GAAG;AACzD,UAAM,gBAAgB,IAAI;AAC1B,QAAI,CAAC,cAAe,QAAO;AAC3B,UAAM,aAAa,OAAO,aAAa;AACvC,QAAI,CAAC,OAAO,SAAS,UAAU,KAAK,cAAc,EAAG,QAAO;AAC5D,UAAM,aAAa;AACnB,QAAI;AACJ,QAAI;AACF,eAAS,MAAM,KAAK;AAAA,QAClB,KAAK,OAAO;AAAA,QACZ;AAAA,UACE,GAAG,KAAK,OAAO;AAAA,UACf;AAAA,UACA;AAAA,UACA;AAAA,UACA,OAAO,UAAU;AAAA,UACjB;AAAA,UACA,IAAI;AAAA,UACJ;AAAA,UACA,IAAI;AAAA,UACJ;AAAA,QACF;AAAA,QACA,EAAE,KAAK,KAAK,aAAa;AAAA,MAC3B;AAAA,IACF,SAAS,KAAK;AAIZ,aAAO;AAAA,QACL,oDAAoD,KAAK,KAAK,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,MAChH;AACA,aAAO;AAAA,IACT;AACA,UAAM,UAAU,OAAO,KAAK;AAC5B,QAAI,CAAC,QAAS,QAAO;AACrB,QAAI;AACJ,QAAI;AACF,eAAS,KAAK,MAAM,OAAO;AAAA,IAC7B,QAAQ;AACN,aAAO;AAAA,IACT;AACA,WAAO,qBAAqB,QAAQ,KAAK;AAAA,EAC3C;AAAA,EAEA,MAAM,kBACJ,OACA,aACkC;AAClC,QACE,CAAC,OAAO,SAAS,WAAW,KAC5B,eAAe,KACf,cAAc,KAAK,IACnB;AACA,YAAM,IAAI;AAAA,QACR;AAAA,QACA;AAAA,MACF;AAAA,IACF;AACA,QAAI,UAAU,QAAQ,CAAC,cAAc,KAAK,KAAK,GAAG;AAChD,YAAM,IAAI,iBAAiB,yCAAyC,GAAG;AAAA,IACzE;AACA,UAAM,OAAO,mBAAmB;AAChC,QAAI,CAAC,WAAW,IAAI,GAAG;AACrB,aAAO,6BAA6B,OAAO,WAAW;AAAA,IACxD;AACA,UAAM,WAAW,KAAK,IAAI,IAAI,cAAc;AAC5C,UAAM,MAAM,MAAM,GAAG,SAAS,MAAM,MAAM;AAC1C,UAAM,UAA8B,CAAC;AACrC,eAAW,QAAQ,IAAI,MAAM,IAAI,GAAG;AAClC,YAAM,UAAU,KAAK,KAAK;AAC1B,UAAI,CAAC,QAAS;AACd,YAAM,MAAM,aAAa,OAAO;AAChC,UAAI,CAAC,IAAK;AACV,UAAI,UAAU,QAAQ,IAAI,UAAU,MAAO;AAC3C,YAAM,KAAK,KAAK,MAAM,IAAI,EAAE;AAC5B,UAAI,CAAC,OAAO,SAAS,EAAE,KAAK,KAAK,SAAU;AAC3C,cAAQ,KAAK,GAAG;AAAA,IAClB;AACA,WAAO,wBAAwB,SAAS,OAAO,WAAW;AAAA,EAC5D;AAAA;AAAA,EAIA,MAAc,YAAY,QAAsC;AAC9D,UAAM,cAAc,KAAK,KAAK,cAAc,WAAW,eAAe;AACtE,QAAI,CAAC,WAAW,WAAW,GAAG;AAC5B,aAAO;AAAA,QACL,oDAAoD,WAAW;AAAA,MACjE;AACA,YAAM,KAAK,MAAM,OAAO,OAAO,QAAQ;AAAA,QACrC,QAAQ;AAAA,QACR,WAAU,oBAAI,KAAK,GAAE,YAAY;AAAA,QACjC,OAAO;AAAA,MACT,CAAC;AACD;AAAA,IACF;AACA,UAAM,KAAK,MAAM,OAAO,OAAO,QAAQ;AAAA,MACrC,QAAQ;AAAA,MACR,aAAY,oBAAI,KAAK,GAAE,YAAY;AAAA,IACrC,CAAC;AACD,UAAM,OAAO;AAAA,MACX;AAAA,MACA;AAAA,MACA;AAAA,MACA,OAAO;AAAA,MACP;AAAA,MACA,OAAO,OAAO,MAAM;AAAA,IACtB;AACA,QAAI;AACF,YAAM,WAAW,MAAM,KAAK;AAAA,QAC1B,OAAO;AAAA,QACP;AAAA,QACA;AAAA,QACA,KAAK;AAAA,QACL;AAAA,UACE,UAAU,OAAO;AAAA,QACnB;AAAA,MACF;AACA,YAAM,aAAa,MAAM,KAAK,yBAAyB,MAAM;AAC7D,UAAI,aAAa,GAAG;AAClB,cAAM,KAAK,MAAM,OAAO,OAAO,QAAQ;AAAA,UACrC,QAAQ;AAAA,UACR,WAAU,oBAAI,KAAK,GAAE,YAAY;AAAA,UACjC,WAAW;AAAA,UACX,kBAAkB;AAAA,QACpB,CAAC;AAAA,MACH,OAAO;AACL,cAAM,KAAK,MAAM,OAAO,OAAO,QAAQ;AAAA,UACrC,QAAQ;AAAA,UACR,WAAU,oBAAI,KAAK,GAAE,YAAY;AAAA,UACjC,WAAW;AAAA,UACX,kBAAkB;AAAA,UAClB,OAAO,kCAAkC,QAAQ;AAAA,QACnD,CAAC;AAAA,MACH;AAAA,IACF,SAAS,KAAK;AACZ,aAAO;AAAA,QACL,8CAA8C,OAAO,MAAM,KAAK,eAAe,QAAS,IAAI,SAAS,IAAI,UAAW,OAAO,GAAG,CAAC;AAAA,MACjI;AACA,YAAM,KAAK,MAAM,OAAO,OAAO,QAAQ;AAAA,QACrC,QAAQ;AAAA,QACR,WAAU,oBAAI,KAAK,GAAE,YAAY;AAAA,QACjC,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAAA,MACxD,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEA,MAAc,yBACZ,QACwB;AACxB,UAAM,YAAY;AAAA,MAChB,KAAK;AAAA,MACL;AAAA,MACA;AAAA,MACA,GAAG,OAAO,QAAQ;AAAA,IACpB;AACA,QAAI,WAAW,SAAS,GAAG;AACzB,UAAI;AACF,cAAM,MAAM,MAAM,GAAG,SAAS,WAAW,MAAM;AAC/C,cAAM,SAAkB,KAAK,MAAM,GAAG;AACtC,YAAI,UAAU,OAAO,WAAW,UAAU;AACxC,gBAAM,MAAM;AACZ,gBAAM,KAAK,IAAI,eAAe,IAAI;AAClC,cAAI,OAAO,OAAO,YAAY,GAAG,KAAK,EAAG,QAAO,GAAG,KAAK;AACxD,cAAI,OAAO,OAAO,SAAU,QAAO,OAAO,EAAE;AAAA,QAC9C;AAAA,MACF,SAAS,KAAK;AACZ,eAAO;AAAA,UACL,yDAAyD,SAAS,KAAK,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,QACzH;AAAA,MACF;AAAA,IACF;AACA,UAAM,UAAU,MAAM,eAAe,OAAO,QAAQ,GAAG;AACvD,eAAW,QAAQ,SAAS;AAC1B,YAAM,QAAQ,0CAA0C,KAAK,IAAI;AACjE,UAAI,MAAO,QAAO,MAAM,CAAC;AAAA,IAC3B;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,WACN,SACA,MACA,SACiB;AACjB,WAAO,WAAW,KAAK,WAAW,SAAS,MAAM,OAAO;AAAA,EAC1D;AAAA,EAEQ,iBACN,OACA,SACA,MACA,KACA,WAA8B,CAAC,GACd;AACjB,WAAO;AAAA,MACL,KAAK;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACF;AAIA,SAAS,eAAe,aAA6B;AACnD,SAAO,GAAG,YAAY,QAAQ,OAAO,GAAG,CAAC;AAC3C;AAEA,SAAS,YAAoB;AAC3B,SAAO,QAAQ,YAAY,CAAC;AAC9B;AAEA,SAAS,cAAsB;AAE7B,QAAM,MAAM,IAAI,WAAW,CAAC;AAC5B,MAAI,OAAO,WAAW,OAAO,oBAAoB,YAAY;AAC3D,eAAW,OAAO,gBAAgB,GAAG;AAAA,EACvC,OAAO;AACL,aAAS,IAAI,GAAG,IAAI,IAAI,QAAQ,KAAK;AACnC,UAAI,CAAC,IAAI,KAAK,MAAM,KAAK,OAAO,IAAI,GAAG;AAAA,EAC3C;AACA,MAAI,IAAI;AACR,aAAW,QAAQ,IAAK,KAAK,KAAK,KAAM,OAAO,IAAI;AACnD,SAAO,EAAE,SAAS,EAAE,EAAE,SAAS,IAAI,GAAG,EAAE,MAAM,GAAG,EAAE;AACrD;AAEA,SAAS,oBAAoB,OAA0C;AACrE,MAAI,CAAC,SAAS,OAAO,UAAU,SAAU,QAAO;AAChD,QAAM,MAAM;AACZ,MACE,OAAO,IAAI,qBAAqB,YAChC,OAAO,IAAI,kBAAkB,YAC7B,OAAO,IAAI,iBAAiB,YAC5B,OAAO,IAAI,eAAe,YAC1B,OAAO,IAAI,SAAS,YACpB,OAAO,IAAI,0BAA0B,UACrC;AACA,WAAO;AAAA,MACL,kBAAkB,IAAI;AAAA,MACtB,eAAe,IAAI;AAAA,MACnB,cAAc,IAAI;AAAA,MAClB,YAAY,IAAI;AAAA,MAChB,MAAM,IAAI;AAAA,MACV,uBAAuB,IAAI;AAAA,IAC7B;AAAA,EACF;AACA,SAAO;AACT;AAEA,SAAS,qBACP,KACA,OACsB;AACtB,MAAI,CAAC,OAAO,OAAO,QAAQ,YAAY,MAAM,QAAQ,GAAG,EAAG,QAAO;AAClE,QAAM,MAAM;AACZ,QAAM,MAAM,CAAC,MAA6B;AACxC,UAAM,IAAI,IAAI,CAAC;AACf,QAAI,OAAO,MAAM,YAAY,CAAC,OAAO,SAAS,CAAC,EAAG,QAAO;AACzD,WAAO;AAAA,EACT;AACA,QAAM,MAAM,CAAC,MAA6B;AACxC,UAAM,IAAI,IAAI,CAAC;AACf,WAAO,OAAO,MAAM,WAAW,IAAI;AAAA,EACrC;AACA,QAAM,OAAO,CAAC,MAA8B;AAC1C,UAAM,IAAI,IAAI,CAAC;AACf,WAAO,OAAO,MAAM,YAAY,IAAI;AAAA,EACtC;AACA,QAAM,aAAa,IAAI,aAAa;AACpC,QAAM,WAAW,IAAI,UAAU;AAC/B,QAAM,UAAU,IAAI,UAAU;AAC9B,QAAM,UAAU,IAAI,UAAU;AAC9B,QAAM,SAAS,IAAI,SAAS;AAC5B,QAAM,QAAQ,IAAI,OAAO;AACzB,QAAM,SAAS,IAAI,gBAAgB;AACnC,QAAM,eAAe,IAAI,eAAe;AACxC,QAAM,MAAM,IAAI,WAAW;AAC3B,QAAM,QAAQ,IAAI,kBAAkB;AACpC,QAAM,YAAY,IAAI,YAAY;AAClC,QAAM,UAAU,IAAI,UAAU;AAC9B,QAAM,WAAW,KAAK,WAAW;AACjC,QAAM,WAAW,KAAK,WAAW;AACjC,MACE,eAAe,QACf,CAAC,YACD,YAAY,QACZ,CAAC,WACD,CAAC,UACD,CAAC,SACD,WAAW,QACX,CAAC,gBACD,QAAQ,QACR,UAAU,QACV,cAAc,QACd,YAAY,QACZ,aAAa,QACb,aAAa,MACb;AACA,WAAO;AAAA,EACT;AAEA,QAAM,UAAU,IAAI;AACpB,QAAM,UAAU,IAAI;AACpB,QAAM,OACJ,OAAO,YAAY,YAAY,OAAO,SAAS,OAAO,IAAI,UAAU;AACtE,QAAM,OACJ,OAAO,YAAY,YAAY,OAAO,SAAS,OAAO,IAAI,UAAU;AACtE,SAAO;AAAA,IACL,QAAQ;AAAA,IACR,aAAa;AAAA,IACb;AAAA,IACA,UAAU;AAAA,IACV,UAAU;AAAA,IACV,UAAU;AAAA,IACV,SAAS;AAAA,IACT;AAAA,IACA,gBAAgB;AAAA,IAChB,eAAe;AAAA,IACf,WAAW;AAAA,IACX,kBAAkB;AAAA,IAClB,cAAc;AAAA,IACd,cAAc;AAAA,IACd,WAAW;AAAA,IACX,WAAW;AAAA,IACX,YAAY;AAAA,EACd;AACF;AAEA,SAAS,mBAAmB,GAAoB;AAC9C,MAAI,OAAO,MAAM,YAAY,CAAC,EAAE,KAAK,EAAG,QAAO;AAC/C,MAAI,EAAE,WAAW,GAAG,KAAK,EAAE,SAAS,IAAI,EAAG,QAAO;AAClD,SAAO;AACT;AAEA,SAAS,qBACP,eACA,cACS;AACT,MAAI,OAAO,kBAAkB,YAAY,CAAC,cAAc,KAAK,EAAG,QAAO;AACvE,QAAM,OAAO,QAAQ,YAAY;AACjC,QAAM,WAAW,QAAQ,aAAa;AACtC,SAAO,aAAa,QAAQ,SAAS,WAAW,GAAG,IAAI,GAAG;AAC5D;","names":["VastServiceError"]}

package/dist/services/vast-inference-stats.d.ts

@@ -0,0 +1,37 @@
+/**
+ * Inference-stats JSONL parsing + windowed aggregation.
+ *
+ * Sister-agent contract for `~/.eliza/inference-stats.jsonl` rows:
+ *   {"ts","label","tokens_per_sec","p50_tpot_ms","p95_tpot_ms",
+ *    "kv_cache_usage_pct","num_requests_running","spec_decode_accept_rate",
+ *    "apc_hit_rate","peak_vram_mb"}
+ * Error rows take the form `{"ts","label","error"}` and are skipped on read.
+ */
+export interface InferenceStatRow {
+    ts: string;
+    label: string;
+    tokens_per_sec: number | null;
+    p50_tpot_ms: number | null;
+    p95_tpot_ms: number | null;
+    kv_cache_usage_pct: number | null;
+    num_requests_running: number | null;
+    spec_decode_accept_rate: number | null;
+    apc_hit_rate: number | null;
+    peak_vram_mb: number | null;
+}
+export interface InferenceStatsAggregate {
+    label: string | null;
+    window_minutes: number;
+    sample_count: number;
+    tokens_per_sec_avg: number | null;
+    p50_tpot_ms_avg: number | null;
+    p95_tpot_ms_max: number | null;
+    kv_cache_usage_pct_avg: number | null;
+    spec_decode_accept_rate_avg: number | null;
+    apc_hit_rate_avg: number | null;
+    peak_vram_mb_max: number | null;
+}
+export declare function parseStatRow(line: string): InferenceStatRow | null;
+export declare function aggregateInferenceStats(samples: InferenceStatRow[], label: string | null, lastMinutes: number): InferenceStatsAggregate;
+export declare function emptyInferenceStatsAggregate(label: string | null, lastMinutes: number): InferenceStatsAggregate;
+//# sourceMappingURL=vast-inference-stats.d.ts.map

package/dist/services/vast-inference-stats.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"vast-inference-stats.d.ts","sourceRoot":"","sources":["../../src/services/vast-inference-stats.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,oBAAoB,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,uBAAuB,EAAE,MAAM,GAAG,IAAI,CAAC;IACvC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;CAC7B;AAED,MAAM,WAAW,uBAAuB;IACtC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,sBAAsB,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC,2BAA2B,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3C,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;CACjC;AAED,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,gBAAgB,GAAG,IAAI,CAuBlE;AAED,wBAAgB,uBAAuB,CACrC,OAAO,EAAE,gBAAgB,EAAE,EAC3B,KAAK,EAAE,MAAM,GAAG,IAAI,EACpB,WAAW,EAAE,MAAM,GAClB,uBAAuB,CAiBzB;AAED,wBAAgB,4BAA4B,CAC1C,KAAK,EAAE,MAAM,GAAG,IAAI,EACpB,WAAW,EAAE,MAAM,GAClB,uBAAuB,CAazB"}

package/dist/services/vast-inference-stats.js

@@ -0,0 +1,81 @@
+function parseStatRow(line) {
+  let raw;
+  try {
+    raw = JSON.parse(line);
+  } catch {
+    return null;
+  }
+  if (!raw || typeof raw !== "object") return null;
+  const obj = raw;
+  if (typeof obj.error === "string") return null;
+  if (typeof obj.ts !== "string" || typeof obj.label !== "string") return null;
+  return {
+    ts: obj.ts,
+    label: obj.label,
+    tokens_per_sec: numericOrNull(obj.tokens_per_sec),
+    p50_tpot_ms: numericOrNull(obj.p50_tpot_ms),
+    p95_tpot_ms: numericOrNull(obj.p95_tpot_ms),
+    kv_cache_usage_pct: numericOrNull(obj.kv_cache_usage_pct),
+    num_requests_running: numericOrNull(obj.num_requests_running),
+    spec_decode_accept_rate: numericOrNull(obj.spec_decode_accept_rate),
+    apc_hit_rate: numericOrNull(obj.apc_hit_rate),
+    peak_vram_mb: numericOrNull(obj.peak_vram_mb)
+  };
+}
+function aggregateInferenceStats(samples, label, lastMinutes) {
+  if (samples.length === 0)
+    return emptyInferenceStatsAggregate(label, lastMinutes);
+  return {
+    label,
+    window_minutes: lastMinutes,
+    sample_count: samples.length,
+    tokens_per_sec_avg: avg(samples.map((s) => s.tokens_per_sec)),
+    p50_tpot_ms_avg: avg(samples.map((s) => s.p50_tpot_ms)),
+    p95_tpot_ms_max: maxOf(samples.map((s) => s.p95_tpot_ms)),
+    kv_cache_usage_pct_avg: avg(samples.map((s) => s.kv_cache_usage_pct)),
+    spec_decode_accept_rate_avg: avg(
+      samples.map((s) => s.spec_decode_accept_rate)
+    ),
+    apc_hit_rate_avg: avg(samples.map((s) => s.apc_hit_rate)),
+    peak_vram_mb_max: maxOf(samples.map((s) => s.peak_vram_mb))
+  };
+}
+function emptyInferenceStatsAggregate(label, lastMinutes) {
+  return {
+    label,
+    window_minutes: lastMinutes,
+    sample_count: 0,
+    tokens_per_sec_avg: null,
+    p50_tpot_ms_avg: null,
+    p95_tpot_ms_max: null,
+    kv_cache_usage_pct_avg: null,
+    spec_decode_accept_rate_avg: null,
+    apc_hit_rate_avg: null,
+    peak_vram_mb_max: null
+  };
+}
+function numericOrNull(value) {
+  return typeof value === "number" && Number.isFinite(value) ? value : null;
+}
+function avg(values) {
+  const nums = values.filter((v) => v !== null);
+  if (nums.length === 0) return null;
+  let sum = 0;
+  for (const n of nums) sum += n;
+  return sum / nums.length;
+}
+function maxOf(values) {
+  const nums = values.filter((v) => v !== null);
+  if (nums.length === 0) return null;
+  let m = nums[0];
+  for (let i = 1; i < nums.length; i += 1) {
+    if (nums[i] > m) m = nums[i];
+  }
+  return m;
+}
+export {
+  aggregateInferenceStats,
+  emptyInferenceStatsAggregate,
+  parseStatRow
+};
+//# sourceMappingURL=vast-inference-stats.js.map

package/dist/services/vast-inference-stats.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/services/vast-inference-stats.ts"],"sourcesContent":["/**\n * Inference-stats JSONL parsing + windowed aggregation.\n *\n * Sister-agent contract for `~/.eliza/inference-stats.jsonl` rows:\n *   {\"ts\",\"label\",\"tokens_per_sec\",\"p50_tpot_ms\",\"p95_tpot_ms\",\n *    \"kv_cache_usage_pct\",\"num_requests_running\",\"spec_decode_accept_rate\",\n *    \"apc_hit_rate\",\"peak_vram_mb\"}\n * Error rows take the form `{\"ts\",\"label\",\"error\"}` and are skipped on read.\n */\n\nexport interface InferenceStatRow {\n  ts: string;\n  label: string;\n  tokens_per_sec: number | null;\n  p50_tpot_ms: number | null;\n  p95_tpot_ms: number | null;\n  kv_cache_usage_pct: number | null;\n  num_requests_running: number | null;\n  spec_decode_accept_rate: number | null;\n  apc_hit_rate: number | null;\n  peak_vram_mb: number | null;\n}\n\nexport interface InferenceStatsAggregate {\n  label: string | null;\n  window_minutes: number;\n  sample_count: number;\n  tokens_per_sec_avg: number | null;\n  p50_tpot_ms_avg: number | null;\n  p95_tpot_ms_max: number | null;\n  kv_cache_usage_pct_avg: number | null;\n  spec_decode_accept_rate_avg: number | null;\n  apc_hit_rate_avg: number | null;\n  peak_vram_mb_max: number | null;\n}\n\nexport function parseStatRow(line: string): InferenceStatRow | null {\n  let raw: unknown;\n  try {\n    raw = JSON.parse(line);\n  } catch {\n    return null;\n  }\n  if (!raw || typeof raw !== \"object\") return null;\n  const obj = raw as Record<string, unknown>;\n  if (typeof obj.error === \"string\") return null; // error rows are skipped per spec\n  if (typeof obj.ts !== \"string\" || typeof obj.label !== \"string\") return null;\n  return {\n    ts: obj.ts,\n    label: obj.label,\n    tokens_per_sec: numericOrNull(obj.tokens_per_sec),\n    p50_tpot_ms: numericOrNull(obj.p50_tpot_ms),\n    p95_tpot_ms: numericOrNull(obj.p95_tpot_ms),\n    kv_cache_usage_pct: numericOrNull(obj.kv_cache_usage_pct),\n    num_requests_running: numericOrNull(obj.num_requests_running),\n    spec_decode_accept_rate: numericOrNull(obj.spec_decode_accept_rate),\n    apc_hit_rate: numericOrNull(obj.apc_hit_rate),\n    peak_vram_mb: numericOrNull(obj.peak_vram_mb),\n  };\n}\n\nexport function aggregateInferenceStats(\n  samples: InferenceStatRow[],\n  label: string | null,\n  lastMinutes: number,\n): InferenceStatsAggregate {\n  if (samples.length === 0)\n    return emptyInferenceStatsAggregate(label, lastMinutes);\n  return {\n    label,\n    window_minutes: lastMinutes,\n    sample_count: samples.length,\n    tokens_per_sec_avg: avg(samples.map((s) => s.tokens_per_sec)),\n    p50_tpot_ms_avg: avg(samples.map((s) => s.p50_tpot_ms)),\n    p95_tpot_ms_max: maxOf(samples.map((s) => s.p95_tpot_ms)),\n    kv_cache_usage_pct_avg: avg(samples.map((s) => s.kv_cache_usage_pct)),\n    spec_decode_accept_rate_avg: avg(\n      samples.map((s) => s.spec_decode_accept_rate),\n    ),\n    apc_hit_rate_avg: avg(samples.map((s) => s.apc_hit_rate)),\n    peak_vram_mb_max: maxOf(samples.map((s) => s.peak_vram_mb)),\n  };\n}\n\nexport function emptyInferenceStatsAggregate(\n  label: string | null,\n  lastMinutes: number,\n): InferenceStatsAggregate {\n  return {\n    label,\n    window_minutes: lastMinutes,\n    sample_count: 0,\n    tokens_per_sec_avg: null,\n    p50_tpot_ms_avg: null,\n    p95_tpot_ms_max: null,\n    kv_cache_usage_pct_avg: null,\n    spec_decode_accept_rate_avg: null,\n    apc_hit_rate_avg: null,\n    peak_vram_mb_max: null,\n  };\n}\n\nfunction numericOrNull(value: unknown): number | null {\n  return typeof value === \"number\" && Number.isFinite(value) ? value : null;\n}\n\nfunction avg(values: Array<number | null>): number | null {\n  const nums = values.filter((v): v is number => v !== null);\n  if (nums.length === 0) return null;\n  let sum = 0;\n  for (const n of nums) sum += n;\n  return sum / nums.length;\n}\n\nfunction maxOf(values: Array<number | null>): number | null {\n  const nums = values.filter((v): v is number => v !== null);\n  if (nums.length === 0) return null;\n  let m = nums[0];\n  for (let i = 1; i < nums.length; i += 1) {\n    if (nums[i] > m) m = nums[i];\n  }\n  return m;\n}\n"],"mappings":"AAoCO,SAAS,aAAa,MAAuC;AAClE,MAAI;AACJ,MAAI;AACF,UAAM,KAAK,MAAM,IAAI;AAAA,EACvB,QAAQ;AACN,WAAO;AAAA,EACT;AACA,MAAI,CAAC,OAAO,OAAO,QAAQ,SAAU,QAAO;AAC5C,QAAM,MAAM;AACZ,MAAI,OAAO,IAAI,UAAU,SAAU,QAAO;AAC1C,MAAI,OAAO,IAAI,OAAO,YAAY,OAAO,IAAI,UAAU,SAAU,QAAO;AACxE,SAAO;AAAA,IACL,IAAI,IAAI;AAAA,IACR,OAAO,IAAI;AAAA,IACX,gBAAgB,cAAc,IAAI,cAAc;AAAA,IAChD,aAAa,cAAc,IAAI,WAAW;AAAA,IAC1C,aAAa,cAAc,IAAI,WAAW;AAAA,IAC1C,oBAAoB,cAAc,IAAI,kBAAkB;AAAA,IACxD,sBAAsB,cAAc,IAAI,oBAAoB;AAAA,IAC5D,yBAAyB,cAAc,IAAI,uBAAuB;AAAA,IAClE,cAAc,cAAc,IAAI,YAAY;AAAA,IAC5C,cAAc,cAAc,IAAI,YAAY;AAAA,EAC9C;AACF;AAEO,SAAS,wBACd,SACA,OACA,aACyB;AACzB,MAAI,QAAQ,WAAW;AACrB,WAAO,6BAA6B,OAAO,WAAW;AACxD,SAAO;AAAA,IACL;AAAA,IACA,gBAAgB;AAAA,IAChB,cAAc,QAAQ;AAAA,IACtB,oBAAoB,IAAI,QAAQ,IAAI,CAAC,MAAM,EAAE,cAAc,CAAC;AAAA,IAC5D,iBAAiB,IAAI,QAAQ,IAAI,CAAC,MAAM,EAAE,WAAW,CAAC;AAAA,IACtD,iBAAiB,MAAM,QAAQ,IAAI,CAAC,MAAM,EAAE,WAAW,CAAC;AAAA,IACxD,wBAAwB,IAAI,QAAQ,IAAI,CAAC,MAAM,EAAE,kBAAkB,CAAC;AAAA,IACpE,6BAA6B;AAAA,MAC3B,QAAQ,IAAI,CAAC,MAAM,EAAE,uBAAuB;AAAA,IAC9C;AAAA,IACA,kBAAkB,IAAI,QAAQ,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC;AAAA,IACxD,kBAAkB,MAAM,QAAQ,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC;AAAA,EAC5D;AACF;AAEO,SAAS,6BACd,OACA,aACyB;AACzB,SAAO;AAAA,IACL;AAAA,IACA,gBAAgB;AAAA,IAChB,cAAc;AAAA,IACd,oBAAoB;AAAA,IACpB,iBAAiB;AAAA,IACjB,iBAAiB;AAAA,IACjB,wBAAwB;AAAA,IACxB,6BAA6B;AAAA,IAC7B,kBAAkB;AAAA,IAClB,kBAAkB;AAAA,EACpB;AACF;AAEA,SAAS,cAAc,OAA+B;AACpD,SAAO,OAAO,UAAU,YAAY,OAAO,SAAS,KAAK,IAAI,QAAQ;AACvE;AAEA,SAAS,IAAI,QAA6C;AACxD,QAAM,OAAO,OAAO,OAAO,CAAC,MAAmB,MAAM,IAAI;AACzD,MAAI,KAAK,WAAW,EAAG,QAAO;AAC9B,MAAI,MAAM;AACV,aAAW,KAAK,KAAM,QAAO;AAC7B,SAAO,MAAM,KAAK;AACpB;AAEA,SAAS,MAAM,QAA6C;AAC1D,QAAM,OAAO,OAAO,OAAO,CAAC,MAAmB,MAAM,IAAI;AACzD,MAAI,KAAK,WAAW,EAAG,QAAO;AAC9B,MAAI,IAAI,KAAK,CAAC;AACd,WAAS,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK,GAAG;AACvC,QAAI,KAAK,CAAC,IAAI,EAAG,KAAI,KAAK,CAAC;AAAA,EAC7B;AACA,SAAO;AACT;","names":[]}

package/dist/services/vast-job-store.d.ts

@@ -0,0 +1,74 @@
+/**
+ * Vast.ai training job + inference-stats persistence.
+ *
+ * Mirrors the in-memory + JSONL pattern used elsewhere in this app:
+ * `TrainingService` keeps records in process memory, while we additionally
+ * append every state transition to a JSONL log so jobs survive restarts.
+ *
+ * Storage layout (under `resolveStateDir()`):
+ *   ~/.eliza/training-vast/jobs.jsonl       — append-only job event log.
+ *   ~/.eliza/training-vast/logs/<id>.log    — per-job stdout/stderr tail.
+ *
+ * The inference-stats reader points at `~/.eliza/inference-stats.jsonl`,
+ * the path the inference side writes to (overridable via env).
+ */
+export type VastJobStatus = "queued" | "provisioning" | "running" | "completed" | "failed" | "cancelled";
+export interface VastJobRecord {
+    job_id: string;
+    run_name: string;
+    registry_key: string;
+    status: VastJobStatus;
+    epochs: number;
+    created_at: string;
+    updated_at: string;
+    started_at: string | null;
+    ended_at: string | null;
+    vast_instance_id: string | null;
+    exit_code: number | null;
+    error: string | null;
+}
+export interface VastJobUpdate {
+    status?: VastJobStatus;
+    vast_instance_id?: string | null;
+    exit_code?: number | null;
+    error?: string | null;
+    started_at?: string | null;
+    ended_at?: string | null;
+}
+export declare function jobLogPath(jobId: string): string;
+export declare function inferenceStatsPath(): string;
+/**
+ * Append-only job store. The cache is the source of truth at runtime;
+ * the JSONL file is the durability log we replay on first read.
+ */
+export declare class VastJobStore {
+    private cache;
+    private hydratePromise;
+    list(): Promise<VastJobRecord[]>;
+    get(jobId: string): Promise<VastJobRecord | null>;
+    insert(record: VastJobRecord): Promise<VastJobRecord>;
+    update(jobId: string, patch: VastJobUpdate): Promise<VastJobRecord>;
+    /** Test-only: drop in-memory cache so the next read re-hydrates from disk. */
+    resetCacheForTests(): void;
+    private ensureHydrated;
+    private hydrate;
+    private appendEvent;
+}
+/** Read the tail of a job's stdout/stderr log file. Empty array if no log. */
+export declare function readJobLogTail(jobId: string, tailLines: number): Promise<string[]>;
+/**
+ * Append a chunk of subprocess output (stdout + stderr merged) to the
+ * per-job log file, creating the directory as needed.
+ */
+export declare function appendJobLog(jobId: string, chunk: string): Promise<void>;
+/** Inference endpoint registry — single JSON file under the vast root. */
+export interface InferenceEndpointRecord {
+    id: string;
+    label: string;
+    base_url: string;
+    registry_key: string;
+    created_at: string;
+}
+export declare function readInferenceEndpoints(): Promise<InferenceEndpointRecord[]>;
+export declare function writeInferenceEndpoints(endpoints: InferenceEndpointRecord[]): Promise<void>;
+//# sourceMappingURL=vast-job-store.d.ts.map

package/dist/services/vast-job-store.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"vast-job-store.d.ts","sourceRoot":"","sources":["../../src/services/vast-job-store.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAMH,MAAM,MAAM,aAAa,GACrB,QAAQ,GACR,cAAc,GACd,SAAS,GACT,WAAW,GACX,QAAQ,GACR,WAAW,CAAC;AAEhB,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,aAAa,CAAC;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;CACtB;AAED,MAAM,WAAW,aAAa;IAC5B,MAAM,CAAC,EAAE,aAAa,CAAC;IACvB,gBAAgB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B;AAcD,wBAAgB,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAEhD;AAED,wBAAgB,kBAAkB,IAAI,MAAM,CAI3C;AAQD;;;GAGG;AACH,qBAAa,YAAY;IACvB,OAAO,CAAC,KAAK,CAA2C;IACxD,OAAO,CAAC,cAAc,CAA8B;IAE9C,IAAI,IAAI,OAAO,CAAC,aAAa,EAAE,CAAC;IAOhC,GAAG,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC;IAKjD,MAAM,CAAC,MAAM,EAAE,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC;IAUrD,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC;IAyBzE,8EAA8E;IAC9E,kBAAkB,IAAI,IAAI;YAKZ,cAAc;YAYd,OAAO;YAiBP,WAAW;CAK1B;AAuCD,8EAA8E;AAC9E,wBAAsB,cAAc,CAClC,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,MAAM,EAAE,CAAC,CAUnB;AAED;;;GAGG;AACH,wBAAsB,YAAY,CAChC,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,IAAI,CAAC,CAIf;AAED,0EAA0E;AAC1E,MAAM,WAAW,uBAAuB;IACtC,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;CACpB;AAQD,wBAAsB,sBAAsB,IAAI,OAAO,CACrD,uBAAuB,EAAE,CAC1B,CAgCA;AAED,wBAAsB,uBAAuB,CAC3C,SAAS,EAAE,uBAAuB,EAAE,GACnC,OAAO,CAAC,IAAI,CAAC,CAIf"}

package/dist/services/vast-job-store.js

@@ -0,0 +1,194 @@
+import { existsSync, promises as fs, mkdirSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { resolveStateDir } from "@elizaos/core";
+const VAST_DIR_NAME = "training-vast";
+const JOBS_LOG_NAME = "jobs.jsonl";
+const LOGS_SUBDIR = "logs";
+function vastRoot() {
+  return join(resolveStateDir(), VAST_DIR_NAME);
+}
+function jobsLogPath() {
+  return join(vastRoot(), JOBS_LOG_NAME);
+}
+function jobLogPath(jobId) {
+  return join(vastRoot(), LOGS_SUBDIR, `${jobId}.log`);
+}
+function inferenceStatsPath() {
+  const override = process.env.ELIZA_INFERENCE_STATS_PATH;
+  if (override?.trim()) return override.trim();
+  return join(resolveStateDir(), "inference-stats.jsonl");
+}
+function ensureDir(path) {
+  if (!existsSync(path)) {
+    mkdirSync(path, { recursive: true });
+  }
+}
+class VastJobStore {
+  cache = null;
+  hydratePromise = null;
+  async list() {
+    const cache = await this.ensureHydrated();
+    return Array.from(cache.values()).sort(
+      (a, b) => b.created_at.localeCompare(a.created_at)
+    );
+  }
+  async get(jobId) {
+    const cache = await this.ensureHydrated();
+    return cache.get(jobId) ?? null;
+  }
+  async insert(record) {
+    const cache = await this.ensureHydrated();
+    if (cache.has(record.job_id)) {
+      throw new Error(`Job ${record.job_id} already exists`);
+    }
+    cache.set(record.job_id, record);
+    await this.appendEvent(record);
+    return record;
+  }
+  async update(jobId, patch) {
+    const cache = await this.ensureHydrated();
+    const existing = cache.get(jobId);
+    if (!existing) throw new Error(`Job ${jobId} not found`);
+    const next = {
+      ...existing,
+      ...patch,
+      vast_instance_id: patch.vast_instance_id !== void 0 ? patch.vast_instance_id : existing.vast_instance_id,
+      exit_code: patch.exit_code !== void 0 ? patch.exit_code : existing.exit_code,
+      error: patch.error !== void 0 ? patch.error : existing.error,
+      started_at: patch.started_at !== void 0 ? patch.started_at : existing.started_at,
+      ended_at: patch.ended_at !== void 0 ? patch.ended_at : existing.ended_at,
+      updated_at: (/* @__PURE__ */ new Date()).toISOString()
+    };
+    cache.set(jobId, next);
+    await this.appendEvent(next);
+    return next;
+  }
+  /** Test-only: drop in-memory cache so the next read re-hydrates from disk. */
+  resetCacheForTests() {
+    this.cache = null;
+    this.hydratePromise = null;
+  }
+  async ensureHydrated() {
+    if (this.cache) return this.cache;
+    if (!this.hydratePromise) {
+      this.hydratePromise = this.hydrate();
+    }
+    await this.hydratePromise;
+    if (!this.cache) {
+      throw new Error("VastJobStore failed to hydrate");
+    }
+    return this.cache;
+  }
+  async hydrate() {
+    const cache = /* @__PURE__ */ new Map();
+    const path = jobsLogPath();
+    if (!existsSync(path)) {
+      this.cache = cache;
+      return;
+    }
+    const raw = await fs.readFile(path, "utf8");
+    for (const line of raw.split("\n")) {
+      const trimmed = line.trim();
+      if (!trimmed) continue;
+      const parsed = parseJobRecord(trimmed);
+      if (parsed) cache.set(parsed.job_id, parsed);
+    }
+    this.cache = cache;
+  }
+  async appendEvent(record) {
+    const path = jobsLogPath();
+    ensureDir(dirname(path));
+    await fs.appendFile(path, `${JSON.stringify(record)}
+`, "utf8");
+  }
+}
+function parseJobRecord(line) {
+  let raw;
+  try {
+    raw = JSON.parse(line);
+  } catch {
+    return null;
+  }
+  if (!raw || typeof raw !== "object") return null;
+  const obj = raw;
+  if (typeof obj.job_id !== "string" || typeof obj.run_name !== "string" || typeof obj.registry_key !== "string" || typeof obj.status !== "string" || typeof obj.epochs !== "number" || typeof obj.created_at !== "string" || typeof obj.updated_at !== "string") {
+    return null;
+  }
+  return {
+    job_id: obj.job_id,
+    run_name: obj.run_name,
+    registry_key: obj.registry_key,
+    status: obj.status,
+    epochs: obj.epochs,
+    created_at: obj.created_at,
+    updated_at: obj.updated_at,
+    started_at: typeof obj.started_at === "string" ? obj.started_at : null,
+    ended_at: typeof obj.ended_at === "string" ? obj.ended_at : null,
+    vast_instance_id: typeof obj.vast_instance_id === "string" ? obj.vast_instance_id : null,
+    exit_code: typeof obj.exit_code === "number" ? obj.exit_code : null,
+    error: typeof obj.error === "string" ? obj.error : null
+  };
+}
+async function readJobLogTail(jobId, tailLines) {
+  const path = jobLogPath(jobId);
+  if (!existsSync(path)) return [];
+  const raw = await fs.readFile(path, "utf8");
+  const lines = raw.split("\n");
+  if (lines.length > 0 && lines[lines.length - 1] === "") {
+    lines.pop();
+  }
+  if (tailLines <= 0 || lines.length <= tailLines) return lines;
+  return lines.slice(lines.length - tailLines);
+}
+async function appendJobLog(jobId, chunk) {
+  const path = jobLogPath(jobId);
+  ensureDir(dirname(path));
+  await fs.appendFile(path, chunk, "utf8");
+}
+const ENDPOINTS_FILE = "inference-endpoints.json";
+function endpointsPath() {
+  return join(vastRoot(), ENDPOINTS_FILE);
+}
+async function readInferenceEndpoints() {
+  const path = endpointsPath();
+  if (!existsSync(path)) return [];
+  const raw = await fs.readFile(path, "utf8");
+  let parsed;
+  try {
+    parsed = JSON.parse(raw);
+  } catch {
+    return [];
+  }
+  if (!Array.isArray(parsed)) return [];
+  const out = [];
+  for (const entry of parsed) {
+    if (!entry || typeof entry !== "object") continue;
+    const obj = entry;
+    if (typeof obj.id === "string" && typeof obj.label === "string" && typeof obj.base_url === "string" && typeof obj.registry_key === "string" && typeof obj.created_at === "string") {
+      out.push({
+        id: obj.id,
+        label: obj.label,
+        base_url: obj.base_url,
+        registry_key: obj.registry_key,
+        created_at: obj.created_at
+      });
+    }
+  }
+  return out;
+}
+async function writeInferenceEndpoints(endpoints) {
+  const path = endpointsPath();
+  ensureDir(dirname(path));
+  await fs.writeFile(path, `${JSON.stringify(endpoints, null, 2)}
+`, "utf8");
+}
+export {
+  VastJobStore,
+  appendJobLog,
+  inferenceStatsPath,
+  jobLogPath,
+  readInferenceEndpoints,
+  readJobLogTail,
+  writeInferenceEndpoints
+};
+//# sourceMappingURL=vast-job-store.js.map

package/dist/services/vast-job-store.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/services/vast-job-store.ts"],"sourcesContent":["/**\n * Vast.ai training job + inference-stats persistence.\n *\n * Mirrors the in-memory + JSONL pattern used elsewhere in this app:\n * `TrainingService` keeps records in process memory, while we additionally\n * append every state transition to a JSONL log so jobs survive restarts.\n *\n * Storage layout (under `resolveStateDir()`):\n *   ~/.eliza/training-vast/jobs.jsonl       — append-only job event log.\n *   ~/.eliza/training-vast/logs/<id>.log    — per-job stdout/stderr tail.\n *\n * The inference-stats reader points at `~/.eliza/inference-stats.jsonl`,\n * the path the inference side writes to (overridable via env).\n */\n\nimport { existsSync, promises as fs, mkdirSync } from \"node:fs\";\nimport { dirname, join } from \"node:path\";\nimport { resolveStateDir } from \"@elizaos/core\";\n\nexport type VastJobStatus =\n  | \"queued\"\n  | \"provisioning\"\n  | \"running\"\n  | \"completed\"\n  | \"failed\"\n  | \"cancelled\";\n\nexport interface VastJobRecord {\n  job_id: string;\n  run_name: string;\n  registry_key: string;\n  status: VastJobStatus;\n  epochs: number;\n  created_at: string;\n  updated_at: string;\n  started_at: string | null;\n  ended_at: string | null;\n  vast_instance_id: string | null;\n  exit_code: number | null;\n  error: string | null;\n}\n\nexport interface VastJobUpdate {\n  status?: VastJobStatus;\n  vast_instance_id?: string | null;\n  exit_code?: number | null;\n  error?: string | null;\n  started_at?: string | null;\n  ended_at?: string | null;\n}\n\nconst VAST_DIR_NAME = \"training-vast\";\nconst JOBS_LOG_NAME = \"jobs.jsonl\";\nconst LOGS_SUBDIR = \"logs\";\n\nfunction vastRoot(): string {\n  return join(resolveStateDir(), VAST_DIR_NAME);\n}\n\nfunction jobsLogPath(): string {\n  return join(vastRoot(), JOBS_LOG_NAME);\n}\n\nexport function jobLogPath(jobId: string): string {\n  return join(vastRoot(), LOGS_SUBDIR, `${jobId}.log`);\n}\n\nexport function inferenceStatsPath(): string {\n  const override = process.env.ELIZA_INFERENCE_STATS_PATH;\n  if (override?.trim()) return override.trim();\n  return join(resolveStateDir(), \"inference-stats.jsonl\");\n}\n\nfunction ensureDir(path: string): void {\n  if (!existsSync(path)) {\n    mkdirSync(path, { recursive: true });\n  }\n}\n\n/**\n * Append-only job store. The cache is the source of truth at runtime;\n * the JSONL file is the durability log we replay on first read.\n */\nexport class VastJobStore {\n  private cache: Map<string, VastJobRecord> | null = null;\n  private hydratePromise: Promise<void> | null = null;\n\n  async list(): Promise<VastJobRecord[]> {\n    const cache = await this.ensureHydrated();\n    return Array.from(cache.values()).sort((a, b) =>\n      b.created_at.localeCompare(a.created_at),\n    );\n  }\n\n  async get(jobId: string): Promise<VastJobRecord | null> {\n    const cache = await this.ensureHydrated();\n    return cache.get(jobId) ?? null;\n  }\n\n  async insert(record: VastJobRecord): Promise<VastJobRecord> {\n    const cache = await this.ensureHydrated();\n    if (cache.has(record.job_id)) {\n      throw new Error(`Job ${record.job_id} already exists`);\n    }\n    cache.set(record.job_id, record);\n    await this.appendEvent(record);\n    return record;\n  }\n\n  async update(jobId: string, patch: VastJobUpdate): Promise<VastJobRecord> {\n    const cache = await this.ensureHydrated();\n    const existing = cache.get(jobId);\n    if (!existing) throw new Error(`Job ${jobId} not found`);\n    const next: VastJobRecord = {\n      ...existing,\n      ...patch,\n      vast_instance_id:\n        patch.vast_instance_id !== undefined\n          ? patch.vast_instance_id\n          : existing.vast_instance_id,\n      exit_code:\n        patch.exit_code !== undefined ? patch.exit_code : existing.exit_code,\n      error: patch.error !== undefined ? patch.error : existing.error,\n      started_at:\n        patch.started_at !== undefined ? patch.started_at : existing.started_at,\n      ended_at:\n        patch.ended_at !== undefined ? patch.ended_at : existing.ended_at,\n      updated_at: new Date().toISOString(),\n    };\n    cache.set(jobId, next);\n    await this.appendEvent(next);\n    return next;\n  }\n\n  /** Test-only: drop in-memory cache so the next read re-hydrates from disk. */\n  resetCacheForTests(): void {\n    this.cache = null;\n    this.hydratePromise = null;\n  }\n\n  private async ensureHydrated(): Promise<Map<string, VastJobRecord>> {\n    if (this.cache) return this.cache;\n    if (!this.hydratePromise) {\n      this.hydratePromise = this.hydrate();\n    }\n    await this.hydratePromise;\n    if (!this.cache) {\n      throw new Error(\"VastJobStore failed to hydrate\");\n    }\n    return this.cache;\n  }\n\n  private async hydrate(): Promise<void> {\n    const cache = new Map<string, VastJobRecord>();\n    const path = jobsLogPath();\n    if (!existsSync(path)) {\n      this.cache = cache;\n      return;\n    }\n    const raw = await fs.readFile(path, \"utf8\");\n    for (const line of raw.split(\"\\n\")) {\n      const trimmed = line.trim();\n      if (!trimmed) continue;\n      const parsed = parseJobRecord(trimmed);\n      if (parsed) cache.set(parsed.job_id, parsed);\n    }\n    this.cache = cache;\n  }\n\n  private async appendEvent(record: VastJobRecord): Promise<void> {\n    const path = jobsLogPath();\n    ensureDir(dirname(path));\n    await fs.appendFile(path, `${JSON.stringify(record)}\\n`, \"utf8\");\n  }\n}\n\nfunction parseJobRecord(line: string): VastJobRecord | null {\n  let raw: unknown;\n  try {\n    raw = JSON.parse(line);\n  } catch {\n    return null;\n  }\n  if (!raw || typeof raw !== \"object\") return null;\n  const obj = raw as Record<string, unknown>;\n  if (\n    typeof obj.job_id !== \"string\" ||\n    typeof obj.run_name !== \"string\" ||\n    typeof obj.registry_key !== \"string\" ||\n    typeof obj.status !== \"string\" ||\n    typeof obj.epochs !== \"number\" ||\n    typeof obj.created_at !== \"string\" ||\n    typeof obj.updated_at !== \"string\"\n  ) {\n    return null;\n  }\n  return {\n    job_id: obj.job_id,\n    run_name: obj.run_name,\n    registry_key: obj.registry_key,\n    status: obj.status as VastJobStatus,\n    epochs: obj.epochs,\n    created_at: obj.created_at,\n    updated_at: obj.updated_at,\n    started_at: typeof obj.started_at === \"string\" ? obj.started_at : null,\n    ended_at: typeof obj.ended_at === \"string\" ? obj.ended_at : null,\n    vast_instance_id:\n      typeof obj.vast_instance_id === \"string\" ? obj.vast_instance_id : null,\n    exit_code: typeof obj.exit_code === \"number\" ? obj.exit_code : null,\n    error: typeof obj.error === \"string\" ? obj.error : null,\n  };\n}\n\n/** Read the tail of a job's stdout/stderr log file. Empty array if no log. */\nexport async function readJobLogTail(\n  jobId: string,\n  tailLines: number,\n): Promise<string[]> {\n  const path = jobLogPath(jobId);\n  if (!existsSync(path)) return [];\n  const raw = await fs.readFile(path, \"utf8\");\n  const lines = raw.split(\"\\n\");\n  if (lines.length > 0 && lines[lines.length - 1] === \"\") {\n    lines.pop();\n  }\n  if (tailLines <= 0 || lines.length <= tailLines) return lines;\n  return lines.slice(lines.length - tailLines);\n}\n\n/**\n * Append a chunk of subprocess output (stdout + stderr merged) to the\n * per-job log file, creating the directory as needed.\n */\nexport async function appendJobLog(\n  jobId: string,\n  chunk: string,\n): Promise<void> {\n  const path = jobLogPath(jobId);\n  ensureDir(dirname(path));\n  await fs.appendFile(path, chunk, \"utf8\");\n}\n\n/** Inference endpoint registry — single JSON file under the vast root. */\nexport interface InferenceEndpointRecord {\n  id: string;\n  label: string;\n  base_url: string;\n  registry_key: string;\n  created_at: string;\n}\n\nconst ENDPOINTS_FILE = \"inference-endpoints.json\";\n\nfunction endpointsPath(): string {\n  return join(vastRoot(), ENDPOINTS_FILE);\n}\n\nexport async function readInferenceEndpoints(): Promise<\n  InferenceEndpointRecord[]\n> {\n  const path = endpointsPath();\n  if (!existsSync(path)) return [];\n  const raw = await fs.readFile(path, \"utf8\");\n  let parsed: unknown;\n  try {\n    parsed = JSON.parse(raw);\n  } catch {\n    return [];\n  }\n  if (!Array.isArray(parsed)) return [];\n  const out: InferenceEndpointRecord[] = [];\n  for (const entry of parsed) {\n    if (!entry || typeof entry !== \"object\") continue;\n    const obj = entry as Record<string, unknown>;\n    if (\n      typeof obj.id === \"string\" &&\n      typeof obj.label === \"string\" &&\n      typeof obj.base_url === \"string\" &&\n      typeof obj.registry_key === \"string\" &&\n      typeof obj.created_at === \"string\"\n    ) {\n      out.push({\n        id: obj.id,\n        label: obj.label,\n        base_url: obj.base_url,\n        registry_key: obj.registry_key,\n        created_at: obj.created_at,\n      });\n    }\n  }\n  return out;\n}\n\nexport async function writeInferenceEndpoints(\n  endpoints: InferenceEndpointRecord[],\n): Promise<void> {\n  const path = endpointsPath();\n  ensureDir(dirname(path));\n  await fs.writeFile(path, `${JSON.stringify(endpoints, null, 2)}\\n`, \"utf8\");\n}\n"],"mappings":"AAeA,SAAS,YAAY,YAAY,IAAI,iBAAiB;AACtD,SAAS,SAAS,YAAY;AAC9B,SAAS,uBAAuB;AAkChC,MAAM,gBAAgB;AACtB,MAAM,gBAAgB;AACtB,MAAM,cAAc;AAEpB,SAAS,WAAmB;AAC1B,SAAO,KAAK,gBAAgB,GAAG,aAAa;AAC9C;AAEA,SAAS,cAAsB;AAC7B,SAAO,KAAK,SAAS,GAAG,aAAa;AACvC;AAEO,SAAS,WAAW,OAAuB;AAChD,SAAO,KAAK,SAAS,GAAG,aAAa,GAAG,KAAK,MAAM;AACrD;AAEO,SAAS,qBAA6B;AAC3C,QAAM,WAAW,QAAQ,IAAI;AAC7B,MAAI,UAAU,KAAK,EAAG,QAAO,SAAS,KAAK;AAC3C,SAAO,KAAK,gBAAgB,GAAG,uBAAuB;AACxD;AAEA,SAAS,UAAU,MAAoB;AACrC,MAAI,CAAC,WAAW,IAAI,GAAG;AACrB,cAAU,MAAM,EAAE,WAAW,KAAK,CAAC;AAAA,EACrC;AACF;AAMO,MAAM,aAAa;AAAA,EAChB,QAA2C;AAAA,EAC3C,iBAAuC;AAAA,EAE/C,MAAM,OAAiC;AACrC,UAAM,QAAQ,MAAM,KAAK,eAAe;AACxC,WAAO,MAAM,KAAK,MAAM,OAAO,CAAC,EAAE;AAAA,MAAK,CAAC,GAAG,MACzC,EAAE,WAAW,cAAc,EAAE,UAAU;AAAA,IACzC;AAAA,EACF;AAAA,EAEA,MAAM,IAAI,OAA8C;AACtD,UAAM,QAAQ,MAAM,KAAK,eAAe;AACxC,WAAO,MAAM,IAAI,KAAK,KAAK;AAAA,EAC7B;AAAA,EAEA,MAAM,OAAO,QAA+C;AAC1D,UAAM,QAAQ,MAAM,KAAK,eAAe;AACxC,QAAI,MAAM,IAAI,OAAO,MAAM,GAAG;AAC5B,YAAM,IAAI,MAAM,OAAO,OAAO,MAAM,iBAAiB;AAAA,IACvD;AACA,UAAM,IAAI,OAAO,QAAQ,MAAM;AAC/B,UAAM,KAAK,YAAY,MAAM;AAC7B,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,OAAO,OAAe,OAA8C;AACxE,UAAM,QAAQ,MAAM,KAAK,eAAe;AACxC,UAAM,WAAW,MAAM,IAAI,KAAK;AAChC,QAAI,CAAC,SAAU,OAAM,IAAI,MAAM,OAAO,KAAK,YAAY;AACvD,UAAM,OAAsB;AAAA,MAC1B,GAAG;AAAA,MACH,GAAG;AAAA,MACH,kBACE,MAAM,qBAAqB,SACvB,MAAM,mBACN,SAAS;AAAA,MACf,WACE,MAAM,cAAc,SAAY,MAAM,YAAY,SAAS;AAAA,MAC7D,OAAO,MAAM,UAAU,SAAY,MAAM,QAAQ,SAAS;AAAA,MAC1D,YACE,MAAM,eAAe,SAAY,MAAM,aAAa,SAAS;AAAA,MAC/D,UACE,MAAM,aAAa,SAAY,MAAM,WAAW,SAAS;AAAA,MAC3D,aAAY,oBAAI,KAAK,GAAE,YAAY;AAAA,IACrC;AACA,UAAM,IAAI,OAAO,IAAI;AACrB,UAAM,KAAK,YAAY,IAAI;AAC3B,WAAO;AAAA,EACT;AAAA;AAAA,EAGA,qBAA2B;AACzB,SAAK,QAAQ;AACb,SAAK,iBAAiB;AAAA,EACxB;AAAA,EAEA,MAAc,iBAAsD;AAClE,QAAI,KAAK,MAAO,QAAO,KAAK;AAC5B,QAAI,CAAC,KAAK,gBAAgB;AACxB,WAAK,iBAAiB,KAAK,QAAQ;AAAA,IACrC;AACA,UAAM,KAAK;AACX,QAAI,CAAC,KAAK,OAAO;AACf,YAAM,IAAI,MAAM,gCAAgC;AAAA,IAClD;AACA,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAc,UAAyB;AACrC,UAAM,QAAQ,oBAAI,IAA2B;AAC7C,UAAM,OAAO,YAAY;AACzB,QAAI,CAAC,WAAW,IAAI,GAAG;AACrB,WAAK,QAAQ;AACb;AAAA,IACF;AACA,UAAM,MAAM,MAAM,GAAG,SAAS,MAAM,MAAM;AAC1C,eAAW,QAAQ,IAAI,MAAM,IAAI,GAAG;AAClC,YAAM,UAAU,KAAK,KAAK;AAC1B,UAAI,CAAC,QAAS;AACd,YAAM,SAAS,eAAe,OAAO;AACrC,UAAI,OAAQ,OAAM,IAAI,OAAO,QAAQ,MAAM;AAAA,IAC7C;AACA,SAAK,QAAQ;AAAA,EACf;AAAA,EAEA,MAAc,YAAY,QAAsC;AAC9D,UAAM,OAAO,YAAY;AACzB,cAAU,QAAQ,IAAI,CAAC;AACvB,UAAM,GAAG,WAAW,MAAM,GAAG,KAAK,UAAU,MAAM,CAAC;AAAA,GAAM,MAAM;AAAA,EACjE;AACF;AAEA,SAAS,eAAe,MAAoC;AAC1D,MAAI;AACJ,MAAI;AACF,UAAM,KAAK,MAAM,IAAI;AAAA,EACvB,QAAQ;AACN,WAAO;AAAA,EACT;AACA,MAAI,CAAC,OAAO,OAAO,QAAQ,SAAU,QAAO;AAC5C,QAAM,MAAM;AACZ,MACE,OAAO,IAAI,WAAW,YACtB,OAAO,IAAI,aAAa,YACxB,OAAO,IAAI,iBAAiB,YAC5B,OAAO,IAAI,WAAW,YACtB,OAAO,IAAI,WAAW,YACtB,OAAO,IAAI,eAAe,YAC1B,OAAO,IAAI,eAAe,UAC1B;AACA,WAAO;AAAA,EACT;AACA,SAAO;AAAA,IACL,QAAQ,IAAI;AAAA,IACZ,UAAU,IAAI;AAAA,IACd,cAAc,IAAI;AAAA,IAClB,QAAQ,IAAI;AAAA,IACZ,QAAQ,IAAI;AAAA,IACZ,YAAY,IAAI;AAAA,IAChB,YAAY,IAAI;AAAA,IAChB,YAAY,OAAO,IAAI,eAAe,WAAW,IAAI,aAAa;AAAA,IAClE,UAAU,OAAO,IAAI,aAAa,WAAW,IAAI,WAAW;AAAA,IAC5D,kBACE,OAAO,IAAI,qBAAqB,WAAW,IAAI,mBAAmB;AAAA,IACpE,WAAW,OAAO,IAAI,cAAc,WAAW,IAAI,YAAY;AAAA,IAC/D,OAAO,OAAO,IAAI,UAAU,WAAW,IAAI,QAAQ;AAAA,EACrD;AACF;AAGA,eAAsB,eACpB,OACA,WACmB;AACnB,QAAM,OAAO,WAAW,KAAK;AAC7B,MAAI,CAAC,WAAW,IAAI,EAAG,QAAO,CAAC;AAC/B,QAAM,MAAM,MAAM,GAAG,SAAS,MAAM,MAAM;AAC1C,QAAM,QAAQ,IAAI,MAAM,IAAI;AAC5B,MAAI,MAAM,SAAS,KAAK,MAAM,MAAM,SAAS,CAAC,MAAM,IAAI;AACtD,UAAM,IAAI;AAAA,EACZ;AACA,MAAI,aAAa,KAAK,MAAM,UAAU,UAAW,QAAO;AACxD,SAAO,MAAM,MAAM,MAAM,SAAS,SAAS;AAC7C;AAMA,eAAsB,aACpB,OACA,OACe;AACf,QAAM,OAAO,WAAW,KAAK;AAC7B,YAAU,QAAQ,IAAI,CAAC;AACvB,QAAM,GAAG,WAAW,MAAM,OAAO,MAAM;AACzC;AAWA,MAAM,iBAAiB;AAEvB,SAAS,gBAAwB;AAC/B,SAAO,KAAK,SAAS,GAAG,cAAc;AACxC;AAEA,eAAsB,yBAEpB;AACA,QAAM,OAAO,cAAc;AAC3B,MAAI,CAAC,WAAW,IAAI,EAAG,QAAO,CAAC;AAC/B,QAAM,MAAM,MAAM,GAAG,SAAS,MAAM,MAAM;AAC1C,MAAI;AACJ,MAAI;AACF,aAAS,KAAK,MAAM,GAAG;AAAA,EACzB,QAAQ;AACN,WAAO,CAAC;AAAA,EACV;AACA,MAAI,CAAC,MAAM,QAAQ,MAAM,EAAG,QAAO,CAAC;AACpC,QAAM,MAAiC,CAAC;AACxC,aAAW,SAAS,QAAQ;AAC1B,QAAI,CAAC,SAAS,OAAO,UAAU,SAAU;AACzC,UAAM,MAAM;AACZ,QACE,OAAO,IAAI,OAAO,YAClB,OAAO,IAAI,UAAU,YACrB,OAAO,IAAI,aAAa,YACxB,OAAO,IAAI,iBAAiB,YAC5B,OAAO,IAAI,eAAe,UAC1B;AACA,UAAI,KAAK;AAAA,QACP,IAAI,IAAI;AAAA,QACR,OAAO,IAAI;AAAA,QACX,UAAU,IAAI;AAAA,QACd,cAAc,IAAI;AAAA,QAClB,YAAY,IAAI;AAAA,MAClB,CAAC;AAAA,IACH;AAAA,EACF;AACA,SAAO;AACT;AAEA,eAAsB,wBACpB,WACe;AACf,QAAM,OAAO,cAAc;AAC3B,YAAU,QAAQ,IAAI,CAAC;AACvB,QAAM,GAAG,UAAU,MAAM,GAAG,KAAK,UAAU,WAAW,MAAM,CAAC,CAAC;AAAA,GAAM,MAAM;AAC5E;","names":[]}

package/dist/services/vast-subprocess.d.ts

@@ -0,0 +1,27 @@
+/**
+ * Subprocess primitives for the Vast training service.
+ *
+ * Two helpers, both backed by `node:child_process.spawn` via an injectable
+ * `spawnImpl` so tests can stand in fake children:
+ *
+ *   - `runCapture`    — wait for the child to exit, return stdout, throw
+ *                       `VastServiceError` on non-zero exit.
+ *   - `runDetachedToLog` — stream stdout + stderr into a per-job append-only
+ *                       log, resolve with the exit code (no throw on non-zero;
+ *                       the caller decides what a non-zero exit means).
+ *
+ * Both helpers normalize ENOENT-on-binary into a 503 `VastServiceError` so
+ * routes can return an actionable failure when `bash`, `python`, or `uv`
+ * isn't on PATH.
+ */
+import type { spawn } from "node:child_process";
+export declare class VastServiceError extends Error {
+    readonly status: number;
+    constructor(message: string, status: number);
+}
+export type SpawnImpl = typeof spawn;
+export declare function runCapture(spawnImpl: SpawnImpl, command: string, args: string[], options: {
+    cwd: string;
+}): Promise<string>;
+export declare function runDetachedToLog(spawnImpl: SpawnImpl, jobId: string, command: string, args: string[], cwd: string, extraEnv?: NodeJS.ProcessEnv): Promise<number>;
+//# sourceMappingURL=vast-subprocess.d.ts.map

package/dist/services/vast-subprocess.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"vast-subprocess.d.ts","sourceRoot":"","sources":["../../src/services/vast-subprocess.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAGhD,qBAAa,gBAAiB,SAAQ,KAAK;aAGvB,MAAM,EAAE,MAAM;gBAD9B,OAAO,EAAE,MAAM,EACC,MAAM,EAAE,MAAM;CAKjC;AAED,MAAM,MAAM,SAAS,GAAG,OAAO,KAAK,CAAC;AAErC,wBAAgB,UAAU,CACxB,SAAS,EAAE,SAAS,EACpB,OAAO,EAAE,MAAM,EACf,IAAI,EAAE,MAAM,EAAE,EACd,OAAO,EAAE;IAAE,GAAG,EAAE,MAAM,CAAA;CAAE,GACvB,OAAO,CAAC,MAAM,CAAC,CAqCjB;AAED,wBAAgB,gBAAgB,CAC9B,SAAS,EAAE,SAAS,EACpB,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,MAAM,EACf,IAAI,EAAE,MAAM,EAAE,EACd,GAAG,EAAE,MAAM,EACX,QAAQ,GAAE,MAAM,CAAC,UAAe,GAC/B,OAAO,CAAC,MAAM,CAAC,CAuBjB"}