@ruvector/ruvllm-gpu-loop 0.0.0-stage → 1.0.0

package/src/gpu-loop-cli.mjs

@@ -0,0 +1,67 @@
+#!/usr/bin/env node
+import { setTimeout as wait } from 'node:timers/promises';
+import { tickLoop, readLoopStatus, readLoopConfig } from './gpu-loop.mjs';
+import { loadHostAdapter } from './gpu-host.mjs';
+
+const usage = 'Usage: node src/gpu-loop-cli.mjs tick|status|run --config ABS\n'
+  + '  tick/run require --adapter ABS.mjs --adapter-sha256 SHA256\n'
+  + '  run accepts --max-ticks 1..10000 (default 1); no infinite loop is started.\n'
+  + 'The trusted host resolves ruOS secret references and executes Vast.ai operations.\n'
+  + 'No provider or secret access is supplied by this CLI.\n';
+
+function options(argv, command) {
+  const result = {};
+  const allowed = command === 'status' ? ['--config']
+    : ['--config', '--adapter', '--adapter-sha256', ...(command === 'run' ? ['--max-ticks'] : [])];
+  for (let index = 0; index < argv.length; index += 2) {
+    const key = argv[index], value = argv[index + 1];
+    if (!allowed.includes(key) || !value || value.startsWith('--') || Object.hasOwn(result, key)) throw new Error('INVALID_ARGUMENTS');
+    result[key] = value;
+  }
+  if (!result['--config'] || (command !== 'status' && (!result['--adapter'] || !result['--adapter-sha256']))) throw new Error('INVALID_ARGUMENTS');
+  const maxTicks = Number(result['--max-ticks'] ?? 1);
+  if (!Number.isSafeInteger(maxTicks) || maxTicks < 1 || maxTicks > 10000) throw new Error('INVALID_ARGUMENTS');
+  return { ...result, maxTicks };
+}
+
+const [command, ...argv] = process.argv.slice(2);
+if (command === '--help' || command === 'help') process.stdout.write(usage);
+else if (!['tick', 'status', 'run'].includes(command)) {
+  process.stderr.write(usage); process.exitCode = 2;
+} else {
+  const stop = new AbortController();
+  const stopLoop = () => stop.abort();
+  process.once('SIGINT', stopLoop); process.once('SIGTERM', stopLoop);
+  try {
+    const flags = options(argv, command);
+    const configPath = flags['--config'];
+    if (command === 'status') process.stdout.write(JSON.stringify(await readLoopStatus({ configPath })) + '\n');
+    else {
+      const ticks = command === 'run' ? flags.maxTicks : 1;
+      for (let index = 0; index < ticks && !stop.signal.aborted; index++) {
+        // Revalidate code and immutable config before each tick. Tick locks and
+        // durable submission markers own duplicate prevention across processes.
+        const config = await readLoopConfig({ configPath });
+        const adapter = await loadHostAdapter({ modulePath: flags['--adapter'], sha256: flags['--adapter-sha256'] });
+        const state = await tickLoop({ configPath, adapter });
+        process.stdout.write(JSON.stringify(state) + '\n');
+        if (index + 1 < ticks && !stop.signal.aborted) {
+          // The scheduler may shorten nextTickAt to a job deadline or immediate
+          // cleanup. Never sleep the full cadence across those boundaries.
+          const delayMs = Math.max(10, Math.min(config.intervalSeconds * 1000, state.nextTickAt - Date.now()));
+          try { await wait(delayMs, undefined, { signal: stop.signal }); }
+          catch (error) { if (error.name !== 'AbortError') throw error; }
+        }
+      }
+    }
+  } catch (error) {
+    // Raw host exceptions can carry credentials. Surface only scheduler-owned
+    // fixed codes; detailed provider diagnostics remain with the host.
+    const code = typeof error.code === 'string' && /^[A-Z][A-Z0-9_]{0,63}$/.test(error.code)
+      ? error.code : 'GPU_LOOP_FAILED';
+    process.stderr.write(JSON.stringify({ ok: false, code, productionActivation: false }) + '\n');
+    process.exitCode = 1;
+  } finally {
+    process.removeListener('SIGINT', stopLoop); process.removeListener('SIGTERM', stopLoop);
+  }
+}

package/src/gpu-loop.mjs

@@ -0,0 +1,184 @@
+// Persistent orchestration only. A trusted host adapter owns provider APIs and secret resolution.
+import { randomUUID, createHash } from 'node:crypto';
+import { constants } from 'node:fs';
+import { open, lstat, mkdir, rename, rm } from 'node:fs/promises';
+import { join, isAbsolute, resolve } from 'node:path';
+import { checkedPath, parseJson } from './engine.mjs';
+
+const fail = (ok, message) => { if (!ok) throw new Error(message); };
+const id = value => typeof value === 'string' && /^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$/.test(value);
+const sha = bytes => createHash('sha256').update(bytes).digest('hex');
+const amount = value => typeof value === 'number' && Number.isFinite(value) && value >= 0
+  && Number.isSafeInteger(Math.round(value * 1e6)) && Math.round(value * 1e6) / 1e6 === value;
+const exact = (value, required, optional = []) => fail(value && typeof value === 'object' && !Array.isArray(value)
+  && required.every(key => Object.hasOwn(value, key)) && Object.keys(value).every(key => [...required, ...optional].includes(key)), 'Invalid or unknown fields');
+const privateOwner = stat => fail((stat.mode & 0o077) === 0 && stat.uid === process.getuid(), 'Owner-only paths required');
+const freeze = value => {
+  if (value && typeof value === 'object') {
+    for (const nested of Object.values(value)) freeze(nested);
+    Object.freeze(value);
+  }
+  return value;
+};
+
+async function readJson(path) {
+  const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW);
+  try {
+    const stat = await handle.stat(); privateOwner(stat);
+    fail(stat.isFile() && stat.size <= 1024 * 1024, 'Bounded regular JSON file required');
+    const bytes = await handle.readFile(); fail(bytes.length === stat.size, 'JSON file changed');
+    return { value: parseJson(bytes), digest: sha(bytes) };
+  } finally { await handle.close(); }
+}
+
+export async function readLoopConfig({ configPath }) {
+  await checkedPath(configPath);
+  const { value: config, digest } = await readJson(configPath);
+  exact(config, ['schema', 'stateRoot', 'intervalSeconds', 'maxAttempts', 'budgetUsd', 'jobs']);
+  fail(config.schema === 'ruvllm-gpu-loop/v1', 'Unsupported loop config');
+  fail(Number.isInteger(config.intervalSeconds) && config.intervalSeconds >= 1 && config.intervalSeconds <= 86400, 'Invalid interval');
+  fail(Number.isInteger(config.maxAttempts) && config.maxAttempts >= 1 && config.maxAttempts <= 10, 'Invalid attempts');
+  fail(amount(config.budgetUsd), 'Invalid loop budget');
+  await checkedPath(config.stateRoot, { directory: true }); privateOwner(await lstat(config.stateRoot));
+  fail(configPath !== join(config.stateRoot, 'state.json') && configPath !== join(config.stateRoot, 'tick.lock'), 'Config collides with scheduler state');
+  fail(Array.isArray(config.jobs) && config.jobs.length >= 1 && config.jobs.length <= 128, 'Expected 1..128 jobs');
+  const seen = new Set();
+  for (const job of config.jobs) {
+    exact(job, ['id', 'provider', 'model', 'task', 'secretRef', 'resource', 'maxCostUsd', 'timeoutSeconds'], ['experiment']);
+    fail(id(job.id) && !seen.has(job.id), 'Invalid or duplicate job ID'); seen.add(job.id);
+    fail(job.provider === 'vast.ai' && ['evaluate', 'train'].includes(job.task), 'Unsupported provider/task');
+    exact(job.model, ['repo', 'revision']);
+    fail(typeof job.model.repo === 'string' && /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(job.model.repo)
+      && /^[a-f0-9]{40}$/.test(job.model.revision), 'Pinned model revision required');
+    fail(id(job.secretRef), 'Opaque host secret reference required');
+    exact(job.resource, [], ['instanceId', 'offerId']);
+    fail(Object.keys(job.resource).length === 1 && Object.values(job.resource).every(id), 'Exactly one instanceId or offerId required');
+    fail(amount(job.maxCostUsd) && job.maxCostUsd > 0, 'Invalid job cost ceiling');
+    fail(Number.isInteger(job.timeoutSeconds) && job.timeoutSeconds >= 1 && job.timeoutSeconds <= 86400, 'Invalid job timeout');
+    if (job.experiment) {
+      exact(job.experiment, ['configPath', 'planPath']);
+      fail(Object.values(job.experiment).every(path => typeof path === 'string' && isAbsolute(path) && path === resolve(path)), 'Absolute experiment paths required');
+    }
+  }
+  return freeze({ ...config, configSha256: digest });
+}
+
+function initial(config) {
+  return { schema: 'ruvllm-gpu-loop-state/v1', configSha256: config.configSha256,
+    nextTickAt: 0, reservedUsd: 0, jobs: config.jobs.map(job => ({ id: job.id, status: 'queued', attempts: 0 })), productionActivation: false };
+}
+async function load(config) {
+  const path = join(config.stateRoot, 'state.json');
+  let state;
+  try { state = (await readJson(path)).value; }
+  catch (error) { if (error.code === 'ENOENT') return initial(config); throw error; }
+  fail(state.schema === 'ruvllm-gpu-loop-state/v1' && state.configSha256 === config.configSha256, 'State/config binding mismatch');
+  fail(Array.isArray(state.jobs) && state.jobs.length === config.jobs.length && state.jobs.every((job, index) =>
+    job.id === config.jobs[index].id && ['queued', 'submitting', 'running', 'cleanup', 'succeeded', 'failed', 'blocked'].includes(job.status)
+    && Number.isInteger(job.attempts) && job.attempts >= 0 && job.attempts <= config.maxAttempts), 'Invalid state records');
+  fail(amount(state.reservedUsd) && state.reservedUsd <= config.budgetUsd
+    && Number.isSafeInteger(state.nextTickAt) && state.nextTickAt >= 0 && state.productionActivation === false, 'Invalid state budget/timing');
+  fail(Math.round(state.reservedUsd * 1e6) === state.jobs.reduce((sum, entry, index) =>
+    sum + entry.attempts * Math.round(config.jobs[index].maxCostUsd * 1e6), 0), 'State reservation ledger mismatch');
+  for (const entry of state.jobs) {
+    if (['submitting', 'running', 'cleanup', 'succeeded', 'blocked'].includes(entry.status)) {
+      fail(entry.attempts > 0 && id(entry.idempotencyKey) && Number.isSafeInteger(entry.startedAt) && entry.startedAt >= 0, 'Missing attempt identity');
+    }
+    if (['running', 'cleanup', 'succeeded'].includes(entry.status)) fail(id(entry.runId), 'Missing provider run identity');
+    if (entry.status === 'cleanup') fail(['succeeded', 'failed'].includes(entry.targetState) && typeof entry.retryable === 'boolean', 'Invalid cleanup intent');
+    if (entry.status === 'succeeded') fail(entry.cleanupConfirmed === true, 'Completion requires cleanup');
+  }
+  return state;
+}
+async function persist(config, state) {
+  const temporary = join(config.stateRoot, `.state-${randomUUID()}.json`);
+  const handle = await open(temporary, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600);
+  try { await handle.writeFile(JSON.stringify(state, null, 2) + '\n'); await handle.sync(); } finally { await handle.close(); }
+  await rename(temporary, join(config.stateRoot, 'state.json'));
+  const directory = await open(config.stateRoot, constants.O_RDONLY); try { await directory.sync(); } finally { await directory.close(); }
+}
+async function call(method, input, milliseconds) {
+  const controller = new AbortController(); let timer;
+  try {
+    return await Promise.race([Promise.resolve().then(() => method({ ...input, signal: controller.signal })),
+      new Promise((_, reject) => { timer = setTimeout(() => { controller.abort(); reject(new Error('ADAPTER_TIMEOUT')); }, milliseconds); })]);
+  } finally { clearTimeout(timer); }
+}
+function verified(completion, job, entry) {
+  return completion && completion.completionVersion === 1 && completion.completionVerified === true
+    && completion.status === 'ok' && completion.exitCode === 0 && completion.assertionsVerified === true
+    && completion.evidenceComplete === true && completion.truncated === false
+    && completion.jobId === job.id && completion.modelRevision === job.model.revision
+    && completion.modelRepo === job.model.repo && completion.runId === entry.runId
+    && completion.idempotencyKey === entry.idempotencyKey;
+}
+
+export async function readLoopStatus({ configPath }) { return load(await readLoopConfig({ configPath })); }
+
+export async function tickLoop({ configPath, adapter, now = Date.now() }) {
+  fail(Number.isSafeInteger(now) && now >= 0, 'Invalid clock');
+  fail(adapter && ['submit', 'inspect', 'cleanup'].every(method => typeof adapter[method] === 'function'), 'Host adapter required');
+  const config = await readLoopConfig({ configPath }), lock = join(config.stateRoot, 'tick.lock');
+  await mkdir(lock, { mode: 0o700 }); // Never break an existing lock: operator reconciliation is required after a crash.
+  try {
+    const state = await load(config);
+    const deadlineDue = state.jobs.some(entry => entry.status === 'running'
+      && now >= entry.startedAt + config.jobs.find(job => job.id === entry.id).timeoutSeconds * 1000);
+    if (state.jobs.some(job => job.status === 'blocked') || (now < state.nextTickAt && !deadlineDue)) return state;
+    state.nextTickAt = now + config.intervalSeconds * 1000;
+    const entry = state.jobs.find(job => ['submitting', 'running', 'cleanup'].includes(job.status))
+      ?? state.jobs.find(job => job.status === 'queued');
+    if (!entry) { await persist(config, state); return state; }
+    const job = config.jobs.find(job => job.id === entry.id);
+    if (entry.status === 'submitting') { entry.status = 'blocked'; entry.reason = 'ambiguous_submission_reconcile'; }
+    else if (entry.status === 'queued') {
+      const reserved = Math.round(state.reservedUsd * 1e6) + Math.round(job.maxCostUsd * 1e6);
+      if (entry.attempts >= config.maxAttempts || reserved > Math.round(config.budgetUsd * 1e6)) {
+        entry.status = 'failed'; entry.reason = 'attempt_or_budget_limit';
+      } else {
+        entry.attempts++; state.reservedUsd = reserved / 1e6;
+        for (const key of ['runId', 'cleanupConfirmed', 'targetState', 'retryable', 'completionSha256']) delete entry[key];
+        entry.status = 'submitting'; entry.idempotencyKey = randomUUID(); entry.startedAt = now;
+        await persist(config, state); // Persist intent and reservation before the provider effect.
+        try {
+          const submitted = await call(adapter.submit.bind(adapter), { job, idempotencyKey: entry.idempotencyKey }, Math.min(30000, job.timeoutSeconds * 1000));
+          fail(submitted && id(submitted.runId), 'Invalid provider run ID');
+          entry.runId = submitted.runId; entry.status = 'running'; entry.reason = 'submitted';
+        } catch { entry.status = 'blocked'; entry.reason = 'ambiguous_submission_reconcile'; }
+      }
+    } else if (entry.status === 'running') {
+      let result;
+      if (now >= entry.startedAt + job.timeoutSeconds * 1000) {
+        entry.status = 'cleanup'; entry.targetState = 'failed'; entry.retryable = false; entry.reason = 'job_deadline';
+      } else {
+        try {
+          result = await call(adapter.inspect.bind(adapter), { job, runId: entry.runId, idempotencyKey: entry.idempotencyKey }, Math.min(30000, entry.startedAt + job.timeoutSeconds * 1000 - now));
+          fail(result && ['running', 'completed', 'failed'].includes(result.status), 'Invalid provider inspection');
+          if (result.costUsd !== undefined) fail(amount(result.costUsd) && result.costUsd <= job.maxCostUsd, 'Provider cost breach');
+          if (result.status !== 'running') {
+            entry.status = 'cleanup'; entry.targetState = result.status === 'completed' && verified(result.completion, job, entry) ? 'succeeded' : 'failed';
+            entry.retryable = result.status === 'failed'; entry.reason = entry.targetState === 'succeeded' ? 'verified_completion' : result.status === 'failed' ? 'provider_failed' : 'unverified_completion';
+            if (entry.targetState === 'succeeded') entry.completionSha256 = sha(JSON.stringify(result.completion));
+          }
+        } catch { entry.status = 'cleanup'; entry.targetState = 'failed'; entry.retryable = false; entry.reason = 'inspection_failed_or_invalid'; }
+      }
+    } else if (entry.status === 'cleanup') {
+      await persist(config, state); // Cleanup intent is retained until resource absence is confirmed by the host.
+      try {
+        const result = await call(adapter.cleanup.bind(adapter), { job, runId: entry.runId }, 30000);
+        if (result?.released === true) {
+          entry.history ??= [];
+          entry.history.push({ attempt: entry.attempts, runId: entry.runId, idempotencyKey: entry.idempotencyKey,
+            outcome: entry.targetState, reason: entry.reason, cleanupConfirmed: true,
+            ...(entry.completionSha256 ? { completionSha256: entry.completionSha256 } : {}) });
+          entry.status = entry.targetState;
+          if (entry.status === 'failed' && entry.retryable && entry.attempts < config.maxAttempts) entry.status = 'queued';
+          entry.cleanupConfirmed = true;
+        } else entry.reason = 'cleanup_unconfirmed';
+      } catch { entry.reason = 'cleanup_unconfirmed'; }
+    }
+    if (entry.status === 'running') state.nextTickAt = Math.min(state.nextTickAt, entry.startedAt + job.timeoutSeconds * 1000);
+    if (entry.status === 'cleanup' && entry.reason !== 'cleanup_unconfirmed') state.nextTickAt = now;
+    await persist(config, state); return state;
+  } finally { await rm(lock, { recursive: true }); }
+}

package/src/gpu-spool-adapter.mjs

@@ -0,0 +1,80 @@
+// Optional host handoff: writes job requests, never contacts a GPU provider or resolves secrets.
+import { constants } from 'node:fs';
+import { open, lstat, realpath, writeFile } from 'node:fs/promises';
+import { join, resolve, isAbsolute } from 'node:path';
+import { parseJson } from './engine.mjs';
+
+const ensure = (test, message) => { if (!test) throw new Error(message); };
+const safeId = value => typeof value === 'string' && /^[A-Za-z0-9][A-Za-z0-9_.-]{0,159}$/.test(value);
+
+async function spoolRoot() {
+  const root = process.env.RUVLLM_GPU_SPOOL_ROOT;
+  ensure(typeof root === 'string' && isAbsolute(root) && resolve(root) === root, 'SPOOL_NOT_CONFIGURED');
+  ensure(await realpath(root) === root, 'SPOOL_PATH_NOT_CANONICAL');
+  const stat = await lstat(root);
+  ensure(stat.isDirectory() && !stat.isSymbolicLink() && (stat.mode & 0o077) === 0
+    && stat.uid === process.getuid(), 'SPOOL_NOT_PRIVATE');
+  return root;
+}
+
+async function writeOnce(path, value) {
+  const text = JSON.stringify(value, null, 2) + '\n';
+  ensure(Buffer.byteLength(text) <= 65536, 'SPOOL_REQUEST_TOO_LARGE');
+  try { await writeFile(path, text, { flag: 'wx', mode: 0o600 }); }
+  catch (error) {
+    if (error.code !== 'EEXIST') throw new Error('SPOOL_WRITE_FAILED');
+    const prior = await readReply(path);
+    ensure(JSON.stringify(prior) === JSON.stringify(value), 'SPOOL_IDENTITY_CONFLICT');
+  }
+}
+
+async function readReply(path) {
+  let handle;
+  try { handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); }
+  catch (error) { if (error.code === 'ENOENT') return null; throw new Error('SPOOL_READ_FAILED'); }
+  try {
+    const stat = await handle.stat();
+    ensure(stat.isFile() && stat.size <= 65536 && (stat.mode & 0o077) === 0
+      && stat.uid === process.getuid(), 'SPOOL_REPLY_NOT_PRIVATE');
+    const bytes = Buffer.alloc(stat.size);
+    const { bytesRead } = await handle.read(bytes, 0, bytes.length, 0);
+    ensure(bytesRead === bytes.length, 'SPOOL_REPLY_CHANGED');
+    return parseJson(bytes);
+  } finally { await handle.close(); }
+}
+
+export async function submit({ job, idempotencyKey, signal }) {
+  ensure(!signal?.aborted && safeId(idempotencyKey), 'SPOOL_INVALID_REQUEST');
+  const root = await spoolRoot();
+  await writeOnce(join(root, `${idempotencyKey}.request.json`), {
+    schema: 'ruvllm-gpu-spool-request/v1', runId: idempotencyKey, job,
+    credentialHandling: 'host-resolves-secretRef-no-secret-values-in-spool',
+  });
+  return { runId: idempotencyKey };
+}
+
+export async function inspect({ runId, signal }) {
+  ensure(!signal?.aborted && safeId(runId), 'SPOOL_INVALID_RUN_ID');
+  const root = await spoolRoot();
+  const reply = await readReply(join(root, `${runId}.status.json`));
+  if (!reply) return { status: 'running' };
+  ensure(reply.schema === 'ruvllm-gpu-spool-status/v1' && reply.runId === runId
+    && ['running', 'completed', 'failed'].includes(reply.status)
+    && Object.keys(reply).every(key => ['schema', 'runId', 'status', 'costUsd', 'completion'].includes(key)), 'SPOOL_INVALID_STATUS');
+  return { status: reply.status, ...(reply.costUsd === undefined ? {} : { costUsd: reply.costUsd }),
+    ...(reply.completion === undefined ? {} : { completion: reply.completion }) };
+}
+
+export async function cleanup({ runId, job, signal }) {
+  ensure(!signal?.aborted && safeId(runId), 'SPOOL_INVALID_RUN_ID');
+  const root = await spoolRoot();
+  await writeOnce(join(root, `${runId}.cleanup-request.json`), {
+    schema: 'ruvllm-gpu-spool-cleanup-request/v1', runId, jobId: job.id,
+  });
+  const reply = await readReply(join(root, `${runId}.cleanup.json`));
+  if (!reply) return { released: false };
+  ensure(reply.schema === 'ruvllm-gpu-spool-cleanup/v1' && reply.runId === runId
+    && typeof reply.released === 'boolean'
+    && Object.keys(reply).every(key => ['schema', 'runId', 'released'].includes(key)), 'SPOOL_INVALID_CLEANUP');
+  return { released: reply.released };
+}

package/src/vast-host-adapter.mjs

@@ -0,0 +1,274 @@
+// Vast.ai host adapter for the scheduled GPU loop (ADR-015, ADR-017). Trusted
+// operator code: it resolves the job's secretRef from an owner-only host config,
+// rents exactly the configured offer, never reuses a shared instance, and only
+// reports a verified completion after re-scoring the retrieved raw outputs here.
+// No effect happens at import; `node vast-host-adapter.mjs guard ID REF` is the
+// independent destroy timer armed for every rental.
+import { spawn } from 'node:child_process';
+import { createHash } from 'node:crypto';
+import { constants, createWriteStream } from 'node:fs';
+import { mkdir, open, readFile, rm, stat, writeFile } from 'node:fs/promises';
+import { isDeepStrictEqual } from 'node:util';
+import { isAbsolute, join, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+export const HOST_CONFIG_ENV = 'RUVLLM_VAST_HOST_CONFIG';
+const SELF = fileURLToPath(import.meta.url);
+const INSPECT_BUDGET_MS = 24000;
+const fail = code => { const error = new Error(code); error.code = code; throw error; };
+const need = (ok, code) => { if (!ok) fail(code); };
+const abs = value => typeof value === 'string' && isAbsolute(value) && resolve(value) === value;
+const ceilMicro = usd => Math.ceil(usd * 1e6) / 1e6;
+
+async function privateJson(path) {
+  need(abs(path), 'VAST_HOST_CONFIG_PATH');
+  const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW);
+  try {
+    const info = await handle.stat();
+    need(info.isFile() && (info.mode & 0o077) === 0 && info.uid === process.getuid() && info.size <= 65536, 'VAST_HOST_CONFIG_UNSAFE');
+    const bytes = await handle.readFile();
+    return { value: JSON.parse(bytes), sha256: createHash('sha256').update(bytes).digest('hex') };
+  } finally { await handle.close(); }
+}
+
+export async function readHostConfig(path = process.env[HOST_CONFIG_ENV]) {
+  const { value: config, sha256 } = await privateJson(path);
+  need(config?.schema === 'ruvllm-vast-host/v1', 'VAST_HOST_CONFIG_SCHEMA');
+  need(/^https:\/\/[a-z0-9.-]+$/.test(config.apiBase), 'VAST_HOST_CONFIG_API');
+  need(/^[a-z0-9./_-]+@sha256:[a-f0-9]{64}$/.test(config.image), 'VAST_HOST_CONFIG_IMAGE');
+  need(/^https:\/\/github\.com\/[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+\.git$/.test(config.repoUrl) && /^[a-f0-9]{40}$/.test(config.commit), 'VAST_HOST_CONFIG_SOURCE');
+  need(/^v[0-9]+$/.test(config.bundle), 'VAST_HOST_CONFIG_BUNDLE');
+  need(Number.isInteger(config.diskGb) && config.diskGb >= 20 && config.diskGb <= 200, 'VAST_HOST_CONFIG_DISK');
+  need(Number.isInteger(config.armDeadlineSeconds) && config.armDeadlineSeconds >= 60 && config.armDeadlineSeconds <= 43200, 'VAST_HOST_CONFIG_DEADLINE');
+  need(Number.isInteger(config.guardGraceSeconds) && config.guardGraceSeconds >= 60 && config.guardGraceSeconds <= 3600, 'VAST_HOST_CONFIG_GUARD');
+  need(['resultsRoot', 'sshKey', 'repoRoot', 'python', 'node'].every(key => abs(config[key])), 'VAST_HOST_CONFIG_PATHS');
+  need(config.adapterDir === undefined || (/^[a-z0-9][a-z0-9._-]{0,63}$/.test(config.adapterDir) && !config.adapterDir.includes('..')), 'VAST_HOST_CONFIG_ADAPTER');
+  need(config.trainPayload === undefined || /^experiments\/model-improvement\/training\/[a-z0-9][a-z0-9._-]{0,63}$/.test(config.trainPayload), 'VAST_HOST_CONFIG_PAYLOAD');
+  need(config.secrets && typeof config.secrets === 'object' && Object.values(config.secrets).every(entry => abs(entry?.file)), 'VAST_HOST_CONFIG_SECRETS');
+  return Object.freeze({ ...config, sha256 });
+}
+
+async function apiKey(config, ref) {
+  const entry = Object.hasOwn(config.secrets, ref) ? config.secrets[ref] : null;
+  need(entry, 'VAST_SECRET_REF_UNKNOWN');
+  const { mode, uid } = await stat(entry.file);
+  need((mode & 0o077) === 0 && uid === process.getuid(), 'VAST_SECRET_FILE_UNSAFE');
+  const key = (await readFile(entry.file, 'utf8')).trim();
+  need(/^[A-Za-z0-9_-]{16,256}$/.test(key), 'VAST_SECRET_INVALID');
+  return key;
+}
+
+// Bounded child process without a shell. Output is collected, never echoed.
+function defaultRun(command, args, { timeoutMs, cwd, stdoutFile, input } = {}) {
+  return new Promise(resolvePromise => {
+    const child = spawn(command, args, { cwd, stdio: ['pipe', 'pipe', 'pipe'] });
+    const chunks = [], errors = [];
+    let size = 0;
+    const sink = stdoutFile ? createWriteStream(stdoutFile, { flags: 'wx', mode: 0o600 }) : null;
+    const flushed = sink ? new Promise(done => { sink.on('close', done); sink.on('error', done); }) : Promise.resolve();
+    const timer = setTimeout(() => child.kill('SIGKILL'), Math.max(1, timeoutMs ?? 10000));
+    if (sink) child.stdout.pipe(sink);
+    else child.stdout.on('data', chunk => { size += chunk.length; if (size <= 8 << 20) chunks.push(chunk); });
+    child.stderr.on('data', chunk => { if (errors.length < 64) errors.push(chunk); });
+    child.on('error', () => {});
+    child.on('close', async code => {
+      clearTimeout(timer);
+      await flushed;
+      resolvePromise({ code, stdout: Buffer.concat(chunks).toString('utf8'), stderr: Buffer.concat(errors).toString('utf8') });
+    });
+    child.stdin.end(input ?? '');
+  });
+}
+
+async function writePrivate(path, value) {
+  await writeFile(path, typeof value === 'string' ? value : JSON.stringify(value, null, 2) + '\n', { mode: 0o600 });
+}
+
+export function onstartScript(config, task = 'evaluate') {
+  const scripts = '/workspace/ruvllm/experiments/model-improvement/vast';
+  if (task === 'train') need(config.trainPayload, 'VAST_HOST_CONFIG_PAYLOAD');
+  const run = task === 'train'
+    ? `${scripts}/remote-train.sh ${config.commit} ${config.armDeadlineSeconds} ${config.trainPayload}`
+    : `${scripts}/remote-run.sh ${config.commit} ${config.armDeadlineSeconds} ${config.bundle} ${config.adapterDir ?? 'coding-4b'}`;
+  const failed = `grep -q '"failed"' /workspace/run/status.json 2>/dev/null || printf '{"schema":"ruvllm-vast-remote-status/v1","state":"failed","step":"bootstrap"}\\n' > /workspace/run/status.json`;
+  return ['#!/bin/bash', 'mkdir -p /workspace/run',
+    `nohup bash -c '(export DEBIAN_FRONTEND=noninteractive; (command -v git >/dev/null || (apt-get update -qq && apt-get install -y -qq git >/dev/null)) && git clone --quiet ${config.repoUrl} /workspace/ruvllm && git -C /workspace/ruvllm checkout --quiet ${config.commit} && bash ${run}) >/workspace/run/remote.log 2>&1 || { ${failed.replaceAll("'", "'\\''")}; }' >/dev/null 2>&1 &`, ''].join('\n');
+}
+
+export function createVastAdapter({ fetchImpl = globalThis.fetch, run = defaultRun, now = () => Date.now(), readConfig = readHostConfig } = {}) {
+  async function api(config, key, method, path, { body, query, signal } = {}) {
+    const url = new URL(config.apiBase + path);
+    for (const [name, value] of Object.entries(query ?? {})) url.searchParams.set(name, JSON.stringify(value));
+    const response = await fetchImpl(url, { method, signal, headers: { Authorization: `Bearer ${key}`, 'Content-Type': 'application/json' },
+      ...(body === undefined ? {} : { body: JSON.stringify(body) }) });
+    if (response.status === 404) return null;
+    need(response.ok, `VAST_HTTP_${response.status}`);
+    return response.json();
+  }
+  async function instances(config, key, signal) {
+    const rows = [];
+    let after;
+    for (let page = 0; page < 40; page++) {
+      const data = await api(config, key, 'GET', '/api/v1/instances/', { signal,
+        query: { select_filters: {}, order_by: [{ col: 'id', dir: 'asc' }], limit: 25, ...(after ? { after_token: after } : {}) } });
+      rows.push(...(data?.instances ?? []));
+      if (!data?.next_token) return rows;
+      after = data.next_token;
+    }
+    fail('VAST_LIST_UNBOUNDED');
+  }
+  const attemptDir = (config, key) => join(config.resultsRoot, key);
+  const ssh = (config, instance, remote) => ['-i', config.sshKey, '-p', String(instance.ssh_port), '-o', 'BatchMode=yes', '-o', 'ConnectTimeout=8',
+    '-o', 'StrictHostKeyChecking=accept-new', '-o', `UserKnownHostsFile=${join(config.resultsRoot, `known_hosts-${instance.id}`)}`, '-o', 'IdentitiesOnly=yes',
+    `root@${instance.ssh_host}`, remote];
+
+  async function submit({ job, idempotencyKey, signal }) {
+    const config = await readConfig(), key = await apiKey(config, job.secretRef);
+    need(typeof job.resource.offerId === 'string' && /^[0-9]{1,12}$/.test(job.resource.offerId), 'VAST_OFFER_REQUIRED');
+    const label = `ruvllm-${idempotencyKey}`;
+    const existing = (await instances(config, key, signal)).find(row => row.label === label);
+    if (existing) return { runId: String(existing.id) };
+    // The bundles search ignores `id`; `ask_contract_id` is the offer identity filter.
+    const offers = (await api(config, key, 'POST', '/api/v0/bundles/', { signal, body: { ask_contract_id: { eq: Number(job.resource.offerId) },
+      rentable: { eq: true }, type: 'on-demand', allocated_storage: config.diskGb, limit: 2 } }))?.offers ?? [];
+    need(offers.length === 1 && String(offers[0].id) === job.resource.offerId
+      && Number.isFinite(offers[0].dph_total) && offers[0].dph_total > 0, 'VAST_OFFER_UNAVAILABLE');
+    const offer = offers[0];
+    // Spending ceiling: the worst case (full timeout at the offer's total hourly
+    // price, storage included) must fit the job's reserved ceiling before renting.
+    const worstCaseUsd = ceilMicro(offer.dph_total * job.timeoutSeconds / 3600);
+    need(worstCaseUsd <= job.maxCostUsd, 'VAST_COST_CEILING');
+    await mkdir(attemptDir(config, idempotencyKey), { mode: 0o700 });
+    const created = await api(config, key, 'PUT', `/api/v0/asks/${offer.id}/`, { signal, body: { client_id: 'me', image: config.image,
+      env: {}, price: null, disk: config.diskGb, label, onstart: onstartScript(config, job.task), runtype: 'ssh', force: false, cancel_unavail: true } });
+    need(created?.success === true && Number.isSafeInteger(created.new_contract), 'VAST_CREATE_FAILED');
+    const instanceId = created.new_contract;
+    // Independent of this scheduler: a user timer destroys the rental after
+    // timeout + grace even if no further tick ever runs.
+    const guard = await run('systemd-run', ['--user', '--quiet', `--unit=ruvllm-vast-guard-${instanceId}`,
+      `--on-active=${job.timeoutSeconds + config.guardGraceSeconds}s`, `--setenv=${HOST_CONFIG_ENV}=${process.env[HOST_CONFIG_ENV] ?? ''}`,
+      config.node, SELF, 'guard', String(instanceId), job.secretRef], { timeoutMs: 8000 });
+    await writePrivate(join(attemptDir(config, idempotencyKey), 'submission.json'), { schema: 'ruvllm-vast-submission/v1',
+      jobId: job.id, idempotencyKey, label, instanceId, offerId: offer.id, gpuName: offer.gpu_name, dphTotal: offer.dph_total,
+      worstCaseUsd, maxCostUsd: job.maxCostUsd, timeoutSeconds: job.timeoutSeconds, image: config.image, commit: config.commit,
+      task: job.task, bundle: config.bundle, adapterDir: config.adapterDir ?? 'coding-4b', trainPayload: config.trainPayload ?? null, hostConfigSha256: config.sha256, guardArmed: guard.code === 0, submittedAt: new Date(now()).toISOString() });
+    if (guard.code !== 0) {
+      await api(config, key, 'DELETE', `/api/v0/instances/${instanceId}/`, { body: {} });
+      fail('VAST_GUARD_UNAVAILABLE');
+    }
+    return { runId: String(instanceId) };
+  }
+
+  async function retrieve(config, instance, dir, started) {
+    const left = () => INSPECT_BUDGET_MS - (now() - started);
+    const archive = join(dir, 'evidence.tar');
+    await rm(archive, { force: true }); await rm(join(dir, 'evidence'), { recursive: true, force: true });
+    if (left() < 4000) return false;
+    const pulled = await run('ssh', ssh(config, instance, 'tar -C /workspace -cf - out run'), { timeoutMs: left() - 3000, stdoutFile: archive });
+    if (pulled.code !== 0) return false;
+    await mkdir(join(dir, 'evidence'), { mode: 0o700 });
+    return (await run('tar', ['-xf', archive, '-C', join(dir, 'evidence'), '--no-same-owner', '--no-same-permissions'], { timeoutMs: 5000 })).code === 0;
+  }
+
+  async function verify(config, job, dir) {
+    const evidence = join(dir, 'evidence', 'out'), scored = join(dir, 'local-scored');
+    await rm(scored, { recursive: true, force: true });
+    const bundle = join(config.repoRoot, 'experiments/model-improvement/frozen', config.bundle);
+    const result = await run(config.python, [join(config.repoRoot, 'experiments/model-improvement/measurement.py'), '--bundle', bundle,
+      '--base-run', join(evidence, 'base'), '--adapter-run', join(evidence, 'adapter'), '--out', scored], { timeoutMs: 15000, cwd: config.repoRoot });
+    if (result.code !== 0) return { ok: false, reason: 'local_rescore_failed' };
+    const local = JSON.parse(await readFile(join(scored, 'report.json'), 'utf8'));
+    const remote = JSON.parse(await readFile(join(evidence, 'scored', 'report.json'), 'utf8'));
+    const ok = isDeepStrictEqual(local, remote) && local.evidence_mode === 'gpu-inference';
+    await writePrivate(join(dir, 'verification.json'), { schema: 'ruvllm-vast-verification/v1', jobId: job.id,
+      localRescoreMatchesRemote: isDeepStrictEqual(local, remote), evidenceMode: local.evidence_mode,
+      researchImprovementConfirmed: local.research_improvement_confirmed, reasons: local.reasons });
+    return { ok, reason: ok ? 'verified' : 'report_mismatch' };
+  }
+
+  // Training evidence: the trainer's own manifest must hash every retrieved file,
+  // bind the committed payload, and match the completed event in the event log.
+  async function verifyTraining(config, job, dir) {
+    const out = join(dir, 'evidence', 'out', 'training-v1');
+    const sha = async path => createHash('sha256').update(await readFile(path)).digest('hex');
+    const checks = {};
+    try {
+      const bridge = JSON.parse(await readFile(join(config.repoRoot, config.trainPayload, 'bridge.json'), 'utf8'));
+      const manifestBytes = await readFile(join(out, 'manifest.json'));
+      const manifest = JSON.parse(manifestBytes);
+      const events = (await readFile(join(dir, 'evidence', 'run', 'train.log'), 'utf8')).split('\n')
+        .filter(line => line.startsWith('RUFLO_TRAIN ')).map(line => JSON.parse(line.slice(12)));
+      const completed = events.find(event => event.kind === 'completed');
+      checks.payloadBound = manifest.training_sha256 === bridge.training_sha256;
+      checks.manifestEvent = completed?.data?.manifest_sha256 === createHash('sha256').update(manifestBytes).digest('hex');
+      checks.steps = manifest.steps === 96 && manifest.revision === job.model.revision && manifest.model === job.model.repo;
+      const files = Object.entries(manifest.files ?? {});
+      checks.files = files.length >= 6 && (await Promise.all(files.map(async ([name, digest]) =>
+        !name.split('/').includes('..') && await sha(join(out, name)) === digest))).every(Boolean);
+      checks.validation = events.filter(event => event.kind === 'validation').map(event => event.data);
+    } catch (error) { checks.error = error.code ?? error.name; }
+    const ok = Boolean(checks.payloadBound && checks.manifestEvent && checks.steps && checks.files && !checks.error);
+    await writePrivate(join(dir, 'verification.json'), { schema: 'ruvllm-vast-training-verification/v1', jobId: job.id, ...checks, ok });
+    return { ok };
+  }
+
+  async function inspect({ job, runId, idempotencyKey, signal }) {
+    const started = now(), config = await readConfig(), key = await apiKey(config, job.secretRef);
+    const instance = (await instances(config, key, signal)).find(row => String(row.id) === runId);
+    if (!instance) return { status: 'failed' };
+    const costUsd = ceilMicro(instance.dph_total * Math.max(0, now() / 1000 - (instance.start_date ?? now() / 1000)) / 3600);
+    if (costUsd > job.maxCostUsd) return { status: 'failed', costUsd };
+    // A host whose container runtime cannot start the image never recovers; fail now instead of at the deadline.
+    if (instance.actual_status !== 'running' && /error response from daemon|oci runtime|invalid runtime/i.test(instance.status_msg ?? '')) return { status: 'failed', costUsd };
+    if (instance.actual_status !== 'running' || !instance.ssh_host || !instance.ssh_port) return { status: 'running', costUsd };
+    const dir = attemptDir(config, idempotencyKey);
+    const probe = await run('ssh', ssh(config, instance, `cat /workspace/run/status.json && echo && { tail -c 400 /workspace/run/${job.task === 'train' ? 'train' : 'run-pair'}.log 2>/dev/null || true; }`), { timeoutMs: 10000 });
+    if (probe.code !== 0) return { status: 'running', costUsd };
+    let remote;
+    try { remote = JSON.parse(probe.stdout.split('\n')[0]); } catch { return { status: 'running', costUsd }; }
+    await writePrivate(join(dir, 'progress.json'), { at: new Date(now()).toISOString(), costUsd, remote, tail: probe.stdout.split('\n').slice(1).join('\n').slice(-400) });
+    if (remote.state === 'running') return { status: 'running', costUsd };
+    if (remote.state === 'failed') { await retrieve(config, instance, dir, started); return { status: 'failed', costUsd }; }
+    if (remote.state !== 'done') return { status: 'running', costUsd };
+    // Retain raw outputs locally before any terminal report; retry next tick on a slow pull.
+    if (!await retrieve(config, instance, dir, started)) return { status: 'running', costUsd };
+    const checked = job.task === 'train' ? await verifyTraining(config, job, dir) : await verify(config, job, dir);
+    return { status: 'completed', costUsd, completion: { completionVersion: 1, completionVerified: checked.ok, status: checked.ok ? 'ok' : 'error',
+      exitCode: checked.ok ? 0 : 1, assertionsVerified: checked.ok, evidenceComplete: checked.ok, truncated: false, jobId: job.id,
+      modelRepo: job.model.repo, modelRevision: job.model.revision, runId, idempotencyKey } };
+  }
+
+  async function destroyConfirmed(config, key, instanceId, signal) {
+    await api(config, key, 'DELETE', `/api/v0/instances/${instanceId}/`, { body: {}, signal });
+    for (let check = 0; check < 6; check++) {
+      if (!(await instances(config, key, signal)).some(row => String(row.id) === String(instanceId))) return true;
+      await new Promise(done => setTimeout(done, 2500));
+    }
+    return false;
+  }
+
+  async function cleanup({ job, runId, signal }) {
+    const config = await readConfig(), key = await apiKey(config, job.secretRef);
+    const released = await destroyConfirmed(config, key, runId, signal);
+    if (released) await run('systemctl', ['--user', 'stop', `ruvllm-vast-guard-${runId}.timer`], { timeoutMs: 5000 });
+    await writePrivate(join(config.resultsRoot, `cleanup-${runId}.json`), { schema: 'ruvllm-vast-cleanup/v1', jobId: job.id,
+      instanceId: runId, confirmedAbsent: released, at: new Date(now()).toISOString() });
+    return { released };
+  }
+
+  return { submit, inspect, cleanup, destroyConfirmed, instances };
+}
+
+const defaults = createVastAdapter();
+export const submit = input => defaults.submit(input);
+export const inspect = input => defaults.inspect(input);
+export const cleanup = input => defaults.cleanup(input);
+
+if (process.argv[1] === SELF && process.argv[2] === 'guard') {
+  const [instanceId, ref] = process.argv.slice(3);
+  const config = await readHostConfig();
+  const released = await defaults.destroyConfirmed(config, await apiKey(config, ref), instanceId);
+  await writePrivate(join(config.resultsRoot, `guard-${instanceId}.json`), { schema: 'ruvllm-vast-guard/v1', instanceId,
+    confirmedAbsent: released, at: new Date().toISOString() });
+  process.exitCode = released ? 0 : 1;
+}