@ruvector/ruvllm-gpu-loop 1.0.0 → 1.1.1

package/GPU-LOOP.md

@@ -159,12 +159,18 @@ the spool adapter; recalculate and review its pin after any code change.
 | `diskGb`, `armDeadlineSeconds` | Rental disk; per-arm worker deadline |
 | `guardGraceSeconds` | Extra time after `timeoutSeconds` before the independent destroy timer fires |
 | `resultsRoot`, `sshKey`, `repoRoot`, `python`, `node` | Absolute local paths for receipts, retrieval and local re-scoring |
+| `sshKnownHosts` | Absolute path to an owner-managed `known_hosts` file (not group/world-writable) holding the pinned host keys; ssh runs with `StrictHostKeyChecking=yes` and refuses unknown or changed keys |
+| `sshTrustOnFirstUse` | `true` to accept a rental's first host key into a fresh per-instance file (the pre-1.1.0 behaviour). One of `sshKnownHosts` or `sshTrustOnFirstUse: true` is required; the config is rejected (`VAST_HOST_CONFIG_SSH_TRUST`) before anything is rented otherwise |
+| `maxEvidenceBytes`, `maxEvidenceEntries` | Optional caps on the retrieved evidence archive (defaults 2 GiB and 20000 entries) |
 
 Jobs must use `resource.offerId`. Submit refuses the offer when its hourly price
 times `timeoutSeconds` exceeds `maxCostUsd`, labels the rental with the attempt's
 idempotency key, and arms a `systemd-run --user` destroy timer. Inspect reports
 accrued cost, pulls raw outputs when the remote run is done, and re-scores them
-locally; only an identical report counts as verified completion. Cleanup returns
+locally. The pull stops at `maxEvidenceBytes`, and every tar header is checked
+before extraction: only regular files and directories, only relative paths under
+`out/` or `run/` with no `..`, within the entry and size caps. A rejected archive
+is never extracted, fails the attempt and is recorded in `evidence-rejected.json`; only an identical report counts as verified completion. Cleanup returns
 `released: true` only after the instance is absent from the account listing.
 Receipts (`submission.json`, `progress.json`, `verification.json`,
 `cleanup-ID.json`, `guard-ID.json`) contain no credential. Copy the module to a

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ruvector/ruvllm-gpu-loop",
-  "version": "1.0.0",
+  "version": "1.1.1",
   "type": "module",
   "description": "RuVLLM GPU job loop: bounded weight experiments, a durable GPU job scheduler with a Vast.ai host adapter, adapter archives and verified paired measurements (split out of @ruvector/ruvllm 4.0.x)",
   "license": "Apache-2.0",
@@ -64,7 +64,7 @@
     "access": "public"
   },
   "dependencies": {
-    "@metaharness/darwin": "^0.12.0",
+    "@metaharness/darwin": "^0.13.0",
     "@metaharness/flywheel": "^0.1.13",
     "@metaharness/weight-eft": "^0.1.2"
   }

package/src/engine.mjs

@@ -87,6 +87,23 @@ export async function fileDigest(path) {
   } finally { await file.close(); }
 }
 
+// Digest cache for the pinned native binary: re-hash only when the file's
+// identity or change metadata differs (dev, inode, size, mtime, ctime in ns).
+// Any write to the file updates ctime, which callers cannot set.
+const digestCache = new Map();
+async function verifiedDigest(path) {
+  const info = await lstat(path, { bigint: true });
+  assert(info.isFile(), 'Expected regular file');
+  const identity = [info.dev, info.ino, info.size, info.mtimeNs, info.ctimeNs].join(':');
+  const cached = digestCache.get(path);
+  if (cached?.identity === identity) return cached.digest;
+  const digest = await fileDigest(path);
+  const after = await lstat(path, { bigint: true });
+  if ([after.dev, after.ino, after.size, after.mtimeNs, after.ctimeNs].join(':') === identity) digestCache.set(path, { identity, digest });
+  else digestCache.delete(path);
+  return digest;
+}
+
 async function identicalFiles(a, b) {
   const first = await open(a, constants.O_RDONLY | constants.O_NOFOLLOW), second = await open(b, constants.O_RDONLY | constants.O_NOFOLLOW);
   try {
@@ -118,12 +135,12 @@ async function runtime(configPath) {
   assert(!inside(config.writeRoot, configPath) && !inside(config.writeRoot, config.binaryPath), 'Runtime config and binary must be outside write root');
   const stat = await lstat(configPath); assert((stat.mode & 0o077) === 0, 'Runtime config must be owner-only');
   assert(((await lstat(config.writeRoot)).mode & 0o077) === 0, 'Write root must be private');
-  assert(await fileDigest(config.binaryPath) === config.binarySha256, 'Binary digest mismatch');
+  assert(await verifiedDigest(config.binaryPath) === config.binarySha256, 'Binary digest mismatch');
   return { ...config, configSha256: sha256(bytes) };
 }
 
 async function native(config, request) {
-  assert(await fileDigest(config.binaryPath) === config.binarySha256, 'Binary digest mismatch');
+  assert(await verifiedDigest(config.binaryPath) === config.binarySha256, 'Binary digest mismatch');
   const input = JSON.stringify(request); assert(Buffer.byteLength(input) <= 64 * 1024, 'Native request limit');
   return new Promise((accept, reject) => {
     const environment = Object.fromEntries(['PATH', 'HOME', 'TMPDIR', 'LANG', 'LC_ALL'].filter(k => process.env[k]).map(k => [k, process.env[k]]));

package/src/gpu-host.mjs

@@ -3,11 +3,61 @@
 import { createHash } from 'node:crypto';
 import { constants } from 'node:fs';
 import { open } from 'node:fs/promises';
+import nodeModule from 'node:module';
 import { pathToFileURL } from 'node:url';
 import { checkedPath } from './engine.mjs';
 
 function failure(code) { const error = new Error(code); error.code = code; return error; }
 
+// Import the entry module from the bytes that were hashed, not by re-reading the
+// path: a loader hook answers the pinned URL with those exact bytes, so a file
+// swapped after the check is never executed. Relative imports still resolve
+// next to the original file (they are trusted operator code, as before).
+const ASYNC_HOOKS = 'data:text/javascript,' + encodeURIComponent(`
+const sources = new Map();
+export async function initialize({ port }) {
+  port.on('message', ({ href, source }) => { sources.set(href, source); port.postMessage(href); });
+  port.unref();
+}
+export async function load(url, context, nextLoad) {
+  const source = sources.get(url);
+  if (source === undefined) return nextLoad(url, context);
+  sources.delete(url);
+  return { format: 'module', source, shortCircuit: true };
+}`);
+let asyncPort;
+const pending = new Map();
+
+function asyncHooks() {
+  if (!asyncPort) {
+    const { port1, port2 } = new MessageChannel();
+    port1.on('message', href => { pending.get(href)?.(); pending.delete(href); });
+    port1.unref();
+    nodeModule.register(ASYNC_HOOKS, { data: { port: port2 }, transferList: [port2] });
+    asyncPort = port1;
+  }
+  return asyncPort;
+}
+
+async function importVerified(href, bytes) {
+  if (typeof nodeModule.registerHooks === 'function') {
+    const hooks = nodeModule.registerHooks({
+      load(url, context, nextLoad) {
+        if (url === href) return { format: 'module', source: bytes, shortCircuit: true };
+        return nextLoad(url, context);
+      },
+    });
+    try { return await import(href); } finally { hooks.deregister(); }
+  }
+  // Node 22 before module.registerHooks (22.15): off-thread loader hooks.
+  const port = asyncHooks();
+  port.ref();
+  try {
+    await new Promise(resolve => { pending.set(href, resolve); port.postMessage({ href, source: bytes }); });
+  } finally { port.unref(); }
+  return import(href);
+}
+
 export async function loadHostAdapter({ modulePath, sha256 }) {
   if (typeof sha256 !== 'string' || !/^[a-f0-9]{64}$/.test(sha256)) throw failure('HOST_ADAPTER_INVALID_PIN');
   let path, bytes;
@@ -35,7 +85,7 @@ export async function loadHostAdapter({ modulePath, sha256 }) {
     // The entry point and its transitive imports are trusted operator code. The
     // digest is a cache identity as well as a pin; never accept tool-supplied code.
     const url = pathToFileURL(path); url.searchParams.set('sha256', sha256);
-    module = await import(url.href);
+    module = await importVerified(url.href, bytes);
   } catch { throw failure('HOST_ADAPTER_UNAVAILABLE'); }
   if (!['submit', 'inspect', 'cleanup'].every(name => typeof module[name] === 'function')) {
     throw failure('HOST_ADAPTER_INVALID_CONTRACT');

package/src/vast-host-adapter.mjs

@@ -15,6 +15,10 @@ import { fileURLToPath } from 'node:url';
 export const HOST_CONFIG_ENV = 'RUVLLM_VAST_HOST_CONFIG';
 const SELF = fileURLToPath(import.meta.url);
 const INSPECT_BUDGET_MS = 24000;
+// Evidence comes from a rented host and is untrusted until re-scored here.
+const DEFAULT_MAX_EVIDENCE_BYTES = 2 * 1024 ** 3;
+const DEFAULT_MAX_EVIDENCE_ENTRIES = 20000;
+const EVIDENCE_ROOTS = new Set(['out', 'run']);
 const fail = code => { const error = new Error(code); error.code = code; throw error; };
 const need = (ok, code) => { if (!ok) fail(code); };
 const abs = value => typeof value === 'string' && isAbsolute(value) && resolve(value) === value;
@@ -45,6 +49,17 @@ export async function readHostConfig(path = process.env[HOST_CONFIG_ENV]) {
   need(config.adapterDir === undefined || (/^[a-z0-9][a-z0-9._-]{0,63}$/.test(config.adapterDir) && !config.adapterDir.includes('..')), 'VAST_HOST_CONFIG_ADAPTER');
   need(config.trainPayload === undefined || /^experiments\/model-improvement\/training\/[a-z0-9][a-z0-9._-]{0,63}$/.test(config.trainPayload), 'VAST_HOST_CONFIG_PAYLOAD');
   need(config.secrets && typeof config.secrets === 'object' && Object.values(config.secrets).every(entry => abs(entry?.file)), 'VAST_HOST_CONFIG_SECRETS');
+  need(config.maxEvidenceBytes === undefined || (Number.isSafeInteger(config.maxEvidenceBytes) && config.maxEvidenceBytes >= 1 << 20 && config.maxEvidenceBytes <= 64 * 1024 ** 3), 'VAST_HOST_CONFIG_EVIDENCE');
+  need(config.maxEvidenceEntries === undefined || (Number.isInteger(config.maxEvidenceEntries) && config.maxEvidenceEntries >= 1 && config.maxEvidenceEntries <= 1000000), 'VAST_HOST_CONFIG_EVIDENCE');
+  // SSH host keys: pinned in an operator-managed known_hosts file, or trust on
+  // first use only when explicitly enabled. Checked before anything is rented.
+  need(config.sshTrustOnFirstUse === undefined || typeof config.sshTrustOnFirstUse === 'boolean', 'VAST_HOST_CONFIG_SSH_TRUST');
+  need(config.sshKnownHosts === undefined || abs(config.sshKnownHosts), 'VAST_HOST_CONFIG_SSH_TRUST');
+  need(config.sshKnownHosts !== undefined || config.sshTrustOnFirstUse === true, 'VAST_HOST_CONFIG_SSH_TRUST');
+  if (config.sshKnownHosts !== undefined) {
+    const info = await stat(config.sshKnownHosts);
+    need(info.isFile() && (info.mode & 0o022) === 0 && info.uid === process.getuid() && info.size > 0, 'VAST_SSH_KNOWN_HOSTS_UNSAFE');
+  }
   return Object.freeze({ ...config, sha256 });
 }
 
@@ -59,27 +74,109 @@ async function apiKey(config, ref) {
 }
 
 // Bounded child process without a shell. Output is collected, never echoed.
-function defaultRun(command, args, { timeoutMs, cwd, stdoutFile, input } = {}) {
+function defaultRun(command, args, { timeoutMs, cwd, stdoutFile, input, maxStdoutBytes } = {}) {
   return new Promise(resolvePromise => {
     const child = spawn(command, args, { cwd, stdio: ['pipe', 'pipe', 'pipe'] });
     const chunks = [], errors = [];
-    let size = 0;
+    let size = 0, overflow = false;
     const sink = stdoutFile ? createWriteStream(stdoutFile, { flags: 'wx', mode: 0o600 }) : null;
     const flushed = sink ? new Promise(done => { sink.on('close', done); sink.on('error', done); }) : Promise.resolve();
     const timer = setTimeout(() => child.kill('SIGKILL'), Math.max(1, timeoutMs ?? 10000));
-    if (sink) child.stdout.pipe(sink);
+    if (sink) {
+      child.stdout.on('data', chunk => {
+        size += chunk.length;
+        if (maxStdoutBytes !== undefined && size > maxStdoutBytes && !overflow) { overflow = true; child.kill('SIGKILL'); }
+      });
+      child.stdout.pipe(sink);
+    }
     else child.stdout.on('data', chunk => { size += chunk.length; if (size <= 8 << 20) chunks.push(chunk); });
     child.stderr.on('data', chunk => { if (errors.length < 64) errors.push(chunk); });
     child.on('error', () => {});
     child.on('close', async code => {
       clearTimeout(timer);
       await flushed;
-      resolvePromise({ code, stdout: Buffer.concat(chunks).toString('utf8'), stderr: Buffer.concat(errors).toString('utf8') });
+      resolvePromise({ code: overflow ? null : code, overflow, stdout: Buffer.concat(chunks).toString('utf8'), stderr: Buffer.concat(errors).toString('utf8') });
     });
     child.stdin.end(input ?? '');
   });
 }
 
+// Tar numeric field: octal text, or GNU base-256 when the high bit is set.
+function tarNumber(header, offset, length) {
+  const field = header.subarray(offset, offset + length);
+  if (field[0] & 0x80) {
+    let value = field[0] & 0x7f;
+    for (const byte of field.subarray(1)) value = value * 256 + byte;
+    need(Number.isSafeInteger(value), 'VAST_EVIDENCE_ARCHIVE');
+    return value;
+  }
+  const text = field.toString('latin1').replace(/[\0 ]+$/, '').trim();
+  need(/^[0-7]*$/.test(text), 'VAST_EVIDENCE_ARCHIVE');
+  return text ? parseInt(text, 8) : 0;
+}
+
+function evidenceName(name) {
+  const clean = name.replace(/\/+$/, '');
+  const parts = clean.split('/');
+  need(clean.length > 0 && clean.length <= 4096 && !clean.includes('\0') && !clean.includes('\\') && !isAbsolute(clean)
+    && EVIDENCE_ROOTS.has(parts[0]) && parts.every(part => part !== '..' && part !== ''), 'VAST_EVIDENCE_UNSAFE_PATH');
+}
+
+// Validate every header of a retrieved tar before extracting it: regular files
+// and directories only (no links or devices), relative paths under out/ or
+// run/ with no `..`, and bounded entry count and total size.
+async function checkEvidenceArchive(path, { maxBytes, maxEntries }) {
+  const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW);
+  try {
+    const { size } = await handle.stat();
+    need(size <= maxBytes, 'VAST_EVIDENCE_TOO_LARGE');
+    need(size > 0 && size % 512 === 0, 'VAST_EVIDENCE_ARCHIVE');
+    const header = Buffer.alloc(512);
+    const read = async (buffer, position) => {
+      const { bytesRead } = await handle.read(buffer, 0, buffer.length, position);
+      need(bytesRead === buffer.length, 'VAST_EVIDENCE_ARCHIVE');
+    };
+    let offset = 0, entries = 0, total = 0, longName = null, paxPath = null, ended = false;
+    while (offset + 512 <= size) {
+      await read(header, offset); offset += 512;
+      if (header.every(byte => byte === 0)) { ended = true; break; }
+      let sum = 0;
+      for (let i = 0; i < 512; i++) sum += i >= 148 && i < 156 ? 32 : header[i];
+      need(sum === tarNumber(header, 148, 8), 'VAST_EVIDENCE_ARCHIVE');
+      const type = header[156] === 0 ? '0' : String.fromCharCode(header[156]);
+      const bodySize = tarNumber(header, 124, 12);
+      const padded = Math.ceil(bodySize / 512) * 512;
+      need(offset + padded <= size, 'VAST_EVIDENCE_ARCHIVE');
+      if (type === 'L' || type === 'x') {
+        need(bodySize <= 65536, 'VAST_EVIDENCE_ARCHIVE');
+        const body = Buffer.alloc(bodySize); await read(body, offset);
+        if (type === 'L') longName = body.toString('utf8').replace(/\0.*$/s, '');
+        else {
+          for (const record of body.toString('utf8').split('\n').filter(Boolean)) {
+            const match = /^[0-9]+ ([^=]+)=(.*)$/s.exec(record);
+            need(match, 'VAST_EVIDENCE_ARCHIVE');
+            if (match[1] === 'path') paxPath = match[2];
+          }
+        }
+        offset += padded;
+        continue;
+      }
+      need(type === '0' || type === '7' || type === '5', 'VAST_EVIDENCE_ENTRY_TYPE');
+      const ustar = header.subarray(257, 263).toString('latin1') === 'ustar\0';
+      const field = (start, length) => header.subarray(start, start + length).toString('utf8').replace(/\0.*$/s, '');
+      const prefix = ustar ? field(345, 155) : '';
+      const name = paxPath ?? longName ?? (prefix ? `${prefix}/${field(0, 100)}` : field(0, 100));
+      longName = null; paxPath = null;
+      evidenceName(name);
+      entries += 1; total += bodySize;
+      need(entries <= maxEntries, 'VAST_EVIDENCE_TOO_MANY_ENTRIES');
+      need(total <= maxBytes, 'VAST_EVIDENCE_TOO_LARGE');
+      offset += padded;
+    }
+    need(ended && entries > 0, 'VAST_EVIDENCE_ARCHIVE');
+  } finally { await handle.close(); }
+}
+
 async function writePrivate(path, value) {
   await writeFile(path, typeof value === 'string' ? value : JSON.stringify(value, null, 2) + '\n', { mode: 0o600 });
 }
@@ -118,8 +215,14 @@ export function createVastAdapter({ fetchImpl = globalThis.fetch, run = defaultR
     fail('VAST_LIST_UNBOUNDED');
   }
   const attemptDir = (config, key) => join(config.resultsRoot, key);
+  // Host keys: verified against the operator's pinned known_hosts file; an
+  // unknown or changed key is refused. Trust on first use (a fresh per-instance
+  // file) only when the host config sets sshTrustOnFirstUse: true.
+  const hostKeyOptions = (config, instance) => config.sshKnownHosts !== undefined
+    ? ['-o', 'StrictHostKeyChecking=yes', '-o', `UserKnownHostsFile=${config.sshKnownHosts}`, '-o', 'GlobalKnownHostsFile=/dev/null']
+    : ['-o', 'StrictHostKeyChecking=accept-new', '-o', `UserKnownHostsFile=${join(config.resultsRoot, `known_hosts-${instance.id}`)}`];
   const ssh = (config, instance, remote) => ['-i', config.sshKey, '-p', String(instance.ssh_port), '-o', 'BatchMode=yes', '-o', 'ConnectTimeout=8',
-    '-o', 'StrictHostKeyChecking=accept-new', '-o', `UserKnownHostsFile=${join(config.resultsRoot, `known_hosts-${instance.id}`)}`, '-o', 'IdentitiesOnly=yes',
+    ...hostKeyOptions(config, instance), '-o', 'IdentitiesOnly=yes',
     `root@${instance.ssh_host}`, remote];
 
   async function submit({ job, idempotencyKey, signal }) {
@@ -159,15 +262,28 @@ export function createVastAdapter({ fetchImpl = globalThis.fetch, run = defaultR
     return { runId: String(instanceId) };
   }
 
+  // 'ok' | 'retry' (slow or failed pull; next tick tries again) | 'rejected'
+  // (archive over the caps or with unsafe entries; never extracted).
   async function retrieve(config, instance, dir, started) {
     const left = () => INSPECT_BUDGET_MS - (now() - started);
     const archive = join(dir, 'evidence.tar');
+    const maxBytes = config.maxEvidenceBytes ?? DEFAULT_MAX_EVIDENCE_BYTES;
+    const maxEntries = config.maxEvidenceEntries ?? DEFAULT_MAX_EVIDENCE_ENTRIES;
     await rm(archive, { force: true }); await rm(join(dir, 'evidence'), { recursive: true, force: true });
-    if (left() < 4000) return false;
-    const pulled = await run('ssh', ssh(config, instance, 'tar -C /workspace -cf - out run'), { timeoutMs: left() - 3000, stdoutFile: archive });
-    if (pulled.code !== 0) return false;
+    if (left() < 4000) return 'retry';
+    const pulled = await run('ssh', ssh(config, instance, 'tar -C /workspace -cf - out run'), { timeoutMs: left() - 3000, stdoutFile: archive, maxStdoutBytes: maxBytes });
+    const reject = async reason => {
+      await rm(archive, { force: true });
+      await writePrivate(join(dir, 'evidence-rejected.json'), { schema: 'ruvllm-vast-evidence-rejected/v1', reason, maxBytes, maxEntries, at: new Date(now()).toISOString() });
+      return 'rejected';
+    };
+    if (pulled.overflow) return reject('VAST_EVIDENCE_TOO_LARGE');
+    if (pulled.code !== 0) return 'retry';
+    try { await checkEvidenceArchive(archive, { maxBytes, maxEntries }); }
+    catch (error) { if (error.code?.startsWith('VAST_EVIDENCE_')) return reject(error.code); throw error; }
     await mkdir(join(dir, 'evidence'), { mode: 0o700 });
-    return (await run('tar', ['-xf', archive, '-C', join(dir, 'evidence'), '--no-same-owner', '--no-same-permissions'], { timeoutMs: 5000 })).code === 0;
+    const extracted = await run('tar', ['-xf', archive, '-C', join(dir, 'evidence'), '--no-same-owner', '--no-same-permissions'], { timeoutMs: 5000 });
+    return extracted.code === 0 ? 'ok' : 'retry';
   }
 
   async function verify(config, job, dir) {
@@ -230,8 +346,11 @@ export function createVastAdapter({ fetchImpl = globalThis.fetch, run = defaultR
     if (remote.state === 'running') return { status: 'running', costUsd };
     if (remote.state === 'failed') { await retrieve(config, instance, dir, started); return { status: 'failed', costUsd }; }
     if (remote.state !== 'done') return { status: 'running', costUsd };
-    // Retain raw outputs locally before any terminal report; retry next tick on a slow pull.
-    if (!await retrieve(config, instance, dir, started)) return { status: 'running', costUsd };
+    // Retain raw outputs locally before any terminal report; retry next tick on a
+    // slow pull; an oversized or unsafe archive fails the job.
+    const retrieved = await retrieve(config, instance, dir, started);
+    if (retrieved === 'rejected') return { status: 'failed', costUsd };
+    if (retrieved !== 'ok') return { status: 'running', costUsd };
     const checked = job.task === 'train' ? await verifyTraining(config, job, dir) : await verify(config, job, dir);
     return { status: 'completed', costUsd, completion: { completionVersion: 1, completionVerified: checked.ok, status: checked.ok ? 'ok' : 'error',
       exitCode: checked.ok ? 0 : 1, assertionsVerified: checked.ok, evidenceComplete: checked.ok, truncated: false, jobId: job.id,
@@ -264,11 +383,18 @@ export const submit = input => defaults.submit(input);
 export const inspect = input => defaults.inspect(input);
 export const cleanup = input => defaults.cleanup(input);
 
-if (process.argv[1] === SELF && process.argv[2] === 'guard') {
-  const [instanceId, ref] = process.argv.slice(3);
+// The destroy timer entry point. No top-level await, so this module can also be
+// loaded with require() where Node supports require(esm).
+async function guard(instanceId, ref) {
   const config = await readHostConfig();
   const released = await defaults.destroyConfirmed(config, await apiKey(config, ref), instanceId);
   await writePrivate(join(config.resultsRoot, `guard-${instanceId}.json`), { schema: 'ruvllm-vast-guard/v1', instanceId,
     confirmedAbsent: released, at: new Date().toISOString() });
-  process.exitCode = released ? 0 : 1;
+  return released;
+}
+
+if (process.argv[1] === SELF && process.argv[2] === 'guard') {
+  const [instanceId, ref] = process.argv.slice(3);
+  guard(instanceId, ref).then(released => { process.exitCode = released ? 0 : 1; },
+    error => { console.error(error.code ?? 'VAST_GUARD_FAILED'); process.exitCode = 1; });
 }