@north-light/cli 0.3.580 → 0.3.582

package/scripts/sandbox-start.mjs

@@ -9,10 +9,13 @@
 import { spawn, spawnSync } from 'node:child_process';
 import { createServer } from 'node:http';
 import { randomBytes } from 'node:crypto';
-import { existsSync, lstatSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, readlinkSync, realpathSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';
+import { chmodSync, closeSync, constants as fsConstants, createWriteStream, existsSync, fsyncSync, lstatSync, mkdirSync, mkdtempSync, openSync, readFileSync, readSync, readdirSync, readlinkSync, realpathSync, renameSync, rmSync, symlinkSync, unlinkSync, writeFileSync, writeSync } from 'node:fs';
 import { tmpdir } from 'node:os';
-import { dirname, join, resolve, sep } from 'node:path';
+import { basename, dirname, join, resolve, sep } from 'node:path';
 import { fileURLToPath } from 'node:url';
+import { Readable } from 'node:stream';
+import { pipeline } from 'node:stream/promises';
+import { createZstdDecompress } from 'node:zlib';
 
 export const EXIT_NO_BROKER = 3;
 const here = dirname(fileURLToPath(import.meta.url));
@@ -134,6 +137,171 @@ function gitAsync(cwd, args, timeoutMs, env) {
   });
 }
 
+const archiveInvalid = (detail) => new StartFailure('import_archive_invalid', detail);
+
+/** Create `dir` (which must lie lexically in `root`) one component at a time; no existing component may resolve outside `rootReal`. */
+function ensureDir(root, rootReal, dir) {
+  let current = root;
+  for (const segment of dir.slice(root.length).split(sep).filter(Boolean)) {
+    current = join(current, segment);
+    if (!lstatExists(current)) mkdirSync(current);
+    if (!within(rootReal, realpathSync(current))) throw archiveInvalid(`${current} resolves outside ${root}`);
+  }
+}
+
+/** Each entry of a tar file: `{name, type, size, mode, linkname, offset}`; `offset` is where the data starts. Handles ustar, pax (`x`) and GNU long names (`L`, `K`). */
+export function* tarEntries(fd, fileSize) {
+  const block = Buffer.alloc(512);
+  const text = (b, from, to) => { const slice = b.subarray(from, to); const end = slice.indexOf(0); return slice.subarray(0, end < 0 ? slice.length : end).toString('utf8'); };
+  const octal = (b, from, to) => {
+    if (b[from] & 0x80) { let n = 0n; for (let i = from + 1; i < to; i++) n = (n << 8n) | BigInt(b[i]); return Number(n); }
+    const t = text(b, from, to).trim();
+    return t === '' ? 0 : parseInt(t, 8);
+  };
+  const readAt = (buffer, position) => { const n = readSync(fd, buffer, 0, buffer.length, position); if (n !== buffer.length) throw archiveInvalid('the archive is truncated'); };
+  const readText = (position, size) => { const b = Buffer.alloc(size); readAt(b, position); return b; };
+  let position = 0;
+  let overrides = {};
+  let zeroBlocks = 0;
+  while (position + 512 <= fileSize) {
+    readAt(block, position);
+    position += 512;
+    if (block.every((x) => x === 0)) { if (++zeroBlocks === 2) return; continue; }
+    zeroBlocks = 0;
+    let sum = 0;
+    for (let i = 0; i < 512; i++) sum += i >= 148 && i < 156 ? 32 : block[i];
+    if (sum !== parseInt(text(block, 148, 156).trim() || 'x', 8)) throw archiveInvalid('a tar header fails its checksum');
+    const type = String.fromCharCode(block[156] || 48);
+    const size = octal(block, 124, 136);
+    if (!Number.isSafeInteger(size) || size < 0 || position + size > fileSize) throw archiveInvalid('a tar entry runs past the end of the archive');
+    const dataAt = position;
+    position += Math.ceil(size / 512) * 512;
+    if (type === 'x') {
+      const pax = readText(dataAt, size).toString('utf8');
+      for (let i = 0; i < pax.length;) {
+        const space = pax.indexOf(' ', i);
+        const length = Number(pax.slice(i, space));
+        if (!(length > 0)) throw archiveInvalid('a pax header is malformed');
+        const [key, ...value] = Buffer.from(pax).subarray(i, i + length - 1).toString('utf8').slice(space - i + 1).split('=');
+        overrides[key] = value.join('=');
+        i += length;
+      }
+      continue;
+    }
+    if (type === 'g') continue;
+    if (type === 'L') { overrides.path = text(readText(dataAt, size), 0, size); continue; }
+    if (type === 'K') { overrides.linkpath = text(readText(dataAt, size), 0, size); continue; }
+    const prefix = text(block, 345, 500);
+    const ustar = text(block, 257, 262) === 'ustar';
+    const name = overrides.path ?? (ustar && prefix ? `${prefix}/${text(block, 0, 100)}` : text(block, 0, 100));
+    const linkname = overrides.linkpath ?? text(block, 157, 257);
+    overrides = {};
+    yield { name, type, size, mode: octal(block, 100, 108) & 0o7777, linkname, offset: dataAt };
+  }
+  throw archiveInvalid('the archive has no end-of-archive marker');
+}
+
+/** The target of an archive entry: the path map entry whose `from` (without its leading slash) is the longest leading run of the entry's segments, then
+ *  the rest under that entry's `to`. Throws `import_archive_invalid` for an absolute name, a `..` segment, or a name under no prefix of the map. */
+export function archiveTarget(pathMap, name) {
+  if (name.startsWith('/')) throw archiveInvalid(`entry ${name} has an absolute path`);
+  const segments = name.split('/').filter((x) => x !== '' && x !== '.');
+  if (segments.includes('..')) throw archiveInvalid(`entry ${name} escapes its prefix`);
+  let best = null;
+  for (const entry of pathMap) {
+    const from = entry.from.split('/').filter(Boolean);
+    if (from.length <= segments.length && from.every((x, i) => segments[i] === x) && (best === null || from.length > best.length)) best = { entry, length: from.length };
+  }
+  if (best === null) throw archiveInvalid(`entry ${name} lies under no prefix of the path map`);
+  const rest = segments.slice(best.length);
+  return { root: best.entry.to, path: join(best.entry.to, ...rest), rest };
+}
+
+/** Where an archive's entries for `root` are written first, beside it (so the final move is a rename on one filesystem). */
+const stagingOf = (root) => join(dirname(root), `.${basename(root)}.import-staging`);
+
+/** Move what was staged for `root` into it, replacing a same-named entry a retried or interrupted earlier attempt left. */
+function placeStaged(stage, root) {
+  mkdirSync(root, { recursive: true });
+  for (const child of readdirSync(stage)) {
+    const to = join(root, child);
+    if (lstatExists(to)) rmSync(to, { recursive: true, force: true });
+    renameSync(join(stage, child), to);
+  }
+  rmSync(stage, { recursive: true, force: true });
+}
+
+/** Download the node's archive and unpack every entry under its path-map target; nothing is written outside the targets, whatever the archive names.
+ *  Entries are written to a staging directory beside each target and moved in only after the whole archive has been read, so an attempt that fails
+ *  partway leaves no half-unpacked target and a retry starts clean. */
+export async function unpackImportedFiles(files, scratch) {
+  const response = await fetch(files.url, { signal: AbortSignal.timeout(600_000) });
+  if (!response.ok || response.body === null) throw new StartFailure('import_archive_unreachable', `the archive download answered ${response.status}`);
+  const tarFile = join(scratch, 'import.tar');
+  await pipeline(Readable.fromWeb(response.body), createZstdDecompress(), createWriteStream(tarFile));
+  const targets = [...new Set(files.path_map.map((m) => m.to))];
+  for (const root of targets) rmSync(stagingOf(root), { recursive: true, force: true });
+  const fd = openSync(tarFile, 'r');
+  try {
+    const size = lstatSync(tarFile).size;
+    const stages = new Map();
+    const stageOf = (root) => {
+      if (!stages.has(root)) {
+        const stage = stagingOf(root);
+        mkdirSync(stage, { recursive: true });
+        stages.set(root, { stage, real: realpathSync(stage) });
+      }
+      return stages.get(root);
+    };
+    for (const entry of tarEntries(fd, size)) {
+      const { root, rest } = archiveTarget(files.path_map, entry.name);
+      const { stage, real } = stageOf(root);
+      const path = join(stage, ...rest);
+      if (entry.type === '5') { ensureDir(stage, real, path); continue; }
+      if (rest.length === 0) throw archiveInvalid(`entry ${entry.name} is a ${entry.type === '0' ? 'file' : 'link'} where the prefix directory belongs`);
+      ensureDir(stage, real, dirname(path));
+      if (entry.type === '0') {
+        if (lstatExists(path)) unlinkSync(path);
+        const out = openSync(path, fsConstants.O_WRONLY | fsConstants.O_CREAT | fsConstants.O_EXCL | fsConstants.O_NOFOLLOW, 0o600);
+        try {
+          const buffer = Buffer.alloc(Math.min(1 << 20, Math.max(entry.size, 1)));
+          for (let done = 0; done < entry.size;) {
+            const n = readSync(fd, buffer, 0, Math.min(buffer.length, entry.size - done), entry.offset + done);
+            if (n === 0) throw archiveInvalid('the archive is truncated');
+            writeSync(out, buffer, 0, n);
+            done += n;
+          }
+        } finally { closeSync(out); }
+        chmodSync(path, entry.mode & 0o777);
+      } else if (entry.type === '2') {
+        // A link is kept only when it resolves, from where it lies, inside the same target (an absolute one is checked against the final target).
+        const resolved = resolve(dirname(path), entry.linkname);
+        if (entry.linkname.startsWith('/') ? !within(root, entry.linkname) : !within(stage, resolved)) throw archiveInvalid(`symlink ${entry.name} points outside its prefix`);
+        if (lstatExists(path)) unlinkSync(path);
+        symlinkSync(entry.linkname, path);
+      } else {
+        throw archiveInvalid(`entry ${entry.name} has type ${JSON.stringify(entry.type)}, which an import archive does not hold`);
+      }
+    }
+    for (const [root, { stage }] of stages) placeStaged(stage, root);
+  } finally {
+    closeSync(fd);
+    for (const root of targets) rmSync(stagingOf(root), { recursive: true, force: true });
+  }
+}
+
+/** The sandbox's own record that this node's archive was unpacked here: written after a complete unpack, synced to disk, so a sandbox that stopped and
+ *  woke (or a restarted start script) never unpacks over the work done since. A fresh sandbox has none. */
+const unpackMarker = (root, nodeId) => join(root, 'import-unpacked', nodeId);
+
+function writeUnpackMarker(path) {
+  mkdirSync(dirname(path), { recursive: true });
+  const fd = openSync(path, 'w');
+  try { writeSync(fd, `${new Date().toISOString()}\n`); fsyncSync(fd); } finally { closeSync(fd); }
+  const dir = openSync(dirname(path), 'r');
+  try { fsyncSync(dir); } finally { closeSync(dir); }
+}
+
 async function start(env) {
   const need = (name) => { const v = env[name]; if (!v) throw new StartFailure('bad_environment', `${name} is not set`); return v; };
   const apiUrl = need('CRTR_API_URL');
@@ -225,6 +393,21 @@ async function start(env) {
     } finally { await relay?.close(); }
   }
 
+  // 5.6b a hosted node's imported files (design-import, "Sandbox import step"). The control plane names the archive for as long as it exists, so a
+  // replacement sandbox gets it too; this sandbox unpacks only when it holds no marker of its own. The marker is written last: a start that fails
+  // anywhere before it (the download, a hostile entry, the report) leaves nothing half-unpacked in place and unpacks again on retry.
+  if (bundle.import_node_files) {
+    const marker = unpackMarker(root, bundle.node.id);
+    if (!existsSync(marker)) {
+      const scratch = mkdtempSync(join(tmpdir(), 'crtr-import-'));
+      try { await unpackImportedFiles(bundle.import_node_files, scratch); }
+      finally { rmSync(scratch, { recursive: true, force: true }); }
+      const reported = await api(apiUrl, token, 'POST', `/v1/executions/${executionId}/import-unpacked`, {}).catch((error) => ({ ok: false, status: String(error?.message ?? error) }));
+      if (!reported.ok) throw new StartFailure('import_report_failed', `reporting the unpack: ${reported.status}`);
+      writeUnpackMarker(marker);
+    }
+  }
+
   // 5.7 the managed worktree request (D102): the node's branch in the recipe repository's one clone, never a second checkout.
   const worktree = bundle.launch.worktree;
   if (worktree) {