arcway 0.1.13 → 0.1.15

package/server/jobs/worker-pool.js

@@ -0,0 +1,253 @@
+import { Worker } from 'node:worker_threads';
+import { fileURLToPath } from 'node:url';
+import path from 'node:path';
+
+const WORKER_ENTRY_URL = new URL('./worker-entry.js', import.meta.url);
+
+// Sent from workers so the pool can correlate replies back to the matching
+// in-flight task. Tasks time out (terminating the worker) if the worker never
+// produces a settlement message for this id.
+let taskCounter = 0;
+function nextTaskId() {
+  taskCounter = (taskCounter + 1) >>> 0;
+  return taskCounter;
+}
+
+// Structured-clone-safe snapshot of an Error. Worker threads can structured-
+// clone Error instances, but across `postMessage` non-enumerable fields (cause,
+// custom props) survive unevenly across Node versions — normalise explicitly.
+function serializeError(err) {
+  if (!(err instanceof Error)) return { name: 'Error', message: String(err) };
+  return { name: err.name, message: err.message, stack: err.stack };
+}
+
+function deserializeError(raw) {
+  const err = new Error(raw?.message ?? 'Worker task failed');
+  if (raw?.name) err.name = raw.name;
+  if (raw?.stack) err.stack = raw.stack;
+  return err;
+}
+
+// Each pending task carries its own resolve/reject plus a timeout timer that
+// nukes the worker if the handler wedges (esbuild-style infinite loops, deadlocks
+// against an external service, etc.). staleTimeoutMs defaults to the dispatcher's
+// stale timeout but can be overridden per-job.
+class WorkerPool {
+  _entryUrl;
+  _size;
+  _workerData;
+  _workers = new Set();
+  _idle = [];
+  _pending = [];
+  _closed = false;
+  _defaultTimeoutMs;
+
+  constructor(options = {}) {
+    this._entryUrl = options.entryUrl ?? WORKER_ENTRY_URL;
+    const requested = options.size ?? 1;
+    this._size = Math.max(1, requested | 0);
+    this._workerData = options.workerData ?? null;
+    this._defaultTimeoutMs = options.defaultTimeoutMs ?? null;
+  }
+
+  get size() {
+    return this._size;
+  }
+
+  get workerCount() {
+    return this._workers.size;
+  }
+
+  get idleCount() {
+    return this._idle.length;
+  }
+
+  get pendingCount() {
+    return this._pending.length;
+  }
+
+  async run(task, options = {}) {
+    if (this._closed) throw new Error('Worker pool is closed');
+    const timeoutMs = options.timeoutMs ?? this._defaultTimeoutMs;
+    return new Promise((resolve, reject) => {
+      const entry = {
+        id: nextTaskId(),
+        task,
+        timeoutMs,
+        resolve,
+        reject,
+        worker: null,
+        timer: null,
+      };
+      const worker = this._idle.pop();
+      if (worker) {
+        this._dispatch(worker, entry);
+        return;
+      }
+      if (this._workers.size < this._size) {
+        const fresh = this._spawnWorker();
+        this._dispatch(fresh, entry);
+        return;
+      }
+      this._pending.push(entry);
+    });
+  }
+
+  _spawnWorker() {
+    const entryPath =
+      typeof this._entryUrl === 'string' ? this._entryUrl : fileURLToPath(this._entryUrl);
+    const worker = new Worker(entryPath, {
+      workerData: this._workerData,
+    });
+    worker._activeTask = null;
+    worker.on('message', (msg) => this._onMessage(worker, msg));
+    worker.on('error', (err) => this._onError(worker, err));
+    worker.on('exit', (code) => this._onExit(worker, code));
+    this._workers.add(worker);
+    return worker;
+  }
+
+  _dispatch(worker, entry) {
+    entry.worker = worker;
+    worker._activeTask = entry;
+    if (entry.timeoutMs && entry.timeoutMs > 0) {
+      entry.timer = setTimeout(() => this._onTimeout(entry), entry.timeoutMs);
+      // Don't let a pending timer keep the process alive during shutdown.
+      if (typeof entry.timer.unref === 'function') entry.timer.unref();
+    }
+    worker.postMessage({ id: entry.id, task: entry.task });
+  }
+
+  _onMessage(worker, msg) {
+    const entry = worker._activeTask;
+    // Worker sent a reply after we already timed it out or terminated it.
+    if (!entry || entry.id !== msg?.id) return;
+    this._clearTimer(entry);
+    worker._activeTask = null;
+    if (msg.status === 'ok') entry.resolve(msg.result);
+    else entry.reject(deserializeError(msg.error));
+    this._drainOrPark(worker);
+  }
+
+  _onError(worker, err) {
+    const entry = worker._activeTask;
+    if (entry) {
+      this._clearTimer(entry);
+      worker._activeTask = null;
+      entry.reject(err instanceof Error ? err : new Error(String(err)));
+    }
+    // An 'error' event means the worker is going down — Node will emit 'exit'
+    // shortly. Don't park or reuse it.
+    this._workers.delete(worker);
+    this._removeFromIdle(worker);
+    if (!this._closed) this._fillFromPending();
+  }
+
+  _onExit(worker) {
+    const entry = worker._activeTask;
+    if (entry) {
+      this._clearTimer(entry);
+      worker._activeTask = null;
+      entry.reject(new Error('Worker exited before producing a result'));
+    }
+    this._workers.delete(worker);
+    this._removeFromIdle(worker);
+    if (!this._closed) this._fillFromPending();
+  }
+
+  _onTimeout(entry) {
+    const worker = entry.worker;
+    if (!worker || worker._activeTask !== entry) return;
+    worker._activeTask = null;
+    entry.reject(new Error(`Worker task timed out after ${entry.timeoutMs}ms`));
+    // A wedged handler will never return; terminate the worker so it doesn't
+    // keep holding a DB connection / file handle / etc. A replacement spawns
+    // lazily on the next `run()`.
+    this._workers.delete(worker);
+    this._removeFromIdle(worker);
+    worker.terminate().catch(() => {});
+    if (!this._closed) this._fillFromPending();
+  }
+
+  _drainOrPark(worker) {
+    const next = this._pending.shift();
+    if (next) {
+      this._dispatch(worker, next);
+    } else {
+      this._idle.push(worker);
+    }
+  }
+
+  _fillFromPending() {
+    while (this._pending.length > 0 && this._workers.size < this._size) {
+      const entry = this._pending.shift();
+      const worker = this._spawnWorker();
+      this._dispatch(worker, entry);
+    }
+  }
+
+  _removeFromIdle(worker) {
+    const idx = this._idle.indexOf(worker);
+    if (idx !== -1) this._idle.splice(idx, 1);
+  }
+
+  _clearTimer(entry) {
+    if (entry.timer) {
+      clearTimeout(entry.timer);
+      entry.timer = null;
+    }
+  }
+
+  // Ask a worker to tear down its infrastructure (DB pool, redis) and exit
+  // cleanly. Workers that don't ack within `gracefulExitMs` are terminated.
+  _gracefulShutdownWorker(worker, gracefulExitMs) {
+    return new Promise((resolve) => {
+      let settled = false;
+      const done = () => {
+        if (settled) return;
+        settled = true;
+        clearTimeout(timer);
+        resolve();
+      };
+      const timer = setTimeout(() => {
+        worker.terminate().catch(() => {}).finally(done);
+      }, gracefulExitMs);
+      if (typeof timer.unref === 'function') timer.unref();
+      worker.once('exit', done);
+      // Zero-id control message — workers react to task.__shutdown.
+      try {
+        worker.postMessage({ id: 0, task: { __shutdown: true } });
+      } catch {
+        // Worker already down or channel closed — terminate to be safe.
+        worker.terminate().catch(() => {}).finally(done);
+      }
+    });
+  }
+
+  async shutdown({ gracefulExitMs = 2000 } = {}) {
+    this._closed = true;
+    for (const entry of this._pending) {
+      entry.reject(new Error('Worker pool is closed'));
+    }
+    this._pending = [];
+    // Reject in-flight tasks — the worker will be torn down either via the
+    // graceful shutdown message or via terminate() below.
+    for (const worker of this._workers) {
+      const entry = worker._activeTask;
+      if (entry) {
+        this._clearTimer(entry);
+        worker._activeTask = null;
+        entry.reject(new Error('Worker pool is closed'));
+      }
+    }
+    const workers = [...this._workers];
+    const exits = workers.map((w) => this._gracefulShutdownWorker(w, gracefulExitMs));
+    this._workers.clear();
+    this._idle = [];
+    await Promise.all(exits);
+  }
+}
+
+// Exposed for tests and for callers who need to refer to the default entry
+// path (e.g. alternate workerData).
+export { WorkerPool, WORKER_ENTRY_URL, serializeError, deserializeError };

package/server/mcp/index.js

@@ -74,4 +74,4 @@ class McpRouter {
   }
 }
 
-export { McpRouter, createDebugHandler, createMcpRuntime, isDebugRequest, startMcpServer };
+export { McpRouter };

package/server/pages/build-cache.js

@@ -0,0 +1,274 @@
+import path from 'node:path';
+import fs from 'node:fs/promises';
+import crypto from 'node:crypto';
+import { createRequire } from 'node:module';
+
+const require_ = createRequire(import.meta.url);
+const ESBUILD_VERSION = require_('esbuild/package.json').version;
+
+function sha256(data) {
+  return crypto.createHash('sha256').update(data).digest('hex');
+}
+
+function computeConfigHash({ target, minify, devMode, kind, nodeEnv }) {
+  return sha256(
+    JSON.stringify({
+      esbuildVersion: ESBUILD_VERSION,
+      target: target ?? '',
+      minify: !!minify,
+      devMode: !!devMode,
+      kind: kind ?? '',
+      nodeEnv: nodeEnv ?? process.env.NODE_ENV ?? '',
+    }),
+  );
+}
+
+function cacheRoot(rootDir) {
+  return path.join(rootDir, 'node_modules', '.cache', 'arcway-pages');
+}
+
+function entryKey(entryPath) {
+  return sha256(path.resolve(entryPath));
+}
+
+function bucketDir(rootDir, kind) {
+  return path.join(cacheRoot(rootDir), kind);
+}
+
+async function hashFileContent(filePath) {
+  try {
+    const buf = await fs.readFile(filePath);
+    return sha256(buf);
+  } catch {
+    return null;
+  }
+}
+
+function resolveInputPath(p, rootDir) {
+  if (path.isAbsolute(p)) return p;
+  // esbuild metafile paths may be relative to cwd; try that first, then rootDir.
+  const abs = path.resolve(p);
+  return abs;
+}
+
+async function hashInputs(inputs) {
+  const sorted = [...inputs].sort();
+  const parts = await Promise.all(
+    sorted.map(async (p) => {
+      const h = await hashFileContent(p);
+      return h === null ? null : `${p}:${h}`;
+    }),
+  );
+  if (parts.some((p) => p === null)) return null;
+  return sha256(parts.join('\n'));
+}
+
+async function readJson(filePath) {
+  try {
+    return JSON.parse(await fs.readFile(filePath, 'utf-8'));
+  } catch {
+    return null;
+  }
+}
+
+async function writeJsonAtomic(filePath, value) {
+  const tmp = filePath + '.tmp-' + crypto.randomBytes(4).toString('hex');
+  await fs.writeFile(tmp, JSON.stringify(value));
+  await fs.rename(tmp, filePath);
+}
+
+// fs.cp with an existing destination is not atomic (internally unlink + copy),
+// so a concurrent reader can observe ENOENT mid-write. Copy into a tmp sibling
+// and rename into place — rename is atomic on the same filesystem.
+async function cpAtomic(src, dst) {
+  const tmp = dst + '.tmp-' + crypto.randomBytes(4).toString('hex');
+  try {
+    await fs.cp(src, tmp);
+    await fs.rename(tmp, dst);
+  } catch (err) {
+    await fs.rm(tmp, { force: true }).catch(() => {});
+    throw err;
+  }
+}
+
+async function lookupBundle({ rootDir, entryPath, configHash, kind }) {
+  const bucket = bucketDir(rootDir, kind);
+  const key = entryKey(entryPath);
+  const metaPath = path.join(bucket, `${key}.meta.json`);
+  const jsPath = path.join(bucket, `${key}.js`);
+  const mapPath = path.join(bucket, `${key}.js.map`);
+
+  const meta = await readJson(metaPath);
+  if (!meta || meta.configHash !== configHash) {
+    return { hit: false, key, bucket, metaPath, jsPath, mapPath };
+  }
+  // Verify every recorded input still hashes to the stored value.
+  const currentHash = await hashInputs(meta.inputs);
+  if (currentHash === null || currentHash !== meta.inputsHash) {
+    return { hit: false, key, bucket, metaPath, jsPath, mapPath };
+  }
+  // Confirm cached artifact actually exists on disk.
+  try {
+    await fs.access(jsPath);
+  } catch {
+    return { hit: false, key, bucket, metaPath, jsPath, mapPath };
+  }
+  return { hit: true, key, bucket, metaPath, jsPath, mapPath };
+}
+
+async function storeBundle({ bucket, key, outFile, inputs, configHash, sourcemap }) {
+  await fs.mkdir(bucket, { recursive: true });
+  const cachedJs = path.join(bucket, `${key}.js`);
+  await cpAtomic(outFile, cachedJs);
+  if (sourcemap) {
+    const cachedMap = path.join(bucket, `${key}.js.map`);
+    try {
+      await cpAtomic(outFile + '.map', cachedMap);
+    } catch {
+      // Source map may be absent; non-fatal.
+    }
+  }
+  const inputsHash = await hashInputs(inputs);
+  await writeJsonAtomic(path.join(bucket, `${key}.meta.json`), {
+    configHash,
+    inputs,
+    inputsHash,
+    mtime: Date.now(),
+  });
+}
+
+async function restoreBundle({ cacheJs, cacheMap, outFile, sourcemap }) {
+  await fs.mkdir(path.dirname(outFile), { recursive: true });
+  await fs.cp(cacheJs, outFile);
+  if (sourcemap) {
+    try {
+      await fs.cp(cacheMap, outFile + '.map');
+    } catch {
+      // Source map may be absent in the cache entry; non-fatal.
+    }
+  }
+}
+
+async function buildWithCache({
+  rootDir,
+  kind,
+  entryPath,
+  outFile,
+  esbuildOptions,
+  esbuild,
+  devMode,
+}) {
+  const absEntry = path.resolve(entryPath);
+  const configHash = computeConfigHash({
+    target: esbuildOptions.target,
+    minify: esbuildOptions.minify,
+    devMode,
+    kind,
+  });
+
+  const lookup = await lookupBundle({ rootDir, entryPath: absEntry, configHash, kind });
+  if (lookup.hit) {
+    await restoreBundle({
+      cacheJs: lookup.jsPath,
+      cacheMap: lookup.mapPath,
+      outFile,
+      sourcemap: !!esbuildOptions.sourcemap,
+    });
+    return { cacheHit: true };
+  }
+
+  const result = await esbuild.build({ ...esbuildOptions, metafile: true });
+  const inputs = Object.keys(result.metafile?.inputs ?? {}).map((p) =>
+    resolveInputPath(p, rootDir),
+  );
+
+  await storeBundle({
+    bucket: lookup.bucket,
+    key: lookup.key,
+    outFile,
+    inputs,
+    configHash,
+    sourcemap: !!esbuildOptions.sourcemap,
+  });
+
+  return { cacheHit: false, metafile: result.metafile };
+}
+
+// Multi-file cache for bundles with several output files (client build with
+// code-splitting). Each cache entry lives under <bucket>/<coarseKey>/ and the
+// index is a sidecar <bucket>/<coarseKey>.meta.json.
+async function lookupMultiFileCache({ rootDir, kind, coarseKey }) {
+  const bucket = bucketDir(rootDir, kind);
+  const metaPath = path.join(bucket, `${coarseKey}.meta.json`);
+  const cacheDir = path.join(bucket, coarseKey);
+  const meta = await readJson(metaPath);
+  if (!meta) return { hit: false, bucket, cacheDir, metaPath };
+  const currentHash = await hashInputs(meta.inputs);
+  if (currentHash === null || currentHash !== meta.inputsHash) {
+    return { hit: false, bucket, cacheDir, metaPath };
+  }
+  // Sanity-check every recorded output file still exists in the cache dir.
+  for (const rel of meta.outputs) {
+    try {
+      await fs.access(path.join(cacheDir, rel));
+    } catch {
+      return { hit: false, bucket, cacheDir, metaPath };
+    }
+  }
+  return { hit: true, bucket, cacheDir, metaPath, meta };
+}
+
+async function restoreMultiFileCache({ cacheDir, outputs, destDir }) {
+  await fs.mkdir(destDir, { recursive: true });
+  await Promise.all(
+    outputs.map(async (rel) => {
+      const src = path.join(cacheDir, rel);
+      const dst = path.join(destDir, rel);
+      await fs.mkdir(path.dirname(dst), { recursive: true });
+      await fs.cp(src, dst);
+    }),
+  );
+}
+
+async function storeMultiFileCache({
+  bucket,
+  coarseKey,
+  destDir,
+  outputs,
+  inputs,
+  metadata,
+}) {
+  const cacheDir = path.join(bucket, coarseKey);
+  await fs.rm(cacheDir, { recursive: true, force: true });
+  await fs.mkdir(cacheDir, { recursive: true });
+  await Promise.all(
+    outputs.map(async (rel) => {
+      const src = path.join(destDir, rel);
+      const dst = path.join(cacheDir, rel);
+      await fs.mkdir(path.dirname(dst), { recursive: true });
+      await cpAtomic(src, dst);
+    }),
+  );
+  const inputsHash = await hashInputs(inputs);
+  const metaPath = path.join(bucket, `${coarseKey}.meta.json`);
+  await writeJsonAtomic(metaPath, {
+    inputs,
+    inputsHash,
+    outputs,
+    metadata,
+    mtime: Date.now(),
+  });
+}
+
+export {
+  buildWithCache,
+  computeConfigHash,
+  cacheRoot,
+  entryKey,
+  ESBUILD_VERSION,
+  sha256,
+  hashInputs,
+  lookupMultiFileCache,
+  restoreMultiFileCache,
+  storeMultiFileCache,
+};