@thomas.f/prnext 0.1.0-alpha.1

package/runtime/transport.mjs

@@ -0,0 +1,108 @@
+export const MAX_STREAM_CHUNK = 64 * 1024;
+const MAX_CONTROL_BYTES = 64 * 1024;
+const NO_BODY = new Set([204, 205, 304]);
+
+function bytes(value) {
+  if (typeof value === 'string') return Buffer.from(value);
+  if (value instanceof ArrayBuffer) return Buffer.from(value);
+  if (ArrayBuffer.isView(value)) return Buffer.from(value.buffer, value.byteOffset, value.byteLength);
+  throw new TypeError('A response stream must yield bytes');
+}
+
+async function* responseChunks(body) {
+  if (body === null || body === undefined) return;
+  if (typeof body === 'string' || body instanceof ArrayBuffer || ArrayBuffer.isView(body)) {
+    const value = bytes(body);
+    for (let offset = 0; offset < value.length; offset += MAX_STREAM_CHUNK) yield value.subarray(offset, offset + MAX_STREAM_CHUNK);
+    return;
+  }
+  if (typeof body[Symbol.asyncIterator] !== 'function' && typeof body[Symbol.iterator] !== 'function') {
+    throw new TypeError('Invalid response stream');
+  }
+  for await (const item of body) {
+    const value = bytes(item);
+    for (let offset = 0; offset < value.length; offset += MAX_STREAM_CHUNK) yield value.subarray(offset, offset + MAX_STREAM_CHUNK);
+  }
+}
+
+// A resolved write means the pipe has accepted the bytes. Awaiting every frame
+// prevents Node's writable queue from growing with a slow Rust/HTTP consumer.
+export function createProtocolOutput(write, { cork, uncork } = {}) {
+  const send = value => new Promise((resolve, reject) => {
+    write(value, error => error ? reject(error) : resolve());
+  });
+  const controlLine = frame => {
+    const value = JSON.stringify(frame) + '\n';
+    if (Buffer.byteLength(value) > MAX_CONTROL_BYTES) throw new Error('Response headers exceed the 64 KiB transport limit');
+    return value;
+  };
+  const control = frame => send(controlLine(frame));
+  const sendParts = parts => new Promise((resolve, reject) => {
+    cork();
+    try {
+      for (let index = 0; index < parts.length; index++) {
+        write(parts[index], error => {
+          if (error) reject(error);
+          else if (index === parts.length - 1) resolve();
+        });
+      }
+    } catch (error) { reject(error); }
+    finally { uncork(); }
+  });
+  const sendChunk = (id, chunk) => {
+    const line = controlLine({ id, type: 'chunk', length: chunk.length });
+    if (!cork || !uncork) return send(line).then(() => send(chunk));
+    // writev sends metadata and bytes without joining/copying the payload.
+    return sendParts([line, chunk]);
+  };
+
+  return async function output(id, result, { stream = false, compactResponse = false, method = 'GET', production = true, signalController } = {}) {
+    const responseBody = result.bufferedBody ?? result.body;
+    const pageError = { ...(result.pageError ? { pageError: result.pageError } : {}),
+      ...(result.pageFailure ? { pageFailure: result.pageFailure } : {}) };
+    if (stream && !result.isr && (method === 'HEAD' || NO_BODY.has(result.status))) {
+      await result.cancel?.();
+      if (result.body?.cancel && !result.body.locked) await result.body.cancel();
+      await send(JSON.stringify({ id, status: result.status, headers: result.headers || {}, body: '', ...pageError }) + '\n');
+      return;
+    }
+    if (!stream && !result.isr) {
+      await send(JSON.stringify({ id, status: result.status, headers: result.headers || {}, body: Buffer.from(result.body || '').toString('base64'), ...pageError }) + '\n');
+      return;
+    }
+    const headers = Object.fromEntries(Object.entries(result.headers || {}).filter(([name]) => name.toLowerCase() !== 'content-length'));
+    if (compactResponse && !result.isr && (typeof responseBody === 'string' || responseBody instanceof ArrayBuffer || ArrayBuffer.isView(responseBody))) {
+      const body = bytes(responseBody);
+      if (body.length <= 16 * 1024) {
+        const complete = controlLine({ id, type: 'complete', status: result.status, headers, length: body.length, ...pageError });
+        if (cork && uncork) await sendParts(body.length ? [complete, body] : [complete]);
+        else { await send(complete); if (body.length) await send(body); }
+        return;
+      }
+    }
+    const head = controlLine({ id, type: 'head', status: result.status, headers, ...pageError, ...(result.isr ? { isr: result.isr } : {}) });
+    if (cork && uncork && (typeof responseBody === 'string' || responseBody instanceof ArrayBuffer || ArrayBuffer.isView(responseBody))) {
+      const body = bytes(responseBody);
+      if (body.length <= MAX_STREAM_CHUNK) {
+        // Pages HTML and buffered action responses already know their entire
+        // small body. Send their frames together, keeping the same wire format.
+        await sendParts([head, ...(body.length ? [controlLine({id,type:'chunk',length:body.length}),body] : []), controlLine({id,type:'end'})]);
+        return;
+      }
+    }
+    await send(head);
+    try {
+      for await (const chunk of responseChunks(responseBody)) {
+        await sendChunk(id, chunk);
+      }
+      await control({ id, type: 'end' });
+    } catch (error) {
+      signalController?.abort(error);
+      await result.cancel?.(error);
+      console.error('[prnext] Response stream failed:', error?.stack || error);
+      // Headers are already on the wire: terminate this response, never emit a
+      // replacement HTTP response or retry application code after a mutation.
+      await control({ id, type: 'error', message: production ? 'Response stream failed' : String(error?.message || error).slice(0, 4096) });
+    }
+  };
+}

package/runtime/watch-project.mjs

@@ -0,0 +1,130 @@
+import { watch as watchDirectory } from 'node:fs';
+import { readdir } from 'node:fs/promises';
+import path from 'node:path';
+import { shouldWatchProjectFile } from './env.mjs';
+
+const missing = error => error.code === 'ENOENT' || error.code === 'ENOTDIR';
+
+// On Linux Node implements recursive fs.watch by scanning every descendant,
+// including ignored build trees. Atomic build swaps can delete a descendant
+// during that scan. Watch only source directories, and handle removal races
+// in our own traversal. No file contents or node_modules watches are retained.
+export async function watchProject(root, { onChange, onError, ignore = () => false, watch = watchDirectory, maxDirectories = 8192, nativeRecursive = process.platform === 'darwin' }) {
+  // macOS has native recursive FSEvents support. Retain that backend: separate
+  // directory subscriptions can lose coalesced events during rapid renames.
+  if (nativeRecursive) {
+    const watcher = watch(root, { recursive: true }, (_event, filename) => {
+      const relative = filename && String(filename).replaceAll(path.sep, '/');
+      if (relative && !ignore(relative) && shouldWatchProjectFile(relative)) onChange(relative);
+    });
+    watcher.on('error', error => { watcher.close(); onError(error); });
+    return { close: () => watcher.close(), async refresh() {} };
+  }
+  const directories = new Map();
+  const pending = new Set();
+  let closed = false, ready = false, timer, running, refreshAll = false;
+  const allowedDirectory = relative => !relative || (!ignore(relative) && (relative === '.contentlayer' || shouldWatchProjectFile(relative)));
+  const allowedFile = relative => relative && !ignore(relative) && shouldWatchProjectFile(relative);
+  function remove(relative) {
+    for (const [name, watcher] of directories) if (name === relative || name.startsWith(relative + '/')) {
+      directories.delete(name);
+      watcher.close();
+    }
+  }
+  function close() {
+    closed = true;
+    clearTimeout(timer);
+    pending.clear();
+    for (const watcher of directories.values()) watcher.close();
+    directories.clear();
+  }
+  function fail(error) {
+    if (closed) return;
+    close();
+    onError(error);
+  }
+  function queue(relative) {
+    if (closed) return;
+    pending.add(relative);
+    clearTimeout(timer);
+    timer = setTimeout(() => { void drain().catch(fail); }, 20);
+  }
+  function disappeared(relative) {
+    remove(relative);
+    // A rapid rename can be coalesced by macOS. If a scheduled scan observes
+    // the old path disappearing, reconcile its parent to discover the new one.
+    queue(path.posix.dirname(relative) === '.' ? '' : path.posix.dirname(relative));
+  }
+  async function scan(relative, recursive = false) {
+    if (closed) return;
+    if (!allowedDirectory(relative)) { remove(relative); return; }
+    if (!directories.has(relative)) {
+      if (directories.size >= maxDirectories) throw new Error(`Development watcher exceeds ${maxDirectories} source directories. Narrow the application directory.`);
+      let watcher;
+      try {
+        watcher = watch(path.join(root, relative), { recursive: false }, (event, filename) => {
+          if (closed) return;
+          if (relative && event === 'rename' && String(filename) === path.posix.basename(relative)) {
+            // inotify subscriptions follow an inode. Reopen this subtree even
+            // when an atomic source replacement already recreated its path.
+            disappeared(relative);
+            return;
+          }
+          const changed = filename ? path.posix.join(relative, String(filename).replaceAll(path.sep, '/')) : relative;
+          if (allowedFile(changed)) onChange(changed);
+          // The .contentlayer container itself is ignored as a source, but
+          // its generated directory may be created after the watcher starts.
+          // Platforms can coalesce a directory rename into a change event.
+          // Rescan only this directory, rather than relying on the event name.
+          if (allowedDirectory(changed) || allowedFile(changed)) queue(relative);
+        });
+      } catch (error) { if (relative && missing(error)) { disappeared(relative); return; } throw error; }
+      directories.set(relative, watcher);
+      // Files may already exist by the time a newly created directory is
+      // subscribed. Its discovery must also invalidate the source snapshot.
+      if (ready && allowedFile(relative)) onChange(relative);
+      watcher.on('error', error => {
+        if (closed) return;
+        if (relative && missing(error)) disappeared(relative);
+        else fail(error);
+      });
+    }
+    let entries;
+    try { entries = await readdir(path.join(root, relative), { withFileTypes: true }); }
+    catch (error) { if (missing(error) && relative) { disappeared(relative); return; } throw error; }
+    if (closed) return;
+    const children = new Set();
+    for (const entry of entries) {
+      const child = path.posix.join(relative, entry.name);
+      if (!entry.isDirectory() || !allowedDirectory(child)) continue;
+      children.add(child);
+      if (recursive || !directories.has(child)) await scan(child, recursive);
+    }
+    for (const name of directories.keys()) {
+      if (name && (path.posix.dirname(name) === '.' ? '' : path.posix.dirname(name)) === relative && !children.has(name)) remove(name);
+    }
+  }
+  function drain() {
+    if (running) return running;
+    running = (async () => {
+      while (!closed && pending.size) {
+        const batch = [...pending]; pending.clear();
+        const recursive = refreshAll; refreshAll = false;
+        for (const relative of batch) await scan(relative, recursive);
+      }
+    })().finally(() => { running = undefined; });
+    return running;
+  }
+  try { await scan('', true); }
+  catch (error) { close(); throw error; }
+  ready = true;
+  return {
+    close,
+    async refresh() {
+      if (closed) return;
+      refreshAll = true;
+      pending.add('');
+      await drain();
+    },
+  };
+}

package/runtime/worker-artifacts.mjs

@@ -0,0 +1,22 @@
+// Immutable build artefacts only. IDs and evictions use the request's MessagePort
+// so a receiver resolves an ID before a later eviction can remove its entry.
+export function workerArtifacts(port, { maxEntries = 8, maxBytes = 256 * 1024, maxEntryBytes = 64 * 1024 } = {}) {
+  const entries = new Map();
+  let sequence = 0, bytes = 0;
+  return flight => {
+    if (typeof flight !== 'string') return undefined;
+    const size = flight.length * 2;
+    if (!maxEntries || size > maxBytes || size > maxEntryBytes) return undefined;
+    const old = entries.get(flight);
+    if (old) { entries.delete(flight); entries.set(flight, old); return old.id; }
+    while (entries.size && (entries.size >= maxEntries || bytes + size > maxBytes)) {
+      const [key, item] = entries.entries().next().value;
+      port.postMessage({ type: 'artifact-drop', id: item.id });
+      entries.delete(key); bytes -= item.size;
+    }
+    const id = ++sequence;
+    port.postMessage({ type: 'artifact', id, value: flight });
+    entries.set(flight, { id, size }); bytes += size;
+    return id;
+  };
+}

package/runtime/worker-metadata.mjs

@@ -0,0 +1,31 @@
+// Only build manifests belong here. Never register headers, models, closures or
+// request contexts. Eviction is ordered on the same MessagePort as requests.
+export function workerMetadata(port, limit = 32) {
+  const entries = new Map();
+  let sequence = 0;
+  return value => {
+    if (!value || typeof value !== 'object') return undefined;
+    let id = entries.get(value);
+    if (id !== undefined) { entries.delete(value); entries.set(value, id); return id; }
+    id = ++sequence;
+    if (entries.size >= limit) {
+      const [old, oldId] = entries.entries().next().value;
+      port.postMessage({ type: 'metadata-drop', id: oldId });
+      entries.delete(old);
+    }
+    port.postMessage({ type: 'metadata', id, value });
+    entries.set(value, id);
+    return id;
+  };
+}
+
+const bundlers = new WeakMap();
+const empty = Object.freeze({});
+export function flightBundler(clientModules = empty, browser = true) {
+  let pair = bundlers.get(clientModules);
+  if (!pair) { pair = []; bundlers.set(clientModules, pair); }
+  const slot = browser ? 1 : 0;
+  if (!pair[slot]) pair[slot] = Object.fromEntries(Object.entries(clientModules).map(([id, item]) =>
+    [id, { id, chunks: browser && item.browserModule ? [id, item.browserModule] : [], name: '*' }]));
+  return pair[slot];
+}

package/runtime/worker.mjs

@@ -0,0 +1,197 @@
+// prnext-transport:socket-v2
+// prnext-concurrency:512
+import { requestFrames } from './request-transport.mjs';
+import { requestWork } from './request-work.mjs';
+import { AsyncResource } from 'node:async_hooks';
+const workScope = new AsyncResource('PRNextWorkObserver', { triggerAsyncId: 0 });
+import { readFile } from 'node:fs/promises';
+import path from 'node:path';
+import { pathToFileURL } from 'node:url';
+import readline from 'node:readline';
+import net from 'node:net';
+import { timingSafeEqual } from 'node:crypto';
+import { Console } from 'node:console';
+import { navigationResponse } from './navigation.mjs';
+import { createProtocolOutput } from './transport.mjs';
+import { loadEnvConfig } from './env.mjs';
+
+// stdout is exclusively the transport, including when user packages log during import.
+const writeProtocol = process.stdout.write.bind(process.stdout);
+process.stdout.write = process.stderr.write.bind(process.stderr);
+globalThis.console = new Console({ stdout: process.stderr, stderr: process.stderr });
+
+const projectRoot = path.resolve(process.argv[2] || process.cwd());
+const distDir = path.resolve(projectRoot, process.argv[3] || '.prnext');
+process.chdir(projectRoot);
+let manifest;
+try { manifest = JSON.parse(await readFile(path.join(distDir, 'manifest.json'), 'utf8')); }
+catch (error) { console.error('PRNext worker could not read build manifest:', error.message); process.exit(1); }
+process.env.NODE_ENV ??= manifest.dev ? 'development' : 'production';
+loadEnvConfig(projectRoot, { dev: !!manifest.dev });
+// NODE_ENV=test selects .env.test above. React itself must match the compiled
+// browser graph and the RSC worker's mode, including dev launched from CI.
+process.env.NODE_ENV = manifest.dev ? 'development' : 'production';
+const production = !manifest.dev;
+// Load each renderer beside the application only when that request needs it.
+// Middleware and API workers do not pay for React or the Pages/App renderers.
+const runtimeModule = name => import(pathToFileURL(path.join(distDir, `runtime/${name}.mjs`)).href);
+let errorResponse;
+try { ({ errorResponse } = await runtimeModule('http')); }
+catch (error) { console.error('PRNext worker could not load the project runtime:', error.message); process.exit(1); }
+const routes = new Map(manifest.routes.map(route => [route.id, route]));
+const prerendered = new Set((manifest.prerendered || []).map(item => item.path));
+let appRuntime;
+let appStaticRuntime;
+let routeStaticRuntime;
+let middlewareRuntime;
+let pagesRuntime;
+let apiRuntime;
+let incrementalRuntime;
+
+async function handleRequest(input, output, signalController) {
+  let responseStarted = false;
+  try {
+    if (input.renderMode === 'incremental-cache' && input.routeId === '__prnext_incremental_cache') {
+      incrementalRuntime ??= await runtimeModule('incremental-cache');
+      const result = await incrementalRuntime.runIncrementalCache({ ...input, manifest, distDir, production, signal: signalController.signal });
+      input.body = undefined;
+      responseStarted = true;
+      await output(input.id, result, { ...input, production, signalController });
+      return;
+    }
+    const isMiddleware = input.renderMode === 'middleware' && input.routeId === '__prnext_middleware';
+    const route = isMiddleware && manifest.middleware
+      ? { ...manifest.middleware, id: '__prnext_middleware', kind: 'middleware' }
+      : routes.get(input.routeId);
+    if (!route) { await output(input.id, { status: 404, body: 'Not Found' }, input); return; }
+    const requestedPath = new URL(input.url).pathname;
+    if (route.router !== 'app' && !['isr', 'error404', 'error500'].includes(input.renderMode) && route.fallback === false && !prerendered.has(requestedPath) && !prerendered.has(requestedPath.replace(/\/$/, '') || '/')) {
+      await output(input.id, { status: 404, body: 'Not Found' }, input); return;
+    }
+    const options = { ...input, modulePath: path.resolve(distDir, route.module), route, production, manifest, distDir, signal: signalController.signal };
+    let result;
+    if (isMiddleware) {
+      middlewareRuntime ??= await import(pathToFileURL(path.join(distDir, 'runtime/middleware.mjs')).href);
+      result = await middlewareRuntime.runMiddleware(options);
+    } else if (route.router === 'app' && route.kind === 'page') {
+      if (input.renderMode === 'isr') {
+        appStaticRuntime ??= await import(pathToFileURL(path.join(distDir, 'runtime/app-static.mjs')).href);
+        result = await appStaticRuntime.renderAppIsrPage(options);
+      } else {
+        appRuntime ??= await import(pathToFileURL(path.join(distDir, 'runtime/app-render.mjs')).href);
+        result = await appRuntime.renderAppPage(options);
+      }
+    } else if (route.router === 'app' && route.kind === 'api' && input.renderMode === 'isr') {
+      routeStaticRuntime ??= await import(pathToFileURL(path.join(distDir, 'runtime/route-static.mjs')).href);
+      result = await routeStaticRuntime.renderRouteHandlerIsr(options);
+    } else if (route.kind === 'api') {
+      apiRuntime ??= await runtimeModule('api');
+      result = await apiRuntime.runApi(options);
+    } else {
+      pagesRuntime ??= await runtimeModule('render');
+      result = input.renderMode === 'isr' ? await pagesRuntime.renderIsrPage({ ...options, capturePageFailure: true })
+        : await pagesRuntime.renderPageRequest({ ...options, nativeErrors: true });
+    }
+    // Long responses should retain only the application's decoded request,
+    // not the original JSON line and an extra base64 copy of its body.
+    input.body = undefined;
+    options.body = undefined;
+    responseStarted = true;
+    try { await output(input.id, result, { ...input, production, signalController }); }
+    finally { await result.finalizeCache?.(); }
+  } catch (error) {
+    signalController.abort(error);
+    if (responseStarted) {
+      // A broken pipe or invalid transport cannot be repaired with a second
+      // response on the same request. Rust replaces this failed worker.
+      throw error;
+    }
+    await output(input?.id ?? null, navigationResponse(error) || errorResponse(error, production), { ...input, production, signalController });
+  }
+}
+
+// Native lanes use separate sockets, so backpressure or cancellation on one
+// response never blocks another response's bytes or aborts its request context.
+const laneLimit = Math.min(512, Math.max(1, Number(process.env.PRNEXT_WORKER_SOCKET) || 1));
+const token = process.env.PRNEXT_WORKER_TOKEN;
+delete process.env.PRNEXT_WORKER_TOKEN;
+delete process.env.PRNEXT_WORKER_SOCKET;
+if (token && laneLimit > 1) {
+  const expected = Buffer.from(token);
+  const sockets = new Set();
+  const invocations = new Set();
+  const server = net.createServer(socket => {
+    if (sockets.size >= laneLimit) { socket.destroy(); return; }
+    sockets.add(socket); socket.setNoDelay(true);
+    let controller;
+    const authTimer = setTimeout(() => socket.destroy(), 5000);
+    socket.on('error', () => {});
+    socket.on('close', () => { clearTimeout(authTimer); sockets.delete(socket); });
+    // A handler may stop consuming the async iterator while awaiting npm code.
+    // Observe EOF even in paused mode without draining or buffering more input.
+    socket.on('readable', () => { if (socket.readableLength === 0) socket.read(0); });
+    const disconnected = () => {
+      const abandoned = controller;
+      if (!abandoned || abandoned.signal.aborted) return;
+      abandoned.abort(new Error('Request transport closed'));
+
+    };
+    socket.on('end', disconnected);
+    socket.on('close', disconnected);
+    const output = createProtocolOutput(socket.write.bind(socket), { cork: () => socket.cork(), uncork: () => socket.uncork() });
+    void (async () => {
+      try {
+        for await (const input of requestFrames(socket, { authenticate(line) {
+          const candidate = Buffer.from(line);
+          if (candidate.length !== expected.length || !timingSafeEqual(candidate, expected)) throw new Error('Worker authentication failed');
+          clearTimeout(authTimer); socket.write('ready\n');
+        } })) {
+          if (invocations.size >= laneLimit) throw new Error('Worker invocation capacity exceeded');
+          controller = new AbortController();
+          const active = controller;
+          let pendingWork = 0, finished = false;
+          invocations.add(active);
+          active.signal[requestWork] = promise => {
+            pendingWork++;
+            const settled = () => { if (--pendingWork === 0 && finished) invocations.delete(active); };
+            workScope.runInAsyncScope(() => Promise.resolve(promise).then(settled, settled));
+          };
+          const retireIfAbandoned = () => {
+            const timer = setTimeout(() => { if (invocations.has(active)) process.exit(70); }, 5000);
+            timer.unref();
+          };
+          active.signal.addEventListener('abort', retireIfAbandoned, { once: true });
+          try { await handleRequest(input, output, active); }
+          finally {
+            finished = true;
+            if (!pendingWork) invocations.delete(active);
+            else if (!active.signal.aborted) {
+              // Background code after sending a complete response still has a
+              // finite lifetime, matching the native request deadline.
+              const timer = setTimeout(() => { if (invocations.has(active)) process.exit(70); }, 30000);
+              timer.unref();
+            }
+            controller = undefined;
+          }
+        }
+      } catch (error) {
+        if (!socket.destroyed) console.error('[prnext] Request lane failed:', error?.message || error);
+      } finally { socket.destroy(); }
+    })();
+  });
+  // The stdin pipe is a parent-lifetime guard even though requests use sockets.
+  process.stdin.resume();
+  process.stdin.once('end', () => { for (const socket of sockets) socket.destroy(); server.close(); process.exit(0); });
+  server.listen(0, '127.0.0.1', () => {
+    writeProtocol(JSON.stringify({ port: server.address().port, binary: true, heartbeat: true }) + '\n');
+    setInterval(() => writeProtocol('alive\n'), 1000).unref();
+  });
+} else {
+  const output = createProtocolOutput(writeProtocol, { cork: () => process.stdout.cork(), uncork: () => process.stdout.uncork() });
+  for await (let line of readline.createInterface({ input: process.stdin, crlfDelay: Infinity })) {
+    if (!line.trim()) continue;
+    const input = JSON.parse(line); line = '';
+    try { await handleRequest(input, output, new AbortController()); }
+    catch (error) { console.error('[prnext] Worker transport failed:', error?.stack || error); process.exit(1); }
+  }
+}