@pygmalionjs/pygmalion 0.2.44 → 0.5.0

package/node/preview-capture-worker.mjs

@@ -0,0 +1,529 @@
+import { spawn } from 'node:child_process';
+import fs from 'node:fs/promises';
+import { validateRoutePreviewArtifactBundle } from './route-preview-artifact-v3.mjs';
+
+const DEFAULT_IDLE_MS = 300_000;
+const DEFAULT_JOB_TIMEOUT_MS = 1_200_000;
+const DEFAULT_MAX_RESTARTS = 3;
+const DEFAULT_RESTART_WINDOW_MS = 600_000;
+const DEFAULT_PING_TIMEOUT_MS = 10_000;
+const MAX_RESTART_BACKOFF_MS = 8_000;
+const STOP_KILL_GRACE_MS = 5_000;
+const STDERR_TAIL_LIMIT = 8_000;
+
+function assertPositiveNumber(value, name, fallback) {
+  if (value == null) return fallback;
+  if (typeof value !== 'number' || !Number.isFinite(value) || value <= 0) {
+    throw new TypeError(`Preview capture worker ${name} must be a positive number.`);
+  }
+  return value;
+}
+
+function assertNonNegativeInteger(value, name, fallback) {
+  if (value == null) return fallback;
+  if (!Number.isInteger(value) || value < 0) {
+    throw new TypeError(
+      `Preview capture worker ${name} must be a non-negative integer.`,
+    );
+  }
+  return value;
+}
+
+function normalizeArgs(args) {
+  if (args == null) return [];
+  if (
+    !Array.isArray(args) ||
+    args.some((entry) => typeof entry !== 'string')
+  ) {
+    throw new TypeError(
+      'Preview capture worker args must be an array of strings.',
+    );
+  }
+  return [...args];
+}
+
+function normalizeRequest(request) {
+  if (!request || typeof request !== 'object') {
+    throw new TypeError('Preview capture request must be an object.');
+  }
+  const { namespace, sourceRevision, frames, captureBaseUrl } = request;
+  if (typeof namespace !== 'string' || !namespace.trim()) {
+    throw new TypeError('Preview capture request requires a namespace.');
+  }
+  if (typeof sourceRevision !== 'string' || !sourceRevision.trim()) {
+    throw new TypeError('Preview capture request requires a source revision.');
+  }
+  if (frames != null && !Array.isArray(frames)) {
+    throw new TypeError('Preview capture request frames must be an array.');
+  }
+  if (captureBaseUrl != null && typeof captureBaseUrl !== 'string') {
+    throw new TypeError(
+      'Preview capture request captureBaseUrl must be a string.',
+    );
+  }
+  return {
+    namespace,
+    sourceRevision,
+    ...(frames == null ? {} : { frames: frames.map((frame) => ({ ...frame })) }),
+    ...(captureBaseUrl == null ? {} : { captureBaseUrl }),
+  };
+}
+
+function attachLineParser(stream, onMessage) {
+  let buffer = '';
+  stream.setEncoding('utf8');
+  stream.on('data', (chunk) => {
+    buffer += chunk;
+    let newline = buffer.indexOf('\n');
+    while (newline >= 0) {
+      const line = buffer.slice(0, newline).trim();
+      buffer = buffer.slice(newline + 1);
+      newline = buffer.indexOf('\n');
+      if (!line) continue;
+      let message;
+      try {
+        message = JSON.parse(line);
+      } catch {
+        continue; // Not a protocol line; the worker owns its own noise.
+      }
+      if (message && typeof message === 'object') onMessage(message);
+    }
+  });
+}
+
+function attachStderrTail(stream, worker) {
+  stream.setEncoding('utf8');
+  stream.on('data', (chunk) => {
+    worker.stderrTail = `${worker.stderrTail}${chunk}`.slice(-STDERR_TAIL_LIMIT);
+  });
+}
+
+async function consumeArtifactFile(response) {
+  const file = response?.artifactFile;
+  if (typeof file !== 'string' || !file.trim()) {
+    throw new Error('Preview capture worker returned no artifact file.');
+  }
+  try {
+    const bundle = JSON.parse(await fs.readFile(file, 'utf8'));
+    const validation = validateRoutePreviewArtifactBundle(bundle);
+    if (!validation.valid) {
+      throw new Error(
+        `Preview capture worker artifact is invalid: ${(
+          validation.errors ?? ['unknown validation error']
+        ).join(', ')}`,
+      );
+    }
+    return bundle;
+  } finally {
+    // The hand-off file is single-use: the bundle now lives in memory and the
+    // plugin persists it through the artifact store, so the temp file would
+    // only accumulate.
+    await fs.rm(file, { force: true }).catch(() => undefined);
+  }
+}
+
+function workerFailure(worker, fallbackMessage) {
+  const tail = worker.stderrTail.trim();
+  return new Error(tail ? `${fallbackMessage}\n${tail}` : fallbackMessage);
+}
+
+/**
+ * Long-lived capture worker channel: one child process that keeps its expensive
+ * boot (runtime preflight, case bundling, browser launch) warm across artifact
+ * requests, speaking JSON lines over stdio.
+ *
+ * Protocol: the channel writes `{ id, type: 'capture', payload }` — payload is
+ * `{ namespace, sourceRevision, frames?, captureBaseUrl? }` — and the worker
+ * answers `{ id, ok: true, artifactFile }` with the bundle in a temp file the
+ * channel reads, validates, and deletes, or `{ id, ok: false, error }`.
+ * `{ id, type: 'ping' }` must be answered immediately even while booting; the
+ * channel uses it to detect a wedged worker before reusing it.
+ *
+ * Failure policy: a crash rejects the in-flight job and the worker restarts on
+ * the next job with capped backoff. After `maxRestarts` crashes inside
+ * `restartWindowMs` the channel degrades permanently to spawning the script
+ * per request with `--once` — slower, but never dark.
+ */
+export function createPreviewCaptureWorkerChannel({
+  script,
+  args = [],
+  env,
+  cwd,
+  idleMs = DEFAULT_IDLE_MS,
+  jobTimeoutMs = DEFAULT_JOB_TIMEOUT_MS,
+  maxRestarts = DEFAULT_MAX_RESTARTS,
+  restartWindowMs = DEFAULT_RESTART_WINDOW_MS,
+  pingTimeoutMs = DEFAULT_PING_TIMEOUT_MS,
+} = {}) {
+  if (typeof script !== 'string' || !script.trim()) {
+    throw new TypeError(
+      'Preview capture worker script must be a non-empty string.',
+    );
+  }
+  const workerArgs = normalizeArgs(args);
+  if (env != null && (typeof env !== 'object' || Array.isArray(env))) {
+    throw new TypeError('Preview capture worker env must be an object.');
+  }
+  if (cwd != null && (typeof cwd !== 'string' || !cwd.trim())) {
+    throw new TypeError('Preview capture worker cwd must be a non-empty string.');
+  }
+  const resolvedIdleMs = assertPositiveNumber(idleMs, 'idleMs', DEFAULT_IDLE_MS);
+  const resolvedJobTimeoutMs = assertPositiveNumber(
+    jobTimeoutMs,
+    'jobTimeoutMs',
+    DEFAULT_JOB_TIMEOUT_MS,
+  );
+  const resolvedMaxRestarts = assertNonNegativeInteger(
+    maxRestarts,
+    'maxRestarts',
+    DEFAULT_MAX_RESTARTS,
+  );
+  const resolvedRestartWindowMs = assertPositiveNumber(
+    restartWindowMs,
+    'restartWindowMs',
+    DEFAULT_RESTART_WINDOW_MS,
+  );
+  const resolvedPingTimeoutMs = assertPositiveNumber(
+    pingTimeoutMs,
+    'pingTimeoutMs',
+    DEFAULT_PING_TIMEOUT_MS,
+  );
+
+  const spawnEnv = env ? { ...process.env, ...env } : undefined;
+  let worker = null;
+  let inFlight = false;
+  let degraded = false;
+  let disposed = false;
+  let idleTimer = null;
+  let requestCounter = 0;
+  const crashTimestamps = [];
+
+  function nextRequestId() {
+    requestCounter += 1;
+    return `capture-${process.pid}-${requestCounter}`;
+  }
+
+  function recordCrash(now = Date.now()) {
+    while (
+      crashTimestamps.length > 0 &&
+      now - crashTimestamps[0] > resolvedRestartWindowMs
+    ) {
+      crashTimestamps.shift();
+    }
+    crashTimestamps.push(now);
+    if (crashTimestamps.length > resolvedMaxRestarts) degraded = true;
+  }
+
+  function restartBackoffMs() {
+    const recent = crashTimestamps.length;
+    if (recent === 0) return 0;
+    return Math.min(500 * 2 ** (recent - 1), MAX_RESTART_BACKOFF_MS);
+  }
+
+  function clearIdleTimer() {
+    if (idleTimer) clearTimeout(idleTimer);
+    idleTimer = null;
+  }
+
+  function scheduleIdleShutdown() {
+    clearIdleTimer();
+    if (!worker || disposed) return;
+    idleTimer = setTimeout(() => {
+      // Idle means no job is in flight, so the worker drains instantly.
+      void stopWorker();
+    }, resolvedIdleMs);
+    idleTimer.unref?.();
+  }
+
+  function settlePending(current, settle) {
+    const pending = current.pending;
+    if (!pending) return;
+    current.pending = null;
+    clearTimeout(pending.timer);
+    settle(pending);
+  }
+
+  function spawnWorker() {
+    const proc = spawn(process.execPath, [script, ...workerArgs], {
+      ...(cwd ? { cwd } : {}),
+      ...(spawnEnv ? { env: spawnEnv } : {}),
+      stdio: ['pipe', 'pipe', 'pipe'],
+    });
+    const current = {
+      proc,
+      pending: null,
+      stopping: false,
+      stderrTail: '',
+      exited: new Promise((resolve) => {
+        proc.once('exit', resolve);
+        proc.once('error', resolve);
+      }),
+    };
+    attachLineParser(proc.stdout, (message) => {
+      const pending = current.pending;
+      if (!pending || message.id !== pending.id) return;
+      if (pending.kind === 'ping') {
+        settlePending(current, ({ resolve }) => resolve(message));
+        return;
+      }
+      if (message.ok === true) {
+        settlePending(current, ({ resolve }) => resolve(message));
+      } else {
+        // A refused job is the worker doing its job — the process stays warm.
+        settlePending(current, ({ reject }) =>
+          reject(
+            new Error(
+              typeof message.error === 'string' && message.error
+                ? message.error
+                : 'Preview capture worker reported a failure.',
+            ),
+          ),
+        );
+      }
+    });
+    attachStderrTail(proc.stderr, current);
+    const onGone = () => {
+      if (worker === current) worker = null;
+      if (current.stopping) return;
+      current.stopping = true;
+      recordCrash();
+      settlePending(current, ({ reject }) =>
+        reject(workerFailure(current, 'Preview capture worker exited unexpectedly.')),
+      );
+    };
+    proc.once('exit', onGone);
+    proc.once('error', onGone);
+    return current;
+  }
+
+  async function stopWorker() {
+    const current = worker;
+    if (!current) return;
+    worker = null;
+    current.stopping = true;
+    settlePending(current, ({ reject }) =>
+      reject(new Error('Preview capture worker channel is shutting down.')),
+    );
+    try {
+      current.proc.stdin.end();
+    } catch {
+      // Already closed; the signal below still applies.
+    }
+    current.proc.kill('SIGTERM');
+    const killTimer = setTimeout(() => {
+      current.proc.kill('SIGKILL');
+    }, STOP_KILL_GRACE_MS);
+    killTimer.unref?.();
+    await current.exited;
+    clearTimeout(killTimer);
+  }
+
+  function killWorker(current) {
+    if (worker === current) worker = null;
+    current.stopping = true;
+    current.proc.kill('SIGKILL');
+  }
+
+  function dispatch(current, message, { kind, timeoutMs, onTimeout }) {
+    return new Promise((resolve, reject) => {
+      const timer = setTimeout(() => {
+        current.pending = null;
+        onTimeout();
+        reject(
+          workerFailure(
+            current,
+            kind === 'ping'
+              ? 'Preview capture worker did not answer a health check.'
+              : `Preview capture job timed out after ${timeoutMs}ms.`,
+          ),
+        );
+      }, timeoutMs);
+      timer.unref?.();
+      current.pending = { id: message.id, kind, resolve, reject, timer };
+      try {
+        current.proc.stdin.write(`${JSON.stringify(message)}\n`);
+      } catch (error) {
+        settlePending(current, () => undefined);
+        reject(
+          workerFailure(
+            current,
+            `Preview capture worker is not accepting requests: ${
+              error?.message ?? error
+            }`,
+          ),
+        );
+      }
+    });
+  }
+
+  async function pingWorker(current) {
+    await dispatch(
+      current,
+      { id: nextRequestId(), type: 'ping' },
+      {
+        kind: 'ping',
+        timeoutMs: resolvedPingTimeoutMs,
+        onTimeout: () => {
+          // A worker that cannot answer a ping cannot run a job either.
+          recordCrash();
+          killWorker(current);
+        },
+      },
+    );
+  }
+
+  /** Returns a live worker, or null when the channel degraded on the way. */
+  async function ensureWorker() {
+    if (worker) {
+      const current = worker;
+      try {
+        await pingWorker(current);
+        return current;
+      } catch {
+        // The wedged worker was killed and the crash recorded. When that
+        // crash exhausted the restart budget, the caller takes the degraded
+        // path; otherwise fall through to a fresh spawn below.
+        if (degraded) return null;
+      }
+    }
+    const backoff = restartBackoffMs();
+    if (backoff > 0) {
+      await new Promise((resolve) => {
+        const timer = setTimeout(resolve, backoff);
+        timer.unref?.();
+      });
+    }
+    if (disposed) {
+      throw new Error('Preview capture worker channel is disposed.');
+    }
+    worker = spawnWorker();
+    return worker;
+  }
+
+  function runWorkerJob(current, payload) {
+    return dispatch(
+      current,
+      { id: nextRequestId(), type: 'capture', payload },
+      {
+        kind: 'capture',
+        timeoutMs: resolvedJobTimeoutMs,
+        onTimeout: () => {
+          recordCrash();
+          killWorker(current);
+        },
+      },
+    );
+  }
+
+  /**
+   * Degraded path: one spawn per request with `--once`, the same protocol over
+   * the child's own stdio. Slow again, but on-demand capture keeps answering.
+   */
+  function runOnceJob(payload) {
+    return new Promise((resolve, reject) => {
+      const proc = spawn(
+        process.execPath,
+        [script, ...workerArgs, '--once'],
+        {
+          ...(cwd ? { cwd } : {}),
+          ...(spawnEnv ? { env: spawnEnv } : {}),
+          stdio: ['pipe', 'pipe', 'pipe'],
+        },
+      );
+      const once = { stderrTail: '' };
+      let settled = false;
+      const settle = (fn) => {
+        if (settled) return;
+        settled = true;
+        clearTimeout(timer);
+        fn();
+      };
+      const timer = setTimeout(() => {
+        settle(() => {
+          proc.kill('SIGKILL');
+          reject(
+            new Error(`Preview capture job timed out after ${resolvedJobTimeoutMs}ms.`),
+          );
+        });
+      }, resolvedJobTimeoutMs);
+      timer.unref?.();
+      const id = nextRequestId();
+      attachLineParser(proc.stdout, (message) => {
+        if (message.id !== id) return;
+        if (message.ok === true) {
+          settle(() => resolve(message));
+        } else {
+          settle(() =>
+            reject(
+              new Error(
+                typeof message.error === 'string' && message.error
+                  ? message.error
+                  : 'Preview capture worker reported a failure.',
+              ),
+            ),
+          );
+        }
+        proc.stdin.end();
+      });
+      attachStderrTail(proc.stderr, once);
+      const onGone = () => {
+        settle(() =>
+          reject(
+            workerFailure(once, 'Preview capture worker exited without a response.'),
+          ),
+        );
+      };
+      proc.once('exit', onGone);
+      proc.once('error', onGone);
+      try {
+        proc.stdin.write(
+          `${JSON.stringify({ id, type: 'capture', payload })}\n`,
+        );
+      } catch (error) {
+        settle(() =>
+          reject(
+            new Error(
+              `Preview capture worker is not accepting requests: ${
+                error?.message ?? error
+              }`,
+            ),
+          ),
+        );
+      }
+    });
+  }
+
+  async function generateArtifact(request) {
+    if (disposed) {
+      throw new Error('Preview capture worker channel is disposed.');
+    }
+    // The artifact plugin serializes per identity; overlap here means a caller
+    // bypassed that queue, and interleaving two captures on one warm browser
+    // would let them poison each other.
+    if (inFlight) {
+      throw new Error('Preview capture worker accepts one job at a time.');
+    }
+    const payload = normalizeRequest(request);
+    inFlight = true;
+    clearIdleTimer();
+    try {
+      const current = degraded ? null : await ensureWorker();
+      const response = current
+        ? await runWorkerJob(current, payload)
+        : await runOnceJob(payload);
+      return await consumeArtifactFile(response);
+    } finally {
+      inFlight = false;
+      scheduleIdleShutdown();
+    }
+  }
+
+  async function dispose() {
+    if (disposed) return;
+    disposed = true;
+    clearIdleTimer();
+    await stopWorker();
+  }
+
+  return { generateArtifact, dispose };
+}

package/node/route-dependency-digest.mjs

@@ -1,3 +1,7 @@
+import { createHash } from 'node:crypto';
+import fs from 'node:fs/promises';
+import path from 'node:path';
+
 function normalizedRoutePath(value) {
   if (typeof value !== 'string' || !value.trim()) return null;
   try {
@@ -60,3 +64,150 @@ export function resolvePreviewRouteDependencyDigest(route, entries) {
   );
   return candidates[0]?.digest;
 }
+
+// Observed dependencies — the source files a captured frame actually rendered.
+//
+// Freshness used to be decided by the source revision, so every commit retired
+// every frame even when a frame's own content could not have changed. A route
+// digest is not a usable replacement in a single-page application: nearly every
+// screen lives on one or two routes, so a route's dependency closure is the
+// whole application and any edit invalidates everything.
+//
+// A frozen snapshot stamps the file each element came from, so the set of files
+// a screen really rendered is recoverable from the capture itself. That set is
+// finer than a route and cannot drift from the truth, because it is an
+// observation rather than a declaration.
+
+/** Attributes the DOM importer stamps a source file onto. */
+const SOURCE_ATTRIBUTES = ['data-pygmalion-source', 'data-pygmalion-own-source'];
+
+/**
+ * Minimum stamped elements before an observed set is trusted.
+ *
+ * A frame whose stamps are missing (a host that never wired sourceComponents,
+ * or a capture that failed early) would otherwise read as "depends on nothing"
+ * and never be invalidated again. Too few stamps therefore means unknown, and
+ * unknown must fall back to the coarse key rather than claim precision.
+ */
+export const OBSERVED_DEPENDENCY_MIN_STAMPS = 8;
+
+/** Files whose content decides a screen without ever being stamped. */
+const DEFAULT_ALWAYS_INCLUDED = Object.freeze([]);
+
+function normalizedSourcePath(value) {
+  if (typeof value !== 'string') return null;
+  // A stamp is "<file>|<hash>|<tag>"; only the file participates in identity.
+  const file = value.split('|', 1)[0].trim();
+  if (!file || file.startsWith('/') || file.includes('..')) return null;
+  return file.split(path.sep).join('/');
+}
+
+/**
+ * Source files stamped into a serialized snapshot.
+ *
+ * Parsing the stored HTML rather than collecting in the page keeps the answer
+ * true to what was actually stored: a serializer that prunes or rewrites nodes
+ * cannot make the recorded set disagree with the artifact it describes. It also
+ * means one function serves every producer, since all of them publish HTML.
+ */
+export function extractObservedSourceFiles(snapshot) {
+  // A stored snapshot is `{ document, head, body }`, but callers also hold the
+  // raw body string; accept both so one function serves every producer.
+  const html =
+    typeof snapshot === 'string'
+      ? snapshot
+      : snapshot && typeof snapshot === 'object'
+        ? [snapshot.document, snapshot.head, snapshot.body]
+            .filter((part) => typeof part === 'string')
+            .join('\n')
+        : '';
+  if (!html) {
+    return { files: [], stamps: 0 };
+  }
+  const files = new Set();
+  let stamps = 0;
+  for (const attribute of SOURCE_ATTRIBUTES) {
+    const pattern = new RegExp(`${attribute}="([^"]*)"`, 'g');
+    for (const match of html.matchAll(pattern)) {
+      stamps += 1;
+      const file = normalizedSourcePath(match[1]);
+      if (file) files.add(file);
+    }
+  }
+  return { files: [...files].sort(), stamps };
+}
+
+async function fileContentHash(root, file) {
+  try {
+    const resolved = path.resolve(root, file);
+    if (!resolved.startsWith(path.resolve(root) + path.sep)) return null;
+    const source = await fs.readFile(resolved);
+    return createHash('sha256').update(source).digest('hex').slice(0, 32);
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Records the observed set as `[path, contentHash]` pairs.
+ *
+ * Pairs rather than one digest: freshness is then a comparison against the
+ * files as they are now, so recording the set never changes the key the entry
+ * was written under. Storing a digest in the key instead would retire the very
+ * entry that just produced it.
+ */
+export async function recordObservedDependencies({
+  snapshot,
+  sourceRoot,
+  alwaysInclude = DEFAULT_ALWAYS_INCLUDED,
+  minStamps = OBSERVED_DEPENDENCY_MIN_STAMPS,
+}) {
+  if (typeof sourceRoot !== 'string' || !sourceRoot) return null;
+  const { files, stamps } = extractObservedSourceFiles(snapshot);
+  if (stamps < minStamps || files.length === 0) return null;
+  const wanted = [
+    ...new Set([
+      ...files,
+      ...alwaysInclude
+        .map((file) => normalizedSourcePath(file))
+        .filter((file) => file != null),
+    ]),
+  ].sort();
+  const recorded = [];
+  for (const file of wanted) {
+    const hash = await fileContentHash(sourceRoot, file);
+    // A stamped file we cannot read leaves the set unverifiable, and an
+    // unverifiable set must not be treated as fresh later.
+    if (hash == null) return null;
+    recorded.push([file, hash]);
+  }
+  return recorded;
+}
+
+/** True when every recorded file still hashes to what it did at capture time. */
+export async function observedDependenciesUnchanged({ recorded, sourceRoot }) {
+  if (!Array.isArray(recorded) || recorded.length === 0) return false;
+  if (typeof sourceRoot !== 'string' || !sourceRoot) return false;
+  for (const pair of recorded) {
+    if (!Array.isArray(pair) || pair.length !== 2) return false;
+    const [file, hash] = pair;
+    if (typeof file !== 'string' || typeof hash !== 'string') return false;
+    const current = await fileContentHash(sourceRoot, file);
+    if (current !== hash) return false;
+  }
+  return true;
+}
+
+/** Normalizes a persisted set, dropping anything malformed. */
+export function normalizeObservedDependencies(value) {
+  if (!Array.isArray(value) || value.length === 0) return null;
+  const pairs = [];
+  for (const pair of value) {
+    if (!Array.isArray(pair) || pair.length !== 2) return null;
+    const file = normalizedSourcePath(pair[0]);
+    const hash = typeof pair[1] === 'string' ? pair[1].trim() : '';
+    if (!file || !hash) return null;
+    pairs.push([file, hash]);
+  }
+  return pairs.sort((left, right) => left[0].localeCompare(right[0]));
+}

package/node/route-preview-artifact-v3.mjs

@@ -652,11 +652,16 @@ export function selectRoutePreviewArtifactFrames(bundle, wanted, options = {}) {
     }
     const fingerprint = isPlainRecord(request) ? request.fingerprint : undefined;
     const frameSourceRevision = frame.sourceRevision ?? bundle.sourceRevision;
-    if (
-      (options.sourceRevision != null &&
-        frameSourceRevision !== options.sourceRevision) ||
-      (fingerprint != null && frame.fingerprint !== fingerprint)
-    ) {
+    // A fingerprint is content identity, so it decides on its own and the
+    // revision stays provenance. Requiring both retired every frame on every
+    // commit, even when the fingerprint proved the frame could not differ.
+    // Without a fingerprint the revision is the only identity available.
+    const staleForRequest =
+      fingerprint != null
+        ? frame.fingerprint !== fingerprint
+        : options.sourceRevision != null &&
+          frameSourceRevision !== options.sourceRevision;
+    if (staleForRequest) {
       stale.push(id);
       if (options.includeStale === true) frames[id] = frame;
       continue;