@pygmalionjs/pygmalion 0.6.31 → 0.6.33

package/node/preview-artifact-store.mjs

@@ -141,30 +141,164 @@ function processIsAlive(pid) {
   }
 }
 
+async function acquireFilesystemReclaimMutex(lockPath, deadline) {
+  const reclaimPath = `${lockPath}.reclaim`;
+  const token = randomUUID();
+  const payload = `${JSON.stringify({
+    token,
+    pid: process.pid,
+    createdAt: new Date().toISOString(),
+  })}\n`;
+  while (true) {
+    const candidate = `${reclaimPath}.candidate-${randomUUID()}`;
+    try {
+      await fs.writeFile(candidate, payload, { encoding: 'utf8', flag: 'wx' });
+      await fs.link(candidate, reclaimPath);
+      return async () => {
+        const current = await readJson(reclaimPath)
+          .then((owner) => owner?.token)
+          .catch(() => null);
+        if (current === token) {
+          await fs.unlink(reclaimPath).catch(() => undefined);
+        }
+      };
+    } catch (error) {
+      if (error?.code !== 'EEXIST') throw error;
+      // Recovery of this guard would recreate the same ABA race one level down.
+      // A dead guard therefore fails closed until it is removed manually.
+      if (Date.now() >= deadline) {
+        throw new Error('Timed out waiting for the preview artifact store lock.');
+      }
+      await new Promise((resolve) => setTimeout(resolve, LOCK_POLL_MS));
+    } finally {
+      await fs.rm(candidate, { force: true }).catch(() => undefined);
+    }
+  }
+}
+
+async function reclaimStaleFilesystemLock({
+  lockPath,
+  deadline,
+  staleMs,
+  onStaleLockConfirmed,
+}) {
+  const releaseReclaim = await acquireFilesystemReclaimMutex(lockPath, deadline);
+  try {
+    const staleDeadOwner = async () => {
+      const stat = await fs.stat(lockPath).catch(() => null);
+      if (!stat || Date.now() - stat.mtimeMs <= staleMs) return false;
+      const owner = await readJson(path.join(lockPath, 'owner.json')).catch(
+        () => null,
+      );
+      // An absent or malformed owner cannot prove that its initializer died.
+      // New locks are atomically published complete, so keep legacy/corrupt
+      // shapes fail-closed instead of guessing and deleting a live directory.
+      if (
+        !owner ||
+        typeof owner.token !== 'string' ||
+        !owner.token ||
+        !Number.isSafeInteger(owner.pid) ||
+        owner.pid <= 0
+      ) {
+        return false;
+      }
+      return processIsAlive(owner.pid) === false;
+    };
+    if (!(await staleDeadOwner())) return false;
+    await onStaleLockConfirmed?.({ lockPath });
+    // Re-read after any delay while every other reclaimer is excluded. The
+    // fixed path may now name a new live generation, which must be preserved.
+    if (!(await staleDeadOwner())) return false;
+    await fs.rm(lockPath, { recursive: true, force: true });
+    return true;
+  } finally {
+    await releaseReclaim();
+  }
+}
+
+async function filesystemLockPathExists(lockPath) {
+  try {
+    await fs.lstat(lockPath);
+    return true;
+  } catch (error) {
+    if (error?.code === 'ENOENT') return false;
+    throw error;
+  }
+}
+
+async function filesystemLockPublishConflict(error, lockPath, candidate) {
+  if (['EEXIST', 'ENOTEMPTY'].includes(error?.code)) return true;
+  if (!['EACCES', 'EPERM'].includes(error?.code)) return false;
+  // Windows can report an existing destination directory as a permission
+  // failure. Treat that shape as contention, but propagate a genuine parent or
+  // candidate permission failure when no competing fixed directory exists.
+  const [fixedExists, candidateExists] = await Promise.all([
+    filesystemLockPathExists(lockPath),
+    fs
+      .lstat(candidate)
+      .then((stat) => stat.isDirectory())
+      .catch(() => false),
+  ]);
+  return fixedExists && candidateExists;
+}
+
 async function withFilesystemLock(lockPath, task, options = {}) {
   const timeoutMs = options.lockTimeoutMs ?? LOCK_TIMEOUT_MS;
   const staleMs = options.lockStaleMs ?? LOCK_STALE_MS;
   const deadline = Date.now() + timeoutMs;
+  const ownerToken = randomUUID();
+  const ownerPayload = `${JSON.stringify({
+    token: ownerToken,
+    pid: process.pid,
+    createdAt: new Date().toISOString(),
+  })}\n`;
+  let owner;
   await fs.mkdir(path.dirname(lockPath), { recursive: true });
-  while (true) {
+  while (!owner) {
+    const candidate = `${lockPath}.candidate-${randomUUID()}`;
+    let publicationConflict = false;
+    let publicationError;
     try {
-      await fs.mkdir(lockPath);
-      break;
+      await fs.mkdir(candidate);
+      await fs.writeFile(path.join(candidate, 'owner.json'), ownerPayload, 'utf8');
+      await options.onLockOwnerReady?.({ candidate, lockPath });
+      if (await filesystemLockPathExists(lockPath)) {
+        publicationConflict = true;
+      } else {
+        try {
+          await fs.rename(candidate, lockPath);
+          owner = path.join(lockPath, 'owner.json');
+        } catch (error) {
+          publicationConflict = await filesystemLockPublishConflict(
+            error,
+            lockPath,
+            candidate,
+          );
+          if (!publicationConflict) publicationError = error;
+        }
+      }
     } catch (error) {
-      if (error?.code !== 'EEXIST') throw error;
+      publicationError = error;
+    } finally {
+      await fs.rm(candidate, { recursive: true, force: true }).catch(
+        () => undefined,
+      );
+    }
+    if (owner) continue;
+    if (publicationError) throw publicationError;
+    if (publicationConflict) {
       const stale = await fs
         .stat(lockPath)
         .then((stat) => Date.now() - stat.mtimeMs > staleMs)
         .catch(() => false);
       if (stale) {
-        const ownerState = await readJson(path.join(lockPath, 'owner.json'))
-          .then((owner) => processIsAlive(owner?.pid))
-          .catch(() => null);
-        // A delayed but live writer must never lose its lock: it may resume and
-        // publish a manifest read before another writer. Recover only when the
-        // owning process is confirmed dead, or an old lock never got an owner.
-        if (ownerState === false || ownerState === null) {
-          await fs.rm(lockPath, { recursive: true, force: true });
+        const reclaimed = await reclaimStaleFilesystemLock({
+          lockPath,
+          deadline,
+          staleMs,
+          onStaleLockConfirmed: options.onStaleLockConfirmed,
+        });
+        if (reclaimed) {
           continue;
         }
       }
@@ -172,24 +306,9 @@ async function withFilesystemLock(lockPath, task, options = {}) {
         throw new Error('Timed out waiting for the preview artifact store lock.');
       }
       await new Promise((resolve) => setTimeout(resolve, LOCK_POLL_MS));
+      continue;
     }
-  }
-
-  const owner = path.join(lockPath, 'owner.json');
-  const ownerToken = randomUUID();
-  try {
-    await fs.writeFile(
-      owner,
-      `${JSON.stringify({
-        token: ownerToken,
-        pid: process.pid,
-        createdAt: new Date().toISOString(),
-      })}\n`,
-      'utf8',
-    );
-  } catch (error) {
-    await fs.rm(lockPath, { recursive: true, force: true });
-    throw error;
+    throw new Error('Preview artifact store lock publication did not settle.');
   }
   const heartbeat = setInterval(() => {
     const now = new Date();
@@ -254,12 +373,8 @@ export function createPreviewArtifactStore({
   lockTimeoutMs,
   lockStaleMs,
   onLockAcquired,
-  /**
-   * Application source root. With it, a frame records the files it actually
-   * rendered and freshness becomes a question about their content instead of
-   * about the revision that happened to capture them.
-   */
-  sourceRoot,
+  onLockOwnerReady,
+  onStaleLockConfirmed,
   observedAlwaysInclude,
   /** Byte budget for stored objects. See the retention comment above. */
   maxBytes = PYGMALION_PREVIEW_ARTIFACT_STORE_RETENTION.maxBytes,
@@ -271,13 +386,12 @@ export function createPreviewArtifactStore({
   }
   const legacyFile = path.resolve(artifactFile);
   const root = path.resolve(storeDirectory);
-  const currentSourceRoot = () => {
-    const candidate =
-      typeof sourceRoot === 'function' ? sourceRoot() : sourceRoot;
-    return typeof candidate === 'string' && candidate
-      ? path.resolve(candidate)
+  const sourceRootForRevision = (sourceContext, sourceRevision) =>
+    sourceContext?.sourceRevision === sourceRevision &&
+    typeof sourceContext?.sourceRoot === 'string' &&
+    sourceContext.sourceRoot
+      ? path.resolve(sourceContext.sourceRoot)
       : undefined;
-  };
   let legacyCache = null;
   const objectCache = new Map();
   let objectCacheBytes = 0;
@@ -457,12 +571,17 @@ export function createPreviewArtifactStore({
     valid: 2,
   });
 
-  const classifyFrameEntry = async (entry) => {
+  const classifyFrameEntry = async (entry, sourceContext) => {
     const observed = normalizeObservedDependencies(entry.sourceFiles);
     if (!observed) return 'unknown';
+    const exactSourceRoot = sourceRootForRevision(
+      sourceContext,
+      sourceContext?.sourceRevision,
+    );
+    if (!exactSourceRoot) return 'unknown';
     return (await observedDependenciesUnchanged({
       recorded: observed,
-      sourceRoot: currentSourceRoot(),
+      sourceRoot: exactSourceRoot,
     }))
       ? 'valid'
       : 'invalid';
@@ -477,7 +596,7 @@ export function createPreviewArtifactStore({
    * is what keeps a concurrent reader from losing a bundle it is about to
    * fetch.
    */
-  const evictToBudget = async (manifest, { at }) => {
+  const evictToBudget = async (manifest, { at, sourceContext } = {}) => {
     const referenced = new Map();
     const remember = async (hash) => {
       if (!referenced.has(hash)) referenced.set(hash, await objectByteSize(hash));
@@ -508,7 +627,7 @@ export function createPreviewArtifactStore({
       candidates.push({
         kind: 'frame',
         entry,
-        rank: EVICTION_RANK[await classifyFrameEntry(entry)],
+        rank: EVICTION_RANK[await classifyFrameEntry(entry, sourceContext)],
         // Protection within a rank, never across it: the last copy of a frame
         // is worth more than a surplus copy, but an invalid sole entry is still
         // garbage and goes before a valid one.
@@ -632,6 +751,40 @@ export function createPreviewArtifactStore({
     };
   };
 
+  /**
+   * Records every frame's observed source set before publication can wait on the
+   * namespace lock. A Map entry is present even when observation is unavailable,
+   * so `null` is an explicit pin rather than permission to inspect another root.
+   */
+  const observeArtifactDependencies = async (artifact, explicitSourceRoot) => {
+    const validation = validateRoutePreviewArtifactBundle(artifact);
+    if (!validation.valid) {
+      throw new Error('Preview artifact store refused an invalid artifact.');
+    }
+    const observedByFrame = new Map();
+    if (artifact.version !== 3) return observedByFrame;
+    const pinnedSourceRoot =
+      typeof explicitSourceRoot === 'string' && explicitSourceRoot
+        ? path.resolve(explicitSourceRoot)
+        : undefined;
+    for (const [id, frame] of Object.entries(artifact.frames)) {
+      const selected = selectRoutePreviewArtifactFrames(artifact, [{ id }]);
+      // Artifact frames are an open record in the v3 wire format, so an
+      // unvalidated `sourceFiles` property supplied by a generator cannot be a
+      // freshness proof. Recompute it from the serialized snapshot and the
+      // explicitly pinned root every time.
+      const observed = await recordObservedDependencies({
+        snapshot: selected.bundle?.frames?.[id]?.snapshot ?? frame.snapshot,
+        sourceRoot: pinnedSourceRoot,
+        ...(observedAlwaysInclude
+          ? { alwaysInclude: observedAlwaysInclude }
+          : {}),
+      });
+      observedByFrame.set(id, observed);
+    }
+    return observedByFrame;
+  };
+
   const publishArtifact = async (
     artifact,
     {
@@ -639,6 +792,8 @@ export function createPreviewArtifactStore({
       sourceRevision = artifact?.sourceRevision,
       materializeLegacy = true,
       frameRequests = [],
+      observedDependenciesByFrame,
+      sourceContext,
     } = {},
   ) => {
     const validation = validateRoutePreviewArtifactBundle(artifact);
@@ -652,7 +807,51 @@ export function createPreviewArtifactStore({
     ) {
       throw new Error('Preview artifact store refused a mismatched revision.');
     }
+    if (
+      observedDependenciesByFrame != null &&
+      !(observedDependenciesByFrame instanceof Map)
+    ) {
+      throw new TypeError(
+        'Preview artifact observed dependencies must be provided as a Map.',
+      );
+    }
+    let pinnedObservedDependenciesByFrame;
+    if (observedDependenciesByFrame instanceof Map && artifact.version === 3) {
+      const frameIds = new Set(Object.keys(artifact.frames));
+      if (
+        observedDependenciesByFrame.size !== frameIds.size ||
+        [...frameIds].some((id) => !observedDependenciesByFrame.has(id)) ||
+        [...observedDependenciesByFrame.keys()].some((id) => !frameIds.has(id))
+      ) {
+        throw new TypeError(
+          'Preview artifact observed dependencies must cover every frame exactly.',
+        );
+      }
+      pinnedObservedDependenciesByFrame = new Map();
+      for (const observed of observedDependenciesByFrame.values()) {
+        if (observed !== null && normalizeObservedDependencies(observed) == null) {
+          throw new TypeError(
+            'Preview artifact observed dependencies contain an invalid entry.',
+          );
+        }
+      }
+      for (const id of frameIds) {
+        const normalized = normalizeObservedDependencies(
+          observedDependenciesByFrame.get(id),
+        );
+        pinnedObservedDependenciesByFrame.set(
+          id,
+          normalized?.map(([file, hash]) => [file, hash]) ?? null,
+        );
+      }
+    }
     const namespace = artifact.namespace;
+    const exactSourceContext = sourceRootForRevision(
+      sourceContext,
+      sourceRevision,
+    )
+      ? sourceContext
+      : undefined;
     return withFilesystemLock(
       lockPath(namespace),
       async () => {
@@ -701,16 +900,13 @@ export function createPreviewArtifactStore({
             const selected = selectRoutePreviewArtifactFrames(artifact, [{ id }]);
             const object = await writeObject(selected.bundle);
             // Recorded from the stored snapshot, so the set describes exactly
-            // the artifact it travels with. Null keeps today's behaviour.
-            const observed =
-              normalizeObservedDependencies(frame.sourceFiles) ??
-              (await recordObservedDependencies({
-                snapshot: selected.bundle?.frames?.[id]?.snapshot ?? frame.snapshot,
-                sourceRoot: currentSourceRoot(),
-                ...(observedAlwaysInclude
-                  ? { alwaysInclude: observedAlwaysInclude }
-                  : {}),
-              }));
+            // the artifact it travels with. Null keeps the entry on the
+            // revision/fingerprint-only fallback.
+            const observed = pinnedObservedDependenciesByFrame
+              ? normalizeObservedDependencies(
+                  pinnedObservedDependenciesByFrame.get(id),
+                )
+              : null;
             const nextFrame = {
               id,
               fingerprint:
@@ -742,7 +938,10 @@ export function createPreviewArtifactStore({
         // reclaim whatever has been unreferenced long enough to be safe. No
         // human is asked when the store should be cleaned.
         const at = now();
-        const trimmed = await evictToBudget(manifest, { at });
+        const trimmed = await evictToBudget(manifest, {
+          at,
+          sourceContext: exactSourceContext,
+        });
         manifest = trimmed.dropped
           ? { ...trimmed.manifest, lastEviction: trimmed.dropped }
           : trimmed.manifest;
@@ -808,7 +1007,13 @@ export function createPreviewArtifactStore({
         }
         return artifact;
       },
-      { lockTimeoutMs, lockStaleMs, onLockAcquired },
+      {
+        lockTimeoutMs,
+        lockStaleMs,
+        onLockAcquired,
+        onLockOwnerReady,
+        onStaleLockConfirmed,
+      },
     );
   };
 
@@ -911,9 +1116,17 @@ export function createPreviewArtifactStore({
    * it survives an edit to a file the frame never rendered, which a route-wide
    * digest cannot express in a single-page application.
    */
-  const frameIsFresh = async (entry, request, sourceRevision) => {
+  const frameFreshness = async (
+    entry,
+    request,
+    sourceRevision,
+    sourceContext,
+  ) => {
     if (request.fingerprint != null && entry.fingerprint === request.fingerprint) {
-      return true;
+      return 'fresh';
+    }
+    if (request.fingerprint == null) {
+      return entry.sourceRevision === sourceRevision ? 'fresh' : 'stale';
     }
     // Source hashes cannot prove that viewport, environment, or interaction
     // inputs stayed the same. They may bridge a changed source fingerprint only
@@ -924,30 +1137,45 @@ export function createPreviewArtifactStore({
         entry.recipeFingerprint == null ||
         entry.recipeFingerprint !== request.recipeFingerprint)
     ) {
-      return false;
+      return 'stale';
     }
     const observed = normalizeObservedDependencies(entry.sourceFiles);
     if (observed) {
-      return observedDependenciesUnchanged({
+      const exactSourceRoot = sourceRootForRevision(
+        sourceContext,
+        sourceRevision,
+      );
+      if (!exactSourceRoot) return 'source-pending';
+      return (await observedDependenciesUnchanged({
         recorded: observed,
-        sourceRoot: currentSourceRoot(),
-      });
+        sourceRoot: exactSourceRoot,
+      }))
+        ? 'fresh'
+        : 'stale';
     }
     // Nothing content-based to go on: fall back to the old rule so a frame is
     // never treated as fresher than it was before.
     return (
       entry.sourceRevision === sourceRevision &&
-      (request.fingerprint == null || entry.fingerprint === request.fingerprint)
-    );
+      entry.fingerprint === request.fingerprint
+    )
+      ? 'fresh'
+      : 'stale';
   };
 
   /** Resolves freshness from the compact manifest without loading frame payloads. */
-  const resolveFrameSelection = async (namespace, sourceRevision, wanted) => {
+  const resolveFrameSelection = async (
+    namespace,
+    sourceRevision,
+    wanted,
+    { sourceContext } = {},
+  ) => {
     const manifest = await readManifest(namespace);
     const candidatesById = indexFrameCandidates(manifest, wanted);
     const exact = [];
     const missing = [];
     const stale = [];
+    const sourcePending = [];
     for (const request of wanted) {
       const candidates = candidatesById.get(request.id) ?? [];
       if (!candidates.length) {
@@ -955,20 +1183,41 @@ export function createPreviewArtifactStore({
         continue;
       }
       let current = false;
+      let mayBecomeCurrent = false;
       for (const entry of candidates) {
-        if (await frameIsFresh(entry, request, sourceRevision)) {
+        const freshness = await frameFreshness(
+          entry,
+          request,
+          sourceRevision,
+          sourceContext,
+        );
+        if (freshness === 'fresh') {
           current = true;
           break;
         }
+        if (freshness === 'source-pending') mayBecomeCurrent = true;
       }
+      if (!current && mayBecomeCurrent) sourcePending.push(request.id);
       (current ? exact : stale).push(request.id);
     }
-    return { exact, missing, stale };
+    return {
+      exact,
+      missing,
+      stale,
+      ...(sourcePending.length ? { sourcePending } : {}),
+    };
   };
 
-  const readFrameSelection = async (namespace, sourceRevision, wanted) => {
+  const readFrameSelection = async (
+    namespace,
+    sourceRevision,
+    wanted,
+    { sourceContext } = {},
+  ) => {
     const manifest = await readManifest(namespace);
     let aggregate = null;
+    const contentFreshIds = new Set();
+    const sourcePendingIds = new Set();
     if (manifest) {
       // Index once per read. The earlier path filtered the complete manifest for
       // every wanted frame, which made a storyboard probe O(wanted × entries).
@@ -977,7 +1226,16 @@ export function createPreviewArtifactStore({
         const candidates = candidatesById.get(request.id) ?? [];
         const fresh = [];
         for (const entry of candidates) {
-          if (await frameIsFresh(entry, request, sourceRevision)) fresh.push(entry);
+          const freshness = await frameFreshness(
+            entry,
+            request,
+            sourceRevision,
+            sourceContext,
+          );
+          if (freshness === 'fresh') fresh.push(entry);
+          if (freshness === 'source-pending') {
+            sourcePendingIds.add(request.id);
+          }
         }
         // Several revisions can now be fresh at once, so provenance decides
         // which one is served: the requested revision's own capture first, then
@@ -1016,16 +1274,30 @@ export function createPreviewArtifactStore({
               aggregate,
               withProvenance,
             );
+            if (exactSet.has(entry)) {
+              contentFreshIds.add(request.id);
+              sourcePendingIds.delete(request.id);
+            }
             break;
           }
         }
       }
     }
-    return selectRoutePreviewArtifactFrames(
+    const selection = selectRoutePreviewArtifactFrames(
       aggregate ?? emptyBundle(namespace, sourceRevision),
       wanted,
       { includeStale: true, sourceRevision },
     );
+    return {
+      ...selection,
+      // The wire-level selector only knows fingerprints. A frame whose recipe
+      // and observed files were verified against the requested source context is
+      // also current, even when its coarse source fingerprint changed.
+      stale: selection.stale.filter((id) => !contentFreshIds.has(id)),
+      ...(sourcePendingIds.size
+        ? { sourcePending: [...sourcePendingIds] }
+        : {}),
+    };
   };
 
   /**
@@ -1035,7 +1307,7 @@ export function createPreviewArtifactStore({
    * provenance-only" is the split that explains both the size and the next
    * eviction — so it is reported rather than left to be inferred.
    */
-  const inspect = async (namespace) => {
+  const inspect = async (namespace, { sourceContext } = {}) => {
     const manifest = await readManifest(namespace);
     if (!manifest) {
       return {
@@ -1057,7 +1329,7 @@ export function createPreviewArtifactStore({
     }
     const counts = { valid: 0, unknown: 0, invalid: 0 };
     for (const entry of manifest.frames) {
-      counts[await classifyFrameEntry(entry)] += 1;
+      counts[await classifyFrameEntry(entry, sourceContext)] += 1;
     }
     return {
       namespace,
@@ -1082,7 +1354,7 @@ export function createPreviewArtifactStore({
    * merely being read would stay over budget until the next capture. Callers
    * run this when the server starts.
    */
-  const applyRetention = async (namespace) => {
+  const applyRetention = async (namespace, { sourceContext } = {}) => {
     // No namespace means every namespace: at server start the caller has not
     // seen a request yet and so cannot name one.
     if (namespace === undefined) {
@@ -1099,7 +1371,9 @@ export function createPreviewArtifactStore({
           () => null,
         );
         if (typeof stored?.namespace !== 'string') continue;
-        const result = await applyRetention(stored.namespace).catch(() => null);
+        const result = await applyRetention(stored.namespace, {
+          sourceContext,
+        }).catch(() => null);
         if (result) dropped.push({ namespace: stored.namespace, ...result });
       }
       return dropped;
@@ -1110,7 +1384,7 @@ export function createPreviewArtifactStore({
         const stored = await readManifest(namespace);
         if (!stored) return null;
         const at = now();
-        const trimmed = await evictToBudget(stored, { at });
+        const trimmed = await evictToBudget(stored, { at, sourceContext });
         let manifest = trimmed.dropped
           ? { ...trimmed.manifest, lastEviction: trimmed.dropped }
           : trimmed.manifest;
@@ -1123,7 +1397,13 @@ export function createPreviewArtifactStore({
         }
         return trimmed.dropped;
       },
-      { lockTimeoutMs, lockStaleMs, onLockAcquired },
+      {
+        lockTimeoutMs,
+        lockStaleMs,
+        onLockAcquired,
+        onLockOwnerReady,
+        onStaleLockConfirmed,
+      },
     );
   };
 
@@ -1131,6 +1411,7 @@ export function createPreviewArtifactStore({
     artifactFile: legacyFile,
     storeDirectory: root,
     importLegacyArtifact,
+    observeArtifactDependencies,
     publishArtifact,
     readExactArtifact,
     readFrameSelection,