@pygmalionjs/pygmalion 0.4.0 → 0.5.0

package/node/preview-artifact-store.mjs

@@ -1,6 +1,11 @@
 import { createHash, randomUUID } from 'node:crypto';
 import fs from 'node:fs/promises';
 import path from 'node:path';
+import {
+  normalizeObservedDependencies,
+  observedDependenciesUnchanged,
+  recordObservedDependencies,
+} from './route-dependency-digest.mjs';
 import {
   mergeRoutePreviewArtifactV3,
   selectRoutePreviewArtifactFrames,
@@ -10,10 +15,34 @@ import {
 export const PYGMALION_PREVIEW_ARTIFACT_STORE_VERSION = 1;
 
 const MANIFEST_ENTRY_LIMIT = 131_072;
+/**
+ * Retention is one policy, not a lifecycle.
+ *
+ * Freshness is decided by content now, so an entry is no longer tied to the
+ * revision that produced it and there are no "generations" to prune. What is
+ * left is an ordinary cache, and the only thing a cache has to be told is how
+ * much room it may use.
+ *
+ * The budget is in bytes because bytes are what hurt: a measured 66-frame
+ * catalog held 444 objects in 375 MB, and object sizes ranged from 0.1 to
+ * 1.5 MB, so an entry count bounds nothing. The per-frame and per-namespace
+ * counts below survive only as manifest-size guards.
+ */
 export const PYGMALION_PREVIEW_ARTIFACT_STORE_RETENTION = Object.freeze({
   revisionsPerNamespace: 16,
   variantsPerFrame: 32,
+  maxBytes: 256 * 1024 * 1024,
+  /**
+   * An unreferenced object is unlinked only after it has been unreferenced
+   * this long. A reader that read the manifest just before an eviction
+   * committed still holds the old hash, and deleting immediately would hand it
+   * a miss it did not earn.
+   */
+  reclaimGraceMs: 60_000,
 });
+const ACCESS_JOURNAL_VERSION = 1;
+const ACCESS_JOURNAL_LIMIT = 65_536;
+const ACCESS_FLUSH_DEBOUNCE_MS = 2_000;
 const LOCK_TIMEOUT_MS = 30_000;
 const LOCK_STALE_MS = 120_000;
 const LOCK_POLL_MS = 40;
@@ -50,6 +79,16 @@ function validManifestEntry(entry, kind) {
   );
 }
 
+function validPendingEntry(entry) {
+  return (
+    entry &&
+    typeof entry === 'object' &&
+    !Array.isArray(entry) &&
+    OBJECT_HASH.test(entry.object) &&
+    Number.isFinite(entry.since)
+  );
+}
+
 function validateManifest(value, namespace) {
   if (
     !value ||
@@ -63,7 +102,11 @@ function validateManifest(value, namespace) {
     !Array.isArray(value.frames) ||
     value.revisions.length + value.frames.length > MANIFEST_ENTRY_LIMIT ||
     !value.revisions.every((entry) => validManifestEntry(entry, 'revision')) ||
-    !value.frames.every((entry) => validManifestEntry(entry, 'frame'))
+    !value.frames.every((entry) => validManifestEntry(entry, 'frame')) ||
+    // Written by newer stores only, so absence is normal and must not throw —
+    // a manifest from before retention became a budget stays readable.
+    (value.pending !== undefined &&
+      (!Array.isArray(value.pending) || !value.pending.every(validPendingEntry)))
   ) {
     throw new Error('Preview artifact store manifest is invalid.');
   }
@@ -209,6 +252,17 @@ export function createPreviewArtifactStore({
   lockTimeoutMs,
   lockStaleMs,
   onLockAcquired,
+  /**
+   * Application source root. With it, a frame records the files it actually
+   * rendered and freshness becomes a question about their content instead of
+   * about the revision that happened to capture them.
+   */
+  sourceRoot,
+  observedAlwaysInclude,
+  /** Byte budget for stored objects. See the retention comment above. */
+  maxBytes = PYGMALION_PREVIEW_ARTIFACT_STORE_RETENTION.maxBytes,
+  reclaimGraceMs = PYGMALION_PREVIEW_ARTIFACT_STORE_RETENTION.reclaimGraceMs,
+  now = () => Date.now(),
 } = {}) {
   if (typeof artifactFile !== 'string' || !artifactFile.trim()) {
     throw new TypeError('Preview artifact store requires an artifact file.');
@@ -245,6 +299,75 @@ export function createPreviewArtifactStore({
     return [stat.dev, stat.ino, stat.size, stat.mtimeMs, stat.ctimeMs].join(':');
   };
 
+  // ── Access journal ────────────────────────────────────────────────────────
+  // Recency has to come from reads, not writes: a frame captured once and
+  // opened every day must outlive one captured today and never opened, and
+  // the manifest's generation counter only ever knew about writes.
+  //
+  // The journal is deliberately lock-free. It is a heuristic, so a lost update
+  // costs a slightly worse eviction choice and never correctness — whereas
+  // taking the namespace lock on the read path would put write contention in
+  // front of every frame fetch. Flushes merge with what is on disk, so two
+  // processes reading at once do not erase each other.
+  const accessJournalPath = path.join(root, 'accesses.json');
+  const pendingAccess = new Map();
+  let accessFlushTimer = null;
+  let accessFlushChain = Promise.resolve();
+
+  const readAccessJournal = async () => {
+    try {
+      const value = await readJson(accessJournalPath);
+      if (
+        !value ||
+        typeof value !== 'object' ||
+        value.version !== ACCESS_JOURNAL_VERSION ||
+        !value.entries ||
+        typeof value.entries !== 'object'
+      ) {
+        return new Map();
+      }
+      const entries = new Map();
+      for (const [hash, at] of Object.entries(value.entries)) {
+        if (OBJECT_HASH.test(hash) && Number.isFinite(at)) entries.set(hash, at);
+      }
+      return entries;
+    } catch {
+      return new Map();
+    }
+  };
+
+  const flushAccessJournal = async () => {
+    if (pendingAccess.size === 0) return;
+    const mine = new Map(pendingAccess);
+    pendingAccess.clear();
+    accessFlushChain = accessFlushChain.then(async () => {
+      const stored = await readAccessJournal();
+      for (const [hash, at] of mine) {
+        if ((stored.get(hash) ?? 0) < at) stored.set(hash, at);
+      }
+      // Bounded so a long-lived store cannot grow a journal without end. The
+      // oldest accesses are the ones eviction cares least about.
+      const kept = [...stored.entries()]
+        .sort((left, right) => right[1] - left[1])
+        .slice(0, ACCESS_JOURNAL_LIMIT);
+      await writeJsonAtomic(accessJournalPath, {
+        version: ACCESS_JOURNAL_VERSION,
+        entries: Object.fromEntries(kept),
+      }).catch(() => undefined);
+    });
+    return accessFlushChain;
+  };
+
+  const touchObject = (hash) => {
+    pendingAccess.set(hash, now());
+    if (accessFlushTimer) return;
+    accessFlushTimer = setTimeout(() => {
+      accessFlushTimer = null;
+      void flushAccessJournal();
+    }, ACCESS_FLUSH_DEBOUNCE_MS);
+    accessFlushTimer.unref?.();
+  };
+
   const namespaceHash = (namespace) => sha256(namespace);
   const manifestPath = (namespace) =>
     path.join(root, 'namespaces', `${namespaceHash(namespace)}.json`);
@@ -286,6 +409,7 @@ export function createPreviewArtifactStore({
     if (cached) {
       objectCache.delete(hash);
       objectCache.set(hash, cached);
+      touchObject(hash);
       return cached.artifact;
     }
     try {
@@ -295,12 +419,181 @@ export function createPreviewArtifactStore({
       if (sha256(serialized) !== hash) return null;
       if (!validateRoutePreviewArtifactBundle(artifact).valid) return null;
       rememberObject(hash, artifact, Buffer.byteLength(serialized));
+      touchObject(hash);
       return artifact;
     } catch {
       return null;
     }
   };
 
+  const objectByteSize = async (hash) => {
+    try {
+      return (await fs.stat(objectPath(hash))).size;
+    } catch {
+      return 0;
+    }
+  };
+
+  /**
+   * Ranks a frame entry by how much keeping it is worth.
+   *
+   * Validity comes first: an entry whose observed files still hash the same is
+   * a cache hit waiting to happen, while an entry whose files moved is garbage
+   * that only survives as provenance. Recency only breaks ties inside a rank,
+   * so a busy but invalid entry never outranks a quiet valid one.
+   */
+  const EVICTION_RANK = Object.freeze({
+    invalid: 0,
+    unknown: 1,
+    valid: 2,
+  });
+
+  const classifyFrameEntry = async (entry) => {
+    const observed = normalizeObservedDependencies(entry.sourceFiles);
+    if (!observed) return 'unknown';
+    return (await observedDependenciesUnchanged({ recorded: observed, sourceRoot }))
+      ? 'valid'
+      : 'invalid';
+  };
+
+  /**
+   * Trims the namespace to the byte budget.
+   *
+   * Runs under the namespace lock, so no other writer can be mutating the
+   * manifest. Objects are not unlinked here: their hashes move to `pending`
+   * and are reclaimed by a later pass once the grace window has passed, which
+   * is what keeps a concurrent reader from losing a bundle it is about to
+   * fetch.
+   */
+  const evictToBudget = async (manifest, { at }) => {
+    const referenced = new Map();
+    const remember = async (hash) => {
+      if (!referenced.has(hash)) referenced.set(hash, await objectByteSize(hash));
+      return referenced.get(hash);
+    };
+    for (const entry of [...manifest.revisions, ...manifest.frames]) {
+      await remember(entry.object);
+    }
+    let bytes = [...referenced.values()].reduce((total, size) => total + size, 0);
+    if (bytes <= maxBytes) return { manifest, dropped: null, bytes };
+
+    const journal = await readAccessJournal();
+    for (const [hash, seen] of pendingAccess) {
+      if ((journal.get(hash) ?? 0) < seen) journal.set(hash, seen);
+    }
+    const accessOf = (entry) => journal.get(entry.object) ?? 0;
+
+    const perFrame = new Map();
+    for (const entry of manifest.frames) {
+      perFrame.set(entry.id, (perFrame.get(entry.id) ?? 0) + 1);
+    }
+    const newestRevision = [...manifest.revisions].sort(
+      (left, right) => right.generation - left.generation,
+    )[0];
+
+    const candidates = [];
+    for (const entry of manifest.frames) {
+      candidates.push({
+        kind: 'frame',
+        entry,
+        rank: EVICTION_RANK[await classifyFrameEntry(entry)],
+        // Protection within a rank, never across it: the last copy of a frame
+        // is worth more than a surplus copy, but an invalid sole entry is still
+        // garbage and goes before a valid one.
+        sole: perFrame.get(entry.id) === 1 ? 1 : 0,
+      });
+    }
+    for (const entry of manifest.revisions) {
+      candidates.push({
+        kind: 'revision',
+        entry,
+        // A whole-revision object is a full bundle kept for provenance and the
+        // compatibility file; its content is never verified, so it can only be
+        // unknown. Only the newest earns within-rank protection.
+        rank: EVICTION_RANK.unknown,
+        sole: entry === newestRevision ? 1 : 0,
+      });
+    }
+    candidates.sort(
+      (left, right) =>
+        left.rank - right.rank ||
+        left.sole - right.sole ||
+        accessOf(left.entry) - accessOf(right.entry) ||
+        left.entry.generation - right.entry.generation,
+    );
+
+    const evicted = new Set();
+    let droppedBytes = 0;
+    for (const candidate of candidates) {
+      if (bytes <= maxBytes) break;
+      // Within a rank the last copy of a frame is reached only after every
+      // surplus copy: giving it up turns a recapture-and-keep into a recapture
+      // that also loses the frame's provenance.
+      evicted.add(candidate.entry);
+      const stillReferenced = [...manifest.revisions, ...manifest.frames].some(
+        (entry) => !evicted.has(entry) && entry.object === candidate.entry.object,
+      );
+      if (!stillReferenced) {
+        const size = referenced.get(candidate.entry.object) ?? 0;
+        bytes -= size;
+        droppedBytes += size;
+      }
+    }
+    if (evicted.size === 0) return { manifest, dropped: null, bytes };
+
+    const keptFrames = manifest.frames.filter((entry) => !evicted.has(entry));
+    const keptRevisions = manifest.revisions.filter((entry) => !evicted.has(entry));
+    const stillReferencedObjects = new Set(
+      [...keptFrames, ...keptRevisions].map((entry) => entry.object),
+    );
+    const unreferenced = [...evicted]
+      .map((entry) => entry.object)
+      .filter((hash) => !stillReferencedObjects.has(hash));
+    return {
+      manifest: {
+        ...manifest,
+        frames: keptFrames,
+        revisions: keptRevisions,
+        pending: [
+          ...(manifest.pending ?? []),
+          ...[...new Set(unreferenced)].map((object) => ({ object, since: at })),
+        ],
+      },
+      dropped: {
+        at: new Date(at).toISOString(),
+        entries: evicted.size,
+        bytes: droppedBytes,
+        reason: 'byte-budget',
+      },
+      bytes,
+    };
+  };
+
+  /** Unlinks objects whose grace window has passed and that nothing references. */
+  const reclaimPending = async (manifest, { at }) => {
+    const pending = manifest.pending ?? [];
+    if (pending.length === 0) return manifest;
+    const live = new Set(
+      [...manifest.revisions, ...manifest.frames].map((entry) => entry.object),
+    );
+    const keep = [];
+    const remove = [];
+    for (const entry of pending) {
+      if (live.has(entry.object)) continue;
+      if (at - entry.since >= reclaimGraceMs) remove.push(entry.object);
+      else keep.push(entry);
+    }
+    await Promise.all(
+      [...new Set(remove)].map(async (hash) => {
+        await fs.rm(objectPath(hash), { force: true }).catch(() => undefined);
+        forgetObject(hash);
+      }),
+    );
+    if (keep.length === pending.length) return manifest;
+    const { pending: _dropped, ...rest } = manifest;
+    return keep.length ? { ...rest, pending: keep } : rest;
+  };
+
   const materializeManifest = async (manifest, sourceRevision) => {
     const latestById = new Map();
     for (const entry of manifest.frames) {
@@ -391,6 +684,17 @@ export function createPreviewArtifactStore({
           for (const [id, frame] of Object.entries(artifact.frames)) {
             const selected = selectRoutePreviewArtifactFrames(artifact, [{ id }]);
             const object = await writeObject(selected.bundle);
+            // Recorded from the stored snapshot, so the set describes exactly
+            // the artifact it travels with. Null keeps today's behaviour.
+            const observed =
+              normalizeObservedDependencies(frame.sourceFiles) ??
+              (await recordObservedDependencies({
+                snapshot: selected.bundle?.frames?.[id]?.snapshot ?? frame.snapshot,
+                sourceRoot,
+                ...(observedAlwaysInclude
+                  ? { alwaysInclude: observedAlwaysInclude }
+                  : {}),
+              }));
             const nextFrame = {
               id,
               fingerprint:
@@ -398,6 +702,7 @@ export function createPreviewArtifactStore({
               sourceRevision: sourceRevision ?? null,
               object,
               generation,
+              ...(observed ? { sourceFiles: observed } : {}),
             };
             const alreadyRecorded = frames.some(
               (entry) =>
@@ -411,6 +716,37 @@ export function createPreviewArtifactStore({
           manifest = { ...manifest, frames };
         }
         manifest = { ...manifest, generation };
+        // One policy, applied on every write: trim to the byte budget, then
+        // reclaim whatever has been unreferenced long enough to be safe. No
+        // human is asked when the store should be cleaned.
+        const at = now();
+        const trimmed = await evictToBudget(manifest, { at });
+        manifest = trimmed.dropped
+          ? { ...trimmed.manifest, lastEviction: trimmed.dropped }
+          : trimmed.manifest;
+        // Objects the structural upserts just made unreachable join the same
+        // queue rather than being unlinked here. Immediate deletion is what
+        // could pull a bundle out from under a reader that read the previous
+        // manifest and has not fetched yet.
+        const retainedObjectReferences = new Set(
+          [...manifest.revisions, ...manifest.frames].map((entry) => entry.object),
+        );
+        const orphanedByUpsert = [...previousObjectReferences].filter(
+          (hash) => !retainedObjectReferences.has(hash),
+        );
+        if (orphanedByUpsert.length) {
+          const queued = new Set((manifest.pending ?? []).map((entry) => entry.object));
+          manifest = {
+            ...manifest,
+            pending: [
+              ...(manifest.pending ?? []),
+              ...orphanedByUpsert
+                .filter((hash) => !queued.has(hash))
+                .map((object) => ({ object, since: at })),
+            ],
+          };
+        }
+        manifest = await reclaimPending(manifest, { at });
         if (
           manifest.revisions.length + manifest.frames.length >
           MANIFEST_ENTRY_LIMIT
@@ -419,23 +755,6 @@ export function createPreviewArtifactStore({
         }
         await writeJsonAtomic(manifestPath(namespace), manifest);
 
-        // Objects contain the namespace, so an identical hash cannot be owned
-        // by another namespace. Under this namespace lock it is safe to remove
-        // entries retention just made unreachable, after the manifest commit.
-        const retainedObjectReferences = new Set(
-          [...manifest.revisions, ...manifest.frames].map(
-            (entry) => entry.object,
-          ),
-        );
-        await Promise.all(
-          [...previousObjectReferences]
-            .filter((hash) => !retainedObjectReferences.has(hash))
-            .map(async (hash) => {
-              await fs.rm(objectPath(hash), { force: true });
-              forgetObject(hash);
-            }),
-        );
-
         if (materializeLegacy) {
           // Never stamp a cross-revision frame aggregate with the current
           // revision. The compatibility file remains an honest exact snapshot;
@@ -561,6 +880,31 @@ export function createPreviewArtifactStore({
     return candidatesById;
   };
 
+  /**
+   * A stored frame is fresh when its content identity still holds.
+   *
+   * The revision is provenance, not identity: gating on revision equality
+   * retired every frame on every commit even when the fingerprint proved the
+   * frame could not have changed. A recorded observed set is stronger still —
+   * it survives an edit to a file the frame never rendered, which a route-wide
+   * digest cannot express in a single-page application.
+   */
+  const frameIsFresh = async (entry, request, sourceRevision) => {
+    if (request.fingerprint != null && entry.fingerprint === request.fingerprint) {
+      return true;
+    }
+    const observed = normalizeObservedDependencies(entry.sourceFiles);
+    if (observed) {
+      return observedDependenciesUnchanged({ recorded: observed, sourceRoot });
+    }
+    // Nothing content-based to go on: fall back to the old rule so a frame is
+    // never treated as fresher than it was before.
+    return (
+      entry.sourceRevision === sourceRevision &&
+      (request.fingerprint == null || entry.fingerprint === request.fingerprint)
+    );
+  };
+
   /** Resolves freshness from the compact manifest without loading frame payloads. */
   const resolveFrameSelection = async (namespace, sourceRevision, wanted) => {
     const manifest = await readManifest(namespace);
@@ -574,12 +918,13 @@ export function createPreviewArtifactStore({
         missing.push(request.id);
         continue;
       }
-      const current = candidates.some(
-        (entry) =>
-          entry.sourceRevision === sourceRevision &&
-          (request.fingerprint == null ||
-            entry.fingerprint === request.fingerprint),
-      );
+      let current = false;
+      for (const entry of candidates) {
+        if (await frameIsFresh(entry, request, sourceRevision)) {
+          current = true;
+          break;
+        }
+      }
       (current ? exact : stale).push(request.id);
     }
     return { exact, missing, stale };
@@ -594,11 +939,18 @@ export function createPreviewArtifactStore({
       const candidatesById = indexFrameCandidates(manifest, wanted);
       for (const request of wanted) {
         const candidates = candidatesById.get(request.id) ?? [];
-        const exact = candidates.filter((entry) =>
-          entry.sourceRevision === sourceRevision &&
-          (request.fingerprint == null ||
-            entry.fingerprint === request.fingerprint),
-        );
+        const fresh = [];
+        for (const entry of candidates) {
+          if (await frameIsFresh(entry, request, sourceRevision)) fresh.push(entry);
+        }
+        // Several revisions can now be fresh at once, so provenance decides
+        // which one is served: the requested revision's own capture first, then
+        // the newest. Without this a caller asking about revision A would be
+        // handed revision B's bundle merely because it was captured later.
+        const exact = [
+          ...fresh.filter((entry) => entry.sourceRevision === sourceRevision),
+          ...fresh.filter((entry) => entry.sourceRevision !== sourceRevision),
+        ];
         const exactSet = new Set(exact);
         const ordered = [
           ...exact,
@@ -640,6 +992,105 @@ export function createPreviewArtifactStore({
     );
   };
 
+  /**
+   * What the cache is holding, in the terms the policy decides by.
+   *
+   * A cache nobody can look at becomes a cache nobody trusts, and "valid vs
+   * provenance-only" is the split that explains both the size and the next
+   * eviction — so it is reported rather than left to be inferred.
+   */
+  const inspect = async (namespace) => {
+    const manifest = await readManifest(namespace);
+    if (!manifest) {
+      return {
+        namespace,
+        bytes: 0,
+        maxBytes,
+        objects: 0,
+        frames: { entries: 0, ids: 0, valid: 0, unknown: 0, invalid: 0 },
+        revisions: 0,
+        pending: 0,
+        lastEviction: null,
+      };
+    }
+    const sizes = new Map();
+    for (const entry of [...manifest.revisions, ...manifest.frames]) {
+      if (!sizes.has(entry.object)) {
+        sizes.set(entry.object, await objectByteSize(entry.object));
+      }
+    }
+    const counts = { valid: 0, unknown: 0, invalid: 0 };
+    for (const entry of manifest.frames) {
+      counts[await classifyFrameEntry(entry)] += 1;
+    }
+    return {
+      namespace,
+      bytes: [...sizes.values()].reduce((total, size) => total + size, 0),
+      maxBytes,
+      objects: sizes.size,
+      frames: {
+        entries: manifest.frames.length,
+        ids: new Set(manifest.frames.map((entry) => entry.id)).size,
+        ...counts,
+      },
+      revisions: manifest.revisions.length,
+      pending: (manifest.pending ?? []).length,
+      lastEviction: manifest.lastEviction ?? null,
+    };
+  };
+
+  /**
+   * Applies retention without publishing anything.
+   *
+   * Eviction otherwise only runs on a write, so a store that is over budget and
+   * merely being read would stay over budget until the next capture. Callers
+   * run this when the server starts.
+   */
+  const applyRetention = async (namespace) => {
+    // No namespace means every namespace: at server start the caller has not
+    // seen a request yet and so cannot name one.
+    if (namespace === undefined) {
+      const dropped = [];
+      let names = [];
+      try {
+        names = await fs.readdir(path.join(root, 'namespaces'));
+      } catch {
+        return dropped;
+      }
+      for (const file of names) {
+        if (!file.endsWith('.json')) continue;
+        const stored = await readJson(path.join(root, 'namespaces', file)).catch(
+          () => null,
+        );
+        if (typeof stored?.namespace !== 'string') continue;
+        const result = await applyRetention(stored.namespace).catch(() => null);
+        if (result) dropped.push({ namespace: stored.namespace, ...result });
+      }
+      return dropped;
+    }
+    return withFilesystemLock(
+      lockPath(namespace),
+      async () => {
+        const stored = await readManifest(namespace);
+        if (!stored) return null;
+        const at = now();
+        const trimmed = await evictToBudget(stored, { at });
+        let manifest = trimmed.dropped
+          ? { ...trimmed.manifest, lastEviction: trimmed.dropped }
+          : trimmed.manifest;
+        manifest = await reclaimPending(manifest, { at });
+        if (manifest !== stored) {
+          await writeJsonAtomic(manifestPath(namespace), {
+            ...manifest,
+            generation: stored.generation + 1,
+          });
+        }
+        return trimmed.dropped;
+      },
+      { lockTimeoutMs, lockStaleMs, onLockAcquired },
+    );
+  };
+
   return {
     artifactFile: legacyFile,
     storeDirectory: root,
@@ -649,5 +1100,8 @@ export function createPreviewArtifactStore({
     readFrameSelection,
     readLatestArtifact,
     resolveFrameSelection,
+    inspect,
+    applyRetention,
+    flushAccessJournal,
   };
 }