@pygmalionjs/pygmalion 0.2.38 → 0.2.40

package/node/preview-artifact-store.mjs

@@ -0,0 +1,653 @@
+import { createHash, randomUUID } from 'node:crypto';
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import {
+  mergeRoutePreviewArtifactV3,
+  selectRoutePreviewArtifactFrames,
+  validateRoutePreviewArtifactBundle,
+} from './route-preview-artifact-v3.mjs';
+
+export const PYGMALION_PREVIEW_ARTIFACT_STORE_VERSION = 1;
+
+const MANIFEST_ENTRY_LIMIT = 131_072;
+export const PYGMALION_PREVIEW_ARTIFACT_STORE_RETENTION = Object.freeze({
+  revisionsPerNamespace: 16,
+  variantsPerFrame: 32,
+});
+const LOCK_TIMEOUT_MS = 30_000;
+const LOCK_STALE_MS = 120_000;
+const LOCK_POLL_MS = 40;
+const OBJECT_HASH = /^[a-f0-9]{64}$/u;
+const OBJECT_CACHE_MAX_BYTES = 64 * 1024 * 1024;
+
+function sha256(value) {
+  return createHash('sha256').update(value).digest('hex');
+}
+
+function emptyBundle(namespace, sourceRevision) {
+  return {
+    version: 3,
+    namespace,
+    ...(sourceRevision ? { sourceRevision } : {}),
+    assets: { head: {}, stylesheets: {}, screenshots: {} },
+    frames: {},
+  };
+}
+
+function validManifestEntry(entry, kind) {
+  if (!entry || typeof entry !== 'object' || Array.isArray(entry)) return false;
+  if (!OBJECT_HASH.test(entry.object) || !Number.isSafeInteger(entry.generation)) {
+    return false;
+  }
+  if (kind === 'revision') {
+    return typeof entry.sourceRevision === 'string' && entry.sourceRevision.length > 0;
+  }
+  return (
+    typeof entry.id === 'string' &&
+    entry.id.length > 0 &&
+    (entry.fingerprint == null || typeof entry.fingerprint === 'string') &&
+    (entry.sourceRevision == null || typeof entry.sourceRevision === 'string')
+  );
+}
+
+function validateManifest(value, namespace) {
+  if (
+    !value ||
+    typeof value !== 'object' ||
+    Array.isArray(value) ||
+    value.version !== PYGMALION_PREVIEW_ARTIFACT_STORE_VERSION ||
+    value.namespace !== namespace ||
+    !Number.isSafeInteger(value.generation) ||
+    value.generation < 0 ||
+    !Array.isArray(value.revisions) ||
+    !Array.isArray(value.frames) ||
+    value.revisions.length + value.frames.length > MANIFEST_ENTRY_LIMIT ||
+    !value.revisions.every((entry) => validManifestEntry(entry, 'revision')) ||
+    !value.frames.every((entry) => validManifestEntry(entry, 'frame'))
+  ) {
+    throw new Error('Preview artifact store manifest is invalid.');
+  }
+  return value;
+}
+
+async function readJson(file) {
+  return JSON.parse(await fs.readFile(file, 'utf8'));
+}
+
+async function writeJsonAtomic(file, value) {
+  await fs.mkdir(path.dirname(file), { recursive: true });
+  const temporary = `${file}.tmp-${process.pid}-${randomUUID()}`;
+  try {
+    await fs.writeFile(temporary, `${JSON.stringify(value)}\n`, 'utf8');
+    await fs.rename(temporary, file);
+  } catch (error) {
+    await fs.rm(temporary, { force: true }).catch(() => undefined);
+    throw error;
+  }
+}
+
+function processIsAlive(pid) {
+  if (!Number.isSafeInteger(pid) || pid <= 0) return null;
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (error) {
+    return error?.code === 'ESRCH' ? false : true;
+  }
+}
+
+async function withFilesystemLock(lockPath, task, options = {}) {
+  const timeoutMs = options.lockTimeoutMs ?? LOCK_TIMEOUT_MS;
+  const staleMs = options.lockStaleMs ?? LOCK_STALE_MS;
+  const deadline = Date.now() + timeoutMs;
+  await fs.mkdir(path.dirname(lockPath), { recursive: true });
+  while (true) {
+    try {
+      await fs.mkdir(lockPath);
+      break;
+    } catch (error) {
+      if (error?.code !== 'EEXIST') throw error;
+      const stale = await fs
+        .stat(lockPath)
+        .then((stat) => Date.now() - stat.mtimeMs > staleMs)
+        .catch(() => false);
+      if (stale) {
+        const ownerState = await readJson(path.join(lockPath, 'owner.json'))
+          .then((owner) => processIsAlive(owner?.pid))
+          .catch(() => null);
+        // A delayed but live writer must never lose its lock: it may resume and
+        // publish a manifest read before another writer. Recover only when the
+        // owning process is confirmed dead, or an old lock never got an owner.
+        if (ownerState === false || ownerState === null) {
+          await fs.rm(lockPath, { recursive: true, force: true });
+          continue;
+        }
+      }
+      if (Date.now() >= deadline) {
+        throw new Error('Timed out waiting for the preview artifact store lock.');
+      }
+      await new Promise((resolve) => setTimeout(resolve, LOCK_POLL_MS));
+    }
+  }
+
+  const owner = path.join(lockPath, 'owner.json');
+  const ownerToken = randomUUID();
+  try {
+    await fs.writeFile(
+      owner,
+      `${JSON.stringify({
+        token: ownerToken,
+        pid: process.pid,
+        createdAt: new Date().toISOString(),
+      })}\n`,
+      'utf8',
+    );
+  } catch (error) {
+    await fs.rm(lockPath, { recursive: true, force: true });
+    throw error;
+  }
+  const heartbeat = setInterval(() => {
+    const now = new Date();
+    void fs.utimes(lockPath, now, now).catch(() => undefined);
+  }, Math.min(2_000, Math.max(100, Math.floor(staleMs / 4))));
+  heartbeat.unref();
+  try {
+    await options.onLockAcquired?.();
+    return await task();
+  } finally {
+    clearInterval(heartbeat);
+    const stillOwned = await readJson(owner)
+      .then((value) => value?.token === ownerToken)
+      .catch(() => false);
+    if (stillOwned) {
+      await fs.rm(lockPath, { recursive: true, force: true });
+    }
+  }
+}
+
+function upsertRevision(entries, next) {
+  return [
+    ...entries.filter((entry) => entry.sourceRevision !== next.sourceRevision),
+    next,
+  ]
+    .sort((a, b) => b.generation - a.generation)
+    .slice(
+      0,
+      PYGMALION_PREVIEW_ARTIFACT_STORE_RETENTION.revisionsPerNamespace,
+    );
+}
+
+function upsertFrame(entries, next) {
+  const updated = [
+    ...entries.filter(
+      (entry) =>
+        entry.id !== next.id ||
+        entry.fingerprint !== next.fingerprint ||
+        entry.sourceRevision !== next.sourceRevision,
+    ),
+    next,
+  ];
+  const keptForFrame = new Set(
+    updated
+      .filter((entry) => entry.id === next.id)
+      .sort((a, b) => b.generation - a.generation)
+      .slice(0, PYGMALION_PREVIEW_ARTIFACT_STORE_RETENTION.variantsPerFrame),
+  );
+  return updated.filter(
+    (entry) => entry.id !== next.id || keptForFrame.has(entry),
+  );
+}
+
+/** The persistent sharded store derived from the compatible artifactFile path. */
+export function resolvePreviewArtifactStoreDirectory(artifactFile) {
+  return `${path.resolve(artifactFile)}.store`;
+}
+
+export function createPreviewArtifactStore({
+  artifactFile,
+  storeDirectory = resolvePreviewArtifactStoreDirectory(artifactFile),
+  lockTimeoutMs,
+  lockStaleMs,
+  onLockAcquired,
+} = {}) {
+  if (typeof artifactFile !== 'string' || !artifactFile.trim()) {
+    throw new TypeError('Preview artifact store requires an artifact file.');
+  }
+  const legacyFile = path.resolve(artifactFile);
+  const root = path.resolve(storeDirectory);
+  let legacyCache = null;
+  const objectCache = new Map();
+  let objectCacheBytes = 0;
+
+  const rememberObject = (hash, artifact, bytes) => {
+    const previous = objectCache.get(hash);
+    if (previous) objectCacheBytes -= previous.bytes;
+    objectCache.delete(hash);
+    objectCache.set(hash, { artifact, bytes });
+    objectCacheBytes += bytes;
+    while (objectCacheBytes > OBJECT_CACHE_MAX_BYTES && objectCache.size > 1) {
+      const oldestHash = objectCache.keys().next().value;
+      const oldest = objectCache.get(oldestHash);
+      objectCache.delete(oldestHash);
+      objectCacheBytes -= oldest?.bytes ?? 0;
+    }
+  };
+
+  const forgetObject = (hash) => {
+    const cached = objectCache.get(hash);
+    if (!cached) return;
+    objectCache.delete(hash);
+    objectCacheBytes -= cached.bytes;
+  };
+
+  const legacyFileSignature = async () => {
+    const stat = await fs.stat(legacyFile);
+    return [stat.dev, stat.ino, stat.size, stat.mtimeMs, stat.ctimeMs].join(':');
+  };
+
+  const namespaceHash = (namespace) => sha256(namespace);
+  const manifestPath = (namespace) =>
+    path.join(root, 'namespaces', `${namespaceHash(namespace)}.json`);
+  const lockPath = (namespace) =>
+    path.join(root, 'locks', `${namespaceHash(namespace)}.lock`);
+  const objectPath = (hash) =>
+    path.join(root, 'objects', hash.slice(0, 2), `${hash}.json`);
+
+  const readManifest = async (namespace) => {
+    try {
+      return validateManifest(await readJson(manifestPath(namespace)), namespace);
+    } catch (error) {
+      if (error?.code === 'ENOENT') return null;
+      throw error;
+    }
+  };
+
+  const writeObject = async (artifact) => {
+    const validation = validateRoutePreviewArtifactBundle(artifact);
+    if (!validation.valid) {
+      throw new Error('Preview artifact store refused an invalid artifact.');
+    }
+    const serialized = JSON.stringify(artifact);
+    const hash = sha256(serialized);
+    const target = objectPath(hash);
+    rememberObject(hash, artifact, Buffer.byteLength(serialized));
+    try {
+      await fs.access(target);
+      return hash;
+    } catch {
+      await writeJsonAtomic(target, artifact);
+      return hash;
+    }
+  };
+
+  const readObject = async (hash) => {
+    if (!OBJECT_HASH.test(hash)) return null;
+    const cached = objectCache.get(hash);
+    if (cached) {
+      objectCache.delete(hash);
+      objectCache.set(hash, cached);
+      return cached.artifact;
+    }
+    try {
+      const raw = await fs.readFile(objectPath(hash), 'utf8');
+      const artifact = JSON.parse(raw);
+      const serialized = JSON.stringify(artifact);
+      if (sha256(serialized) !== hash) return null;
+      if (!validateRoutePreviewArtifactBundle(artifact).valid) return null;
+      rememberObject(hash, artifact, Buffer.byteLength(serialized));
+      return artifact;
+    } catch {
+      return null;
+    }
+  };
+
+  const materializeManifest = async (manifest, sourceRevision) => {
+    const latestById = new Map();
+    for (const entry of manifest.frames) {
+      const current = latestById.get(entry.id);
+      if (!current || current.generation < entry.generation) {
+        latestById.set(entry.id, entry);
+      }
+    }
+    let aggregate = null;
+    for (const entry of [...latestById.values()].sort((a, b) =>
+      a.id.localeCompare(b.id),
+    )) {
+      const artifact = await readObject(entry.object);
+      if (!artifact || artifact.version !== 3 || artifact.namespace !== manifest.namespace) {
+        continue;
+      }
+      aggregate = mergeRoutePreviewArtifactV3(aggregate, artifact);
+    }
+    if (!aggregate) return null;
+    const { sourceRevision: _mixedRevision, ...withoutRevision } = aggregate;
+    return {
+      ...withoutRevision,
+      ...(sourceRevision ? { sourceRevision } : {}),
+    };
+  };
+
+  const publishArtifact = async (
+    artifact,
+    {
+      recordRevision = true,
+      sourceRevision = artifact?.sourceRevision,
+      materializeLegacy = true,
+    } = {},
+  ) => {
+    const validation = validateRoutePreviewArtifactBundle(artifact);
+    if (!validation.valid || typeof artifact.namespace !== 'string') {
+      throw new Error('Preview artifact store refused an invalid artifact.');
+    }
+    if (
+      recordRevision &&
+      sourceRevision &&
+      artifact.sourceRevision !== sourceRevision
+    ) {
+      throw new Error('Preview artifact store refused a mismatched revision.');
+    }
+    const namespace = artifact.namespace;
+    return withFilesystemLock(
+      lockPath(namespace),
+      async () => {
+        // Cross-process correctness depends on this read happening after the
+        // lock is held. Every writer merges its entries into the newest manifest.
+        const stored = await readManifest(namespace);
+        const generation = (stored?.generation ?? 0) + 1;
+        let manifest = stored ?? {
+          version: PYGMALION_PREVIEW_ARTIFACT_STORE_VERSION,
+          namespace,
+          generation: 0,
+          revisions: [],
+          frames: [],
+        };
+        const previousObjectReferences = new Set(
+          [
+            ...(stored?.revisions ?? []),
+            ...(stored?.frames ?? []),
+          ].map((entry) => entry.object),
+        );
+        if (recordRevision && sourceRevision) {
+          const wholeObject = await writeObject(artifact);
+          const nextRevision = {
+            sourceRevision,
+            object: wholeObject,
+            generation,
+          };
+          const alreadyRecorded = manifest.revisions.some(
+            (entry) =>
+              entry.sourceRevision === nextRevision.sourceRevision &&
+              entry.object === nextRevision.object,
+          );
+          if (!alreadyRecorded) {
+            manifest = {
+              ...manifest,
+              revisions: upsertRevision(manifest.revisions, nextRevision),
+            };
+          }
+        }
+        if (artifact.version === 3) {
+          let frames = manifest.frames;
+          for (const [id, frame] of Object.entries(artifact.frames)) {
+            const selected = selectRoutePreviewArtifactFrames(artifact, [{ id }]);
+            const object = await writeObject(selected.bundle);
+            const nextFrame = {
+              id,
+              fingerprint:
+                typeof frame.fingerprint === 'string' ? frame.fingerprint : null,
+              sourceRevision: sourceRevision ?? null,
+              object,
+              generation,
+            };
+            const alreadyRecorded = frames.some(
+              (entry) =>
+                entry.id === nextFrame.id &&
+                entry.fingerprint === nextFrame.fingerprint &&
+                entry.sourceRevision === nextFrame.sourceRevision &&
+                entry.object === nextFrame.object,
+            );
+            if (!alreadyRecorded) frames = upsertFrame(frames, nextFrame);
+          }
+          manifest = { ...manifest, frames };
+        }
+        manifest = { ...manifest, generation };
+        if (
+          manifest.revisions.length + manifest.frames.length >
+          MANIFEST_ENTRY_LIMIT
+        ) {
+          throw new Error('Preview artifact store manifest is too large.');
+        }
+        await writeJsonAtomic(manifestPath(namespace), manifest);
+
+        // Objects contain the namespace, so an identical hash cannot be owned
+        // by another namespace. Under this namespace lock it is safe to remove
+        // entries retention just made unreachable, after the manifest commit.
+        const retainedObjectReferences = new Set(
+          [...manifest.revisions, ...manifest.frames].map(
+            (entry) => entry.object,
+          ),
+        );
+        await Promise.all(
+          [...previousObjectReferences]
+            .filter((hash) => !retainedObjectReferences.has(hash))
+            .map(async (hash) => {
+              await fs.rm(objectPath(hash), { force: true });
+              forgetObject(hash);
+            }),
+        );
+
+        if (materializeLegacy) {
+          // Never stamp a cross-revision frame aggregate with the current
+          // revision. The compatibility file remains an honest exact snapshot;
+          // endpoint frame selection is served from the manifest above.
+          let materialized = recordRevision ? artifact : null;
+          if (artifact.version === 3 && !recordRevision) {
+            const exactRevision =
+              manifest.revisions.find(
+                (entry) => entry.sourceRevision === sourceRevision,
+              ) ??
+              [...manifest.revisions].sort(
+                (a, b) => b.generation - a.generation,
+              )[0];
+            materialized = exactRevision
+              ? await readObject(exactRevision.object)
+              : null;
+          }
+          // A partial frame capture is never written as a whole revision. If no
+          // exact revision exists yet, leave the compatibility file untouched.
+          if (materialized) {
+            await writeJsonAtomic(legacyFile, materialized);
+            legacyCache = {
+              signature: await legacyFileSignature(),
+              hash: sha256(JSON.stringify(materialized)),
+              status: 'valid',
+              artifact: materialized,
+            };
+          }
+        }
+        return artifact;
+      },
+      { lockTimeoutMs, lockStaleMs, onLockAcquired },
+    );
+  };
+
+  const importLegacyArtifact = async () => {
+    let signature;
+    try {
+      signature = await legacyFileSignature();
+    } catch (error) {
+      if (error?.code === 'ENOENT') {
+        legacyCache = {
+          signature: 'missing',
+          status: 'missing',
+          artifact: null,
+        };
+        return legacyCache;
+      }
+      throw error;
+    }
+    if (legacyCache?.signature === signature) return legacyCache;
+    let raw;
+    try {
+      raw = await fs.readFile(legacyFile, 'utf8');
+    } catch (error) {
+      if (error?.code === 'ENOENT') return { status: 'missing', artifact: null };
+      throw error;
+    }
+    let artifact;
+    try {
+      artifact = JSON.parse(raw);
+    } catch {
+      legacyCache = { signature, status: 'invalid', artifact: null };
+      return legacyCache;
+    }
+    const hash = sha256(JSON.stringify(artifact));
+    if (!validateRoutePreviewArtifactBundle(artifact).valid) {
+      legacyCache = {
+        signature,
+        hash,
+        status: 'invalid',
+        artifact: null,
+      };
+      return legacyCache;
+    }
+    // The compatibility contract treats this file as one exact snapshot. Frame
+    // consumers still decide exactness from their requested fingerprints; the
+    // idempotent entries below ensure restarting Vite cannot promote this seed
+    // above a newer partial capture merely by importing it again.
+    await publishArtifact(artifact, { materializeLegacy: false });
+    legacyCache = { signature, hash, status: 'valid', artifact };
+    return legacyCache;
+  };
+
+  const readExactArtifact = async (namespace, sourceRevision) => {
+    const manifest = await readManifest(namespace);
+    const entry = manifest?.revisions.find(
+      (candidate) => candidate.sourceRevision === sourceRevision,
+    );
+    if (!entry) return null;
+    const artifact = await readObject(entry.object);
+    return artifact?.namespace === namespace &&
+      artifact.sourceRevision === sourceRevision
+      ? artifact
+      : null;
+  };
+
+  const readLatestArtifact = async (namespace) => {
+    const manifest = await readManifest(namespace);
+    if (!manifest) return null;
+    const revision = [...manifest.revisions].sort(
+      (a, b) => b.generation - a.generation,
+    )[0];
+    if (revision) {
+      const artifact = await readObject(revision.object);
+      if (artifact?.namespace === namespace) return artifact;
+    }
+    return materializeManifest(manifest);
+  };
+
+  const indexFrameCandidates = (manifest, wanted) => {
+    const wantedIds = new Set(wanted.map((request) => request.id));
+    const candidatesById = new Map();
+    for (const entry of manifest?.frames ?? []) {
+      if (!wantedIds.has(entry.id)) continue;
+      const candidates = candidatesById.get(entry.id) ?? [];
+      candidates.push(entry);
+      candidatesById.set(entry.id, candidates);
+    }
+    for (const candidates of candidatesById.values()) {
+      candidates.sort((left, right) => right.generation - left.generation);
+    }
+    return candidatesById;
+  };
+
+  /** Resolves freshness from the compact manifest without loading frame payloads. */
+  const resolveFrameSelection = async (namespace, sourceRevision, wanted) => {
+    const manifest = await readManifest(namespace);
+    const candidatesById = indexFrameCandidates(manifest, wanted);
+    const exact = [];
+    const missing = [];
+    const stale = [];
+    for (const request of wanted) {
+      const candidates = candidatesById.get(request.id) ?? [];
+      if (!candidates.length) {
+        missing.push(request.id);
+        continue;
+      }
+      const current = candidates.some(
+        (entry) =>
+          entry.sourceRevision === sourceRevision &&
+          (request.fingerprint == null ||
+            entry.fingerprint === request.fingerprint),
+      );
+      (current ? exact : stale).push(request.id);
+    }
+    return { exact, missing, stale };
+  };
+
+  const readFrameSelection = async (namespace, sourceRevision, wanted) => {
+    const manifest = await readManifest(namespace);
+    let aggregate = null;
+    if (manifest) {
+      // Index once per read. The earlier path filtered the complete manifest for
+      // every wanted frame, which made a storyboard probe O(wanted × entries).
+      const candidatesById = indexFrameCandidates(manifest, wanted);
+      for (const request of wanted) {
+        const candidates = candidatesById.get(request.id) ?? [];
+        const exact = candidates.filter((entry) =>
+          entry.sourceRevision === sourceRevision &&
+          (request.fingerprint == null ||
+            entry.fingerprint === request.fingerprint),
+        );
+        const exactSet = new Set(exact);
+        const ordered = [
+          ...exact,
+          ...candidates.filter((entry) => !exactSet.has(entry)),
+        ];
+        for (const entry of ordered) {
+          const artifact = await readObject(entry.object);
+          if (
+            artifact?.version === 3 &&
+            artifact.namespace === namespace &&
+            artifact.frames[request.id]
+          ) {
+            const frame = artifact.frames[request.id];
+            const withProvenance = {
+              ...artifact,
+              frames: {
+                ...artifact.frames,
+                [request.id]: {
+                  ...frame,
+                  ...(entry.sourceRevision == null
+                    ? {}
+                    : { sourceRevision: entry.sourceRevision }),
+                },
+              },
+            };
+            aggregate = mergeRoutePreviewArtifactV3(
+              aggregate,
+              withProvenance,
+            );
+            break;
+          }
+        }
+      }
+    }
+    return selectRoutePreviewArtifactFrames(
+      aggregate ?? emptyBundle(namespace, sourceRevision),
+      wanted,
+      { includeStale: true, sourceRevision },
+    );
+  };
+
+  return {
+    artifactFile: legacyFile,
+    storeDirectory: root,
+    importLegacyArtifact,
+    publishArtifact,
+    readExactArtifact,
+    readFrameSelection,
+    readLatestArtifact,
+    resolveFrameSelection,
+  };
+}

package/node/preview-vite-cache.mjs

@@ -0,0 +1,26 @@
+import { createHash } from 'node:crypto';
+import path from 'node:path';
+
+/** Returns a stable, isolated Vite optimizeDeps cache for one preview instance. */
+export function resolvePygmalionPreviewViteCacheDir({
+  appRoot,
+  instance,
+  modulesDirectory = 'node_modules',
+}) {
+  if (typeof appRoot !== 'string' || !appRoot.trim()) {
+    throw new TypeError('Preview Vite cache requires an app root.');
+  }
+  if (typeof instance !== 'string' || !instance.trim()) {
+    throw new TypeError('Preview Vite cache requires an instance identity.');
+  }
+  const resolvedRoot = path.resolve(appRoot);
+  const digest = createHash('sha256')
+    .update(`${resolvedRoot}\u0000${instance}`)
+    .digest('hex')
+    .slice(0, 20);
+  return path.join(
+    path.resolve(resolvedRoot, modulesDirectory),
+    '.vite-pygmalion',
+    digest,
+  );
+}