@laurence79/wireit 0.14.13-shared-cache.0

package/lib/caching/local-cache.js

@@ -0,0 +1,78 @@
+/**
+ * @license
+ * Copyright 2022 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import * as fs from '../util/fs.js';
+import * as pathlib from 'path';
+import { createHash } from 'crypto';
+import { getScriptDataDir } from '../util/script-data-dir.js';
+import { copyEntries } from '../util/copy.js';
+import { glob } from '../util/glob.js';
+/**
+ * Caches script output to each package's
+ * ".wireit/<script-name-hex>/cache/<cache-key-sha256-hex>" folder.
+ */
+export class LocalCache {
+    async get(script, fingerprint) {
+        const cacheDir = this.#getCacheDir(script, fingerprint);
+        try {
+            await fs.access(cacheDir);
+        }
+        catch (error) {
+            if (error.code === 'ENOENT') {
+                return;
+            }
+            throw error;
+        }
+        return new LocalCacheHit(cacheDir, script.packageDir);
+    }
+    async set(script, fingerprint, absoluteFiles) {
+        // TODO(aomarks) A script's cache directory currently just grows forever.
+        // We'll have the "clean" command to help with manual cleanup, but we'll
+        // almost certainly want an automated way to limit the size of the cache
+        // directory (e.g. LRU capped to some number of entries).
+        // https://github.com/google/wireit/issues/71
+        const absCacheDir = this.#getCacheDir(script, fingerprint);
+        // Note fs.mkdir returns the first created directory, or undefined if no
+        // directory was created.
+        const existed = (await fs.mkdir(absCacheDir, { recursive: true })) === undefined;
+        if (existed) {
+            // This is an unexpected error because the Executor should already have
+            // checked for an existing cache hit.
+            throw new Error(`Did not expect ${absCacheDir} to already exist.`);
+        }
+        await copyEntries(absoluteFiles, script.packageDir, absCacheDir);
+        return true;
+    }
+    #getCacheDir(script, fingerprint) {
+        return pathlib.join(getScriptDataDir(script), 'cache', createHash('sha256').update(fingerprint.string).digest('hex'));
+    }
+}
+class LocalCacheHit {
+    /**
+     * The folder where the cached output is stored. Assumed to exist.
+     */
+    #source;
+    /**
+     * The folder where the cached output should be written when {@link apply} is
+     * called.
+     */
+    #destination;
+    constructor(source, destination) {
+        this.#source = source;
+        this.#destination = destination;
+    }
+    async apply() {
+        const entries = await glob(['**'], {
+            cwd: this.#source,
+            followSymlinks: false,
+            includeDirectories: true,
+            expandDirectories: true,
+            // Shouldn't ever happen, but would be really weird.
+            throwIfOutsideCwd: true,
+        });
+        await copyEntries(entries, this.#source, this.#destination);
+    }
+}
+//# sourceMappingURL=local-cache.js.map

package/lib/caching/shared-cache.js

@@ -0,0 +1,256 @@
+/**
+ * @license
+ * Copyright 2025 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import * as pathlib from 'path';
+import { createHash } from 'crypto';
+import * as fs from '../util/fs.js';
+import { copyEntries } from '../util/copy.js';
+import { glob } from '../util/glob.js';
+/**
+ * On-disk format version. A future incompatible layout change can bump this and
+ * coexist with older entries in a cache that outlives wireit versions.
+ */
+const FORMAT_VERSION = 'v1';
+/**
+ * Sub-directory of the format-version root where in-flight writes live before
+ * being atomically renamed into place. Kept under the same root so that the
+ * rename stays on one filesystem (and is therefore atomic).
+ */
+const TEMP_DIR_NAME = '.tmp';
+/**
+ * A content-addressed, multi-writer-safe cache that lives at a stable path
+ * outside any working tree, so entries survive across builds and can be shared
+ * between multiple agents on a long-lived machine.
+ *
+ * Unlike {@link LocalCache}, entries are keyed purely by fingerprint hash (no
+ * per-package/per-script namespacing), which gives cross-package/cross-agent
+ * dedup. Writes are published with a temp-dir-then-atomic-rename, which makes
+ * concurrent writers safe (first writer wins; the content is identical by
+ * construction) and means readers never observe a partial entry.
+ *
+ * A shared-cache problem can make builds slower, never broken: all runtime
+ * read/write errors are non-fatal and degrade to "no cache" or "read-only".
+ */
+export class SharedCache {
+    /** The format-version root, i.e. `<root>/v1`. */
+    #versionRoot;
+    /** Where in-flight writes are staged, i.e. `<root>/v1/.tmp`. */
+    #tempDir;
+    #logger;
+    /**
+     * Whether we should skip all writes. Set initially from config, and flipped
+     * on if we hit a permission error while writing (degrade-to-read-only safety
+     * net).
+     */
+    #readonly;
+    /**
+     * Whether we've already logged a write failure. Used to avoid flooding the
+     * output when the same problem recurs for every script in a run.
+     */
+    #writeWarningLogged = false;
+    constructor(versionRoot, readonly, logger) {
+        this.#versionRoot = versionRoot;
+        this.#tempDir = pathlib.join(versionRoot, TEMP_DIR_NAME);
+        this.#readonly = readonly;
+        this.#logger = logger;
+    }
+    /**
+     * Create a {@link SharedCache} rooted at {@link rootDir}, probing whether the
+     * directory is usable and emitting a startup health line (never silent).
+     *
+     * @returns A cache instance, or `undefined` if the cache is unavailable (in
+     * which case caching is disabled for the run, but the build still proceeds).
+     */
+    static async create(rootDir, options, logger) {
+        const versionRoot = pathlib.join(rootDir, FORMAT_VERSION);
+        const tempDir = pathlib.join(versionRoot, TEMP_DIR_NAME);
+        if (options.readonly) {
+            // We only ever read, so there's nothing to set up. A missing directory
+            // simply means every lookup is a miss.
+            console.error(`ℹ️ Shared cache: ${rootDir} (read-only)`);
+            return new SharedCache(versionRoot, true, logger);
+        }
+        try {
+            // Eagerly create the temp directory (and therefore the version root). If
+            // this works, we know we can write.
+            await fs.mkdir(tempDir, { recursive: true });
+        }
+        catch (error) {
+            const code = error.code;
+            // We couldn't set up for writing. Fall back to read-only if we can still
+            // read existing entries, otherwise disable caching entirely. Either way
+            // the build proceeds.
+            if (await exists(rootDir)) {
+                console.error(`⚠️ Shared cache: ${rootDir} not writable — running read-only ` +
+                    `(${code ?? String(error)})`);
+                return new SharedCache(versionRoot, true, logger);
+            }
+            console.error(`⚠️ Shared cache: ${rootDir} unavailable — caching disabled ` +
+                `(${code ?? String(error)})`);
+            return undefined;
+        }
+        console.error(`ℹ️ Shared cache: ${rootDir} (read-write)`);
+        return new SharedCache(versionRoot, false, logger);
+    }
+    async get(script, fingerprint) {
+        const entryDir = this.#entryDir(fingerprint);
+        try {
+            // The only way an entry directory can appear is a completed rename, so its
+            // mere existence means it's a complete, readable hit. No done-marker or
+            // partial-read window to worry about.
+            await fs.access(entryDir);
+        }
+        catch (error) {
+            const code = error.code;
+            if (code === 'ENOENT') {
+                // Miss.
+                return undefined;
+            }
+            // Unexpected read error. Treat as a miss so the build proceeds, but let
+            // the user know.
+            this.#logger.log({
+                script,
+                type: 'info',
+                detail: 'cache-info',
+                message: `Shared cache read error, treating as a miss: ${String(error)}`,
+            });
+            return undefined;
+        }
+        return new SharedCacheHit(entryDir, script.packageDir);
+    }
+    async set(script, fingerprint, absoluteFiles) {
+        if (this.#readonly) {
+            return false;
+        }
+        const entryDir = this.#entryDir(fingerprint);
+        let tempEntryDir;
+        try {
+            // The temp directory could have been reaped externally since startup, so
+            // ensure it exists before staging into it.
+            await fs.mkdir(this.#tempDir, { recursive: true });
+            tempEntryDir = await fs.mkdtemp(pathlib.join(this.#tempDir, 'set-'));
+        }
+        catch (error) {
+            return this.#handleWriteError(error, script);
+        }
+        try {
+            // copyEntries already attempts a reflink (COPYFILE_FICLONE), so staging is
+            // copy-on-write when the cache and working tree share a filesystem.
+            await copyEntries(absoluteFiles, script.packageDir, tempEntryDir);
+            // Create the shard directory (`<root>/v1/<aa>/`) before publishing.
+            await fs.mkdir(pathlib.dirname(entryDir), { recursive: true });
+            // Atomic publish.
+            await fs.rename(tempEntryDir, entryDir);
+            return true;
+        }
+        catch (error) {
+            // The rename failed (or a step before it did). Clean up our temp dir;
+            // a crash-orphaned temp dir is harmless (never read by get()) but we
+            // tidy up on the happy-ish path anyway.
+            await deleteQuietly(tempEntryDir);
+            // If the destination now exists, another agent published the identical
+            // content first. Entries are content-addressed, so theirs is
+            // interchangeable with ours — this is the expected outcome of a publish
+            // race, not an error. (This is the explicit fix for LocalCache's
+            // "Did not expect ... to already exist" throw under concurrency.)
+            if (await exists(entryDir)) {
+                return true;
+            }
+            return this.#handleWriteError(error, script);
+        }
+    }
+    #entryDir(fingerprint) {
+        const hash = createHash('sha256').update(fingerprint.string).digest('hex');
+        // 2-char shard prefix keeps any one directory from accumulating tens of
+        // thousands of entries on filesystems that handle that poorly.
+        return pathlib.join(this.#versionRoot, hash.slice(0, 2), hash);
+    }
+    /**
+     * Handle a write-path error: degrade to read-only on permission problems,
+     * otherwise just skip caching this entry. Always non-fatal.
+     */
+    #handleWriteError(error, script) {
+        const code = error.code;
+        if (code === 'EACCES' || code === 'EPERM') {
+            // Permission problem. Stop trying to write for the rest of this process
+            // so builds keep working without repeated failures.
+            this.#readonly = true;
+            if (!this.#writeWarningLogged) {
+                this.#writeWarningLogged = true;
+                console.warn(`⚠️ Shared cache: not writable — continuing read-only (${code})`);
+            }
+            return false;
+        }
+        // Other I/O error (ENOSPC, EIO, a flaky mount, ...). Best-effort: skip
+        // caching this entry and let the build proceed. Warn once to avoid noise.
+        if (!this.#writeWarningLogged) {
+            this.#writeWarningLogged = true;
+            this.#logger.log({
+                script,
+                type: 'info',
+                detail: 'cache-info',
+                message: `Shared cache write error, not caching: ${String(error)}`,
+            });
+        }
+        return false;
+    }
+}
+/**
+ * Apply a {@link SharedCache} hit by copying the entry's payload into the
+ * script's package directory.
+ */
+class SharedCacheHit {
+    /** The cache entry directory. Assumed to exist. */
+    #source;
+    /** Where the cached output should be written when {@link apply} is called. */
+    #destination;
+    constructor(source, destination) {
+        this.#source = source;
+        this.#destination = destination;
+    }
+    async apply() {
+        const entries = await glob(['**'], {
+            cwd: this.#source,
+            followSymlinks: false,
+            includeDirectories: true,
+            expandDirectories: true,
+            // Shouldn't ever happen, but would be really weird.
+            throwIfOutsideCwd: true,
+        });
+        // Restore is always a copy, never a hardlink: a later in-place write to a
+        // restored file would otherwise corrupt the shared entry and poison every
+        // future restore. copyEntries passes COPYFILE_FICLONE, so this is
+        // copy-on-write automatically when the cache and working tree share a
+        // filesystem, and a plain copy otherwise.
+        await copyEntries(entries, this.#source, this.#destination);
+    }
+}
+/**
+ * Whether a path exists (best-effort; any error other than a clean "yes" is
+ * treated as "no").
+ */
+async function exists(path) {
+    try {
+        await fs.access(path);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Delete a path recursively, swallowing any error. Used for best-effort cleanup
+ * of temp directories.
+ */
+async function deleteQuietly(path) {
+    try {
+        await fs.rm(path, { recursive: true, force: true });
+    }
+    catch {
+        // Best-effort: an orphaned temp dir is harmless (never read by get()) and
+        // is reaped externally.
+    }
+}
+//# sourceMappingURL=shared-cache.js.map