@laurence79/wireit 0.14.13-shared-cache.0

package/lib/util/fs.js

@@ -0,0 +1,258 @@
+/**
+ * @license
+ * Copyright 2023 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+var __addDisposableResource = (this && this.__addDisposableResource) || function (env, value, async) {
+    if (value !== null && value !== void 0) {
+        if (typeof value !== "object" && typeof value !== "function") throw new TypeError("Object expected.");
+        var dispose, inner;
+        if (async) {
+            if (!Symbol.asyncDispose) throw new TypeError("Symbol.asyncDispose is not defined.");
+            dispose = value[Symbol.asyncDispose];
+        }
+        if (dispose === void 0) {
+            if (!Symbol.dispose) throw new TypeError("Symbol.dispose is not defined.");
+            dispose = value[Symbol.dispose];
+            if (async) inner = dispose;
+        }
+        if (typeof dispose !== "function") throw new TypeError("Object not disposable.");
+        if (inner) dispose = function() { try { inner.call(this); } catch (e) { return Promise.reject(e); } };
+        env.stack.push({ value: value, dispose: dispose, async: async });
+    }
+    else if (async) {
+        env.stack.push({ async: true });
+    }
+    return value;
+};
+var __disposeResources = (this && this.__disposeResources) || (function (SuppressedError) {
+    return function (env) {
+        function fail(e) {
+            env.error = env.hasError ? new SuppressedError(e, env.error, "An error was suppressed during disposal.") : e;
+            env.hasError = true;
+        }
+        var r, s = 0;
+        function next() {
+            while (r = env.stack.pop()) {
+                try {
+                    if (!r.async && s === 1) return s = 0, env.stack.push(r), Promise.resolve().then(next);
+                    if (r.dispose) {
+                        var result = r.dispose.call(r.value);
+                        if (r.async) return s |= 2, Promise.resolve(result).then(next, function(e) { fail(e); return next(); });
+                    }
+                    else s |= 1;
+                }
+                catch (e) {
+                    fail(e);
+                }
+            }
+            if (s === 1) return env.hasError ? Promise.reject(env.error) : Promise.resolve();
+            if (env.hasError) throw env.error;
+        }
+        return next();
+    };
+})(typeof SuppressedError === "function" ? SuppressedError : function (error, suppressed, message) {
+    var e = new Error(message);
+    return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
+});
+import * as fs from 'fs/promises';
+import { createReadStream as rawCreateReadStream, createWriteStream as rawCreateWriteStream, } from 'fs';
+import { Deferred } from './deferred.js';
+import './dispose.js';
+export { constants } from 'fs';
+export class Semaphore {
+    #remaining;
+    #waiting = [];
+    constructor(numSlots) {
+        if (numSlots <= 0) {
+            throw new Error(`numSlots must be positive, got ${numSlots}`);
+        }
+        this.#remaining = numSlots;
+    }
+    async reserve() {
+        while (this.#remaining === 0) {
+            const deferred = new Deferred();
+            this.#waiting.push(deferred);
+            await deferred.promise;
+        }
+        this.#remaining--;
+        let disposed = false;
+        return {
+            [Symbol.dispose]: () => {
+                if (disposed) {
+                    return;
+                }
+                disposed = true;
+                this.#remaining++;
+                if (this.#waiting.length > 0) {
+                    this.#waiting.pop()?.resolve();
+                }
+            },
+        };
+    }
+}
+export const fileBudget = (() => {
+    let maxOpenFiles = Number(process.env['WIREIT_MAX_OPEN_FILES']);
+    if (isNaN(maxOpenFiles)) {
+        // This is tricky to get right. There's no simple cross-platform way to
+        // determine what our current limits are. Windows it's 512, on macOS it
+        // defaults to 256, and on Linux it varies a lot.
+        // 200 gives us a bit of headroom for other things that might be using
+        // file descriptors in our process, like node internals.
+        maxOpenFiles = 200;
+    }
+    return new Semaphore(maxOpenFiles);
+})();
+export async function mkdir(path, options) {
+    const reservation = await fileBudget.reserve();
+    try {
+        return await fs.mkdir(path, options);
+    }
+    finally {
+        reservation[Symbol.dispose]();
+    }
+}
+export async function mkdtemp(path) {
+    const reservation = await fileBudget.reserve();
+    try {
+        return await fs.mkdtemp(path);
+    }
+    finally {
+        reservation[Symbol.dispose]();
+    }
+}
+export async function writeFile(path, contents, encoding) {
+    const reservation = await fileBudget.reserve();
+    try {
+        return await fs.writeFile(path, contents, encoding);
+    }
+    finally {
+        reservation[Symbol.dispose]();
+    }
+}
+export async function readFile(path, encoding) {
+    const reservation = await fileBudget.reserve();
+    try {
+        return await fs.readFile(path, encoding);
+    }
+    finally {
+        reservation[Symbol.dispose]();
+    }
+}
+export async function rm(path, options) {
+    const reservation = await fileBudget.reserve();
+    try {
+        return await fs.rm(path, options);
+    }
+    finally {
+        reservation[Symbol.dispose]();
+    }
+}
+export async function lstat(path) {
+    const reservation = await fileBudget.reserve();
+    try {
+        return await fs.lstat(path);
+    }
+    finally {
+        reservation[Symbol.dispose]();
+    }
+}
+export async function stat(path) {
+    const reservation = await fileBudget.reserve();
+    try {
+        return await fs.stat(path);
+    }
+    finally {
+        reservation[Symbol.dispose]();
+    }
+}
+export async function access(path) {
+    const reservation = await fileBudget.reserve();
+    try {
+        return await fs.access(path);
+    }
+    finally {
+        reservation[Symbol.dispose]();
+    }
+}
+export async function createReadStream(path, options) {
+    const reservation = await fileBudget.reserve();
+    const stream = rawCreateReadStream(path, options);
+    stream.on('close', () => reservation[Symbol.dispose]());
+    return stream;
+}
+export async function createWriteStream(path) {
+    const reservation = await fileBudget.reserve();
+    const stream = rawCreateWriteStream(path);
+    stream.on('close', () => reservation[Symbol.dispose]());
+    return stream;
+}
+export async function copyFile(src, dest, flags) {
+    const reservation = await fileBudget.reserve();
+    try {
+        return await fs.copyFile(src, dest, flags);
+    }
+    finally {
+        reservation[Symbol.dispose]();
+    }
+}
+export async function readlink(path, options) {
+    const reservation = await fileBudget.reserve();
+    try {
+        return await fs.readlink(path, options);
+    }
+    finally {
+        reservation[Symbol.dispose]();
+    }
+}
+export async function symlink(target, path, type) {
+    const reservation = await fileBudget.reserve();
+    try {
+        return await fs.symlink(target, path, type);
+    }
+    finally {
+        reservation[Symbol.dispose]();
+    }
+}
+export async function unlink(target) {
+    const reservation = await fileBudget.reserve();
+    try {
+        return await fs.unlink(target);
+    }
+    finally {
+        reservation[Symbol.dispose]();
+    }
+}
+export async function rename(oldPath, newPath) {
+    const reservation = await fileBudget.reserve();
+    try {
+        return await fs.rename(oldPath, newPath);
+    }
+    finally {
+        reservation[Symbol.dispose]();
+    }
+}
+export async function rmdir(target) {
+    const reservation = await fileBudget.reserve();
+    try {
+        return await fs.rmdir(target);
+    }
+    finally {
+        reservation[Symbol.dispose]();
+    }
+}
+export async function readdir(path, options) {
+    const env_1 = { stack: [], error: void 0, hasError: false };
+    try {
+        const _reservation = __addDisposableResource(env_1, await fileBudget.reserve(), false);
+        return await fs.readdir(path, options);
+    }
+    catch (e_1) {
+        env_1.error = e_1;
+        env_1.hasError = true;
+    }
+    finally {
+        __disposeResources(env_1);
+    }
+}
+//# sourceMappingURL=fs.js.map

package/lib/util/glob.js

@@ -0,0 +1,255 @@
+/**
+ * @license
+ * Copyright 2022 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import fastGlob from 'fast-glob';
+import { expand as braces } from 'brace-expansion';
+import * as pathlib from 'path';
+/**
+ * The error raised when {@link glob} matches a path that is outside of
+ * {@link GlobOptions.cwd} when {@link GlobOptions.throwIfOutsideCwd} is `true`.
+ */
+export class GlobOutsideCwdError extends Error {
+    constructor(path, cwd) {
+        super(`${JSON.stringify(path)} was outside ${JSON.stringify(cwd)}`);
+    }
+}
+/**
+ * Match glob patterns against the file system.
+ *
+ * - Input patterns must be / separated.
+ * - Matches are returned with the OS-specific separator.
+ * - Leading `/`s are interpreted as relative to the `cwd`, instead of the root
+ *   of the filesystem.
+ * - Dot (aka hidden) files are always matched.
+ * - Empty or blank patterns throw.
+ * - The order of "!exclusion" patterns matter (i.e. files can be "re-included"
+ *   after exclusion).
+ * - Results are always absolute.
+ *
+ * @param patterns The glob patterns to match. Must use forward-slash separator,
+ * even on Windows.
+ * @params opts See {@link GlobOptions}.
+ */
+export async function glob(patterns, opts) {
+    if (patterns.length === 0) {
+        return [];
+    }
+    const expandedPatterns = []; // New array so we don't mutate input patterns array.
+    for (const pattern of patterns) {
+        // We need to expand `{foo,bar}` style brace patterns ourselves so that we
+        // can reliably interpret the syntax of the pattern. For example, for
+        // re-rooting we need to check for a leading `/`, but we can't do that
+        // directly on `{/foo,/bar}`.
+        const allExpanded = pattern === '' ? [''] : braces(pattern);
+        for (const expanded of allExpanded) {
+            expandedPatterns.push(expanded);
+            if (opts.expandDirectories) {
+                // Also include a recursive-children version of every pattern, in case
+                // the pattern refers to a directory. This gives us behavior similar to
+                // the npm package.json "files" array, where matching a directory
+                // implicitly includes all transitive children.
+                if (!isRecursive(expanded)) {
+                    const isExclusive = pattern[0] === '!';
+                    if (!isExclusive) {
+                        // We use the "ignore" feature of fast-glob for exclusive patterns,
+                        // which already automatically recursively excludes directories by
+                        // not recursing into them at all, so there is no need to also
+                        // generate a recursive version when excluding.
+                        expandedPatterns.push(expanded + (expanded.endsWith('/') ? '**' : '/**'));
+                    }
+                }
+            }
+        }
+    }
+    // fast-glob doesn't pay attention to the order of !excluded patterns. For
+    // example, the following pattern array should include "foo/bar/baz", but
+    // fast-glob excludes it:
+    //
+    //   foo/**
+    //   !foo/bar/**
+    //   foo/bar/baz  <-- wrongly excluded
+    //   !foo/qux
+    //
+    // To fix this behavior, we divide the patterns into groups that can be
+    // evaluated separately and then combined. We create a new group whenever an
+    // !exclude pattern is in front of an include pattern, because that's when
+    // this problem could occur, and we include all subsequent negations (but not
+    // preceding ones) in each group:
+    //
+    //   Group 1:
+    //     include: foo/**
+    //     exclude: foo/bar/**
+    //     exclude: foo/qux
+    //
+    //   Group 2:
+    //     include: foo/bar/baz
+    //     exclude: foo/qux
+    let currentGroup = { include: [], exclude: [] };
+    const groups = [currentGroup];
+    let prevWasInclusive = false;
+    // We want each group to include all subsequent negated patterns. The simplest
+    // way to do that is to build the groups backwards.
+    for (let i = expandedPatterns.length - 1; i >= 0; i--) {
+        let pattern = expandedPatterns[i];
+        const isExclusive = pattern[0] === '!';
+        if (isExclusive) {
+            pattern = pattern.slice(1); // Remove the "!"
+        }
+        // Ignore leading `/`s so that e.g. "/foo" is interpreted relative to the
+        // cwd, instead of relative to the root of the filesystem. We want to include
+        // >1 leading slashes, since those are technically valid paths too.
+        pattern = pattern.replace(/^\/+/, '');
+        if (isExclusive) {
+            if (prevWasInclusive) {
+                // A new group is needed because this exclusion comes before an
+                // inclusion.
+                //
+                //   foo/**
+                //   !foo/bar/** <-- we are here
+                //   foo/bar/baz <-- this is the previous one
+                //   !foo/qux
+                currentGroup = { include: [], exclude: [] };
+                for (const previousGroup of groups) {
+                    // Also include all exclusions we've accumulated so far into the new
+                    // group (since we're iterating backwards, these are the exclusions
+                    // that come after it).
+                    currentGroup.exclude.push(...previousGroup.exclude);
+                }
+                groups.push(currentGroup);
+            }
+            currentGroup.exclude.push(
+            // Trim trailing slashes because fast-glob does not understand trailing
+            // slashes in "ignore" list entries (they have no effect!).
+            pattern.replace(/\/+$/, ''));
+        }
+        else if (pattern.match(/^\s*$/)) {
+            // fast-glob already throws on empty strings, but we also throw on
+            // only-whitespace patterns.
+            //
+            // Note minor optimization here: there is no reason to check this regexp
+            // on exclusive patterns, because by definition they start with a "!" so
+            // can't have been empty/blank.
+            throw new Error(`glob encountered empty or blank pattern: ${JSON.stringify(pattern)}`);
+        }
+        else {
+            currentGroup.include.push(pattern);
+        }
+        prevWasInclusive = !isExclusive;
+    }
+    // Pass each group to fast-glob to match in parallel, and combine into a
+    // single set.
+    const combinedMap = new Map();
+    // Ensure the cwd is absolute and normalized so that we can do path string
+    // comparisons.
+    const normalizedCwd = pathlib.resolve(opts.cwd);
+    const normalizedCwdWithTrailingSep = normalizedCwd + pathlib.sep;
+    await Promise.all(groups.map(async ({ include, exclude }) => {
+        const matches = await fastGlob(include, {
+            ignore: exclude,
+            cwd: normalizedCwd,
+            dot: true,
+            onlyFiles: !opts.includeDirectories,
+            absolute: true,
+            followSymbolicLinks: opts.followSymlinks,
+            // This should have no overhead because fast-glob already uses these
+            // objects for its internal representation:
+            // https://github.com/mrmlnc/fast-glob#objectmode
+            objectMode: true,
+            // Since we append "/**" to patterns above, we will sometimes get
+            // ENOTDIR errors when the path we appended to was not a directory. We
+            // can't know in advance which patterns refer to directories.
+            suppressErrors: true,
+            // We already do brace expansion ourselves. Doing it again would be
+            // inefficient and would also break brace escaping.
+            braceExpansion: false,
+        });
+        const potentiallyProblematicSymlinkParents = new Set();
+        for (const match of matches) {
+            // Normalize the path so that:
+            //
+            // 1. We have native path separators. fast-glob returns "/" even on
+            //    Windows.
+            //
+            // 2. Remnants of input pattern syntax like ".." and trailing "/"s are
+            //    removed (which fast-glob preserves in the results). Note that
+            //    `fs.normalize` does not trim trailing "/"s, so we do that
+            //    ourselves (`fs.resolve` does, but that also makes the path
+            //    absolute).
+            match.path = pathlib.normalize(match.path.replace(/\/+$/g, ''));
+            if (opts.throwIfOutsideCwd) {
+                const absPath = match.path;
+                if (
+                // Match "parent/child" and "parent", but not "parentx".
+                !absPath.startsWith(normalizedCwdWithTrailingSep) &&
+                    absPath !== normalizedCwd) {
+                    // TODO(aomarks) This check could in theory be done before we execute
+                    // the globs, but we'd need to be really sure we account for special
+                    // glob syntax, which could make it not 100% straightforward to do
+                    // path checking. Checking the resulting paths is straightforward
+                    // because we know they don't contain special syntax.
+                    throw new GlobOutsideCwdError(absPath, opts.cwd);
+                }
+            }
+            combinedMap.set(match.path, match);
+            if (opts.expandDirectories &&
+                !opts.followSymlinks &&
+                match.dirent.isSymbolicLink()) {
+                potentiallyProblematicSymlinkParents.add(match.path);
+            }
+        }
+        if (potentiallyProblematicSymlinkParents.size > 0) {
+            // When the user passes a path "foo" and expandDirectories is true, we
+            // convert the pattern to "foo/**" (see above about expandDirectories
+            // for why).
+            //
+            // However, what if "foo" is a symlink to a folder with some children,
+            // and followSymbolicLinks is false (which is the combination of
+            // settings we use when wireit globs output files for caching)?
+            //
+            // Well, if you pass "foo/**" to fast-glob where "foo" is a symlink to a
+            // directory, it's actually going to follow the symlink and return its
+            // children, even if followSymbolicLinks is false. (This seems fairly
+            // reasonable, since otherwise it would always have to check all parent
+            // directories of a path before reading any directory contents in case
+            // there's a symlink somewhere up the tree).
+            //
+            // But that's bad for us, because if a wireit user has a script that
+            // creates a symlink to a directory, and they list that symlink directly
+            // in their output paths, then for the purposes of caching we really
+            // just want to just restore the literal symlink, and not copy its
+            // contents. (Otherwise it would be a symlink when built the first time,
+            // but a regular folder with children when restored from cache).
+            //
+            // So, since we don't know whether we might have problematically
+            // appended a "/**" to a symlink, we will instead filter out child
+            // matches. (Otherwise we'd need to check every given path to see if
+            // it's a symlink, and directly listed symlinks are pretty rare, this
+            // post-hoc approach is probably more efficient on average).
+            for (const match of combinedMap.values()) {
+                // Walk up the file hierarchy to check if any parent was a symlink
+                // that was also directly matched by the glob.
+                let child = match.path;
+                while (true) {
+                    const parent = pathlib.dirname(child);
+                    if (parent === child) {
+                        // Reached the filesystem root.
+                        break;
+                    }
+                    if (potentiallyProblematicSymlinkParents.has(parent)) {
+                        combinedMap.delete(match.path);
+                        break;
+                    }
+                    child = parent;
+                }
+            }
+        }
+    }));
+    return [...combinedMap.values()];
+}
+const isRecursive = (pattern) => pattern === '**' ||
+    pattern === '**/*' ||
+    pattern.endsWith('/**') ||
+    pattern.endsWith('/**/*');
+//# sourceMappingURL=glob.js.map

package/lib/util/line-monitor.js

@@ -0,0 +1,69 @@
+/**
+ * @license
+ * Copyright 2022 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import { Deferred } from './deferred.js';
+/**
+ * Monitors the stdout and stderr of a child process line-by-line searching for
+ * a match of the given regular expression.
+ *
+ * Note we can't use readline here because we want to check lines that haven't
+ * completed yet.
+ */
+export class LineMonitor {
+    #child;
+    #pattern;
+    #matched;
+    #stdout;
+    #stderr;
+    constructor(child, pattern) {
+        this.#matched = new Deferred();
+        this.#stdout = '';
+        this.#stderr = '';
+        /**
+         * Resolves to `{"ok": true}` when a match was found or `{"ok": false}` when
+         * this monitor was aborted.
+         */
+        this.matched = this.#matched.promise;
+        this.#onStdout = (data) => {
+            this.#stdout = this.#check(this.#stdout + String(data));
+        };
+        this.#onStderr = (data) => {
+            this.#stderr = this.#check(this.#stderr + String(data));
+        };
+        this.#child = child;
+        this.#pattern = pattern;
+        child.stdout.on('data', this.#onStdout);
+        child.stderr.on('data', this.#onStderr);
+    }
+    abort() {
+        this.#removeEventListeners();
+        this.#matched.resolve({ ok: false, error: undefined });
+    }
+    #removeEventListeners() {
+        this.#child.stdout.removeListener('data', this.#onStdout);
+        this.#child.stderr.removeListener('data', this.#onStderr);
+    }
+    #onStdout;
+    #onStderr;
+    #check(buffer) {
+        const lines = buffer.split(/\n/g);
+        let end = 0;
+        for (let i = 0; i < lines.length; i++) {
+            const line = lines[i];
+            if (i !== lines.length - 1) {
+                // Don't move beyond the final line, since it might be incomplete, and
+                // we want to match the entire line the next time _check is called.
+                end += line.length + 1;
+            }
+            if (this.#pattern.test(line)) {
+                this.#removeEventListeners();
+                this.#matched.resolve({ ok: true, value: undefined });
+                break;
+            }
+        }
+        return buffer.slice(end);
+    }
+}
+//# sourceMappingURL=line-monitor.js.map

package/lib/util/manifest.js

@@ -0,0 +1,31 @@
+/**
+ * @license
+ * Copyright 2022 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+export function computeManifestEntry(stats) {
+    return {
+        t: stats.isFile()
+            ? 'f'
+            : stats.isDirectory()
+                ? 'd'
+                : stats.isSymbolicLink()
+                    ? 'l'
+                    : stats.isBlockDevice()
+                        ? 'b'
+                        : stats.isCharacterDevice()
+                            ? 'c'
+                            : stats.isFIFO()
+                                ? 'p'
+                                : stats.isSocket()
+                                    ? 's'
+                                    : '?',
+        // Don't include timestamp or size for directories, because they can change
+        // when a child is added or removed. If we are tracking the child, then it
+        // will have its own entry. If we are not tracking the child, then we don't
+        // want it to affect the manifest.
+        m: stats.isDirectory() ? undefined : stats.mtimeMs,
+        s: stats.isDirectory() ? undefined : stats.size,
+    };
+}
+//# sourceMappingURL=manifest.js.map

package/lib/util/optimize-mkdirs.js

@@ -0,0 +1,55 @@
+/**
+ * @license
+ * Copyright 2022 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import { dirname } from 'path';
+/**
+ * Given a set of filesystem directory paths, returns the smallest set of
+ * recursive {@link fs.mkdir} operations required to create all directories.
+ *
+ *  For example, given:
+ *
+ *   a/b/c
+ *   a/b
+ *   d
+ *   d/e/f/g/h
+ *
+ * Returns:
+ *
+ *   a/b/c
+ *   d/e/f/g/h
+ *
+ * Note this function does an in-place sort of the given dirs.
+ */
+export const optimizeMkdirs = (dirs) => {
+    if (dirs.length <= 1) {
+        return dirs;
+    }
+    const ops = [];
+    // Sorting from longest to shortest ensures that child directories come before
+    // parents (e.g. [d/e, d/e/f, a, a/b/c] => [d/e/f, a/b/c, d/e, a]).
+    // Parent/child adjacency doesn't matter.
+    dirs.sort((a, b) => b.length - a.length);
+    const handled = new Set();
+    for (const dir of dirs) {
+        if (handled.has(dir)) {
+            // Skip this directory because it has already been handled by a longer
+            // path we've already seen (e.g. "a/b/c" also creates "a/b" and "a").
+            continue;
+        }
+        ops.push(dir);
+        // Add this directory and all of its parent directories to the "done" set.
+        let cur = dir;
+        while (true) {
+            handled.add(cur);
+            const parent = dirname(cur);
+            if (parent === cur) {
+                break;
+            }
+            cur = parent;
+        }
+    }
+    return ops;
+};
+//# sourceMappingURL=optimize-mkdirs.js.map