@laurence79/wireit 0.14.13-shared-cache.0

package/lib/util/package-json-reader.js

@@ -0,0 +1,61 @@
+/**
+ * @license
+ * Copyright 2022 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import { AsyncCache } from './async-cache.js';
+import { PackageJson } from './package-json.js';
+import * as pathlib from 'path';
+import * as fs from './fs.js';
+import { parseTree } from './ast.js';
+export const astKey = Symbol('ast');
+/**
+ * Reads package.json files and caches them.
+ */
+export class CachingPackageJsonReader {
+    #cache = new AsyncCache();
+    #fs;
+    constructor(filesystem = fs) {
+        this.#fs = filesystem;
+    }
+    async read(packageDir) {
+        return this.#cache.getOrCompute(packageDir, async () => {
+            const path = pathlib.resolve(packageDir, 'package.json');
+            let contents;
+            try {
+                contents = await this.#fs.readFile(path, 'utf8');
+            }
+            catch (error) {
+                if (error.code === 'ENOENT') {
+                    return {
+                        ok: false,
+                        error: {
+                            type: 'failure',
+                            reason: 'missing-package-json',
+                            script: { packageDir },
+                        },
+                    };
+                }
+                throw error;
+            }
+            const astResult = parseTree(path, contents);
+            if (!astResult.ok) {
+                return astResult;
+            }
+            const packageJsonFile = new PackageJson({ contents, path }, astResult.value);
+            return { ok: true, value: packageJsonFile };
+        });
+    }
+    async *getFailures() {
+        const values = await Promise.all([...this.#cache.values]);
+        for (const result of values) {
+            if (!result.ok) {
+                yield result.error;
+                continue;
+            }
+            const packageJson = result.value;
+            yield* packageJson.failures;
+        }
+    }
+}
+//# sourceMappingURL=package-json-reader.js.map

package/lib/util/package-json.js

@@ -0,0 +1,179 @@
+/**
+ * @license
+ * Copyright 2022 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import { findNamedNodeAtLocation } from './ast.js';
+import { failUnlessJsonObject, failUnlessKeyValue, failUnlessNonBlankString, } from '../analyzer.js';
+import { offsetInsideNamedNode, offsetInsideRange } from '../error.js';
+/**
+ * A parsed and minimally analyzed package.json file.
+ *
+ * This does some very basic syntactic analysis of the package.json file,
+ * finding issues like "the scripts section isn't an object mapping strings to
+ * strings" and "the wireit section isn't an object mapping strings to objects".
+ *
+ * Makes it easy to find the syntax nodes for a script.
+ *
+ * Does not do any validation or analysis of the wireit script configs.
+ *
+ * This class exists in part so that we walk the package.json file only once,
+ * and in part so that we generate file-level syntactic diagnostics only once,
+ * so that we can do better deduplication of errors.
+ */
+export class PackageJson {
+    // We keep the file level AST node private to represent the invariant that
+    // we only walk the file once, in this class, and nowhere else.
+    #fileAstNode;
+    #scripts;
+    constructor(jsonFile, fileAstNode) {
+        this.#scripts = new Map();
+        this.scriptsSection = undefined;
+        this.wireitSection = undefined;
+        this.jsonFile = jsonFile;
+        this.#fileAstNode = fileAstNode;
+        const failures = [];
+        this.scriptsSection = this.#analyzeScriptsSection(failures);
+        this.wireitSection = this.#analyzeWireitSection(failures);
+        this.failures = failures;
+    }
+    getScriptInfo(name) {
+        return this.#scripts.get(name);
+    }
+    get scripts() {
+        return this.#scripts.values();
+    }
+    getInfoAboutLocation(offset) {
+        if (this.scriptsSection && offsetInsideRange(offset, this.scriptsSection)) {
+            for (const scriptSyntaxInfo of this.scripts) {
+                if (scriptSyntaxInfo.scriptNode &&
+                    offsetInsideNamedNode(offset, scriptSyntaxInfo.scriptNode)) {
+                    return { kind: 'scripts-section-script', scriptSyntaxInfo };
+                }
+            }
+        }
+        else if (this.wireitSection &&
+            offsetInsideRange(offset, this.wireitSection)) {
+            for (const scriptSyntaxInfo of this.scripts) {
+                if (scriptSyntaxInfo.wireitConfigNode &&
+                    offsetInsideNamedNode(offset, scriptSyntaxInfo.wireitConfigNode)) {
+                    return { kind: 'wireit-section-script', scriptSyntaxInfo };
+                }
+            }
+        }
+    }
+    #getOrMakeScriptInfo(name) {
+        let info = this.#scripts.get(name);
+        if (info === undefined) {
+            info = { name };
+            this.#scripts.set(name, info);
+        }
+        return info;
+    }
+    /**
+     * Do some basic structural validation of the "scripts" section of this
+     * package.json file. Create placeholders for each of the declared scripts and
+     * add them to this._scripts.
+     */
+    #analyzeScriptsSection(failures) {
+        const scriptsSectionResult = findNamedNodeAtLocation(this.#fileAstNode, ['scripts'], this.jsonFile);
+        if (!scriptsSectionResult.ok) {
+            failures.push(scriptsSectionResult.error);
+            return;
+        }
+        const scriptsSection = scriptsSectionResult.value;
+        if (scriptsSection === undefined) {
+            return;
+        }
+        const fail = failUnlessJsonObject(scriptsSection, this.jsonFile);
+        if (fail !== undefined) {
+            failures.push(fail);
+            return;
+        }
+        for (const child of scriptsSection.children ?? []) {
+            if (child.type !== 'property') {
+                continue;
+            }
+            if (child.children === undefined) {
+                continue;
+            }
+            const nameAndValueResult = failUnlessKeyValue(child, child.children, this.jsonFile);
+            if (!nameAndValueResult.ok) {
+                failures.push(nameAndValueResult.error);
+                continue;
+            }
+            const [rawName, rawValue] = nameAndValueResult.value;
+            const nameResult = failUnlessNonBlankString(rawName, this.jsonFile);
+            if (!nameResult.ok) {
+                failures.push(nameResult.error);
+                continue;
+            }
+            const valueResult = failUnlessNonBlankString(rawValue, this.jsonFile);
+            if (!valueResult.ok) {
+                failures.push(valueResult.error);
+                continue;
+            }
+            const scriptAstNode = valueResult.value;
+            scriptAstNode.name = nameResult.value;
+            this.#getOrMakeScriptInfo(nameResult.value.value).scriptNode =
+                scriptAstNode;
+        }
+        return scriptsSectionResult.value;
+    }
+    /**
+     * Do some basic structural validation of the "wireit" section of this
+     * package.json file.
+     *
+     * Create placeholders for each of the declared scripts and
+     * add them to this._scripts.
+     *
+     * Does not do any validation of any wireit configs themselves, that's done
+     * on demand when executing, or all at once when finding all diagnostics.
+     */
+    #analyzeWireitSection(failures) {
+        const wireitSectionResult = findNamedNodeAtLocation(this.#fileAstNode, ['wireit'], this.jsonFile);
+        if (!wireitSectionResult.ok) {
+            failures.push(wireitSectionResult.error);
+            return;
+        }
+        const wireitSection = wireitSectionResult.value;
+        if (wireitSection === undefined) {
+            return;
+        }
+        const fail = failUnlessJsonObject(wireitSection, this.jsonFile);
+        if (fail !== undefined) {
+            failures.push(fail);
+            return;
+        }
+        for (const child of wireitSection.children ?? []) {
+            if (child.type !== 'property') {
+                continue;
+            }
+            if (child.children === undefined) {
+                continue;
+            }
+            const nameAndValueResult = failUnlessKeyValue(child, child.children, this.jsonFile);
+            if (!nameAndValueResult.ok) {
+                failures.push(nameAndValueResult.error);
+                continue;
+            }
+            const [rawName, rawValue] = nameAndValueResult.value;
+            const nameResult = failUnlessNonBlankString(rawName, this.jsonFile);
+            if (!nameResult.ok) {
+                failures.push(nameResult.error);
+                continue;
+            }
+            const fail = failUnlessJsonObject(rawValue, this.jsonFile);
+            if (fail !== undefined) {
+                failures.push(fail);
+                continue;
+            }
+            const wireitConfigNode = rawValue;
+            wireitConfigNode.name = nameResult.value;
+            this.#getOrMakeScriptInfo(nameResult.value.value).wireitConfigNode =
+                wireitConfigNode;
+        }
+        return wireitSectionResult.value;
+    }
+}
+//# sourceMappingURL=package-json.js.map

package/lib/util/script-data-dir.js

@@ -0,0 +1,19 @@
+/**
+ * @license
+ * Copyright 2022 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import * as pathlib from 'path';
+/**
+ * Get the directory name where Wireit data can be saved for a script.
+ */
+export const getScriptDataDir = (script) => pathlib.join(script.packageDir, '.wireit', 
+// Script names can contain any character, so they aren't safe to use
+// directly in a filepath, because certain characters aren't allowed on
+// certain filesystems (e.g. ":" is forbidden on Windows). Hex-encode
+// instead so that we only get safe ASCII characters.
+//
+// Reference:
+// https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file#naming-conventions
+Buffer.from(script.name).toString('hex'));
+//# sourceMappingURL=script-data-dir.js.map

package/lib/util/shuffle.js

@@ -0,0 +1,16 @@
+/**
+ * @license
+ * Copyright 2022 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+/**
+ * Randomize the order of an array in-place.
+ */
+export const shuffle = (array) => {
+    // https://en.wikipedia.org/wiki/Fisher%E2%80%93Yates_shuffle#The_modern_algorithm
+    for (let i = array.length - 1; i > 0; i--) {
+        const j = Math.floor(Math.random() * (i + 1));
+        [array[i], array[j]] = [array[j], array[i]];
+    }
+};
+//# sourceMappingURL=shuffle.js.map

package/lib/util/unreachable.js

@@ -0,0 +1,12 @@
+/**
+ * @license
+ * Copyright 2022 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+/**
+ * TypeScript will error if it believes this function could be invoked. Useful
+ * to check for branches that should never be reached (e.g. that all possible
+ * cases in a switch are handled).
+ */
+export const unreachable = (value) => value;
+//# sourceMappingURL=unreachable.js.map

package/lib/util/windows.js

@@ -0,0 +1,87 @@
+/**
+ * @license
+ * Copyright 2022 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+/**
+ * Whether we're running on Windows.
+ */
+export const IS_WINDOWS = process.platform === 'win32';
+/**
+ * If we're on Windows, convert all back-slashes to forward-slashes (e.g.
+ * "foo\bar" -> "foo/bar").
+ */
+export const posixifyPathIfOnWindows = (path) => IS_WINDOWS ? path.replace(/\\/g, '/') : path;
+/**
+ * Overlay the given environment variables on top of the current process's
+ * environment variables in a way that is reliable on Windows.
+ *
+ * Windows environment variable names are **sort of** case-insensitive. When you
+ * `spawn` a process and pass 2 environment variables that differ only in case,
+ * then the value that actually gets set is ambiguous, and depends on the Node
+ * version. In Node 14 it seems to be the last one in iteration order, in Node
+ * 16 it seems to be the first one after sorting.
+ *
+ * For example, if you run:
+ *
+ * ```ts
+ * spawn('foo', {
+ *   env: {
+ *     PATH: 'C:\\extra;C:\\before',
+ *     Path: 'C:\\before'
+ *   }
+ * });
+ * ```
+ *
+ * Then sometimes the value that the spawned process receives could be
+ * `C:\before`, and other times it could be `C:\extra;C:\before`.
+ *
+ * This function ensures that the values given in `augmentations` will always
+ * win, by normalizing casing to match the casing that was already set in
+ * `process.env`.
+ */
+export const augmentProcessEnvSafelyIfOnWindows = (augmentations) => {
+    if (ENVIRONMENT_VARIABLE_CASINGS_IF_WINDOWS === undefined) {
+        // On Linux and macOS, environment variables are case-sensitive, so there's
+        // nothing special to do here.
+        return { ...process.env, ...augmentations };
+    }
+    const augmented = { ...process.env };
+    for (const [name, value] of Object.entries(augmentations)) {
+        const existingNames = ENVIRONMENT_VARIABLE_CASINGS_IF_WINDOWS.get(name.toLowerCase());
+        if (existingNames === undefined) {
+            augmented[name] = value;
+        }
+        else {
+            for (const existingName of existingNames) {
+                augmented[existingName] = value;
+            }
+        }
+    }
+    return augmented;
+};
+/**
+ * A map from lowercase environment variable name to the specific name casing(s)
+ * that were found in this process's environment variables.
+ *
+ * This is an array because in Node 14 the `process.env` object can actually
+ * contain multiple entries for the same variable with different casings, even
+ * though the values are always the same. In Node 16, there is only one name
+ * casing, even if it was spawned with multiple.
+ */
+const ENVIRONMENT_VARIABLE_CASINGS_IF_WINDOWS = IS_WINDOWS
+    ? (() => {
+        const map = new Map();
+        for (const caseSensitiveName of Object.keys(process.env)) {
+            const lowerCaseName = caseSensitiveName.toLowerCase();
+            let arr = map.get(lowerCaseName);
+            if (arr === undefined) {
+                arr = [];
+                map.set(lowerCaseName, arr);
+            }
+            arr.push(caseSensitiveName);
+        }
+        return map;
+    })()
+    : undefined;
+//# sourceMappingURL=windows.js.map

package/lib/util/worker-pool.js

@@ -0,0 +1,61 @@
+/**
+ * @license
+ * Copyright 2022 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import { Deferred } from './deferred.js';
+/**
+ * A mechanism for ensuring that at most N tasks are taking place at once.
+ *
+ * Useful in Wireit to prevent running too many scripts at once and swamping
+ * the system. For unlimited parallelism, just set numWorkers to Infinity.
+ *
+ * Note that node is still single threaded by default. This is useful for
+ * Wireit because almost all work is happening in script commands which run
+ * in separate processes.
+ *
+ * No guarantee is made about ordering or fairness of scheduling, though
+ * as implemented it's currently LIFO. Deadlocks may occur if there are
+ * dependencies between tasks.
+ */
+export class WorkerPool {
+    #availableWorkers;
+    #waitingWorkers = [];
+    constructor(numWorkers) {
+        if (numWorkers <= 0) {
+            throw new Error(`WorkerPool needs a positive number of workers, got ${numWorkers}`);
+        }
+        this.#availableWorkers = numWorkers;
+    }
+    /**
+     * Calls workFn and returns its result.
+     *
+     * However, no more than `numWorkers` simultaneous calls to workFns will
+     * be running at any given time, to prevent overloading the machine.
+     */
+    async run(workFn) {
+        if (this.#availableWorkers <= 0) {
+            const waiter = new Deferred();
+            this.#waitingWorkers.push(waiter);
+            await waiter.promise;
+            if (this.#availableWorkers <= 0) {
+                throw new Error(`Internal error: expected availableWorkers to be positive after task was awoken, but was ${this.#availableWorkers}`);
+            }
+        }
+        this.#availableWorkers--;
+        try {
+            return await workFn();
+        }
+        finally {
+            this.#availableWorkers++;
+            if (this.#availableWorkers <= 0) {
+                // We intend to override any return or throw with this error in this
+                // case.
+                // eslint-disable-next-line no-unsafe-finally
+                throw new Error(`Internal error: expected availableWorkers to be positive after incrementing, but was ${this.#availableWorkers}`);
+            }
+            this.#waitingWorkers.pop()?.resolve();
+        }
+    }
+}
+//# sourceMappingURL=worker-pool.js.map