@laurence79/wireit 0.14.13-shared-cache.0

package/lib/executor.js

@@ -0,0 +1,249 @@
+/**
+ * @license
+ * Copyright 2022 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import { NoCommandScriptExecution } from './execution/no-command.js';
+import { StandardScriptExecution } from './execution/standard.js';
+import { ServiceScriptExecution } from './execution/service.js';
+import { scriptReferenceToString, } from './config.js';
+import { Deferred } from './util/deferred.js';
+import { convertExceptionToFailure } from './error.js';
+let executorConstructorHook;
+/**
+ * For GC testing only. A function that is called whenever an Executor is
+ * constructed.
+ */
+export function registerExecutorConstructorHook(fn) {
+    executorConstructorHook = fn;
+}
+/**
+ * Executes a script that has been analyzed and validated by the Analyzer.
+ */
+export class Executor {
+    #rootConfig;
+    #executions = new Map();
+    #persistentServices = new Map();
+    #ephemeralServices = [];
+    #previousIterationServices;
+    #logger;
+    #workerPool;
+    #cache;
+    #isWatchMode;
+    #previousWatchIterationFailures;
+    /** Resolves when the first failure occurs in any script. */
+    #failureOccured = new Deferred();
+    /** Resolves when we decide that new scripts should not be started. */
+    #stopStartingNewScripts = new Deferred();
+    /** Resolves when we decide that running scripts should be killed. */
+    #killRunningScripts = new Deferred();
+    /** Resolves when we decide that services should be stopped. */
+    #stopServices = new Deferred();
+    constructor(rootConfig, logger, workerPool, cache, failureMode, previousIterationServices, isWatchMode, previousWatchIterationFailures) {
+        executorConstructorHook?.(this);
+        this.#rootConfig = rootConfig;
+        this.#logger = logger;
+        this.#workerPool = workerPool;
+        this.#cache = cache;
+        this.#previousIterationServices = previousIterationServices;
+        this.#isWatchMode = isWatchMode;
+        this.#previousWatchIterationFailures = previousWatchIterationFailures;
+        // If a failure occurs, then whether we stop starting new scripts or kill
+        // running ones depends on the failure mode setting.
+        void this.#failureOccured.promise.then(() => {
+            switch (failureMode) {
+                case 'continue': {
+                    if (!this.#isWatchMode) {
+                        this.#stopServices.resolve();
+                    }
+                    break;
+                }
+                case 'no-new': {
+                    this.#stopStartingNewScripts.resolve();
+                    if (!this.#isWatchMode) {
+                        this.#stopServices.resolve();
+                    }
+                    break;
+                }
+                case 'kill': {
+                    this.#stopStartingNewScripts.resolve();
+                    this.#killRunningScripts.resolve();
+                    this.#stopServices.resolve();
+                    break;
+                }
+                default: {
+                    const never = failureMode;
+                    throw new Error(`Internal error: unexpected failure mode: ${String(never)}`);
+                }
+            }
+        });
+    }
+    /**
+     * If this entire execution is aborted because e.g. the user sent a SIGINT to
+     * the Wireit process, then dont start new scripts, and kill running ones.
+     */
+    abort() {
+        this.#stopStartingNewScripts.resolve();
+        this.#killRunningScripts.resolve();
+        this.#stopServices.resolve();
+        if (this.#previousIterationServices !== undefined) {
+            for (const service of this.#previousIterationServices.values()) {
+                void service.abort();
+            }
+        }
+    }
+    /**
+     * Execute the root script.
+     */
+    async execute() {
+        if (this.#previousIterationServices !== undefined &&
+            this.#previousIterationServices.size > 0) {
+            // If any services were removed from the graph entirely, or used to be
+            // persistent but are no longer, then stop them now.
+            const currentPersistentServices = new Set();
+            for (const script of findAllScripts(this.#rootConfig)) {
+                if (script.service && script.isPersistent) {
+                    currentPersistentServices.add(scriptReferenceToString(script));
+                }
+            }
+            const abortPromises = [];
+            for (const [key, service] of this.#previousIterationServices) {
+                if (!currentPersistentServices.has(key)) {
+                    abortPromises.push(service.abort());
+                    this.#previousIterationServices.delete(key);
+                }
+            }
+            await Promise.all(abortPromises);
+        }
+        const errors = [];
+        let rootExecutionResult;
+        try {
+            rootExecutionResult = await this.getExecution(this.#rootConfig).execute();
+        }
+        catch (error) {
+            rootExecutionResult = convertExceptionToFailure(error, this.#rootConfig);
+        }
+        if (!rootExecutionResult.ok) {
+            errors.push(...rootExecutionResult.error);
+        }
+        // Wait for all persistent services to start.
+        for (const service of this.#persistentServices.values()) {
+            // Persistent services start automatically, so calling start() here should
+            // be a no-op, but it lets us get the started promise.
+            const result = await service.start();
+            if (!result.ok) {
+                errors.push(result.error);
+            }
+        }
+        // Wait for all ephemeral services to have terminated (either started and
+        // stopped, or never needed to start).
+        const ephemeralServiceResults = await Promise.all(this.#ephemeralServices.map((service) => service.terminated));
+        for (const result of ephemeralServiceResults) {
+            if (!result.ok) {
+                errors.push(result.error);
+            }
+        }
+        // All previous services are either now adopted or stopped. Remove the
+        // reference to this map to allow for garbage collection, otherwise in watch
+        // mode we'll have a chain of references all the way back through every
+        // iteration.
+        this.#previousIterationServices = undefined;
+        return {
+            persistentServices: this.#persistentServices,
+            errors,
+        };
+    }
+    /**
+     * Signal that a script has failed, which will potentially stop starting or
+     * kill other scripts depending on the {@link FailureMode}.
+     *
+     * This method will be called automatically in the normal flow of execution,
+     * but scripts can also call it directly to synchronously signal a failure.
+     */
+    notifyFailure() {
+        this.#failureOccured.resolve();
+    }
+    /**
+     * Synchronously check if new scripts should stop being started.
+     */
+    get shouldStopStartingNewScripts() {
+        return this.#stopStartingNewScripts.settled;
+    }
+    /**
+     * A promise which resolves if we should kill running scripts.
+     */
+    get shouldKillRunningScripts() {
+        return this.#killRunningScripts.promise;
+    }
+    /**
+     * Get the execution instance for a script config, creating one if it doesn't
+     * already exist.
+     */
+    getExecution(config) {
+        const key = scriptReferenceToString(config);
+        let execution = this.#executions.get(key);
+        if (execution === undefined) {
+            if (config.command === undefined) {
+                execution = new NoCommandScriptExecution(config, this, this.#logger);
+            }
+            else if (config.service !== undefined) {
+                execution = new ServiceScriptExecution(config, this, this.#logger, this.#stopServices.promise, this.#previousIterationServices?.get(key), this.#isWatchMode);
+                if (config.isPersistent) {
+                    this.#persistentServices.set(key, execution);
+                }
+                else {
+                    this.#ephemeralServices.push(execution);
+                }
+            }
+            else {
+                execution = new StandardScriptExecution(config, this, this.#workerPool, this.#cache, this.#logger);
+            }
+            this.#executions.set(key, execution);
+        }
+        // Cast needed because our Map type doesn't know about the config ->
+        // execution type guarantees. We could make a smarter Map type, but not
+        // really worth it here.
+        return execution;
+    }
+    /**
+     * If we're in watch mode, check whether in the previous watch iteration the
+     * given script failed with the given fingerprint.
+     */
+    failedInPreviousWatchIteration(script, fingerprint) {
+        if (this.#previousWatchIterationFailures === undefined) {
+            return false;
+        }
+        const previous = this.#previousWatchIterationFailures.get(scriptReferenceToString(script));
+        if (previous === undefined) {
+            return false;
+        }
+        return previous.equal(fingerprint);
+    }
+    /**
+     * If we're in watch mode, record that a script failed for the purpose of
+     * preventing it from running unless its fingerprint changes in the next watch
+     * iteration.
+     */
+    registerWatchIterationFailure(script, fingerprint) {
+        this.#previousWatchIterationFailures?.set(scriptReferenceToString(script), fingerprint);
+    }
+}
+/**
+ * Walk the dependencies of the given root script and return all scripts in the
+ * graph (including the root itself).
+ */
+function findAllScripts(root) {
+    const visited = new Set();
+    const stack = [root];
+    while (stack.length > 0) {
+        const next = stack.pop();
+        visited.add(next);
+        for (const dep of next.dependencies) {
+            if (!visited.has(dep.config)) {
+                stack.push(dep.config);
+            }
+        }
+    }
+    return visited;
+}
+//# sourceMappingURL=executor.js.map

package/lib/fingerprint.js

@@ -0,0 +1,164 @@
+/**
+ * @license
+ * Copyright 2022 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import { createHash } from 'crypto';
+import { createReadStream } from './util/fs.js';
+import { glob } from './util/glob.js';
+import { scriptReferenceToString } from './config.js';
+/**
+ * The fingerprint of a script. Converts lazily between string and data object
+ * forms.
+ */
+export class Fingerprint {
+    static fromString(string) {
+        const fingerprint = new Fingerprint();
+        fingerprint.#str = string;
+        return fingerprint;
+    }
+    /**
+     * Generate the fingerprint data object for a script based on its current
+     * configuration, input files, and the fingerprints of its dependencies.
+     */
+    static async compute(script, dependencyFingerprints) {
+        let allDependenciesAreFullyTracked = true;
+        const filteredDependencyFingerprints = [];
+        for (const [dep, depFingerprint] of dependencyFingerprints) {
+            if (!dep.cascade) {
+                // cascade: false means the fingerprint of the dependency isn't
+                // directly inherited.
+                continue;
+            }
+            if (!depFingerprint.data.fullyTracked) {
+                allDependenciesAreFullyTracked = false;
+            }
+            filteredDependencyFingerprints.push([
+                scriptReferenceToString(dep.config),
+                depFingerprint.hash,
+            ]);
+        }
+        let fileHashes;
+        if (script.files?.values.length) {
+            const files = await glob(script.files.values, {
+                cwd: script.packageDir,
+                followSymlinks: true,
+                // TODO(aomarks) This means that empty directories are not reflected in
+                // the fingerprint, however an empty directory could modify the behavior
+                // of a script. We should probably include empty directories; we'll just
+                // need special handling when we compute the fingerprint, because there
+                // is no hash we can compute.
+                includeDirectories: false,
+                // We must expand directories here, because we need the complete
+                // explicit list of files to hash.
+                expandDirectories: true,
+                throwIfOutsideCwd: false,
+            });
+            // TODO(aomarks) Instead of reading and hashing every input file on every
+            // build, use inode/mtime/ctime/size metadata (which is much faster to
+            // read) as a heuristic to detect files that have likely changed, and
+            // otherwise re-use cached hashes that we store in e.g.
+            // ".wireit/<script>/hashes".
+            const erroredFilePaths = [];
+            fileHashes = await Promise.all(files.map(async (file) => {
+                const absolutePath = file.path;
+                const hash = createHash('sha256');
+                try {
+                    const stream = await createReadStream(absolutePath);
+                    for await (const chunk of stream) {
+                        hash.update(chunk);
+                    }
+                }
+                catch (error) {
+                    // It's possible for a file to be deleted between the
+                    // time it is globbed and the time it is fingerprinted.
+                    const { code } = error;
+                    if (code !== /* does not exist */ 'ENOENT') {
+                        throw error;
+                    }
+                    erroredFilePaths.push(absolutePath);
+                }
+                return [file.path, hash.digest('hex')];
+            }));
+            if (erroredFilePaths.length > 0) {
+                return {
+                    ok: false,
+                    error: {
+                        type: 'failure',
+                        reason: 'input-file-deleted-unexpectedly',
+                        script: script,
+                        filePaths: erroredFilePaths,
+                    },
+                };
+            }
+        }
+        else {
+            fileHashes = [];
+        }
+        const fullyTracked = 
+        // If any any dependency is not fully tracked, then we can't be either,
+        // because we can't know if there was an undeclared input that this script
+        // depends on.
+        allDependenciesAreFullyTracked &&
+            // A no-command script. Doesn't ever do anything itsef, so always fully
+            // tracked.
+            (script.command === undefined ||
+                // A service. Fully tracked if we know its inputs. Can't produce output.
+                (script.service !== undefined && script.files !== undefined) ||
+                // A standard script. Fully tracked if we know both its inputs and
+                // outputs.
+                (script.files !== undefined && script.output !== undefined));
+        const fingerprint = new Fingerprint();
+        // Note: The order of all fields is important so that we can do fast string
+        // comparison.
+        const data = {
+            fullyTracked,
+            platform: process.platform,
+            arch: process.arch,
+            nodeVersion: process.version,
+            command: script.command?.value,
+            extraArgs: script.extraArgs ?? [],
+            clean: script.clean,
+            files: Object.fromEntries(fileHashes.sort(([aFile], [bFile]) => aFile.localeCompare(bFile))),
+            output: script.output?.values ?? [],
+            dependencies: Object.fromEntries(filteredDependencyFingerprints.sort(([aRef], [bRef]) => aRef.localeCompare(bRef))),
+            service: script.service === undefined
+                ? undefined
+                : {
+                    readyWhen: {
+                        lineMatches: script.service.readyWhen.lineMatches?.toString(),
+                    },
+                },
+            env: script.env,
+        };
+        fingerprint.#data = data;
+        return { ok: true, value: fingerprint };
+    }
+    #str;
+    #data;
+    #hash;
+    get string() {
+        if (this.#str === undefined) {
+            this.#str = JSON.stringify(this.#data);
+        }
+        return this.#str;
+    }
+    get data() {
+        if (this.#data === undefined) {
+            this.#data = JSON.parse(this.#str);
+        }
+        return this.#data;
+    }
+    get hash() {
+        if (this.#hash === undefined) {
+            this.#hash = createHash('sha256')
+                .update(this.string)
+                .digest('hex');
+        }
+        return this.#hash;
+    }
+    equal(other) {
+        return this.string === other.string;
+    }
+}
+//# sourceMappingURL=fingerprint.js.map