@laurence79/wireit 0.14.13-shared-cache.0 → 0.14.13-shared-cache.1

package/README.md

@@ -652,6 +652,51 @@ If you have a service that produces output, you should define a _non-service_
 script that depends on it, and which exits when the service's output is
 complete.
 
+### Shared services
+
+By default, every Wireit invocation that needs a service starts its own copy of
+it. If you run two terminals in the same package that both depend on the same
+service — for example a fixed-port emulator — they will each start an instance
+and clash on that resource.
+
+Set `service.shared` to `true` to let concurrent invocations in the same package
+directory **share a single instance** instead. The first invocation that needs
+the service starts it; subsequent invocations attach to the running instance.
+The service is torn down only when its last live consumer exits.
+
+```json
+{
+  "command": "firebase emulators:start",
+  "files": ["firebase.json"],
+  "service": {
+    "shared": true,
+    "readyWhen": {
+      "lineMatches": "All emulators ready"
+    }
+  }
+}
+```
+
+Notes and constraints:
+
+- **Same directory only.** Sharing is keyed on the service's
+  [fingerprint](#fingerprint) within a single package directory. Two checkouts
+  (or worktrees) have separate state and do not share.
+- **Fully tracked services only.** The fingerprint is used as the share key, so a
+  shared service must be fully tracked — meaning Wireit knows all of its inputs
+  (see [incremental build](#incremental-build)). In practice, give it a `files`
+  array (it may be empty). If it isn't fully tracked, `shared` is ignored and
+  each invocation starts its own instance.
+- **A long-lived consumer keeps it warm.** If one terminal runs a persistent
+  consumer (e.g. `npm start`), the shared service stays up across another
+  terminal's repeated `npm test` runs, and is stopped only when the last consumer
+  leaves.
+- **Teardown is liveness-based.** A consumer that is `Ctrl-C`'d, closed, or even
+  `kill -9`'d is detected and dropped; the service is never leaked, and one
+  terminal's `Ctrl-C` never stops a service another terminal still needs.
+- **POSIX only.** On Windows, `shared` degrades to the default per-process
+  behavior (no sharing, no regression).
+
 ## Execution cascade
 
 By default, a script always needs to run (or restart in the case of
@@ -935,20 +980,20 @@ The following syntaxes can be used in the `wireit.<script>.dependencies` array:
 
 The following environment variables affect the behavior of Wireit:
 
-| Variable                | Description                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               |
-| ----------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `WIREIT_CACHE`          | [Caching mode](#caching).<br><br>Defaults to `local` unless `CI` is `true`, in which case defaults to `none`.<br><br>Automatically set to `github` by the [`google/wireit@setup-github-actions-caching/v2`](#github-actions-caching) action.<br><br>Options:<ul><li>[`local`](#local-caching): Cache to local disk.</li><li>[`github`](#github-actions-caching): Cache to GitHub Actions.</li><li>[`shared`](#shared-caching): Cache to a stable, content-addressed directory shared across builds and agents.</li><li>`none`: Disable caching.</li></ul>                                                                                                                                    |
-| `WIREIT_CACHE_SHARED_DIR` | Root directory of the [shared cache](#shared-caching). Only applies when `WIREIT_CACHE=shared`.<br><br>Defaults to `~/.wireit/cache`.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  |
-| `WIREIT_CACHE_SHARED_READONLY` | When `true`, the [shared cache](#shared-caching) restores entries but never writes them. Useful for untrusted writers. Only applies when `WIREIT_CACHE=shared`.<br><br>Must be exactly `true` or `false`. Defaults to `false`.                                                                                                                                                                                                                                                                                                                                                                                                                                                     |
-| `WIREIT_FAILURES`       | [How to handle script failures](#failures-and-errors).<br><br>Options:<br><ul><li>[`no-new`](#failures-and-errors) (default): Allow running scripts to finish, but don't start new ones.</li><li>[`continue`](#continue): Allow running scripts to continue, and start new ones unless any of their dependencies failed.</li><li>[`kill`](#kill): Immediately kill running scripts, and don't start new ones.</li></ul>                                                                                                                                                                                                                                                                   |
-| `WIREIT_LOGGER`         | How to present progress and results on the command line.<br><br>Options:<br><ul><li>`quiet` (default for normal execution): Writes a single dynamically updating line summarizing progress. Only passes along stdout and stderr from commands if there's a failure, or if the command is a service.</li><li>`quiet-ci` (default when `env.CI` or `!stdout.isTTY`): like `quiet` but optimized for non-interactive environments, like GitHub Actions runners.</li><li>`simple`: A verbose logger that presents clear information about the work that Wireit is doing.</li><li>`metrics`: Like `simple`, but also presents a summary table of results once a command is finished.</li></ul> |
-| `WIREIT_DEBUG_LOG_FILE` | Path to a file which will receive detailed event logging.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 |
-| `WIREIT_MAX_OPEN_FILES` | Limits the number of file descriptors Wireit will have open concurrently. Prevents resource exhaustion when checking large numbers of cached files. Set to a lower number if you hit file descriptor limits.                                                                                                                                                                                                                                                                                                                                                                                                                                                                              |
-| `WIREIT_PARALLEL`       | [Maximum number of scripts to run at one time](#parallelism).<br><br>Defaults to 2×logical CPU cores.<br><br>Must be a positive integer or `infinity`.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    |
-| `WIREIT_WATCH`          | Set to `true` to enable [watch mode](#watch-mode).                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        |
-| `WIREIT_WATCH_STRATEGY` | How Wireit determines when a file has changed which warrants a new watch iteration.<br><br>Options:<br><ul><li>`event` (default): Register OS file system watcher callbacks (using [chokidar](https://github.com/paulmillr/chokidar)).</li><li>`poll`: Poll the filesystem every `WIREIT_WATCH_POLL_MS` milliseconds. Less responsive and worse performance than `event`, but a good fallback for when `event` does not work well or at all (e.g. filesystems that don't support filesystem events, or performance and memory problems with large file trees).</li></ul>                                                                                                                  |
-| `WIREIT_WATCH_POLL_MS`  | When `WIREIT_WATCH_STRATEGY` is `poll`, how many milliseconds to wait between each filesystem poll. Defaults to `500`.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    |
-| `CI`                    | Affects the default value of `WIREIT_CACHE`.<br><br>Automatically set to `true` by [GitHub Actions](https://docs.github.com/en/actions/learn-github-actions/environment-variables#default-environment-variables) and most other CI (continuous integration) services.<br><br>Must be exactly `true`. If unset or any other value, interpreted as `false`.                                                                                                                                                                                                                                                                                                                                 |
+| Variable                       | Description                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               |
+| ------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `WIREIT_CACHE`                 | [Caching mode](#caching).<br><br>Defaults to `local` unless `CI` is `true`, in which case defaults to `none`.<br><br>Automatically set to `github` by the [`google/wireit@setup-github-actions-caching/v2`](#github-actions-caching) action.<br><br>Options:<ul><li>[`local`](#local-caching): Cache to local disk.</li><li>[`github`](#github-actions-caching): Cache to GitHub Actions.</li><li>[`shared`](#shared-caching): Cache to a stable, content-addressed directory shared across builds and agents.</li><li>`none`: Disable caching.</li></ul>                                                                                                                                 |
+| `WIREIT_CACHE_SHARED_DIR`      | Root directory of the [shared cache](#shared-caching). Only applies when `WIREIT_CACHE=shared`.<br><br>Defaults to `~/.wireit/cache`.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     |
+| `WIREIT_CACHE_SHARED_READONLY` | When `true`, the [shared cache](#shared-caching) restores entries but never writes them. Useful for untrusted writers. Only applies when `WIREIT_CACHE=shared`.<br><br>Must be exactly `true` or `false`. Defaults to `false`.                                                                                                                                                                                                                                                                                                                                                                                                                                                            |
+| `WIREIT_FAILURES`              | [How to handle script failures](#failures-and-errors).<br><br>Options:<br><ul><li>[`no-new`](#failures-and-errors) (default): Allow running scripts to finish, but don't start new ones.</li><li>[`continue`](#continue): Allow running scripts to continue, and start new ones unless any of their dependencies failed.</li><li>[`kill`](#kill): Immediately kill running scripts, and don't start new ones.</li></ul>                                                                                                                                                                                                                                                                   |
+| `WIREIT_LOGGER`                | How to present progress and results on the command line.<br><br>Options:<br><ul><li>`quiet` (default for normal execution): Writes a single dynamically updating line summarizing progress. Only passes along stdout and stderr from commands if there's a failure, or if the command is a service.</li><li>`quiet-ci` (default when `env.CI` or `!stdout.isTTY`): like `quiet` but optimized for non-interactive environments, like GitHub Actions runners.</li><li>`simple`: A verbose logger that presents clear information about the work that Wireit is doing.</li><li>`metrics`: Like `simple`, but also presents a summary table of results once a command is finished.</li></ul> |
+| `WIREIT_DEBUG_LOG_FILE`        | Path to a file which will receive detailed event logging.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 |
+| `WIREIT_MAX_OPEN_FILES`        | Limits the number of file descriptors Wireit will have open concurrently. Prevents resource exhaustion when checking large numbers of cached files. Set to a lower number if you hit file descriptor limits.                                                                                                                                                                                                                                                                                                                                                                                                                                                                              |
+| `WIREIT_PARALLEL`              | [Maximum number of scripts to run at one time](#parallelism).<br><br>Defaults to 2×logical CPU cores.<br><br>Must be a positive integer or `infinity`.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    |
+| `WIREIT_WATCH`                 | Set to `true` to enable [watch mode](#watch-mode).                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        |
+| `WIREIT_WATCH_STRATEGY`        | How Wireit determines when a file has changed which warrants a new watch iteration.<br><br>Options:<br><ul><li>`event` (default): Register OS file system watcher callbacks (using [chokidar](https://github.com/paulmillr/chokidar)).</li><li>`poll`: Poll the filesystem every `WIREIT_WATCH_POLL_MS` milliseconds. Less responsive and worse performance than `event`, but a good fallback for when `event` does not work well or at all (e.g. filesystems that don't support filesystem events, or performance and memory problems with large file trees).</li></ul>                                                                                                                  |
+| `WIREIT_WATCH_POLL_MS`         | When `WIREIT_WATCH_STRATEGY` is `poll`, how many milliseconds to wait between each filesystem poll. Defaults to `500`.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    |
+| `CI`                           | Affects the default value of `WIREIT_CACHE`.<br><br>Automatically set to `true` by [GitHub Actions](https://docs.github.com/en/actions/learn-github-actions/environment-variables#default-environment-variables) and most other CI (continuous integration) services.<br><br>Must be exactly `true`. If unset or any other value, interpreted as `false`.                                                                                                                                                                                                                                                                                                                                 |
 
 ### Glob patterns
 

package/lib/analyzer.js

@@ -847,7 +847,29 @@ export class Analyzer {
             return undefined;
         }
         let lineMatches = undefined;
+        let shared = false;
         if (serviceNode.type === 'object') {
+            const sharedNode = findNodeAtLocation(serviceNode, ['shared']);
+            if (sharedNode !== undefined) {
+                if (sharedNode.type !== 'boolean') {
+                    placeholder.failures.push({
+                        type: 'failure',
+                        reason: 'invalid-config-syntax',
+                        script: placeholder,
+                        diagnostic: {
+                            severity: 'error',
+                            message: `The "shared" property must be a boolean.`,
+                            location: {
+                                file: packageJson.jsonFile,
+                                range: { length: sharedNode.length, offset: sharedNode.offset },
+                            },
+                        },
+                    });
+                }
+                else {
+                    shared = sharedNode.value;
+                }
+            }
             const waitForNode = findNodeAtLocation(serviceNode, ['readyWhen']);
             if (waitForNode !== undefined) {
                 if (waitForNode.type !== 'object') {
@@ -951,7 +973,7 @@ export class Analyzer {
                 },
             });
         }
-        return { readyWhen: { lineMatches } };
+        return { readyWhen: { lineMatches }, shared };
     }
     async #processPackageLocks(placeholder, packageJson, syntaxInfo, files) {
         if (syntaxInfo.wireitConfigNode === undefined) {

package/lib/execution/service.js

@@ -8,6 +8,8 @@ import { Fingerprint } from '../fingerprint.js';
 import { Deferred } from '../util/deferred.js';
 import { ScriptChildProcess } from '../script-child-process.js';
 import { LineMonitor } from '../util/line-monitor.js';
+import { SharedServiceProcess } from './shared-service-coordinator.js';
+import { IS_WINDOWS } from '../util/windows.js';
 function unknownState(state) {
     return new Error(`Unknown service state ${state.id}`);
 }
@@ -584,15 +586,39 @@ export class ServiceScriptExecution extends BaseExecutionWithCommand {
             case 'depsStarting': {
                 const detached = this.#state.adoptee?.detach();
                 if (detached === undefined) {
-                    const child = new ScriptChildProcess(this._config);
+                    const fingerprint = this.#state.fingerprint;
+                    // A shared service is coordinated across concurrent wireit invocations
+                    // through the filesystem rather than being a child of this invocation.
+                    // It requires POSIX (where a detached process survives its launcher)
+                    // and a reliable fingerprint to use as the share key; otherwise we
+                    // transparently fall back to a normal per-process child.
+                    const shared = this._config.service.shared &&
+                        !IS_WINDOWS &&
+                        fingerprint.data.fullyTracked;
+                    let child;
+                    let readyMonitor;
+                    if (shared) {
+                        const handle = new SharedServiceProcess(this._config, fingerprint, this._logger);
+                        child = handle;
+                        // Readiness for a shared service is published in its coordination
+                        // file (derived by tailing the shared logfile), not observed on a
+                        // local child's pipes, so we always route through its ready monitor.
+                        readyMonitor = handle.readyMonitor;
+                    }
+                    else {
+                        const childProcess = new ScriptChildProcess(this._config);
+                        child = childProcess;
+                        readyMonitor =
+                            this._config.service.readyWhen.lineMatches === undefined
+                                ? undefined
+                                : new LineMonitor(childProcess, this._config.service.readyWhen.lineMatches);
+                    }
                     this.#state = {
                         id: 'starting',
                         child,
                         started: this.#state.started,
-                        fingerprint: this.#state.fingerprint,
-                        readyMonitor: this._config.service.readyWhen.lineMatches === undefined
-                            ? undefined
-                            : new LineMonitor(child, this._config.service.readyWhen.lineMatches),
+                        fingerprint,
+                        readyMonitor,
                     };
                     void this.#state.child.started.then(() => {
                         this.#onChildStarted();

package/lib/execution/shared-service-coordinator.js

@@ -0,0 +1,664 @@
+/**
+ * @license
+ * Copyright 2024 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import * as pathlib from 'path';
+import { closeSync, openSync } from 'fs';
+import * as fs from 'fs/promises';
+import { PassThrough } from 'stream';
+import lockfile from 'proper-lockfile';
+import { Deferred } from '../util/deferred.js';
+import { getScriptDataDir } from '../util/script-data-dir.js';
+import { spawnDetachedServiceProcess } from '../script-child-process.js';
+/**
+ * The basename of the coordination file, the lock sentinel, and the logfile,
+ * all of which live in the service's `.wireit/<script-hex>/` directory.
+ */
+const COORDINATION_FILE = 'service.json';
+const LOCK_FILE = 'service.lock';
+const LOGFILE = 'service.log';
+/**
+ * How often (ms) each observer re-reads the coordination file, polls service
+ * liveness, and tails the logfile.
+ */
+const POLL_INTERVAL_MS = 100;
+/**
+ * proper-lockfile settings, matching the convention used elsewhere in wireit
+ * (see standard.ts). Critical sections here are short (read coord → decide →
+ * write coord), so a hard-crashed holder is reclaimed after `stale` ms.
+ */
+const LOCK_STALE_MS = 10_000;
+const LOCK_UPDATE_MS = 2_000;
+/**
+ * When restarting a shared service to a new definition, how long (ms) to wait
+ * for the previous instance's process group to actually die before giving up
+ * and spawning the replacement anyway.
+ */
+const RESTART_DEATH_TIMEOUT_MS = 5_000;
+/**
+ * The restart rate-limit guard window. Restarts older than this are ignored when
+ * deciding whether concurrent invocations are oscillating the service.
+ */
+const RESTART_WINDOW_MS = 10_000;
+/**
+ * If at least this many restarts happen within {@link RESTART_WINDOW_MS} and
+ * they involve more than one distinct fingerprint, we treat it as conflicting
+ * concurrent definitions and bail instead of continuing to oscillate.
+ */
+const RESTART_LIMIT = 5;
+/**
+ * Whether a pid is currently alive. Treats `EPERM` (the process exists but is
+ * owned by another user) as alive, and `ESRCH` (no such process) as dead.
+ */
+export function isAlive(pid) {
+    if (!Number.isInteger(pid) || pid <= 1) {
+        return false;
+    }
+    try {
+        process.kill(pid, 0);
+        return true;
+    }
+    catch (error) {
+        return error.code === 'EPERM';
+    }
+}
+/** Filter a list of pids down to those still alive. */
+function pruneDead(pids) {
+    return pids.filter((pid) => isAlive(pid));
+}
+/** Send a signal to an entire process group, ignoring "already dead" errors. */
+function signalGroup(pgid, signal) {
+    if (!Number.isInteger(pgid) || pgid <= 1) {
+        return;
+    }
+    try {
+        process.kill(-pgid, signal);
+    }
+    catch {
+        // Already gone, or we lost the race — nothing to do.
+    }
+}
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+/**
+ * Observes a single shared service for one wireit invocation.
+ *
+ * Exposes the same surface that the service state machine
+ * ({@link ServiceScriptExecution}) already consumes from {@link
+ * ScriptChildProcess} — `started`, `completed`, `stdout`, `stderr`, `kill()` —
+ * plus a {@link readyMonitor}, so that the state machine can drive a shared
+ * service almost identically to a per-process one. Internally, whether this
+ * invocation spawned the service or attached to an existing one, it behaves as
+ * an *observer*: it tails the logfile and watches the coordination file, and
+ * tears the service down (by pgid) only once it is the last live consumer.
+ *
+ * POSIX only; the caller is responsible for not constructing this on Windows.
+ */
+export class SharedServiceProcess {
+    #config;
+    #fingerprint;
+    #logger;
+    #myPid;
+    #dir;
+    #coordPath;
+    #lockPath;
+    #logPath;
+    #lineMatches;
+    #startedDeferred;
+    #completedDeferred;
+    #readyDeferred;
+    #stdoutStream;
+    #stderrStream;
+    /** The pgid of the instance we spawned or attached to. */
+    #pgid;
+    /** Display tail starts here: 0 for a spawner, current EOF for an attacher. */
+    #attachOffset;
+    /** How far into the logfile we've read (from 0, for readiness scanning). */
+    #logReadOffset;
+    /** Buffer of the not-yet-newline-terminated tail, for ready-line scanning. */
+    #readyScanBuffer;
+    /** True once `started` has resolved (spawn confirmed, or attach completed). */
+    #handshakeDone;
+    /** True once we've begun (or completed) our own teardown via {@link kill}. */
+    #tearingDown;
+    #closed;
+    #pollTimer;
+    get stdout() {
+        return this.#stdoutStream;
+    }
+    get stderr() {
+        return this.#stderrStream;
+    }
+    constructor(config, fingerprint, logger) {
+        this.#myPid = process.pid;
+        this.#startedDeferred = new Deferred();
+        this.#completedDeferred = new Deferred();
+        this.#readyDeferred = new Deferred();
+        this.#stdoutStream = new PassThrough();
+        this.#stderrStream = new PassThrough();
+        /** Display tail starts here: 0 for a spawner, current EOF for an attacher. */
+        this.#attachOffset = 0;
+        /** How far into the logfile we've read (from 0, for readiness scanning). */
+        this.#logReadOffset = 0;
+        /** Buffer of the not-yet-newline-terminated tail, for ready-line scanning. */
+        this.#readyScanBuffer = '';
+        /** True once `started` has resolved (spawn confirmed, or attach completed). */
+        this.#handshakeDone = false;
+        /** True once we've begun (or completed) our own teardown via {@link kill}. */
+        this.#tearingDown = false;
+        this.#closed = false;
+        /** Resolves when the service is spawned (us) or confirmed running (attach). */
+        this.started = this.#startedDeferred.promise;
+        /** Resolves when, for us, the service is finished (exited, or we detached). */
+        this.completed = this.#completedDeferred.promise;
+        this.readyMonitor = {
+            matched: this.#readyDeferred.promise,
+            abort: () => {
+                if (!this.#readyDeferred.settled) {
+                    this.#readyDeferred.resolve({ ok: false, error: undefined });
+                }
+            },
+        };
+        this.#config = config;
+        this.#fingerprint = fingerprint;
+        this.#logger = logger;
+        this.#lineMatches = config.service.readyWhen.lineMatches;
+        this.#dir = getScriptDataDir(config);
+        this.#coordPath = pathlib.join(this.#dir, COORDINATION_FILE);
+        this.#lockPath = pathlib.join(this.#dir, LOCK_FILE);
+        this.#logPath = pathlib.join(this.#dir, LOGFILE);
+        void this.#init();
+    }
+    /**
+     * Detach this invocation from the shared service. If we are the last live
+     * consumer, the whole service process group is torn down; otherwise the
+     * service keeps running for the remaining consumers. Either way, our
+     * {@link completed} resolves.
+     *
+     * Mirrors {@link ScriptChildProcess.kill}: returns immediately.
+     */
+    kill() {
+        if (this.#tearingDown || this.#completedDeferred.settled) {
+            return;
+        }
+        this.#tearingDown = true;
+        void this.#teardown();
+    }
+    async #init() {
+        try {
+            await fs.mkdir(this.#dir, { recursive: true });
+            const decision = await this.#withLock(() => this.#decideUnderLock());
+            if (decision.kind === 'conflict') {
+                // Concurrent invocations keep restarting this service to conflicting
+                // definitions. Bail with a clear error rather than oscillate it.
+                const failure = {
+                    type: 'failure',
+                    script: this.#config,
+                    reason: 'service-shared-conflict',
+                    message: 'Shared service restarted with conflicting definitions by ' +
+                        'concurrent invocations (for example a different Node version or ' +
+                        'environment per terminal). Wireit stopped restarting it.',
+                };
+                this.#logger.log(failure);
+                this.#markHandshakeDone();
+                this.#failCompleted(failure);
+                return;
+            }
+            if (decision.kind === 'attach') {
+                this.#pgid = decision.pgid;
+                this.#attachOffset = decision.attachOffset;
+                this.#logger.log({
+                    script: this.#config,
+                    type: 'info',
+                    detail: 'service-shared-attached',
+                });
+            }
+            else {
+                // We spawned the detached process under the lock; #pgid is already set.
+                // The display tail starts from the beginning of the (freshly truncated)
+                // logfile.
+                this.#attachOffset = 0;
+                this.#watchSpawnError(decision.child);
+                this.#logger.log({
+                    script: this.#config,
+                    type: 'info',
+                    detail: 'service-shared-spawned',
+                });
+            }
+            // The handshake is complete as soon as the spawn-vs-attach decision is
+            // made: a spawned child already has a pid, and an attach means a live
+            // instance is already running. Resolving `started` here — before the
+            // first poll, and synchronously before the state machine's `started`
+            // callback runs — guarantees the machine leaves its "starting" state
+            // before any exit can be observed.
+            this.#markHandshakeDone();
+            this.#startPolling();
+        }
+        catch (error) {
+            // Anything unexpected during setup fails this consumer's run, but we still
+            // resolve `started` so the state machine advances to a state from which it
+            // can observe the failure via `completed`.
+            this.#markHandshakeDone();
+            this.#failCompleted({
+                type: 'failure',
+                script: this.#config,
+                reason: 'spawn-error',
+                message: `Failed to coordinate shared service: ${error instanceof Error ? error.message : String(error)}`,
+            });
+        }
+    }
+    // ---- Locked decision: spawn vs attach vs restart -----------------------
+    async #decideUnderLock() {
+        const coord = await this.#readCoord();
+        if (coord !== undefined &&
+            coord.state === 'running' &&
+            isAlive(coord.pgid)) {
+            if (coord.fingerprint === this.#fingerprint.string) {
+                return this.#attachUnderLock(coord);
+            }
+            // A concurrent invocation in the same directory wants a different
+            // definition of this service. Restart it so everyone converges.
+            return this.#restartUnderLock(coord);
+        }
+        // No coordination file, or the recorded instance is gone/stale — spawn.
+        // Carry forward any recent restart history (pruned to the window) so that a
+        // crash-and-respawn during an active oscillation is still rate-limited.
+        return this.#spawnUnderLock(this.#recentRestarts(coord?.restarts));
+    }
+    async #attachUnderLock(coord) {
+        const consumers = pruneDead(coord.consumers);
+        if (!consumers.includes(this.#myPid)) {
+            consumers.push(this.#myPid);
+        }
+        const attachOffset = await this.#logSize();
+        await this.#writeCoord({ ...coord, consumers });
+        return { kind: 'attach', pgid: coord.pgid, attachOffset };
+    }
+    async #spawnUnderLock(restarts) {
+        // Truncate the logfile so this instance starts with a clean log.
+        const fd = openSync(this.#logPath, 'w');
+        let child;
+        try {
+            child = spawnDetachedServiceProcess(this.#config, fd);
+        }
+        finally {
+            closeSync(fd);
+        }
+        const pgid = child.pid;
+        if (pgid === undefined) {
+            // Synchronous spawn failure (rare). Record the failure so other waiters
+            // don't try to attach to a non-existent process.
+            await this.#writeCoord({
+                fingerprint: this.#fingerprint.string,
+                pgid: -1,
+                logfile: LOGFILE,
+                readyState: 'failed',
+                state: 'exited',
+                consumers: [],
+                restarts,
+            });
+            throw new Error('service process was spawned without a pid');
+        }
+        this.#pgid = pgid;
+        await this.#writeCoord({
+            fingerprint: this.#fingerprint.string,
+            pgid,
+            logfile: LOGFILE,
+            readyState: 'starting',
+            state: 'running',
+            consumers: [this.#myPid],
+            restarts,
+        });
+        return { kind: 'spawn', child };
+    }
+    /**
+     * Restart-to-latest: a concurrent invocation wants a different definition, so
+     * kill the running instance and spawn a replacement with our fingerprint —
+     * unless the restart rate-limit guard detects that concurrent invocations are
+     * oscillating the service between conflicting definitions, in which case we
+     * bail (§8) rather than continue to flap it.
+     */
+    async #restartUnderLock(coord) {
+        const recent = this.#recentRestarts(coord.restarts) ?? [];
+        const prospective = [
+            ...recent,
+            { at: Date.now(), fingerprint: this.#fingerprint.string },
+        ];
+        const distinctFingerprints = new Set(prospective.map((restart) => restart.fingerprint));
+        if (prospective.length >= RESTART_LIMIT && distinctFingerprints.size >= 2) {
+            // Leave the currently-running instance alone and signal the conflict.
+            return { kind: 'conflict' };
+        }
+        signalGroup(coord.pgid, 'SIGINT');
+        await this.#waitForGroupDeath(coord.pgid, RESTART_DEATH_TIMEOUT_MS);
+        return this.#spawnUnderLock(prospective);
+    }
+    /** Restart records from within the rate-limit window. */
+    #recentRestarts(restarts) {
+        if (restarts === undefined) {
+            return undefined;
+        }
+        const cutoff = Date.now() - RESTART_WINDOW_MS;
+        const recent = restarts.filter((restart) => restart.at >= cutoff);
+        return recent.length > 0 ? recent : undefined;
+    }
+    async #waitForGroupDeath(pgid, timeoutMs) {
+        const deadline = Date.now() + timeoutMs;
+        let escalated = false;
+        while (isAlive(pgid)) {
+            if (!escalated && Date.now() > deadline - timeoutMs / 2) {
+                signalGroup(pgid, 'SIGKILL');
+                escalated = true;
+            }
+            if (Date.now() > deadline) {
+                return;
+            }
+            await sleep(POLL_INTERVAL_MS);
+        }
+    }
+    // ---- Spawn confirmation -------------------------------------------------
+    /**
+     * Fast-path detection of a failure to spawn the detached process (e.g. the
+     * shell itself is missing). This is best-effort — the liveness poller is the
+     * backstop if the `error` event is missed — but it surfaces spawn failures
+     * promptly and marks the instance failed so other waiters don't attach.
+     */
+    #watchSpawnError(child) {
+        child.once('error', (error) => {
+            void this.#mutateCoord((coord) => coord.pgid === this.#pgid
+                ? { ...coord, state: 'exited', readyState: 'failed' }
+                : coord);
+            this.#failCompleted({
+                type: 'failure',
+                script: this.#config,
+                reason: 'spawn-error',
+                message: error.message,
+            });
+        });
+    }
+    #markHandshakeDone() {
+        if (!this.#handshakeDone) {
+            this.#handshakeDone = true;
+            this.#startedDeferred.resolve({ ok: true, value: undefined });
+        }
+    }
+    // ---- Polling loop: readiness, log streaming, exit detection ------------
+    #startPolling() {
+        this.#pollTimer = setTimeout(() => void this.#tick(), 0);
+    }
+    async #tick() {
+        if (this.#closed) {
+            return;
+        }
+        try {
+            await this.#streamNewLog();
+            const coord = await this.#readCoord();
+            this.#evaluate(coord);
+        }
+        catch {
+            // Transient fs error (e.g. a write racing our read); try again next tick.
+        }
+        if (!this.#closed) {
+            this.#pollTimer = setTimeout(() => void this.#tick(), POLL_INTERVAL_MS);
+        }
+    }
+    #evaluate(coord) {
+        if (this.#closed || this.#tearingDown) {
+            return;
+        }
+        // The instance is gone or has been replaced out from under us.
+        if (coord === undefined ||
+            coord.state === 'exited' ||
+            coord.pgid !== this.#pgid ||
+            coord.readyState === 'failed') {
+            this.#onServiceGone();
+            return;
+        }
+        // We believe it's running — verify the process group is actually alive.
+        if (!isAlive(coord.pgid)) {
+            // We're the first observer to notice the crash; record it.
+            void this.#mutateCoord((c) => c.pgid === this.#pgid && c.state === 'running'
+                ? {
+                    ...c,
+                    state: 'exited',
+                    readyState: c.readyState === 'ready' ? 'ready' : 'failed',
+                    exit: { code: null, signal: null },
+                }
+                : c);
+            this.#onServiceGone();
+            return;
+        }
+        // Alive and running — resolve readiness if we can.
+        if (!this.#readyDeferred.settled) {
+            if (coord.readyState === 'ready') {
+                this.#readyDeferred.resolve({ ok: true, value: undefined });
+            }
+            else if (this.#lineMatches === undefined && this.#handshakeDone) {
+                // No ready condition: ready as soon as the spawn is confirmed.
+                this.#markReady();
+            }
+            // Otherwise we have a ready line to wait for; #streamNewLog scans for it.
+        }
+    }
+    #onServiceGone() {
+        if (!this.#readyDeferred.settled) {
+            this.#readyDeferred.resolve({ ok: false, error: undefined });
+        }
+        this.#failCompleted({
+            type: 'failure',
+            script: this.#config,
+            reason: 'service-exited-unexpectedly',
+        });
+    }
+    /**
+     * Read everything appended to the logfile since last tick. Feed it to the
+     * ready-line scanner (from the start of the file) and re-emit the portion at
+     * or after our attach offset to {@link stdout}.
+     */
+    async #streamNewLog() {
+        const start = this.#logReadOffset;
+        const { data, nextOffset } = await this.#readLogFrom(start);
+        if (data.length === 0) {
+            return;
+        }
+        this.#logReadOffset = nextOffset;
+        if (this.#lineMatches !== undefined && !this.#readyDeferred.settled) {
+            this.#scanForReady(data.toString('utf8'));
+        }
+        const end = start + data.length;
+        if (end > this.#attachOffset) {
+            const sliceStart = Math.max(0, this.#attachOffset - start);
+            this.#stdoutStream.write(data.subarray(sliceStart));
+        }
+    }
+    #scanForReady(text) {
+        // Mirror LineMonitor: test every line including the trailing incomplete one,
+        // but only retain the trailing incomplete line for next time.
+        this.#readyScanBuffer += text;
+        const lines = this.#readyScanBuffer.split('\n');
+        for (let i = 0; i < lines.length; i++) {
+            const line = lines[i];
+            if (this.#lineMatches.test(line)) {
+                this.#readyScanBuffer = '';
+                this.#markReady();
+                return;
+            }
+        }
+        this.#readyScanBuffer = lines[lines.length - 1];
+    }
+    /** Resolve readiness, and persist `readyState=ready` for late attachers. */
+    #markReady() {
+        if (this.#readyDeferred.settled) {
+            return;
+        }
+        this.#readyDeferred.resolve({ ok: true, value: undefined });
+        void this.#mutateCoord((coord) => coord.pgid === this.#pgid &&
+            coord.state === 'running' &&
+            coord.readyState === 'starting'
+            ? { ...coord, readyState: 'ready' }
+            : coord);
+    }
+    // ---- Teardown -----------------------------------------------------------
+    async #teardown() {
+        try {
+            await this.#withLock(async () => {
+                const coord = await this.#readCoord();
+                if (coord === undefined ||
+                    coord.pgid !== this.#pgid ||
+                    coord.state !== 'running') {
+                    // The instance is already gone or has been replaced; nothing to do.
+                    return;
+                }
+                const remaining = pruneDead(coord.consumers.filter((pid) => pid !== this.#myPid));
+                if (remaining.length === 0) {
+                    // We're the last live consumer — stop the service and clear the file.
+                    signalGroup(coord.pgid, 'SIGINT');
+                    await this.#deleteCoord();
+                }
+                else {
+                    await this.#writeCoord({ ...coord, consumers: remaining });
+                }
+            });
+        }
+        catch {
+            // Best-effort; a dead consumer is pruned by whoever next holds the lock.
+        }
+        this.#completeOk();
+    }
+    // ---- Completion ---------------------------------------------------------
+    #failCompleted(failure) {
+        if (this.#completedDeferred.settled) {
+            return;
+        }
+        this.#completedDeferred.resolve({ ok: false, error: failure });
+        this.#close();
+    }
+    #completeOk() {
+        if (this.#completedDeferred.settled) {
+            return;
+        }
+        this.#completedDeferred.resolve({ ok: true, value: undefined });
+        this.#close();
+    }
+    #close() {
+        this.#closed = true;
+        if (this.#pollTimer !== undefined) {
+            clearTimeout(this.#pollTimer);
+            this.#pollTimer = undefined;
+        }
+        this.#stdoutStream.end();
+        this.#stderrStream.end();
+    }
+    // ---- Coordination file & lock I/O --------------------------------------
+    async #withLock(fn) {
+        // proper-lockfile locks an existing file by creating an adjacent "<file>.lock"
+        // directory (an atomic mkdir). Ensure the sentinel file exists first.
+        await fs.writeFile(this.#lockPath, '', 'utf8');
+        for (;;) {
+            let release;
+            try {
+                release = await lockfile.lock(this.#lockPath, {
+                    stale: LOCK_STALE_MS,
+                    update: LOCK_UPDATE_MS,
+                    retries: 0,
+                });
+            }
+            catch (error) {
+                if (error.code === 'ELOCKED') {
+                    await sleep(POLL_INTERVAL_MS / 2);
+                    continue;
+                }
+                throw error;
+            }
+            try {
+                return await fn();
+            }
+            finally {
+                await release();
+            }
+        }
+    }
+    /** Read+decide+write the coordination file atomically under the lock. */
+    async #mutateCoord(update) {
+        try {
+            await this.#withLock(async () => {
+                const coord = await this.#readCoord();
+                if (coord === undefined) {
+                    return;
+                }
+                const next = update(coord);
+                if (next !== coord) {
+                    await this.#writeCoord(next);
+                }
+            });
+        }
+        catch {
+            // Best-effort coordination metadata update.
+        }
+    }
+    async #readCoord() {
+        let text;
+        try {
+            text = await fs.readFile(this.#coordPath, 'utf8');
+        }
+        catch (error) {
+            if (error.code === 'ENOENT') {
+                return undefined;
+            }
+            throw error;
+        }
+        try {
+            return JSON.parse(text);
+        }
+        catch {
+            // A torn/partial write — treat as "no usable instance".
+            return undefined;
+        }
+    }
+    async #writeCoord(coord) {
+        await fs.writeFile(this.#coordPath, JSON.stringify(coord, null, 2), 'utf8');
+    }
+    async #deleteCoord() {
+        try {
+            await fs.unlink(this.#coordPath);
+        }
+        catch (error) {
+            if (error.code !== 'ENOENT') {
+                throw error;
+            }
+        }
+    }
+    async #logSize() {
+        try {
+            return (await fs.stat(this.#logPath)).size;
+        }
+        catch {
+            return 0;
+        }
+    }
+    async #readLogFrom(offset) {
+        let handle;
+        try {
+            handle = await fs.open(this.#logPath, 'r');
+        }
+        catch {
+            return { data: Buffer.alloc(0), nextOffset: offset };
+        }
+        try {
+            const { size } = await handle.stat();
+            if (size <= offset) {
+                return { data: Buffer.alloc(0), nextOffset: offset };
+            }
+            const length = size - offset;
+            const buffer = Buffer.alloc(length);
+            await handle.read(buffer, 0, length, offset);
+            return { data: buffer, nextOffset: size };
+        }
+        finally {
+            await handle.close();
+        }
+    }
+}
+//# sourceMappingURL=shared-service-coordinator.js.map

package/lib/logging/quiet/run-tracker.js

@@ -335,6 +335,8 @@ export class QuietRunLogger {
                 this.#running.delete(scriptReferenceToString(event.script));
                 return this.#getStatusLine();
             case 'service-ready':
+            case 'service-shared-spawned':
+            case 'service-shared-attached':
             case 'watch-run-start':
             case 'watch-run-end':
                 return noChange;
@@ -476,6 +478,7 @@ export class QuietRunLogger {
             case 'input-file-deleted-unexpectedly':
             case 'output-file-deleted-unexpectedly':
             case 'service-exited-unexpectedly':
+            case 'service-shared-conflict':
             case 'cycle':
             case 'dependency-invalid':
             case 'dependency-on-missing-package-json':

package/lib/logging/simple-logger.js

@@ -155,6 +155,10 @@ export class SimpleLogger {
                         this.console.error(`❌${prefix} Service exited unexpectedly`);
                         break;
                     }
+                    case 'service-shared-conflict': {
+                        this.console.error(`❌${prefix} ${event.message}`);
+                        break;
+                    }
                     case 'input-file-deleted-unexpectedly': {
                         for (const filePath of event.filePaths) {
                             this.console.error(`❌${prefix} Input file "${filePath}" was deleted unexpectedly. Is another process writing to the same location?`);
@@ -247,6 +251,14 @@ export class SimpleLogger {
                         this.console.log(`⬇️${prefix} Service stopped`);
                         break;
                     }
+                    case 'service-shared-spawned': {
+                        this.console.log(`⬆️${prefix} Shared service started`);
+                        break;
+                    }
+                    case 'service-shared-attached': {
+                        this.console.log(`🔗${prefix} Attached to shared service`);
+                        break;
+                    }
                     case 'analysis-started':
                     case 'analysis-completed': {
                         break;

package/lib/script-child-process.js

@@ -26,6 +26,81 @@ const PATH_ENV_SUFFIX = (() => {
     const endOfNodeModuleBins = entries.findIndex((entry) => !entry.endsWith(nodeModulesBinSuffix));
     return entries.slice(endOfNodeModuleBins).join(pathlib.delimiter);
 })();
+/**
+ * Generates the PATH environment variable that should be set when a script's
+ * command is spawned from the given package directory.
+ */
+function pathEnvironmentVariable(packageDir) {
+    // Given package "/foo/bar", walk up the path hierarchy to generate
+    // "/foo/bar/node_modules/.bin:/foo/node_modules/.bin:/node_modules/.bin".
+    const entries = [];
+    let cur = packageDir;
+    while (true) {
+        entries.push(pathlib.join(cur, 'node_modules', '.bin'));
+        const parent = pathlib.dirname(cur);
+        if (parent === cur) {
+            break;
+        }
+        cur = parent;
+    }
+    // Add the inherited PATH variable, minus any "node_modules/.bin" entries that
+    // were set by the "npm run" command that spawned Wireit.
+    entries.push(PATH_ENV_SUFFIX);
+    // Note the PATH delimiter is platform-dependent.
+    return entries.join(pathlib.delimiter);
+}
+/**
+ * Build the environment that a script's command should be spawned with.
+ */
+function spawnEnvironment(script) {
+    return augmentProcessEnvSafelyIfOnWindows({
+        FORCE_COLOR: process.stdout.isTTY && process.env.FORCE_COLOR === undefined
+            ? 'true'
+            : process.env.FORCE_COLOR,
+        PATH: pathEnvironmentVariable(script.packageDir),
+        ...script.env,
+    });
+}
+/**
+ * Spawn a service as a fully detached process whose stdout and stderr are
+ * redirected to the given open logfile file descriptor, instead of being piped
+ * back to this Wireit invocation. This is the spawn mode for {@link
+ * ServiceConfig.shared | shared services}: because the process is detached,
+ * `unref`'d, and writes its output directly to a file, it survives the Wireit
+ * invocation that launched it, so that other concurrent invocations can attach
+ * to it.
+ *
+ * Returns the raw {@link ChildProcess}. Because we spawn `detached`, the
+ * child is the leader of a new process group and session, so `child.pid` is also
+ * the process-group id (pass its negation to `process.kill` to signal the whole
+ * group). The caller reads `child.pid` synchronously and may listen for the
+ * `spawn` and `error` events; the child is already `unref`'d so it will not by
+ * itself keep this process alive.
+ *
+ * POSIX only — shared services degrade to per-process behavior on Windows, so
+ * this is never called there.
+ */
+export function spawnDetachedServiceProcess(script, logFd) {
+    const child = spawn(script.command.value, script.extraArgs ?? [], {
+        cwd: script.packageDir,
+        // See the comment in the ScriptChildProcess constructor for why we use
+        // "shell: true".
+        shell: true,
+        env: spawnEnvironment(script),
+        // Detached so that the process becomes the leader of its own process group
+        // and session. This both lets us kill the whole group with
+        // "process.kill(-pid)", and ensures a Ctrl-C delivered to a consuming
+        // Wireit invocation's foreground process group does not reach this service.
+        detached: true,
+        // Redirect the child's stdout and stderr straight to the logfile, so that
+        // it keeps producing output (and other invocations keep seeing it) after
+        // the launching invocation exits. stdin is ignored.
+        stdio: ['ignore', logFd, logFd],
+    });
+    // Allow the launching invocation to exit without waiting for this process.
+    child.unref();
+    return child;
+}
 /**
  * A child process spawned during execution of a script.
  */
@@ -76,13 +151,7 @@ export class ScriptChildProcess {
             //   https://nodejs.org/api/child_process.html#default-windows-shell
             //   https://github.com/npm/run-script/blob/a5b03bdfc3a499bf7587d7414d5ea712888bfe93/lib/make-spawn-args.js#L11
             shell: true,
-            env: augmentProcessEnvSafelyIfOnWindows({
-                FORCE_COLOR: process.stdout.isTTY && process.env.FORCE_COLOR === undefined
-                    ? 'true'
-                    : process.env.FORCE_COLOR,
-                PATH: this.#pathEnvironmentVariable,
-                ...this.#script.env,
-            }),
+            env: spawnEnvironment(this.#script),
             // Set "detached" on Linux and macOS so that we create a new process
             // group, instead of being added to the process group for this Wireit
             // process.
@@ -243,28 +312,5 @@ export class ScriptChildProcess {
         }
         this.#state = 'killing';
     }
-    /**
-     * Generates the PATH environment variable that should be set when this
-     * script's command is spawned.
-     */
-    get #pathEnvironmentVariable() {
-        // Given package "/foo/bar", walk up the path hierarchy to generate
-        // "/foo/bar/node_modules/.bin:/foo/node_modules/.bin:/node_modules/.bin".
-        const entries = [];
-        let cur = this.#script.packageDir;
-        while (true) {
-            entries.push(pathlib.join(cur, 'node_modules', '.bin'));
-            const parent = pathlib.dirname(cur);
-            if (parent === cur) {
-                break;
-            }
-            cur = parent;
-        }
-        // Add the inherited PATH variable, minus any "node_modules/.bin" entries
-        // that were set by the "npm run" command that spawned Wireit.
-        entries.push(PATH_ENV_SUFFIX);
-        // Note the PATH delimiter is platform-dependent.
-        return entries.join(pathlib.delimiter);
-    }
 }
 //# sourceMappingURL=script-child-process.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@laurence79/wireit",
-  "version": "0.14.13-shared-cache.0",
+  "version": "0.14.13-shared-cache.1",
   "description": "Upgrade your npm scripts to make them smarter and more efficient",
   "author": "Google LLC",
   "license": "Apache-2.0",
@@ -57,6 +57,7 @@
     "test:parallelism": "wireit",
     "test:quiet": "wireit",
     "test:service": "wireit",
+    "test:shared-service": "wireit",
     "test:watch": "wireit"
   },
   "wireit": {
@@ -111,6 +112,7 @@
         "test:parallelism",
         "test:quiet",
         "test:service",
+        "test:shared-service",
         "test:watch"
       ]
     },
@@ -404,6 +406,17 @@
       "files": [],
       "output": []
     },
+    "test:shared-service": {
+      "command": "node --test --test-reporter=dot lib/test/shared-service.test.js",
+      "env": {
+        "NODE_OPTIONS": "--enable-source-maps"
+      },
+      "dependencies": [
+        "build"
+      ],
+      "files": [],
+      "output": []
+    },
     "test:watch": {
       "command": "node --test --test-reporter=dot lib/test/watch.test.js",
       "env": {

package/schema.json

@@ -90,6 +90,10 @@
                         "type": "string"
                       }
                     }
+                  },
+                  "shared": {
+                    "markdownDescription": "If true, concurrent Wireit invocations in the same package directory share a single instance of this service instead of each starting their own. The first invocation that needs it starts a detached process; later invocations attach to it. The service is torn down when its last live consumer exits.\n\nPOSIX-only: on Windows this is ignored and each invocation starts its own instance.",
+                    "type": "boolean"
                   }
                 }
               }