@bastani/atomic 0.9.60 → 0.9.61-alpha.1

package/dist/builtin/workflows/src/extension/local-worker-entry.mjs

@@ -5135,6 +5135,18 @@ function failedOnInfrastructureFault(result) {
     return true;
   return Array.isArray(result.stages) && result.stages.some((stage) => stage?.status === "failed" && (names(stage.failureCode) || names(stage.error)));
 }
+function infrastructureFaultMessage(result) {
+  const names = (value2) => typeof value2 === "string" && (WORKFLOW_INFRASTRUCTURE_FAULTS.has(value2) || workflowInfrastructureFault(value2) !== undefined);
+  if (names(result.error))
+    return result.error;
+  const stage = Array.isArray(result.stages) ? result.stages.find((candidate) => candidate?.status === "failed" && (names(candidate.failureCode) || names(candidate.error))) : undefined;
+  if (typeof stage?.error === "string")
+    return stage.error;
+  if (typeof stage?.failureCode === "string")
+    return stage.failureCode;
+  const code = result.failure?.code;
+  return typeof code === "string" ? code : typeof result.error === "string" ? result.error : "pool fault";
+}
 var WORKFLOW_INFRASTRUCTURE_ERROR_NAME = "WorkflowInfrastructureError", WORKFLOW_INFRASTRUCTURE_FAULTS, MAX_CAUSE_DEPTH = 8, FAULT_PATTERNS, WorkflowInfrastructureError;
 var init_workflow_infrastructure_error = __esm(() => {
   WORKFLOW_INFRASTRUCTURE_FAULTS = new Set([
@@ -16878,6 +16890,25 @@ function admissionProcessStopped(pid) {
     return error.code === "ESRCH";
   }
 }
+function workerProcessExited(pid, final = false) {
+  if (admissionProcessStopped(pid))
+    return true;
+  if (process.platform === "linux") {
+    try {
+      const stat = readFileSync2(`/proc/${pid}/stat`, "utf8");
+      return stat.slice(stat.lastIndexOf(")") + 2).startsWith("Z");
+    } catch {
+      return admissionProcessStopped(pid);
+    }
+  }
+  if (!final)
+    return false;
+  try {
+    return execFileSync("ps", ["-o", "stat=", "-p", String(pid)], { encoding: "utf8", timeout: 5000 }).trim().startsWith("Z");
+  } catch {
+    return admissionProcessStopped(pid);
+  }
+}
 function scanProcessGroup(group) {
   const members = new Map;
   if (process.platform === "linux") {
@@ -59371,17 +59402,26 @@ function isSameCheckpointValue(stored, next) {
   const hash = (value2) => durableHash(value2);
   return hash(stored) === hash(next);
 }
+function withTimesOf(later, recorded) {
+  const both = (key) => typeof recorded[key] === "number" && typeof later[key] === "number";
+  return {
+    ...later,
+    ...both("startedAt") ? { startedAt: recorded.startedAt } : {},
+    ...both("endedAt") ? { endedAt: recorded.endedAt } : {},
+    ...both("completedAt") ? { completedAt: recorded.completedAt } : {}
+  };
+}
 function isRecordedCheckpointValue(stored, next) {
   const recorded = stored;
   if (isSameCheckpointValue(recorded, next))
     return true;
-  const { completedAt, startedAt } = recorded;
-  if (typeof completedAt !== "number")
+  if (typeof recorded.completedAt !== "number")
     return false;
   const later = next;
-  if (startedAt === undefined !== (later.startedAt === undefined))
+  if (recorded.startedAt === undefined !== (later.startedAt === undefined))
     return false;
-  return isSameCheckpointValue({ ...later, completedAt, ...startedAt === undefined ? {} : { startedAt } }, recorded);
+  const retimed = withTimesOf(later, recorded);
+  return isSameCheckpointValue(recorded.topology !== undefined && later.topology !== undefined ? { ...retimed, topology: withTimesOf(later.topology, recorded.topology) } : retimed, recorded);
 }
 var MIRROR_FRESH_MS = 2000;
 var init_pool_remote_journal = __esm(() => {
@@ -60417,15 +60457,7 @@ class SqlPoolQueue {
   }
 }
 function isRecordedCheckpoint(stored, payload) {
-  if (stored === payload)
-    return true;
-  const { completedAt, startedAt } = JSON.parse(stored);
-  if (typeof completedAt !== "number")
-    return false;
-  const next = JSON.parse(payload);
-  if (startedAt === undefined !== (next.startedAt === undefined))
-    return false;
-  return serializeJson({ ...next, completedAt, ...startedAt === undefined ? {} : { startedAt } }) === stored;
+  return stored === payload || isRecordedCheckpointValue(JSON.parse(stored), JSON.parse(payload));
 }
 function openSqlPoolQueue(journal, remote) {
   const storage = sqlPoolStorage(journal);
@@ -65176,7 +65208,7 @@ var init_startup_timeout = () => {};
 import { fork } from "node:child_process";
 import { randomUUID as randomUUID16 } from "node:crypto";
 import { createWriteStream } from "node:fs";
-import { mkdir as mkdir6, rm as rm3, rmdir } from "node:fs/promises";
+import { appendFile, mkdir as mkdir6, rm as rm3, rmdir } from "node:fs/promises";
 import { dirname as dirname13, join as join29, resolve as resolve12 } from "node:path";
 import { setTimeout as delay2 } from "node:timers/promises";
 function isRootLeaseLost(error) {
@@ -65368,6 +65400,9 @@ class PoolCoordinator {
   incarnations = new Map;
   readyWorkers = new Set;
   logs = new Map;
+  keptLogs = new Set;
+  unconfirmedExits = new WeakSet;
+  failed = false;
   cleanups = new Set;
   controlAcks = new Map;
   lastHeard = new Map;
@@ -65415,7 +65450,6 @@ class PoolCoordinator {
   nextRecoveryAt = 0;
   cleanupFailure;
   held = new Set;
-  quarantined = new Set;
   unreaped = [];
   retained = new Map;
   retainedTransitions = Promise.resolve();
@@ -65608,18 +65642,15 @@ class PoolCoordinator {
     try {
       const { survivors, unprobeable } = await reapProcessGroups(this.journal, this.root, { attempt: work });
       if (survivors.length > 0) {
-        this.quarantined.add(work.id);
         this.unreaped.push({ attempt: work });
-        this.port.send({ kind: "work-failure", workId: work.id, error: unconfirmedCleanup(survivors).message });
-        return;
+        this.reportHealth(undefined, `${unconfirmedCleanup(survivors).message}; child ${work.childRunId} of an earlier incarnation replays`);
       }
       if (unprobeable.length === 0)
         await this.releaseRetained(work);
     } catch (error) {
       this.cleanupFailure ??= error instanceof Error ? error : new Error(String(error));
     } finally {
-      if (!this.quarantined.has(work.id))
-        this.held.delete(work.id);
+      this.held.delete(work.id);
     }
   }
   releaseRetained(work) {
@@ -65750,6 +65781,7 @@ class PoolCoordinator {
     });
   }
   fail(error) {
+    this.failed = true;
     this.port.send({ kind: "failure", error: error.message });
   }
   failWork(work, error) {
@@ -66158,7 +66190,11 @@ class PoolCoordinator {
         terminate();
       });
       const closed = new Promise((resolve) => child.once("close", () => resolve()));
-      child.once("exit", () => {
+      let retired = false;
+      const retire = () => {
+        if (retired)
+          return;
+        retired = true;
         Promise.race([closed, delay2(STDERR_DRAIN_GRACE_MS)]).then(() => reject(diagnosed(timedOut ? startupTimeoutMessage(startupTimeoutMs) : "Local worker exited before readiness")));
         this.children.delete(child);
         this.logs.delete(child);
@@ -66182,7 +66218,7 @@ class PoolCoordinator {
         const held = slot.work;
         if (held)
           this.held.add(held.id);
-        let childFailed = false;
+        let unconfirmed;
         const cleanup = disconnected.then(() => serial).then(async () => {
           try {
             const admission = this.admission;
@@ -66195,18 +66231,13 @@ class PoolCoordinator {
               throw unconfirmedCleanup(alive);
             if (alive.length === 0)
               return;
-            const error = unconfirmedCleanup(alive);
-            if (held && await this.queue.result(this.root, held.id) === undefined) {
-              this.quarantined.add(held.id);
-              slot.work = undefined;
-              childFailed = true;
-              this.port.send({ kind: "work-failure", workId: held.id, error: error.message });
-            }
+            unconfirmed = unconfirmedCleanup(alive);
+            this.reportHealth(logPath, `${unconfirmed.message}; the worker was retired and replaced`);
           } catch (error) {
             this.cleanupFailure = error instanceof Error ? error : new Error(String(error));
             throw this.cleanupFailure;
           } finally {
-            if (held && !this.quarantined.has(held.id))
+            if (held)
               this.held.delete(held.id);
           }
         }).then(async () => {
@@ -66227,8 +66258,8 @@ class PoolCoordinator {
           const settled = attempt !== undefined && await this.whileRootLate(queue, () => exhausted ? queue.failExited(lease, attempt) : queue.releaseExited(lease, attempt));
           const superseded = work !== undefined && !settled && await this.superseded(work);
           if (work !== undefined && exhausted && !superseded)
-            this.failWork(work, diagnosed(childRetryLimit(work, "lost its worker")));
-          if (work === undefined && !childFailed && ++this.replacements > 3)
+            this.failWork(work, diagnosed(`${childRetryLimit(work, "lost its worker")}${unconfirmed === undefined ? "" : `; ${unconfirmed.message}`}`));
+          if (work === undefined && ++this.replacements > 3)
             this.fail(diagnosed("Local worker retry limit"));
           else
             await this.spawn();
@@ -66237,7 +66268,9 @@ class PoolCoordinator {
         });
         this.cleanups.add(cleanup);
         cleanup.finally(() => this.cleanups.delete(cleanup));
-      });
+      };
+      slot.retire = retire;
+      child.once("exit", retire);
       child.on("message", (message) => {
         this.observeResponsiveness();
         this.lastHeard.set(worker, { at: performance.now(), stalled: this.stalledMs });
@@ -66311,10 +66344,11 @@ class PoolCoordinator {
             const fenced = slot.work;
             slot.work = undefined;
             if (fenced && !await this.superseded(fenced)) {
+              this.fence(fenced);
               if (this.attempts(fenced) < MAX_CHILD_ATTEMPTS)
                 await leased(() => queue.releaseExited(lease, fenced));
               else if (await leased(() => queue.failExited(lease, fenced)))
-                this.failWork(fenced, new Error(childRetryLimit(fenced, "was refused by the root as stale on every attempt")));
+                this.failWork(fenced, new Error(childRetryLimit(fenced, message.fault === undefined ? "was refused by the root as stale on every attempt" : `failed on a fault of the pool on every attempt (last: ${message.fault})`)));
             }
             return;
           }
@@ -66475,8 +66509,6 @@ class PoolCoordinator {
         return;
       }
       for (const work of expired) {
-        if (this.quarantined.has(work.id))
-          continue;
         if (this.held.has(work.id)) {
           await queue.extend(lease, work, leaseMs).catch((error) => {
             if (error.message !== STALE_WORK_LEASE)
@@ -66492,9 +66524,9 @@ class PoolCoordinator {
           });
           continue;
         }
-        if (child && !await this.stop(child).then(() => true, () => false))
-          continue;
-        if (this.held.has(work.id) || this.quarantined.has(work.id))
+        if (child && !await this.stop(child))
+          this.retireUnconfirmed(child);
+        if (this.held.has(work.id))
           continue;
         if ((await unconfirmedProcessGroups(this.journal, root, { attempt: work })).length > 0)
           continue;
@@ -66521,17 +66553,49 @@ class PoolCoordinator {
     const heard = this.lastHeard.get(worker);
     return child.exitCode === null && child.signalCode === null && child.connected && heard !== undefined && performance.now() - heard.at - (this.stalledMs - heard.stalled) < this.options.leaseMs;
   }
-  stop(child) {
-    if (child.pid === undefined || child.exitCode !== null || child.signalCode !== null)
-      return Promise.resolve();
-    return new Promise((resolve, reject) => {
-      const timeout = setTimeout(() => reject(new Error("Local worker cleanup was not confirmed")), 5000);
-      child.once("exit", () => {
-        clearTimeout(timeout);
-        resolve();
-      });
-      child.kill("SIGKILL");
-    });
+  async stop(child) {
+    const pid = child.pid;
+    const exited = () => child.exitCode !== null || child.signalCode !== null;
+    if (pid === undefined || exited())
+      return true;
+    const exit = new Promise((resolve) => child.once("exit", () => resolve()));
+    const confirmed = async () => {
+      await Promise.race([exit, delay2(WORKER_EXIT_CONFIRMATION_MS)]);
+      return true;
+    };
+    this.observeResponsiveness();
+    child.kill("SIGKILL");
+    const since = performance.now();
+    const stalledAt = this.stalledMs;
+    for (;; ) {
+      await Promise.race([exit, delay2(WORKER_EXIT_POLL_MS)]);
+      if (exited())
+        return true;
+      if (workerProcessExited(pid))
+        return confirmed();
+      this.observeResponsiveness();
+      if (performance.now() - since - (this.stalledMs - stalledAt) >= WORKER_EXIT_CONFIRMATION_MS)
+        return exited() || workerProcessExited(pid, true) && confirmed();
+    }
+  }
+  retireUnconfirmed(child) {
+    if (this.unconfirmedExits.has(child))
+      return;
+    this.unconfirmedExits.add(child);
+    this.reportHealth(this.logs.get(child), `Local worker cleanup was not confirmed: worker process ${child.pid} was still running ${WORKER_EXIT_CONFIRMATION_MS} ms after SIGKILL; it was retired and replaced`);
+    if (child.connected)
+      child.disconnect();
+    for (const slot of this.slots)
+      if (slot.child === child)
+        slot.retire?.();
+  }
+  reportHealth(log, message) {
+    if (log !== undefined) {
+      this.keptLogs.add(log);
+      appendFile(log, `[atomic ${new Date().toISOString()}] ${message}
+`).catch(() => {});
+    }
+    this.port.send({ kind: "health", message: log === undefined ? message : `${message} (worker log: ${log})` });
   }
   async close() {
     this.closed = true;
@@ -66541,10 +66605,15 @@ class PoolCoordinator {
     this.rateLimitOutbox.close();
     let failure;
     let stopping = [];
+    let confirmed = [];
     try {
       await this.recovering?.catch(() => {});
       stopping = [...this.children].map((child) => ({ child, log: this.logs.get(child) }));
-      await Promise.all(stopping.map(({ child }) => this.stop(child)));
+      confirmed = await Promise.all(stopping.map(({ child }) => this.stop(child)));
+      stopping.forEach(({ child }, index) => {
+        if (!confirmed[index])
+          this.retireUnconfirmed(child);
+      });
       await Promise.all([...this.cleanups]);
       if (this.cleanupFailure)
         throw this.cleanupFailure;
@@ -66556,15 +66625,15 @@ class PoolCoordinator {
       if (alive.length > 0)
         throw unconfirmedCleanup(alive);
       if (this.desiredControl !== "quit" && this.queue !== undefined && this.lease !== undefined)
-        await this.queue.releaseClosed(this.lease, [...this.quarantined]).catch((error) => {
+        await this.queue.releaseClosed(this.lease).catch((error) => {
           if (!(error instanceof StaleLeaseError))
             throw error;
         });
     } catch (error) {
       failure = error;
     }
-    if (failure === undefined) {
-      const stopped = stopping.flatMap(({ child, log }) => log !== undefined && (child.exitCode !== null || child.signalCode !== null) ? [log] : []);
+    if (failure === undefined && !this.failed) {
+      const stopped = stopping.flatMap(({ log }, index) => log !== undefined && !this.keptLogs.has(log) && confirmed[index] === true ? [log] : []);
       await Promise.all(stopped.map((log) => rm3(log, { force: true }).catch(() => {})));
       if (this.logDirectory !== undefined)
         await rmdir(this.logDirectory).catch(() => {});
@@ -66616,7 +66685,7 @@ function runPoolCoordinator() {
   port.onClose(() => process.exit(0));
   return coordinator;
 }
-var CONTROL_ACKNOWLEDGEMENT_STALL_MS = 5000, RECOVERY_INTERVAL_MS = 500, TICK_INTERVAL_MS = 100, STALL_THRESHOLD_MS = 500, COORDINATOR_LEASE_MS, LATE_ROOT_POLL_MS, MIRROR_INTERVAL_MS = 1e4, STDERR_TAIL_CHARACTERS = 8192, STDERR_DRAIN_GRACE_MS = 1000, MAX_CHILD_ATTEMPTS = 3, ROUTE_REGISTRATION_WAIT_MS = 120000, STAGE_WATCH_BACKLOG_EVENTS = 2000, STAGE_WATCH_BACKLOG_BYTES, PENDING_STAGE_MESSAGE_EVENT2 = "atomic:workflow-pending-stage-message", STICKY_LIVE_DELIVERED_EVENT2 = "atomic:workflow-sticky-live-delivered", PENDING_STAGE_UNDELIVERABLE_EVENT2 = "atomic:workflow-pending-stage-undeliverable", ENDED_WATCHES_REMEMBERED = 1024;
+var CONTROL_ACKNOWLEDGEMENT_STALL_MS = 5000, RECOVERY_INTERVAL_MS = 500, TICK_INTERVAL_MS = 100, STALL_THRESHOLD_MS = 500, COORDINATOR_LEASE_MS, LATE_ROOT_POLL_MS, MIRROR_INTERVAL_MS = 1e4, STDERR_TAIL_CHARACTERS = 8192, STDERR_DRAIN_GRACE_MS = 1000, WORKER_EXIT_CONFIRMATION_MS = 5000, WORKER_EXIT_POLL_MS = 50, MAX_CHILD_ATTEMPTS = 3, ROUTE_REGISTRATION_WAIT_MS = 120000, STAGE_WATCH_BACKLOG_EVENTS = 2000, STAGE_WATCH_BACKLOG_BYTES, PENDING_STAGE_MESSAGE_EVENT2 = "atomic:workflow-pending-stage-message", STICKY_LIVE_DELIVERED_EVENT2 = "atomic:workflow-sticky-live-delivered", PENDING_STAGE_UNDELIVERABLE_EVENT2 = "atomic:workflow-pending-stage-undeliverable", ENDED_WATCHES_REMEMBERED = 1024;
 var init_pool_coordinator = __esm(() => {
   init_internal();
   init_pool_remote_journal();
@@ -66631,6 +66700,7 @@ var init_pool_coordinator = __esm(() => {
   init_pending_stage_route_readiness();
   init_store();
   init_workflow_infrastructure_error();
+  init_admission_host();
   init_pool_journal();
   init_pooled_watch_trace();
   init_resource_admission();
@@ -68581,6 +68651,7 @@ class LocalWorkerPool {
   works = new Map;
   attempts = new Map;
   results = new Map;
+  healthEvents = [];
   workFailures = new Map;
   changed = Promise.withResolvers();
   waiting = 0;
@@ -68638,6 +68709,9 @@ class LocalWorkerPool {
     if (options.admission && options.rootId === undefined)
       workflowResourceAdmission.set(backend, options.admission);
   }
+  get health() {
+    return this.healthEvents;
+  }
   get workerPids() {
     return [...this.pids];
   }
@@ -69231,14 +69305,8 @@ class LocalWorkerPool {
     try {
       return await this.dispatch(definition, inputs, options);
     } catch (error) {
-      if (error instanceof WorkflowInfrastructureError && [...this.workFailures.values()].includes(error))
-        throw error;
-      try {
-        if (this.desiredControl !== "quit" || !findWorkflowGracefulQuit(error))
-          await this.shutdown();
-      } catch (closeError) {
-        throw asWorkflowInfrastructureError(closeError) ?? closeError;
-      }
+      if (this.failure !== undefined || this.coordinatorLost !== undefined)
+        await this.shutdown().catch(() => {});
       throw asWorkflowInfrastructureError(error) ?? error;
     }
   }
@@ -69306,8 +69374,13 @@ class LocalWorkerPool {
     try {
       for (;; ) {
         options.signal?.throwIfAborted();
-        if (this.closed)
-          throw this.closedError(root);
+        if (!this.results.has(work.id) && (this.closed || this.failure !== undefined)) {
+          const recorded = await this.queue?.result(root, work.id).catch(() => {
+            return;
+          });
+          if (recorded !== undefined)
+            this.results.set(work.id, recorded);
+        }
         const result = this.results.get(work.id);
         if (result) {
           this.completing++;
@@ -69331,6 +69404,8 @@ class LocalWorkerPool {
           this.endFailedChild(work, failed);
           throw failed;
         }
+        if (this.closed)
+          throw this.closedError(root);
         if (this.failure)
           throw this.failure;
         if (this.budgetBlocked)
@@ -69365,6 +69440,19 @@ class LocalWorkerPool {
     if (this.reconnectingViews.size > 0 || this.outboxes.size > 0)
       this.reconnectStageViews();
   }
+  recordHealth(message) {
+    this.healthEvents.push(message);
+    const root = this.root;
+    if (root === undefined || this.store === undefined)
+      return;
+    this.store.recordNotice({
+      id: `local-worker-health:${root}:${this.healthEvents.length}`,
+      runId: root,
+      level: "warning",
+      message: `Local worker pool: ${message}`,
+      createdAt: Date.now()
+    });
+  }
   evaluateBudget() {
     const decision = this.rootBudget?.checkpoint();
     const blocked = decision?.kind === "exhausted" || decision?.kind === "wrap_up";
@@ -69483,6 +69571,9 @@ class LocalWorkerPool {
         this.failure ??= new Error(event.error);
         this.signalChange();
         return;
+      case "health":
+        this.recordHealth(event.message);
+        return;
       case "stage-events":
         this.stageWatches.get(event.watch)?.({ kind: "events", events: event.events });
         this.notice({ kind: "stage-events-applied", watch: event.watch });
@@ -81314,6 +81405,7 @@ async function runWorker(start) {
       await reader.catchUpWorkflow(start.root);
       const work = await request({ kind: "claim" });
       attemptFenced = false;
+      let poolFault;
       if (!work) {
         await delay4(25 + Math.random() * 25);
         continue;
@@ -81486,6 +81578,11 @@ async function runWorker(start) {
         }
         if (attemptFenced)
           throw new WorkAttemptFencedError("Work attempt fenced before completion");
+        if (failedOnInfrastructureFault(result)) {
+          poolFault = infrastructureFaultMessage(result);
+          attemptFenced = true;
+          throw new WorkAttemptFencedError(poolFault);
+        }
         await request({ kind: "complete", result });
       } catch (error) {
         if (!attemptFenced)
@@ -81503,7 +81600,7 @@ async function runWorker(start) {
         await renewal;
         await report;
         await controlPulse;
-        await request({ kind: "fenced" });
+        await request({ kind: "fenced", ...poolFault === undefined ? {} : { fault: poolFault } });
       }
       if (rootLeaseLost)
         break;