@bastani/atomic 0.9.60-alpha.1 → 0.9.61-alpha.1

package/dist/builtin/workflows/src/index.js

@@ -17001,6 +17001,25 @@ function admissionProcessStopped(pid) {
     return error.code === "ESRCH";
   }
 }
+function workerProcessExited(pid, final = false) {
+  if (admissionProcessStopped(pid))
+    return true;
+  if (process.platform === "linux") {
+    try {
+      const stat = readFileSync3(`/proc/${pid}/stat`, "utf8");
+      return stat.slice(stat.lastIndexOf(")") + 2).startsWith("Z");
+    } catch {
+      return admissionProcessStopped(pid);
+    }
+  }
+  if (!final)
+    return false;
+  try {
+    return execFileSync("ps", ["-o", "stat=", "-p", String(pid)], { encoding: "utf8", timeout: 5000 }).trim().startsWith("Z");
+  } catch {
+    return admissionProcessStopped(pid);
+  }
+}
 function scanProcessGroup(group) {
   const members = new Map;
   if (process.platform === "linux") {
@@ -59494,17 +59513,26 @@ function isSameCheckpointValue(stored, next) {
   const hash = (value2) => durableHash(value2);
   return hash(stored) === hash(next);
 }
+function withTimesOf(later, recorded) {
+  const both = (key) => typeof recorded[key] === "number" && typeof later[key] === "number";
+  return {
+    ...later,
+    ...both("startedAt") ? { startedAt: recorded.startedAt } : {},
+    ...both("endedAt") ? { endedAt: recorded.endedAt } : {},
+    ...both("completedAt") ? { completedAt: recorded.completedAt } : {}
+  };
+}
 function isRecordedCheckpointValue(stored, next) {
   const recorded = stored;
   if (isSameCheckpointValue(recorded, next))
     return true;
-  const { completedAt, startedAt } = recorded;
-  if (typeof completedAt !== "number")
+  if (typeof recorded.completedAt !== "number")
     return false;
   const later = next;
-  if (startedAt === undefined !== (later.startedAt === undefined))
+  if (recorded.startedAt === undefined !== (later.startedAt === undefined))
     return false;
-  return isSameCheckpointValue({ ...later, completedAt, ...startedAt === undefined ? {} : { startedAt } }, recorded);
+  const retimed = withTimesOf(later, recorded);
+  return isSameCheckpointValue(recorded.topology !== undefined && later.topology !== undefined ? { ...retimed, topology: withTimesOf(later.topology, recorded.topology) } : retimed, recorded);
 }
 var MIRROR_FRESH_MS = 2000;
 var init_pool_remote_journal = __esm(() => {
@@ -60540,15 +60568,7 @@ class SqlPoolQueue {
   }
 }
 function isRecordedCheckpoint(stored, payload) {
-  if (stored === payload)
-    return true;
-  const { completedAt, startedAt } = JSON.parse(stored);
-  if (typeof completedAt !== "number")
-    return false;
-  const next = JSON.parse(payload);
-  if (startedAt === undefined !== (next.startedAt === undefined))
-    return false;
-  return serializeJson({ ...next, completedAt, ...startedAt === undefined ? {} : { startedAt } }) === stored;
+  return stored === payload || isRecordedCheckpointValue(JSON.parse(stored), JSON.parse(payload));
 }
 function openSqlPoolQueue(journal, remote) {
   const storage = sqlPoolStorage(journal);
@@ -65299,7 +65319,7 @@ var init_startup_timeout = () => {};
 import { fork } from "node:child_process";
 import { randomUUID as randomUUID17 } from "node:crypto";
 import { createWriteStream } from "node:fs";
-import { mkdir as mkdir6, rm as rm3, rmdir } from "node:fs/promises";
+import { appendFile, mkdir as mkdir6, rm as rm3, rmdir } from "node:fs/promises";
 import { dirname as dirname14, join as join30, resolve as resolve13 } from "node:path";
 import { setTimeout as delay2 } from "node:timers/promises";
 function isRootLeaseLost(error) {
@@ -65491,6 +65511,9 @@ class PoolCoordinator {
   incarnations = new Map;
   readyWorkers = new Set;
   logs = new Map;
+  keptLogs = new Set;
+  unconfirmedExits = new WeakSet;
+  failed = false;
   cleanups = new Set;
   controlAcks = new Map;
   lastHeard = new Map;
@@ -65538,7 +65561,6 @@ class PoolCoordinator {
   nextRecoveryAt = 0;
   cleanupFailure;
   held = new Set;
-  quarantined = new Set;
   unreaped = [];
   retained = new Map;
   retainedTransitions = Promise.resolve();
@@ -65731,18 +65753,15 @@ class PoolCoordinator {
     try {
       const { survivors, unprobeable } = await reapProcessGroups(this.journal, this.root, { attempt: work });
       if (survivors.length > 0) {
-        this.quarantined.add(work.id);
         this.unreaped.push({ attempt: work });
-        this.port.send({ kind: "work-failure", workId: work.id, error: unconfirmedCleanup(survivors).message });
-        return;
+        this.reportHealth(undefined, `${unconfirmedCleanup(survivors).message}; child ${work.childRunId} of an earlier incarnation replays`);
       }
       if (unprobeable.length === 0)
         await this.releaseRetained(work);
     } catch (error) {
       this.cleanupFailure ??= error instanceof Error ? error : new Error(String(error));
     } finally {
-      if (!this.quarantined.has(work.id))
-        this.held.delete(work.id);
+      this.held.delete(work.id);
     }
   }
   releaseRetained(work) {
@@ -65873,6 +65892,7 @@ class PoolCoordinator {
     });
   }
   fail(error) {
+    this.failed = true;
     this.port.send({ kind: "failure", error: error.message });
   }
   failWork(work, error) {
@@ -66281,7 +66301,11 @@ class PoolCoordinator {
         terminate();
       });
       const closed = new Promise((resolve) => child.once("close", () => resolve()));
-      child.once("exit", () => {
+      let retired = false;
+      const retire = () => {
+        if (retired)
+          return;
+        retired = true;
         Promise.race([closed, delay2(STDERR_DRAIN_GRACE_MS)]).then(() => reject(diagnosed(timedOut ? startupTimeoutMessage(startupTimeoutMs) : "Local worker exited before readiness")));
         this.children.delete(child);
         this.logs.delete(child);
@@ -66305,7 +66329,7 @@ class PoolCoordinator {
         const held = slot.work;
         if (held)
           this.held.add(held.id);
-        let childFailed = false;
+        let unconfirmed;
         const cleanup = disconnected.then(() => serial).then(async () => {
           try {
             const admission = this.admission;
@@ -66318,18 +66342,13 @@ class PoolCoordinator {
               throw unconfirmedCleanup(alive);
             if (alive.length === 0)
               return;
-            const error = unconfirmedCleanup(alive);
-            if (held && await this.queue.result(this.root, held.id) === undefined) {
-              this.quarantined.add(held.id);
-              slot.work = undefined;
-              childFailed = true;
-              this.port.send({ kind: "work-failure", workId: held.id, error: error.message });
-            }
+            unconfirmed = unconfirmedCleanup(alive);
+            this.reportHealth(logPath, `${unconfirmed.message}; the worker was retired and replaced`);
           } catch (error) {
             this.cleanupFailure = error instanceof Error ? error : new Error(String(error));
             throw this.cleanupFailure;
           } finally {
-            if (held && !this.quarantined.has(held.id))
+            if (held)
               this.held.delete(held.id);
           }
         }).then(async () => {
@@ -66350,8 +66369,8 @@ class PoolCoordinator {
           const settled = attempt !== undefined && await this.whileRootLate(queue, () => exhausted ? queue.failExited(lease, attempt) : queue.releaseExited(lease, attempt));
           const superseded = work !== undefined && !settled && await this.superseded(work);
           if (work !== undefined && exhausted && !superseded)
-            this.failWork(work, diagnosed(childRetryLimit(work, "lost its worker")));
-          if (work === undefined && !childFailed && ++this.replacements > 3)
+            this.failWork(work, diagnosed(`${childRetryLimit(work, "lost its worker")}${unconfirmed === undefined ? "" : `; ${unconfirmed.message}`}`));
+          if (work === undefined && ++this.replacements > 3)
             this.fail(diagnosed("Local worker retry limit"));
           else
             await this.spawn();
@@ -66360,7 +66379,9 @@ class PoolCoordinator {
         });
         this.cleanups.add(cleanup);
         cleanup.finally(() => this.cleanups.delete(cleanup));
-      });
+      };
+      slot.retire = retire;
+      child.once("exit", retire);
       child.on("message", (message) => {
         this.observeResponsiveness();
         this.lastHeard.set(worker, { at: performance.now(), stalled: this.stalledMs });
@@ -66434,10 +66455,11 @@ class PoolCoordinator {
             const fenced = slot.work;
             slot.work = undefined;
             if (fenced && !await this.superseded(fenced)) {
+              this.fence(fenced);
               if (this.attempts(fenced) < MAX_CHILD_ATTEMPTS)
                 await leased(() => queue.releaseExited(lease, fenced));
               else if (await leased(() => queue.failExited(lease, fenced)))
-                this.failWork(fenced, new Error(childRetryLimit(fenced, "was refused by the root as stale on every attempt")));
+                this.failWork(fenced, new Error(childRetryLimit(fenced, message.fault === undefined ? "was refused by the root as stale on every attempt" : `failed on a fault of the pool on every attempt (last: ${message.fault})`)));
             }
             return;
           }
@@ -66598,8 +66620,6 @@ class PoolCoordinator {
         return;
       }
       for (const work of expired) {
-        if (this.quarantined.has(work.id))
-          continue;
         if (this.held.has(work.id)) {
           await queue.extend(lease, work, leaseMs).catch((error) => {
             if (error.message !== STALE_WORK_LEASE)
@@ -66615,9 +66635,9 @@ class PoolCoordinator {
           });
           continue;
         }
-        if (child && !await this.stop(child).then(() => true, () => false))
-          continue;
-        if (this.held.has(work.id) || this.quarantined.has(work.id))
+        if (child && !await this.stop(child))
+          this.retireUnconfirmed(child);
+        if (this.held.has(work.id))
           continue;
         if ((await unconfirmedProcessGroups(this.journal, root, { attempt: work })).length > 0)
           continue;
@@ -66644,17 +66664,49 @@ class PoolCoordinator {
     const heard = this.lastHeard.get(worker);
     return child.exitCode === null && child.signalCode === null && child.connected && heard !== undefined && performance.now() - heard.at - (this.stalledMs - heard.stalled) < this.options.leaseMs;
   }
-  stop(child) {
-    if (child.pid === undefined || child.exitCode !== null || child.signalCode !== null)
-      return Promise.resolve();
-    return new Promise((resolve, reject) => {
-      const timeout = setTimeout(() => reject(new Error("Local worker cleanup was not confirmed")), 5000);
-      child.once("exit", () => {
-        clearTimeout(timeout);
-        resolve();
-      });
-      child.kill("SIGKILL");
-    });
+  async stop(child) {
+    const pid = child.pid;
+    const exited = () => child.exitCode !== null || child.signalCode !== null;
+    if (pid === undefined || exited())
+      return true;
+    const exit = new Promise((resolve) => child.once("exit", () => resolve()));
+    const confirmed = async () => {
+      await Promise.race([exit, delay2(WORKER_EXIT_CONFIRMATION_MS)]);
+      return true;
+    };
+    this.observeResponsiveness();
+    child.kill("SIGKILL");
+    const since = performance.now();
+    const stalledAt = this.stalledMs;
+    for (;; ) {
+      await Promise.race([exit, delay2(WORKER_EXIT_POLL_MS)]);
+      if (exited())
+        return true;
+      if (workerProcessExited(pid))
+        return confirmed();
+      this.observeResponsiveness();
+      if (performance.now() - since - (this.stalledMs - stalledAt) >= WORKER_EXIT_CONFIRMATION_MS)
+        return exited() || workerProcessExited(pid, true) && confirmed();
+    }
+  }
+  retireUnconfirmed(child) {
+    if (this.unconfirmedExits.has(child))
+      return;
+    this.unconfirmedExits.add(child);
+    this.reportHealth(this.logs.get(child), `Local worker cleanup was not confirmed: worker process ${child.pid} was still running ${WORKER_EXIT_CONFIRMATION_MS} ms after SIGKILL; it was retired and replaced`);
+    if (child.connected)
+      child.disconnect();
+    for (const slot of this.slots)
+      if (slot.child === child)
+        slot.retire?.();
+  }
+  reportHealth(log, message) {
+    if (log !== undefined) {
+      this.keptLogs.add(log);
+      appendFile(log, `[atomic ${new Date().toISOString()}] ${message}
+`).catch(() => {});
+    }
+    this.port.send({ kind: "health", message: log === undefined ? message : `${message} (worker log: ${log})` });
   }
   async close() {
     this.closed = true;
@@ -66664,10 +66716,15 @@ class PoolCoordinator {
     this.rateLimitOutbox.close();
     let failure;
     let stopping = [];
+    let confirmed = [];
     try {
       await this.recovering?.catch(() => {});
       stopping = [...this.children].map((child) => ({ child, log: this.logs.get(child) }));
-      await Promise.all(stopping.map(({ child }) => this.stop(child)));
+      confirmed = await Promise.all(stopping.map(({ child }) => this.stop(child)));
+      stopping.forEach(({ child }, index) => {
+        if (!confirmed[index])
+          this.retireUnconfirmed(child);
+      });
       await Promise.all([...this.cleanups]);
       if (this.cleanupFailure)
         throw this.cleanupFailure;
@@ -66679,15 +66736,15 @@ class PoolCoordinator {
       if (alive.length > 0)
         throw unconfirmedCleanup(alive);
       if (this.desiredControl !== "quit" && this.queue !== undefined && this.lease !== undefined)
-        await this.queue.releaseClosed(this.lease, [...this.quarantined]).catch((error) => {
+        await this.queue.releaseClosed(this.lease).catch((error) => {
           if (!(error instanceof StaleLeaseError))
             throw error;
         });
     } catch (error) {
       failure = error;
     }
-    if (failure === undefined) {
-      const stopped = stopping.flatMap(({ child, log }) => log !== undefined && (child.exitCode !== null || child.signalCode !== null) ? [log] : []);
+    if (failure === undefined && !this.failed) {
+      const stopped = stopping.flatMap(({ log }, index) => log !== undefined && !this.keptLogs.has(log) && confirmed[index] === true ? [log] : []);
       await Promise.all(stopped.map((log) => rm3(log, { force: true }).catch(() => {})));
       if (this.logDirectory !== undefined)
         await rmdir(this.logDirectory).catch(() => {});
@@ -66713,7 +66770,7 @@ function childRetryLimit(work, reason) {
   const exportName = work.definition.slice(work.definition.lastIndexOf("#") + 1);
   return `Local worker retry limit: child ${work.childRunId} (${exportName}) ${reason} ${MAX_CHILD_ATTEMPTS} times`;
 }
-var CONTROL_ACKNOWLEDGEMENT_STALL_MS = 5000, RECOVERY_INTERVAL_MS = 500, TICK_INTERVAL_MS = 100, STALL_THRESHOLD_MS = 500, COORDINATOR_LEASE_MS, LATE_ROOT_POLL_MS, MIRROR_INTERVAL_MS = 1e4, STDERR_TAIL_CHARACTERS = 8192, STDERR_DRAIN_GRACE_MS = 1000, MAX_CHILD_ATTEMPTS = 3, ROUTE_REGISTRATION_WAIT_MS = 120000, STAGE_WATCH_BACKLOG_EVENTS = 2000, STAGE_WATCH_BACKLOG_BYTES, PENDING_STAGE_MESSAGE_EVENT2 = "atomic:workflow-pending-stage-message", STICKY_LIVE_DELIVERED_EVENT2 = "atomic:workflow-sticky-live-delivered", PENDING_STAGE_UNDELIVERABLE_EVENT2 = "atomic:workflow-pending-stage-undeliverable", ENDED_WATCHES_REMEMBERED = 1024;
+var CONTROL_ACKNOWLEDGEMENT_STALL_MS = 5000, RECOVERY_INTERVAL_MS = 500, TICK_INTERVAL_MS = 100, STALL_THRESHOLD_MS = 500, COORDINATOR_LEASE_MS, LATE_ROOT_POLL_MS, MIRROR_INTERVAL_MS = 1e4, STDERR_TAIL_CHARACTERS = 8192, STDERR_DRAIN_GRACE_MS = 1000, WORKER_EXIT_CONFIRMATION_MS = 5000, WORKER_EXIT_POLL_MS = 50, MAX_CHILD_ATTEMPTS = 3, ROUTE_REGISTRATION_WAIT_MS = 120000, STAGE_WATCH_BACKLOG_EVENTS = 2000, STAGE_WATCH_BACKLOG_BYTES, PENDING_STAGE_MESSAGE_EVENT2 = "atomic:workflow-pending-stage-message", STICKY_LIVE_DELIVERED_EVENT2 = "atomic:workflow-sticky-live-delivered", PENDING_STAGE_UNDELIVERABLE_EVENT2 = "atomic:workflow-pending-stage-undeliverable", ENDED_WATCHES_REMEMBERED = 1024;
 var init_pool_coordinator = __esm(() => {
   init_internal();
   init_pool_remote_journal();
@@ -66728,6 +66785,7 @@ var init_pool_coordinator = __esm(() => {
   init_pending_stage_route_readiness();
   init_store();
   init_workflow_infrastructure_error();
+  init_admission_host();
   init_pool_journal();
   init_pooled_watch_trace();
   init_resource_admission();
@@ -68658,6 +68716,7 @@ class LocalWorkerPool {
   works = new Map;
   attempts = new Map;
   results = new Map;
+  healthEvents = [];
   workFailures = new Map;
   changed = Promise.withResolvers();
   waiting = 0;
@@ -68715,6 +68774,9 @@ class LocalWorkerPool {
     if (options.admission && options.rootId === undefined)
       workflowResourceAdmission.set(backend, options.admission);
   }
+  get health() {
+    return this.healthEvents;
+  }
   get workerPids() {
     return [...this.pids];
   }
@@ -69308,14 +69370,8 @@ class LocalWorkerPool {
     try {
       return await this.dispatch(definition, inputs, options);
     } catch (error) {
-      if (error instanceof WorkflowInfrastructureError && [...this.workFailures.values()].includes(error))
-        throw error;
-      try {
-        if (this.desiredControl !== "quit" || !findWorkflowGracefulQuit(error))
-          await this.shutdown();
-      } catch (closeError) {
-        throw asWorkflowInfrastructureError(closeError) ?? closeError;
-      }
+      if (this.failure !== undefined || this.coordinatorLost !== undefined)
+        await this.shutdown().catch(() => {});
       throw asWorkflowInfrastructureError(error) ?? error;
     }
   }
@@ -69383,8 +69439,13 @@ class LocalWorkerPool {
     try {
       for (;; ) {
         options.signal?.throwIfAborted();
-        if (this.closed)
-          throw this.closedError(root);
+        if (!this.results.has(work.id) && (this.closed || this.failure !== undefined)) {
+          const recorded = await this.queue?.result(root, work.id).catch(() => {
+            return;
+          });
+          if (recorded !== undefined)
+            this.results.set(work.id, recorded);
+        }
         const result = this.results.get(work.id);
         if (result) {
           this.completing++;
@@ -69408,6 +69469,8 @@ class LocalWorkerPool {
           this.endFailedChild(work, failed);
           throw failed;
         }
+        if (this.closed)
+          throw this.closedError(root);
         if (this.failure)
           throw this.failure;
         if (this.budgetBlocked)
@@ -69442,6 +69505,19 @@ class LocalWorkerPool {
     if (this.reconnectingViews.size > 0 || this.outboxes.size > 0)
       this.reconnectStageViews();
   }
+  recordHealth(message) {
+    this.healthEvents.push(message);
+    const root = this.root;
+    if (root === undefined || this.store === undefined)
+      return;
+    this.store.recordNotice({
+      id: `local-worker-health:${root}:${this.healthEvents.length}`,
+      runId: root,
+      level: "warning",
+      message: `Local worker pool: ${message}`,
+      createdAt: Date.now()
+    });
+  }
   evaluateBudget() {
     const decision = this.rootBudget?.checkpoint();
     const blocked = decision?.kind === "exhausted" || decision?.kind === "wrap_up";
@@ -69560,6 +69636,9 @@ class LocalWorkerPool {
         this.failure ??= new Error(event.error);
         this.signalChange();
         return;
+      case "health":
+        this.recordHealth(event.message);
+        return;
       case "stage-events":
         this.stageWatches.get(event.watch)?.({ kind: "events", events: event.events });
         this.notice({ kind: "stage-events-applied", watch: event.watch });