@bastani/atomic 0.9.44 → 0.9.45-alpha.1

package/dist/builtin/workflows/src/index.js

@@ -923,6 +923,78 @@ function registerRunJournalConformance(runner, name, open, observeRead) {
       assert.deepEqual(await entries(journal), []);
     });
   }
+  check("optional appendMany settles each append in order and fences a stale lease", async (journal) => {
+    if (journal.appendMany === undefined)
+      return;
+    await journal.createRun(header);
+    const held = await lease(journal);
+    const context = (action) => action();
+    const results = await journal.appendMany(held, [
+      { event: { type: "RunStarted", payload: null }, key: "start", context },
+      { event: { type: "StatusChanged", payload: { status: "waiting" } }, context },
+      { event: { type: "RunFinished", payload: null }, key: "start", context }
+    ]);
+    assert.deepEqual(results.map((result) => result.status === "fulfilled" ? [result.value.status, result.value.entry.seq] : []), [
+      ["appended", 1],
+      ["appended", 2],
+      ["exists", 1]
+    ]);
+    assert.deepEqual((await entries(journal)).map((entry) => entry.event.type), ["RunStarted", "StatusChanged"]);
+    assert.equal((await journal.getRun(header.runId))?.status, "waiting");
+    assert.equal((await journal.takeover(header.runId, "next", held.holderId)).status, "acquired");
+    await assert.rejects(journal.appendMany(held, [{ event: { type: "RunFinished", payload: null }, context }]), StaleLeaseError);
+    assert.equal((await entries(journal)).length, 2);
+  });
+  check("optional appendMany isolates a failed append from its valid neighbours", async (journal) => {
+    if (journal.appendMany === undefined)
+      return;
+    await journal.createRun(header);
+    const held = await lease(journal);
+    const context = (action) => action();
+    const results = await journal.appendMany(held, [
+      { event: { type: "RunStarted", payload: null }, context },
+      { event: { type: "StatusChanged", payload: { status: "bad\x00status" } }, context },
+      { event: { type: "StatusChanged", payload: { status: "waiting" } }, context }
+    ]);
+    assert.deepEqual(results.map((result) => result.status), ["fulfilled", "rejected", "fulfilled"]);
+    const [started, invalid, waiting] = results;
+    assert.ok(invalid?.status === "rejected" && invalid.reason instanceof JournalInvalidTextError);
+    assert.deepEqual([started, waiting].map((result) => result?.status === "fulfilled" ? result.value.entry.seq : undefined), [1, 2]);
+    assert.deepEqual((await entries(journal)).map((entry) => entry.event.type), ["RunStarted", "StatusChanged"]);
+    assert.equal((await journal.getRun(header.runId))?.status, "waiting");
+  });
+  check("optional change tracking reports run versions and snapshot tails", async (journal) => {
+    const members = [journal.changeToken, journal.runVersions, journal.snapshotTail];
+    if (members.every((member) => member === undefined))
+      return;
+    const { changeToken, runVersions, snapshotTail } = journal;
+    if (changeToken === undefined || runVersions === undefined || snapshotTail === undefined)
+      return assert.fail("changeToken, runVersions and snapshotTail must be implemented together");
+    const empty = await changeToken.call(journal);
+    assert.equal(await changeToken.call(journal), empty);
+    await journal.createRun(header);
+    const created = await changeToken.call(journal);
+    assert.notEqual(created, empty);
+    assert.deepEqual(await runVersions.call(journal), new Map([[header.runId, { seq: 0, at: 0 }]]));
+    assert.equal(await snapshotTail.call(journal, header.runId), undefined);
+    const held = await lease(journal);
+    const started = await journal.append(held, { type: "RunStarted", payload: null });
+    await journal.append(held, { type: "CheckpointRecorded", payload: { kind: "tool", completedAt: 5 } });
+    const checkpoint = await journal.append(held, {
+      type: "CheckpointRecorded",
+      payload: { kind: "tool", completedAt: 7 }
+    });
+    assert.notEqual(await changeToken.call(journal), created);
+    const latest = { seq: checkpoint.entry.seq, at: checkpoint.entry.at };
+    assert.deepEqual(await runVersions.call(journal), new Map([[header.runId, latest]]));
+    assert.deepEqual(await snapshotTail.call(journal, header.runId), {
+      version: latest,
+      snapshot: started.entry,
+      laterCheckpoints: 2,
+      lastCheckpointAt: 7
+    });
+    assert.equal(await snapshotTail.call(journal, "absent"), undefined);
+  });
   check("closed journals reject operations and started reads while close remains idempotent", async (journal) => {
     await journal.createRun(header);
     const held = await lease(journal);
@@ -977,17 +1049,21 @@ var init_conformance = __esm(() => {
 });
 
 // dist/builtin/workflows/src/durable/journal/internal.ts
-function registerRunJournalVersions(journal, versions) {
-  versionReaders.set(journal, versions);
-}
 function runJournalVersions(journal) {
-  return versionReaders.get(journal);
-}
-function registerRunJournalAppendMany(journal, appendMany) {
-  appendManyWriters.set(journal, appendMany);
+  const { changeToken, runVersions, snapshotTail } = journal;
+  if (typeof changeToken !== "function" || typeof runVersions !== "function" || typeof snapshotTail !== "function")
+    return;
+  return {
+    changeToken: () => changeToken.call(journal),
+    runVersions: () => runVersions.call(journal),
+    snapshotTail: (runId) => snapshotTail.call(journal, runId)
+  };
 }
 function runJournalAppendMany(journal) {
-  return appendManyWriters.get(journal);
+  const { appendMany } = journal;
+  if (typeof appendMany !== "function")
+    return;
+  return (lease, appends) => appendMany.call(journal, lease, appends);
 }
 function registerSqlPoolStorage(journal, storage) {
   poolStorage.set(journal, storage);
@@ -995,7 +1071,7 @@ function registerSqlPoolStorage(journal, storage) {
 function sqlPoolStorage(journal) {
   return poolStorage.get(journal);
 }
-var JOURNAL_READ_PAGE_SIZE = 256, NON_SNAPSHOT_EVENT_TYPES, versionReaders, appendManyWriters, poolStorage;
+var JOURNAL_READ_PAGE_SIZE = 256, NON_SNAPSHOT_EVENT_TYPES, poolStorage;
 var init_internal = __esm(() => {
   init_types();
   NON_SNAPSHOT_EVENT_TYPES = [
@@ -1003,8 +1079,6 @@ var init_internal = __esm(() => {
     RUN_EVENT_TYPES.messageSettled,
     RUN_EVENT_TYPES.messageDelivered
   ];
-  versionReaders = new WeakMap;
-  appendManyWriters = new WeakMap;
   poolStorage = new WeakMap;
 });
 
@@ -1383,12 +1457,6 @@ class SqlRunJournal {
     this.database = database;
     this.tables = tables;
     this.legacy = legacy;
-    registerRunJournalVersions(this, {
-      changeToken: () => this.changeToken(),
-      runVersions: () => this.runVersions(),
-      snapshotTail: (runId) => this.snapshotTail(runId)
-    });
-    registerRunJournalAppendMany(this, (lease, appends) => this.appendMany(lease, appends));
     registerSqlPoolStorage(this, {
       transaction: (body) => this.operation(() => this.database.transaction(body)),
       withAppendGuard: (guard, action) => this.appendGuard.run(guard, action),
@@ -1808,6 +1876,9 @@ import {
   StorageProvider,
   selectStorage
 } from "@bastani/atomic/storage";
+function provideRunJournalHostModules(modules) {
+  hostModules = modules;
+}
 function selectRunJournal(options = {}) {
   const agentDir = options.agentDir ?? getAgentDir();
   const defaultPath = join(dirname(agentDir), "workflows", "journal.sqlite");
@@ -1876,13 +1947,17 @@ async function withRunJournalConfigSource(selection, open) {
     throw error;
   }
 }
-var provider, origins;
+var hostModules, provider, origins;
 var init_resolution = __esm(() => {
   init_memory_run_journal();
   init_sql_run_journal();
+  hostModules = {};
   provider = new StorageProvider({
     memory: (descriptor) => new MemoryRunJournal(descriptor),
-    sql: SqlRunJournal.open
+    sql: SqlRunJournal.open,
+    get hostModules() {
+      return hostModules;
+    }
   });
   origins = new WeakMap;
 });
@@ -11601,6 +11676,13 @@ var init_journal_backend = __esm(() => {
   entryJson = new WeakMap;
 });
 
+// dist/builtin/workflows/src/durable/journal/workflows-host-module.ts
+var init_workflows_host_module = __esm(() => {
+  init_sdk_surface();
+  init_resolution();
+  provideRunJournalHostModules({ "@bastani/atomic/workflows": exports_sdk_surface });
+});
+
 // dist/builtin/workflows/src/durable/factory.ts
 import { getAgentDir as getAgentDir3, SettingsManager } from "@bastani/atomic";
 import { storageSelectionKey } from "@bastani/atomic/storage";
@@ -11653,6 +11735,7 @@ var init_factory = __esm(() => {
   init_backend_process_owner();
   init_journal_backend();
   init_resolution();
+  init_workflows_host_module();
 });
 
 // dist/builtin/workflows/src/durable/run-result.ts
@@ -63320,6 +63403,23 @@ var init_sql_pool_usage = __esm(() => {
   init_values();
 });
 
+// dist/builtin/workflows/src/workers/startup-timeout.ts
+import { availableParallelism, loadavg as loadavg2 } from "node:os";
+function currentHostCapacity() {
+  return { cpus: availableParallelism(), load: loadavg2()[0] ?? 0 };
+}
+function defaultStartupTimeoutMs(workers, host = currentHostCapacity()) {
+  const cores = Math.max(1, host.cpus);
+  const demand = workers + Math.max(0, host.load);
+  const oversubscription = Math.max(1, demand / cores);
+  return Math.min(MAX_DEFAULT_STARTUP_TIMEOUT_MS, Math.ceil(BASE_STARTUP_TIMEOUT_MS * oversubscription));
+}
+function startupTimeoutMessage(startupTimeoutMs) {
+  return `Local worker startup timed out after ${startupTimeoutMs} ms: workers were too slow to start, usually because the machine is busy or the pool is large for its cores. Raise workflows.localWorkers.startupTimeoutMs (localWorkers.startupTimeoutMs in the SDK) to allow more time`;
+}
+var BASE_STARTUP_TIMEOUT_MS = 30000, MAX_DEFAULT_STARTUP_TIMEOUT_MS = 600000;
+var init_startup_timeout = () => {};
+
 // dist/builtin/workflows/src/workers/pool-coordinator.ts
 import { fork } from "node:child_process";
 import { randomUUID as randomUUID15 } from "node:crypto";
@@ -63860,7 +63960,7 @@ class PoolCoordinator {
       });
       const closed = new Promise((resolve) => child.once("close", () => resolve()));
       child.once("exit", () => {
-        Promise.race([closed, delay2(STDERR_DRAIN_GRACE_MS)]).then(() => reject(diagnosed(timedOut ? `Local worker startup timed out after ${startupTimeoutMs} ms` : "Local worker exited before readiness")));
+        Promise.race([closed, delay2(STDERR_DRAIN_GRACE_MS)]).then(() => reject(diagnosed(timedOut ? startupTimeoutMessage(startupTimeoutMs) : "Local worker exited before readiness")));
         this.children.delete(child);
         this.logs.delete(child);
         if (child.pid !== undefined)
@@ -64149,6 +64249,7 @@ class PoolCoordinator {
 var CONTROL_ACKNOWLEDGEMENT_TIMEOUT_MS = 5000, RECOVERY_INTERVAL_MS = 500, TICK_INTERVAL_MS = 100, STALL_THRESHOLD_MS = 500, COORDINATOR_LEASE_MS, STDERR_TAIL_CHARACTERS = 8192, STDERR_DRAIN_GRACE_MS = 1000, ROUTE_REGISTRATION_WAIT_MS = 120000, PENDING_STAGE_MESSAGE_EVENT2 = "atomic:workflow-pending-stage-message", STICKY_LIVE_DELIVERED_EVENT2 = "atomic:workflow-sticky-live-delivered", PENDING_STAGE_UNDELIVERABLE_EVENT2 = "atomic:workflow-pending-stage-undeliverable";
 var init_pool_coordinator = __esm(() => {
   init_resolution();
+  init_workflows_host_module();
   init_sql_pool_queue();
   init_sql_pool_usage();
   init_pending_stage_intercom();
@@ -64157,6 +64258,7 @@ var init_pool_coordinator = __esm(() => {
   init_pending_stage_route_readiness();
   init_store();
   init_resource_admission();
+  init_startup_timeout();
   COORDINATOR_LEASE_MS = 3 * STAGE_SESSION_HEARTBEAT_INTERVAL_MS;
 });
 
@@ -65013,7 +65115,7 @@ class LocalWorkerPool {
         journalPath: this.options.journalPath,
         workers: this.options.workers,
         leaseMs: this.leaseMs,
-        startupTimeoutMs: this.options.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS,
+        startupTimeoutMs: this.options.startupTimeoutMs ?? defaultStartupTimeoutMs(this.options.workers),
         ...this.options.admission === undefined ? {} : { admission: this.options.admission },
         ...workerEntry === undefined ? {} : { workerEntry }
       },
@@ -65434,7 +65536,7 @@ class LocalWorkerPool {
       throw failure;
   }
 }
-var INTERNAL_WORKFLOW_WORKER_ARG = "--internal-workflow-worker", DEFAULT_WORK_LEASE_MS = 30000, MINIMUM_WORK_LEASE_MS, DEFAULT_STARTUP_TIMEOUT_MS = 30000, MAX_TIMER_DELAY_MS = 2147483647, COORDINATOR_STDERR_TAIL_CHARACTERS = 8192, COORDINATOR_EXIT_GRACE_MS = 5000;
+var INTERNAL_WORKFLOW_WORKER_ARG = "--internal-workflow-worker", DEFAULT_WORK_LEASE_MS = 30000, MINIMUM_WORK_LEASE_MS, MAX_TIMER_DELAY_MS = 2147483647, COORDINATOR_STDERR_TAIL_CHARACTERS = 8192, COORDINATOR_EXIT_GRACE_MS = 5000;
 var init_local_pool = __esm(() => {
   init_internal();
   init_sql_pool_queue();
@@ -65449,6 +65551,7 @@ var init_local_pool = __esm(() => {
   init_host_options();
   init_pool_coordinator();
   init_stage_handoff();
+  init_startup_timeout();
   MINIMUM_WORK_LEASE_MS = 2 * WORKER_HEARTBEAT_INTERVAL_MS;
 });
 
@@ -75705,6 +75808,7 @@ var init_sdk_surface = __esm(() => {
   init_workflow_infrastructure_error();
   init_definitions();
   init_registry();
+  init_workflows_host_module();
   init_types();
 });