@ricsam/r5d-worker 0.0.107 → 0.0.109

package/dist/cjs/recovery-store.cjs

@@ -0,0 +1,229 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var recovery_store_exports = {};
+__export(recovery_store_exports, {
+  WorkerRecoveryStore: () => WorkerRecoveryStore,
+  workerOperationDigest: () => workerOperationDigest
+});
+module.exports = __toCommonJS(recovery_store_exports);
+var import_bun_sqlite = require("bun:sqlite");
+var import_node_fs = __toESM(require("node:fs"), 1);
+var import_node_path = __toESM(require("node:path"), 1);
+var import_node_crypto = require("node:crypto");
+function workerOperationDigest(message) {
+  const canonical = (value) => {
+    if (Array.isArray(value)) return value.map(canonical);
+    if (value && typeof value === "object")
+      return Object.fromEntries(
+        Object.entries(value).filter(([, item]) => item !== void 0).sort(([a], [b]) => a.localeCompare(b)).map(([key, item]) => [key, canonical(item)])
+      );
+    return value;
+  };
+  const { requestId: _requestId, ...identity } = message;
+  return (0, import_node_crypto.createHash)("sha256").update(JSON.stringify(canonical(identity))).digest("hex");
+}
+class WorkerRecoveryStore {
+  db;
+  ledgerId;
+  constructor(filename) {
+    import_node_fs.default.mkdirSync(import_node_path.default.dirname(filename), { recursive: true, mode: 448 });
+    this.db = new import_bun_sqlite.Database(filename, { create: true });
+    import_node_fs.default.chmodSync(filename, 384);
+    this.db.exec(`PRAGMA journal_mode=WAL; PRAGMA synchronous=FULL;
+      CREATE TABLE IF NOT EXISTS metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL);
+      CREATE TABLE IF NOT EXISTS operations (request_id TEXT PRIMARY KEY, run_id TEXT, session_id TEXT, request_type TEXT NOT NULL, digest TEXT NOT NULL, state TEXT NOT NULL, response TEXT);
+      CREATE UNIQUE INDEX IF NOT EXISTS operation_run_start ON operations(run_id) WHERE request_type='exec_start';
+      CREATE TABLE IF NOT EXISTS aliases (request_id TEXT PRIMARY KEY, operation_id TEXT NOT NULL);
+      CREATE TABLE IF NOT EXISTS streams (run_id TEXT NOT NULL, stream TEXT NOT NULL, bytes INTEGER NOT NULL DEFAULT 0, acknowledged INTEGER NOT NULL DEFAULT 0, PRIMARY KEY(run_id, stream));
+      CREATE TABLE IF NOT EXISTS output (run_id TEXT NOT NULL, stream TEXT NOT NULL, offset INTEGER NOT NULL, bytes INTEGER NOT NULL, data TEXT NOT NULL, PRIMARY KEY(run_id, stream, offset));
+      CREATE TABLE IF NOT EXISTS terminals (run_id TEXT PRIMARY KEY, message TEXT NOT NULL);
+      CREATE TABLE IF NOT EXISTS cancelled_requests (request_id TEXT PRIMARY KEY);
+    `);
+    this.db.run("INSERT OR IGNORE INTO metadata(key,value) VALUES ('ledgerId',?)", [crypto.randomUUID()]);
+    this.ledgerId = this.db.query("SELECT value FROM metadata WHERE key='ledgerId'").get().value;
+    this.db.run("UPDATE operations SET state='unknown' WHERE state IN ('accepted','preparing','running')");
+  }
+  close() {
+    this.db.close();
+  }
+  row(requestId) {
+    return this.db.query("SELECT * FROM operations WHERE request_id=? OR request_id=(SELECT operation_id FROM aliases WHERE request_id=?)").get(requestId, requestId);
+  }
+  admit(message) {
+    const digest = workerOperationDigest(message);
+    let existing = this.row(message.requestId);
+    if (!existing && message.type === "exec_start" && typeof message.runId === "string")
+      existing = this.db.query("SELECT * FROM operations WHERE run_id=? AND request_type='exec_start'").get(message.runId);
+    if (existing) {
+      if (existing.digest !== digest) return "conflict";
+      this.db.run("INSERT OR IGNORE INTO aliases(request_id, operation_id) VALUES (?,?)", [message.requestId, existing.request_id]);
+      return existing.state === "unknown" ? "unknown" : "duplicate";
+    }
+    this.db.run("INSERT INTO operations(request_id, run_id, session_id, request_type, digest, state) VALUES (?,?,?,?,?,'accepted')", [
+      message.requestId,
+      typeof message.runId === "string" ? message.runId : null,
+      typeof message.sessionId === "string" ? message.sessionId : null,
+      message.type,
+      digest
+    ]);
+    return "new";
+  }
+  isUnknown(requestId) {
+    return this.row(requestId)?.state === "unknown";
+  }
+  unknown(requestId) {
+    const row = this.row(requestId);
+    if (row) this.db.run("UPDATE operations SET state='unknown' WHERE request_id=?", [row.request_id]);
+  }
+  unknownRun(runId) {
+    this.db.run("UPDATE operations SET state='unknown' WHERE run_id=? AND request_type IN ('exec_start','exec')", [runId]);
+  }
+  record(message) {
+    if (message.type === "exec_output" && typeof message.runId === "string" && (message.stream === "stdout" || message.stream === "stderr") && typeof message.data === "string") {
+      const runId = message.runId, stream = message.stream, data = message.data;
+      const offset = this.db.transaction(() => {
+        this.db.run("INSERT OR IGNORE INTO streams(run_id, stream) VALUES (?,?)", [runId, stream]);
+        const offset2 = this.db.query("SELECT bytes FROM streams WHERE run_id=? AND stream=?").get(runId, stream).bytes;
+        const bytes = Buffer.byteLength(data);
+        if (bytes) {
+          this.db.run("INSERT INTO output(run_id, stream, offset, bytes, data) VALUES (?,?,?,?,?)", [runId, stream, offset2, bytes, data]);
+          this.db.run("UPDATE streams SET bytes=bytes+? WHERE run_id=? AND stream=?", [bytes, runId, stream]);
+        }
+        return offset2;
+      })();
+      return { ...message, offset };
+    }
+    if ((message.type === "exec_exit" || message.type === "exec_error") && typeof message.runId === "string") {
+      const terminal = { ...message, ...this.lengths(message.runId) };
+      this.db.transaction(() => {
+        this.db.run("INSERT OR REPLACE INTO terminals(run_id,message) VALUES (?,?)", [message.runId, JSON.stringify(terminal)]);
+        this.db.run("UPDATE operations SET state='completed' WHERE run_id=? AND request_type='exec_start'", [message.runId]);
+      })();
+      return terminal;
+    }
+    if (typeof message.requestId === "string") {
+      const row = this.row(message.requestId);
+      if (row) {
+        const state = message.type === "exec_accepted" || message.type === "operation_queued" ? "preparing" : message.type === "exec_started" ? "running" : message.type === "operation_received" ? "accepted" : "completed";
+        if (message.type !== "operation_received")
+          this.db.run("UPDATE operations SET state=?, response=? WHERE request_id=?", [state, JSON.stringify(message), row.request_id]);
+      }
+    }
+    return message;
+  }
+  operations() {
+    return this.db.query(
+      "SELECT * FROM operations WHERE response IS NOT NULL OR state!='completed' UNION ALL SELECT aliases.request_id,operations.run_id,operations.session_id,operations.request_type,operations.digest,operations.state,operations.response FROM aliases JOIN operations ON operations.request_id=aliases.operation_id WHERE aliases.request_id!=operations.request_id AND (response IS NOT NULL OR state!='completed')"
+    ).all().map((row) => ({
+      requestId: row.request_id,
+      requestType: row.request_type,
+      state: row.state,
+      ...row.run_id ? { runId: row.run_id } : {},
+      ...row.session_id ? { sessionId: row.session_id } : {}
+    }));
+  }
+  response(requestId) {
+    const row = this.row(requestId);
+    return row?.response && row.state !== "unknown" ? { ...JSON.parse(row.response), requestId } : void 0;
+  }
+  responses(requestIds) {
+    const ids = /* @__PURE__ */ new Set([...requestIds, ...this.operations().map((operation) => operation.requestId)]);
+    return [...ids].flatMap((requestId) => {
+      const response = this.response(requestId);
+      return response ? [response] : [];
+    });
+  }
+  lengths(runId) {
+    const result = { stdoutBytes: 0, stderrBytes: 0 };
+    for (const row of this.db.query("SELECT stream,bytes FROM streams WHERE run_id=?").all(runId))
+      result[row.stream === "stdout" ? "stdoutBytes" : "stderrBytes"] = row.bytes;
+    return result;
+  }
+  outputCursors() {
+    const cursors = /* @__PURE__ */ new Map();
+    for (const stream of this.db.query("SELECT run_id,stream,acknowledged FROM streams").all()) {
+      const cursor = cursors.get(stream.run_id) ?? { runId: stream.run_id, stdoutBytes: 0, stderrBytes: 0 };
+      cursor[stream.stream === "stdout" ? "stdoutBytes" : "stderrBytes"] = stream.acknowledged;
+      cursors.set(stream.run_id, cursor);
+    }
+    return [...cursors.values()];
+  }
+  *output(offsets = []) {
+    const cursors = new Map(offsets.map((offset) => [offset.runId, offset]));
+    for (const stream of this.db.query("SELECT * FROM streams WHERE NOT EXISTS (SELECT 1 FROM operations WHERE operations.run_id=streams.run_id AND operations.state='unknown') ORDER BY rowid").all()) {
+      const supplied = cursors.get(stream.run_id);
+      const cursor = supplied ? stream.stream === "stdout" ? supplied.stdoutBytes : supplied.stderrBytes : stream.acknowledged;
+      if (cursor >= stream.bytes) continue;
+      const firstOffset = this.db.query("SELECT MAX(offset) AS offset FROM output WHERE run_id=? AND stream=? AND offset<=?").get(stream.run_id, stream.stream, cursor)?.offset ?? 0;
+      let nextChunk = firstOffset;
+      while (true) {
+        const row = this.db.query("SELECT offset,data FROM output WHERE run_id=? AND stream=? AND offset>=? AND offset<? AND offset+bytes>? ORDER BY offset LIMIT 1").get(stream.run_id, stream.stream, nextChunk, stream.bytes, cursor);
+        if (!row) break;
+        nextChunk = row.offset + 1;
+        yield { type: "exec_output", runId: stream.run_id, stream: stream.stream, offset: row.offset, data: row.data };
+      }
+    }
+  }
+  terminals() {
+    return this.db.query(
+      "SELECT message FROM terminals WHERE NOT EXISTS (SELECT 1 FROM operations WHERE operations.run_id=terminals.run_id AND operations.state='unknown')"
+    ).all().map((row) => JSON.parse(row.message));
+  }
+  acknowledgeOutput(runId, stream, offset) {
+    if (!Number.isSafeInteger(offset) || offset < 0) return;
+    this.db.run("UPDATE streams SET acknowledged=MAX(acknowledged,MIN(bytes,?)) WHERE run_id=? AND stream=?", [offset, runId, stream]);
+  }
+  acknowledgeResult(requestId) {
+    const row = this.row(requestId);
+    if (row && row.request_type !== "exec_start")
+      this.db.run("UPDATE operations SET response=NULL WHERE request_id=? AND state='completed'", [row.request_id]);
+  }
+  acknowledgeTerminal(runId) {
+    this.db.transaction(() => {
+      this.db.run("DELETE FROM terminals WHERE run_id=?", [runId]);
+      this.db.run("DELETE FROM output WHERE run_id=?", [runId]);
+      this.db.run("UPDATE streams SET acknowledged=bytes WHERE run_id=?", [runId]);
+      this.db.run("UPDATE operations SET response=NULL WHERE run_id=? AND request_type='exec_start' AND state='completed'", [runId]);
+    })();
+  }
+  cancelSession(sessionId) {
+    this.db.run("INSERT OR IGNORE INTO cancelled_requests SELECT request_id FROM operations WHERE session_id=? AND state!='completed'", [
+      sessionId
+    ]);
+  }
+  isRequestCancelled(requestId) {
+    const row = this.row(requestId);
+    return !!row && this.db.query("SELECT request_id FROM cancelled_requests WHERE request_id=?").get(row.request_id) !== null;
+  }
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  WorkerRecoveryStore,
+  workerOperationDigest
+});

package/dist/cjs/supervisor.cjs

@@ -22,6 +22,7 @@ __export(supervisor_exports, {
   WORKER_RECONNECT_DELAY_MS: () => WORKER_RECONNECT_DELAY_MS,
   WORKER_RECONNECT_EXIT_CODE: () => WORKER_RECONNECT_EXIT_CODE,
   WORKER_RECONNECT_JITTER_MS: () => WORKER_RECONNECT_JITTER_MS,
+  WORKER_RECONNECT_MAX_DELAY_MS: () => WORKER_RECONNECT_MAX_DELAY_MS,
   WORKER_RELOAD_EXIT_CODE: () => WORKER_RELOAD_EXIT_CODE,
   WORKER_RUNTIME_ENV: () => WORKER_RUNTIME_ENV,
   isRetryableWorkerServerStatus: () => isRetryableWorkerServerStatus,
@@ -37,6 +38,7 @@ const WORKER_RECONNECT_EXIT_CODE = 76;
 const WORKER_CREDENTIAL_RESTART_EXIT_CODE = 77;
 const WORKER_RECONNECT_DELAY_MS = 2e3;
 const WORKER_RECONNECT_JITTER_MS = 3e3;
+const WORKER_RECONNECT_MAX_DELAY_MS = 3e4;
 const SUPERVISOR_SIGNALS = ["SIGINT", "SIGTERM", "SIGHUP"];
 function defaultRuntimeSpawner(argv, options) {
   return Bun.spawn(argv, options);
@@ -44,9 +46,12 @@ function defaultRuntimeSpawner(argv, options) {
 function defaultRuntimeDelay(delayMs) {
   return new Promise((resolve) => setTimeout(resolve, delayMs));
 }
-function workerReconnectDelayMs(random = Math.random) {
+function workerReconnectDelayMs(random = Math.random, attempt = 0) {
   const sample = Math.max(0, Math.min(1, random()));
-  return WORKER_RECONNECT_DELAY_MS + Math.floor(sample * WORKER_RECONNECT_JITTER_MS);
+  const exponent = Math.min(20, Math.max(0, Math.floor(attempt)));
+  const ceiling = Math.min(WORKER_RECONNECT_MAX_DELAY_MS, (WORKER_RECONNECT_DELAY_MS + WORKER_RECONNECT_JITTER_MS) * 2 ** exponent);
+  const floor = Math.min(WORKER_RECONNECT_DELAY_MS * 2 ** exponent, ceiling * 0.4);
+  return floor + Math.floor(sample * (ceiling - floor));
 }
 async function defaultRuntimeReaper(runtime) {
   await (0, import_process_tree.terminateProcessTree)(runtime);
@@ -60,10 +65,11 @@ function signalExitCode(signal) {
 function osSignalNumber(signal) {
   return import_node_os.constants.signals[signal];
 }
-async function superviseWorkerRuntime(argv, env = process.env, spawnRuntime = defaultRuntimeSpawner, waitBeforeReconnect = defaultRuntimeDelay, reapCredentialRuntime = defaultRuntimeReaper, reconnectDelay = workerReconnectDelayMs) {
+async function superviseWorkerRuntime(argv, env = process.env, spawnRuntime = defaultRuntimeSpawner, waitBeforeReconnect = defaultRuntimeDelay, reapCredentialRuntime = defaultRuntimeReaper, reconnectDelay) {
   if (!process.argv[1]) {
     throw new Error("Cannot locate the r5d-worker entrypoint");
   }
+  let reconnectAttempt = 0;
   while (true) {
     const runtime = spawnRuntime([process.execPath, process.argv[1], ...argv], {
       stdin: "inherit",
@@ -112,7 +118,7 @@ async function superviseWorkerRuntime(argv, env = process.env, spawnRuntime = de
         continue;
       }
       if (exitCode === WORKER_RECONNECT_EXIT_CODE) {
-        const reconnectDelayMs = reconnectDelay();
+        const reconnectDelayMs = reconnectDelay?.() ?? workerReconnectDelayMs(Math.random, reconnectAttempt++);
         process.stderr.write(`[r5d-worker] reconnecting in ${reconnectDelayMs}ms
 `);
         await waitBeforeReconnect(reconnectDelayMs);
@@ -132,6 +138,7 @@ async function superviseWorkerRuntime(argv, env = process.env, spawnRuntime = de
   WORKER_RECONNECT_DELAY_MS,
   WORKER_RECONNECT_EXIT_CODE,
   WORKER_RECONNECT_JITTER_MS,
+  WORKER_RECONNECT_MAX_DELAY_MS,
   WORKER_RELOAD_EXIT_CODE,
   WORKER_RUNTIME_ENV,
   isRetryableWorkerServerStatus,