@bastani/atomic 0.9.41 → 0.9.42-alpha.1

package/dist/builtin/intercom/broker/pending-send-registry.ts

@@ -5,6 +5,12 @@ export { buildSendSignature } from "./send-signature.js";
 const DEFAULT_GENERATION_TTL_MS = 10 * 60 * 1000;
 const DEFAULT_MAX_GENERATIONS = 10_000;
 
+export class PendingSendTimeoutError extends Error {
+  constructor() {
+    super("Send timeout");
+  }
+}
+
 export interface SendOptionsLike extends LogicalSendOptions {}
 
 export interface SendResultLike {
@@ -29,6 +35,7 @@ interface OwnedPendingSend extends PendingSendAttempt {
   resolve(result: SendResultLike): void;
   reject(error: Error): void;
   timer?: ReturnType<typeof setTimeout>;
+  timedOut?: boolean;
 }
 
 export interface PendingSendAcquisition {
@@ -106,7 +113,7 @@ export class PendingSendRegistry {
       generation,
     };
     attempt.timer = setTimeout(() => {
-      this.reject(attempt, new Error("Send timeout"));
+      this.expire(attempt);
     }, timeoutMs);
     this.attempts.set(messageId, attempt);
     return { attempt, owner: true };
@@ -114,7 +121,7 @@ export class PendingSendRegistry {
 
   resolve(messageId: string, attemptId: string, result: SendResultLike): boolean {
     const attempt = this.attempts.get(messageId);
-    if (!attempt || attempt.attemptId !== attemptId) return false;
+    if (!attempt || attempt.timedOut || attempt.attemptId !== attemptId) return false;
     this.attempts.delete(messageId);
     if (attempt.timer) clearTimeout(attempt.timer);
     this.touchGeneration(attempt);
@@ -128,10 +135,10 @@ export class PendingSendRegistry {
    */
   extend(messageId: string, attemptId: string | undefined, timeoutMs: number): boolean {
     const attempt = this.attempts.get(messageId);
-    if (!attempt || (attemptId === undefined ? !attempt.legacyEligible : attempt.attemptId !== attemptId)) return false;
+    if (!attempt || attempt.timedOut || (attemptId === undefined ? !attempt.legacyEligible : attempt.attemptId !== attemptId)) return false;
     if (attempt.timer) clearTimeout(attempt.timer);
     attempt.timer = setTimeout(() => {
-      this.reject(attempt, new Error("Send timeout"));
+      this.expire(attempt);
     }, timeoutMs);
     return true;
   }
@@ -139,10 +146,25 @@ export class PendingSendRegistry {
   /** Resolve a pre-attemptId response only for an ID's first active generation. */
   resolveLegacy(messageId: string, result: SendResultLike): boolean {
     const attempt = this.attempts.get(messageId);
-    if (!attempt?.legacyEligible) return false;
+    if (!attempt?.legacyEligible || attempt.timedOut) return false;
     return this.resolve(messageId, attempt.attemptId, result);
   }
 
+  private expire(attempt: OwnedPendingSend): void {
+    if (this.attempts.get(attempt.messageId) !== attempt) return;
+    attempt.timedOut = true;
+    attempt.timer = undefined;
+    attempt.reject(new PendingSendTimeoutError());
+  }
+
+  release(attempt: PendingSendAttempt): boolean {
+    const current = this.attempts.get(attempt.messageId);
+    if (current !== attempt || !current.timedOut) return false;
+    this.attempts.delete(attempt.messageId);
+    this.touchGeneration(current);
+    return true;
+  }
+
   reject(attempt: PendingSendAttempt, error: Error): boolean {
     const current = this.attempts.get(attempt.messageId);
     if (current !== attempt) return false;

package/dist/builtin/intercom/broker/shared-storage-setting.ts

@@ -0,0 +1,60 @@
+import { createHash } from "node:crypto";
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import type { JsonWireValue } from "./framing.js";
+import { BROKER_STORAGE_ENV } from "./paths.js";
+
+/**
+ * The raw shared `storage` selection, read without the host package so the broker stays self-contained: the
+ * selection the launching process handed this broker in `ATOMIC_STORAGE` (its `--storage`, `--storage-file` or own
+ * `ATOMIC_STORAGE`), otherwise the top-level `storage` value of the agent directory's settings. Relative paths in
+ * either resolve against the agent directory's parent, as for sessions and the journal. `undefined` leaves
+ * multi-node routing off: no selection, a per-process selection that is not a JSON object (the launching process
+ * validated it; settings are never consulted instead), and the selections the storage layer reads as in-memory
+ * (`memory: true` without a path, or `path: ":memory:"`), which no other broker can share. `path: ""` is not one of
+ * them: it selects the default storage file.
+ */
+export function readSharedStorageSetting(
+	agentDir: string,
+	env: NodeJS.ProcessEnv = process.env,
+): Record<string, JsonWireValue> | undefined {
+	const processSelection = env[BROKER_STORAGE_ENV];
+	let parsed: JsonWireValue | undefined;
+	try {
+		parsed =
+			processSelection === undefined || processSelection.length === 0
+				? (JSON.parse(readFileSync(join(agentDir, "settings.json"), "utf8")) as { storage?: JsonWireValue }).storage
+				: JSON.parse(processSelection);
+	} catch {
+		return undefined;
+	}
+	if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) return undefined;
+	const selection = parsed;
+	if (selection.adapter === undefined) {
+		if (selection.memory === true && !selection.path) return undefined;
+		if (selection.path === ":memory:") return undefined;
+	}
+	return selection;
+}
+
+/** Where `readSharedStorageSetting` takes the selection from in this environment. */
+export function sharedStorageSettingSource(env: NodeJS.ProcessEnv = process.env): "env" | "settings" {
+	const processSelection = env[BROKER_STORAGE_ENV];
+	return processSelection === undefined || processSelection.length === 0 ? "settings" : "env";
+}
+
+function sortedKeys(_key: string, value: JsonWireValue): JsonWireValue {
+	if (value === null || typeof value !== "object" || Array.isArray(value)) return value;
+	return Object.fromEntries(Object.entries(value).sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0)));
+}
+
+/** Identifies a shared selection independently of key order, without revealing it: two brokers agree when it matches. */
+export function sharedStorageDigest(setting: Readonly<Record<string, JsonWireValue>>): string {
+	return createHash("sha256").update(JSON.stringify(setting, sortedKeys)).digest("hex");
+}
+
+/** The adapter (`sqlite` for a `path` selection) and table prefix a shared selection names, for logs and status. */
+export function describeSharedStorage(setting: Readonly<Record<string, JsonWireValue>>): { adapter: string; prefix?: string } {
+	const adapter = typeof setting.adapter === "string" ? setting.adapter : "sqlite";
+	return typeof setting.prefix === "string" && setting.prefix.length > 0 ? { adapter, prefix: setting.prefix } : { adapter };
+}

package/dist/builtin/intercom/broker/spawn.ts

@@ -272,13 +272,21 @@ export function getBrokerSpawnOptions(
 }
 
 /**
- * Truncate (or create) the broker log so each spawn starts from a bounded, current file. Returns false when a
+ * Preserve a bounded previous log across truncation. Returns false when a
  * Windows broker that is still starting holds the log through its launcher's stderr redirect.
  */
 function resetBrokerLog(logPath: string = BROKER_LOG): boolean {
   ensurePrivateDirectory(dirname(logPath));
   try {
+    const previous = readBrokerLogTail(logPath);
     closeSync(openSync(logPath, "w"));
+    if (previous.length > 0) {
+      try {
+        const previousPath = `${logPath}.previous`;
+        writeFileSync(previousPath, Buffer.from(previous).subarray(-BROKER_LOG_TAIL_BYTES), { mode: 0o600 });
+        if (process.platform !== "win32") chmodSync(previousPath, 0o600);
+      } catch {}
+    }
     return true;
   } catch (error) {
     const code = (error as NodeJS.ErrnoException).code;
@@ -327,6 +335,7 @@ export async function spawnBrokerIfNeeded(
   brokerCommand: string,
   brokerArgs: string[],
   readyTimeoutMs: number = BROKER_READY_TIMEOUT_MS,
+  extensionDir: string = EXTENSION_DIR,
 ): Promise<void> {
   ensurePrivateDirectory(INTERCOM_DIR);
 
@@ -346,8 +355,8 @@ export async function spawnBrokerIfNeeded(
       return;
     }
 
-    const brokerPath = resolveBrokerEntrypoint(EXTENSION_DIR);
-    const launch = getBrokerLaunchSpec(brokerPath, brokerCommand, brokerArgs);
+    const brokerPath = resolveBrokerEntrypoint(extensionDir);
+    const launch = getBrokerLaunchSpec(brokerPath, brokerCommand, brokerArgs, extensionDir);
     if (launch.kind === "windows-launcher") {
       writeWindowsHiddenLauncher(launch.launcherCommandLine, launch.launcherPath);
     }
@@ -362,7 +371,7 @@ export async function spawnBrokerIfNeeded(
     const logFd = launch.kind === "direct" ? openSync(BROKER_LOG, "a") : undefined;
     let child: ReturnType<typeof spawn>;
     try {
-      child = spawn(launch.command, launch.args, getBrokerSpawnOptions(EXTENSION_DIR, logFd ?? "ignore"));
+      child = spawn(launch.command, launch.args, getBrokerSpawnOptions(extensionDir, logFd ?? "ignore"));
     } finally {
       if (logFd !== undefined) closeSync(logFd);
     }

package/dist/builtin/intercom/broker/sql-lease-store.ts

@@ -62,6 +62,29 @@ function stateFromRow(row: StateRow): LeaseState {
 	};
 }
 
+/**
+ * The host names of the brokers whose membership lease is live in `<prefix>intercom_leases`, read without creating
+ * the table: `undefined` when no broker has created it yet. For status reports; brokers use `listLive`.
+ */
+export async function readLiveBrokerNodes(
+	database: SqliteExecutor,
+	prefix: string | undefined,
+	staleAfterMs: number,
+): Promise<string[] | undefined> {
+	const table = `${validateStoragePrefix(prefix)}intercom_leases`;
+	const exists = await database.get("SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?", table);
+	if (exists === undefined) return undefined;
+	const rows = await database.all<StateRow>(
+		`SELECT kind, key, holder_id, fencing_token, subject_json, ${AGE} AS age_ms FROM ${table}
+		WHERE kind = 'broker' AND holder_id IS NOT NULL AND ${AGE} <= ? ORDER BY first_acquired_order`,
+		staleAfterMs,
+	);
+	return rows.flatMap((row) => {
+		const { subject } = stateFromRow(row);
+		return subject.kind === "broker" ? [subject.nodeId] : [];
+	});
+}
+
 /** Opens the S1 lease store on the shared database's `<prefix>intercom_leases` table; the database stays open. */
 export async function openSqlBrokerLeaseStore(options: IntercomBrokerStateOptions): Promise<BrokerLeaseStore> {
 	const { database } = options;

package/dist/builtin/intercom/broker/watchdog.ts

@@ -0,0 +1,105 @@
+import { Worker } from "node:worker_threads";
+
+export const BROKER_WATCHDOG_INTERVAL_MS = 2_000;
+export const BROKER_WATCHDOG_PROBE_TIMEOUT_MS = 10_000;
+export const BROKER_WATCHDOG_SILENCE_MS = 60_000;
+
+const WATCHDOG_SOURCE = `
+const { workerData } = require("node:worker_threads");
+const { connect } = require("node:net");
+const { randomUUID } = require("node:crypto");
+const { writeFileSync } = require("node:fs");
+let lastHealthyAt = performance.now();
+let consecutiveFailures = 0;
+
+function probe() {
+  return new Promise((resolve) => {
+    const requestId = randomUUID();
+    const socket = connect(workerData.socketPath);
+    let buffer = Buffer.alloc(0);
+    let settled = false;
+    const finish = (failure) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timer);
+      socket.destroy();
+      resolve(failure);
+    };
+    const timer = setTimeout(() => finish("response-timeout"), workerData.probeTimeoutMs);
+    socket.on("error", () => finish("connection-error"));
+    socket.on("close", () => finish("connection-closed"));
+    socket.on("connect", () => {
+      const payload = Buffer.from(JSON.stringify({ type: "health", requestId }));
+      const header = Buffer.alloc(4);
+      header.writeUInt32BE(payload.length);
+      try { socket.write(Buffer.concat([header, payload])); }
+      catch { finish("write-error"); }
+    });
+    socket.on("data", (data) => {
+      if (buffer.length + data.length > 1024) return finish("oversized-response");
+      buffer = Buffer.concat([buffer, data]);
+      if (buffer.length < 4) return;
+      const length = buffer.readUInt32BE(0);
+      if (length > 1020) return finish("oversized-response");
+      if (buffer.length < length + 4) return;
+      try {
+        const reply = JSON.parse(buffer.subarray(4, length + 4).toString("utf8"));
+        if (reply.type !== "health" || reply.requestId !== requestId) return finish("unexpected-response");
+        if (reply.instanceId !== workerData.instanceId) return finish("different-broker-instance");
+        finish();
+      } catch { finish("invalid-response"); }
+    });
+  });
+}
+
+async function check() {
+  const failure = await probe();
+  if (failure === undefined) {
+    lastHealthyAt = performance.now();
+    consecutiveFailures = 0;
+  } else {
+    consecutiveFailures += 1;
+    const silentForMs = Math.round(performance.now() - lastHealthyAt);
+    if (consecutiveFailures >= 3 && silentForMs >= workerData.silenceMs) {
+      try {
+        writeFileSync(workerData.logPath, JSON.stringify({
+          at: new Date().toISOString(), pid: process.pid, instanceId: workerData.instanceId,
+          reason: "Intercom broker stopped answering health requests; terminating for recovery",
+          failure, consecutiveFailures, silentForMs
+        }) + "\\n", { mode: 0o600 });
+      } catch {}
+      process.kill(process.pid, "SIGKILL");
+      return;
+    }
+  }
+  setTimeout(check, workerData.intervalMs);
+}
+setTimeout(check, workerData.intervalMs);
+`;
+
+export function startBrokerWatchdog(socketPath: string, instanceId: string, logPath: string): void {
+	try {
+		const worker = new Worker(WATCHDOG_SOURCE, {
+			eval: true,
+			workerData: {
+				socketPath,
+				instanceId,
+				logPath,
+				intervalMs: BROKER_WATCHDOG_INTERVAL_MS,
+				probeTimeoutMs: BROKER_WATCHDOG_PROBE_TIMEOUT_MS,
+				silenceMs: BROKER_WATCHDOG_SILENCE_MS,
+			},
+		});
+		let failed = false;
+		worker.on("error", (error) => {
+			failed = true;
+			console.error("Intercom broker watchdog unavailable:", error);
+		});
+		worker.on("exit", (code) => {
+			if (!failed) console.error(`Intercom broker watchdog exited unexpectedly with code ${code}`);
+		});
+		worker.unref();
+	} catch (error) {
+		console.error("Intercom broker watchdog unavailable:", error);
+	}
+}