@martintrojer/murmur 0.1.3 → 0.1.4

package/dist/index.js

@@ -4,6 +4,43 @@ import { createRequire } from "module";
 // src/agents.ts
 import { spawnSync } from "child_process";
 
+// src/channel.ts
+import { execFile, execFileSync } from "child_process";
+import { promisify } from "util";
+var execFileAsync = promisify(execFile);
+var CONTROL_PATH = "~/.ssh/control/%r@%h:%p";
+var CONNECT_TIMEOUT_S = 1;
+var EXEC_TIMEOUT_MS = 3e3;
+var SSH_OPTIONS = [
+  "-o",
+  "BatchMode=yes",
+  "-o",
+  "ControlMaster=no",
+  "-o",
+  `ControlPath=${CONTROL_PATH}`,
+  "-o",
+  `ConnectTimeout=${CONNECT_TIMEOUT_S}`
+];
+var MAX_EXPORT_BYTES = 64 * 1024 * 1024;
+var ssh = {
+  async exec(target, argv) {
+    const { stdout } = await execFileAsync("ssh", [...SSH_OPTIONS, target, ...argv], {
+      encoding: "utf8",
+      timeout: EXEC_TIMEOUT_MS,
+      maxBuffer: MAX_EXPORT_BYTES
+    });
+    return stdout;
+  }
+};
+function hasWarmSocket(target) {
+  try {
+    execFileSync("ssh", [...SSH_OPTIONS, "-O", "check", target], { stdio: "ignore" });
+    return true;
+  } catch {
+    return false;
+  }
+}
+
 // src/identity.ts
 import { randomUUID } from "crypto";
 import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
@@ -39,10 +76,10 @@ function ensureIdentity(displayName = hostname()) {
 }
 
 // src/mux.ts
-import { execFileSync } from "child_process";
+import { execFileSync as execFileSync2 } from "child_process";
 function runTmux(args) {
   try {
-    return execFileSync("tmux", args, {
+    return execFileSync2("tmux", args, {
       encoding: "utf8",
       timeout: 3e3,
       stdio: ["ignore", "pipe", "ignore"]
@@ -103,7 +140,7 @@ var tmux = {
   },
   attach(session, window) {
     runTmux(["switch-client", "-t", session]);
-    runTmux(["select-window", "-t", window]);
+    return runTmux(["select-window", "-t", window]) !== null;
   },
   // Window ids are what the log stores, because they are stable; names are
   // what a human recognises in a picker. Names are live tmux state, not
@@ -135,7 +172,10 @@ var tmux = {
     return null;
   },
   selectWindow(window) {
-    runTmux(["select-window", "-t", window]);
+    return runTmux(["select-window", "-t", window]) !== null;
+  },
+  newWindow(name, command) {
+    return runTmux(["new-window", "-n", name, command]) !== null;
   },
   // The window a pane belongs to, for a pane murmur has no event for. Clearing
   // a badge is a tmux operation and does not require murmur to own the pane.
@@ -176,6 +216,21 @@ function terminalText(value) {
 function shellQuote(value) {
   return `'${value.replaceAll("'", `'\\''`)}'`;
 }
+var spawnRunner = (file, args, inherit = false) => {
+  const result = spawnSync(file, args, {
+    encoding: "utf8",
+    timeout: 1e4,
+    ...inherit ? { stdio: "inherit" } : {}
+  });
+  return {
+    status: result.status,
+    stdout: result.stdout ?? "",
+    // spawnSync reports a failure to even start the child in `error`, leaving
+    // status null. Collapsing both here keeps the decision table below reading
+    // as one question rather than two.
+    failed: result.error !== void 0
+  };
+};
 function forgetHostReplica(store, hostId) {
   try {
     const peer = store.peers().find((candidate) => candidate.host_id === hostId);
@@ -198,10 +253,10 @@ function forgetReplica(store, agentId, hostId) {
   } catch {
   }
 }
-function jumpToAgent(store, agent) {
+function jumpToAgent(store, agent, mux = tmux, run = spawnRunner) {
   const identity = loadIdentity();
   if (agent.host_id === identity?.host_id) {
-    const live = tmux.liveWindows();
+    const live = mux.liveWindows();
     if (live && !live.has(agent.window)) {
       forgetReplica(store, agent.agent_id, agent.host_id);
       return {
@@ -210,7 +265,13 @@ function jumpToAgent(store, agent) {
         message: `${agentLabel(agent)} is gone -- its window no longer exists. Cleared.`
       };
     }
-    tmux.attach(agent.session, agent.window);
+    if (!mux.attach(agent.session, agent.window)) {
+      return {
+        ok: false,
+        reason: "attach_failed",
+        message: `could not attach to ${agentLabel(agent)} (tmux select-window failed).`
+      };
+    }
     return { ok: true };
   }
   const peer = store.peers().find((candidate) => candidate.host_id === agent.host_id);
@@ -222,18 +283,18 @@ function jumpToAgent(store, agent) {
       message: `No peer configured for host ${agent.host_id.slice(0, 8)}. Try: murmur peer add <target>`
     };
   }
-  const probe = spawnSync(
-    "ssh",
-    ["-o", "BatchMode=yes", target, `tmux list-windows -a -F ${shellQuote("#{window_id}")}`],
-    { encoding: "utf8", timeout: 1e4 }
-  );
+  const probe = run("ssh", [
+    ...SSH_OPTIONS,
+    target,
+    `tmux list-windows -a -F ${shellQuote("#{window_id}")}`
+  ]);
   if (probe.status !== 0) {
-    const sshFailed = probe.status === 255 || probe.error !== void 0;
+    const sshFailed = probe.status === 255 || probe.failed;
     if (sshFailed) {
       return {
         ok: false,
         reason: "unreachable",
-        message: `cannot reach ${target} over ssh. The collector never prompts for auth, so connect once by hand to warm the connection, then retry.`
+        message: `cannot reach ${target} over ssh. Nothing here ever prompts for auth, so check the host is awake and reachable, or connect once by hand to see the real error.`
       };
     }
     forgetHostReplica(store, agent.host_id);
@@ -243,7 +304,7 @@ function jumpToAgent(store, agent) {
       message: `${target} has no tmux server running, so its agents are gone. Removed them; they will come back when it reports again.`
     };
   }
-  const remoteWindows = new Set((probe.stdout ?? "").split("\n").filter(Boolean));
+  const remoteWindows = new Set(probe.stdout.split("\n").filter(Boolean));
   if (!remoteWindows.has(agent.window)) {
     forgetReplica(store, agent.agent_id, agent.host_id);
     return {
@@ -256,51 +317,26 @@ function jumpToAgent(store, agent) {
   if (process.env.TMUX) {
     const command = `ssh -t ${shellQuote(target)} tmux attach -t ${shellQuote(attachTarget)}`;
     const name = `@${peer?.name ?? target}`;
-    const existing = tmux.windowNamed(name);
+    const existing = mux.windowNamed(name);
     if (existing) {
-      tmux.selectWindow(existing);
-      return { ok: true };
+      return mux.selectWindow(existing) ? { ok: true } : {
+        ok: false,
+        reason: "attach_failed",
+        message: `could not switch to the existing ${name} window.`
+      };
     }
-    spawnSync("tmux", ["new-window", "-n", name, command], { stdio: "ignore" });
-    return { ok: true };
-  }
-  spawnSync("ssh", ["-t", target, "tmux", "attach", "-t", attachTarget], { stdio: "inherit" });
-  return { ok: true };
-}
-
-// src/channel.ts
-import { execFile, execFileSync as execFileSync2 } from "child_process";
-import { promisify } from "util";
-var execFileAsync = promisify(execFile);
-var CONTROL_PATH = "~/.ssh/control/%r@%h:%p";
-var CONNECT_TIMEOUT_S = 2;
-var EXEC_TIMEOUT_MS = 1e4;
-var SSH_OPTIONS = [
-  "-o",
-  "BatchMode=yes",
-  "-o",
-  "ControlMaster=no",
-  "-o",
-  `ControlPath=${CONTROL_PATH}`,
-  "-o",
-  `ConnectTimeout=${CONNECT_TIMEOUT_S}`
-];
-var ssh = {
-  async exec(target, argv) {
-    const { stdout } = await execFileAsync("ssh", [...SSH_OPTIONS, target, ...argv], {
-      encoding: "utf8",
-      timeout: EXEC_TIMEOUT_MS
-    });
-    return stdout;
-  }
-};
-function hasWarmSocket(target) {
-  try {
-    execFileSync2("ssh", [...SSH_OPTIONS, "-O", "check", target], { stdio: "ignore" });
-    return true;
-  } catch {
-    return false;
+    return mux.newWindow(name, command) ? { ok: true } : {
+      ok: false,
+      reason: "attach_failed",
+      message: `could not open a window to attach to ${target}.`
+    };
   }
+  const attach = run("ssh", ["-t", target, "tmux", "attach", "-t", attachTarget], true);
+  return attach.status === 0 && !attach.failed ? { ok: true } : {
+    ok: false,
+    reason: "attach_failed",
+    message: `ssh attach to ${target} failed.`
+  };
 }
 
 // src/types.ts
@@ -489,8 +525,30 @@ function exportJsonl(store, since, isAlive, live) {
 }
 
 // src/collector.ts
-var COLLECT_INTERVAL_MS = 3e4;
-var STALENESS_MS = 2 * COLLECT_INTERVAL_MS;
+var STALENESS_MS = 6e4;
+var MAX_CONCURRENT_PEERS = 8;
+var COLLECT_DEADLINE_MS = 4e3;
+async function mapSettled(items, limit, task, deadline) {
+  const results = new Array(items.length);
+  let cursor = 0;
+  let expired = false;
+  const stop = deadline?.then(() => {
+    expired = true;
+  });
+  const worker = async () => {
+    while (cursor < items.length && !expired) {
+      const index = cursor++;
+      try {
+        results[index] = { status: "fulfilled", value: await task(items[index]) };
+      } catch (reason) {
+        results[index] = { status: "rejected", reason };
+      }
+    }
+  };
+  const pool = Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker));
+  await (stop ? Promise.race([pool, stop]) : pool);
+  return results;
+}
 function parseJsonl(output) {
   const lines = output.trim().split("\n");
   const envelope = JSON.parse(lines.shift() ?? "");
@@ -504,20 +562,35 @@ function parseJsonl(output) {
     events: lines.map((line) => eventFromWire(JSON.parse(line)))
   };
 }
-async function collect(store, channel, now = Date.now()) {
+async function collect(store, channel, now = Date.now(), deadline) {
   const results = [];
+  let timer;
   try {
-    for (const peer of store.peers()) {
+    const peers = store.peers();
+    const bounded = deadline ?? new Promise((resolve) => {
+      timer = setTimeout(resolve, COLLECT_DEADLINE_MS);
+      timer.unref?.();
+    });
+    const fetches = await mapSettled(
+      peers,
+      MAX_CONCURRENT_PEERS,
+      async (peer) => parseJsonl(
+        await channel.exec(peer.target, ["murmur", "export", "--since", String(peer.watermark)])
+      ),
+      bounded
+    );
+    for (const [index, peer] of peers.entries()) {
+      const fetch = fetches[index];
       try {
-        const output = await channel.exec(peer.target, [
-          "murmur",
-          "export",
-          "--since",
-          String(peer.watermark)
-        ]);
-        const { envelope, events } = parseJsonl(output);
+        if (!fetch) throw new Error("collect deadline passed before this peer answered");
+        if (fetch.status === "rejected") throw fetch.reason;
+        const { envelope, events } = fetch.value;
         const ingested = store.ingest(events);
-        const watermark = events.filter((event) => event.host_id === envelope.host_id).reduce((highest, event) => Math.max(highest, event.seq), peer.watermark);
+        const origin = events.filter((event) => event.host_id === envelope.host_id);
+        const watermark = origin.reduce(
+          (highest, event) => Math.max(highest, event.seq),
+          peer.watermark
+        );
         store.upsertPeer({
           name: peer.name,
           target: peer.target,
@@ -530,9 +603,16 @@ async function collect(store, channel, now = Date.now()) {
           // events: an export that returns nothing proves the binary ran, not
           // that tmux is back, which is the distinction that let a dead host
           // look healthy for three hours.
-          tmux_down_at: ingested > 0 ? null : peer.tmux_down_at
+          //
+          // Keyed on the watermark advancing, not on ingest's insert count.
+          // Two reasons the count was wrong. Ingest is INSERT OR IGNORE, so a
+          // retry after a partial apply re-sees the same events and reports
+          // zero -- leaving a recovered host marked down until it happened to
+          // author again. And the count includes rows from other origins that
+          // this peer merely relayed, which say nothing about whether this
+          // peer's tmux is back.
+          tmux_down_at: watermark > peer.watermark ? null : peer.tmux_down_at
         });
-        store.prune();
         results.push({ peer: peer.name, ok: true, ingested });
       } catch (error) {
         const message = error instanceof Error ? error.message : String(error);
@@ -544,6 +624,16 @@ async function collect(store, channel, now = Date.now()) {
   } catch (error) {
     process.stderr.write(
       `murmur: collect: ${error instanceof Error ? error.message : String(error)}
+`
+    );
+  } finally {
+    clearTimeout(timer);
+  }
+  try {
+    store.prune();
+  } catch (error) {
+    process.stderr.write(
+      `murmur: collect: prune: ${error instanceof Error ? error.message : String(error)}
 `
     );
   }
@@ -553,16 +643,6 @@ async function collect(store, channel, now = Date.now()) {
 // src/glance.ts
 import { execFileSync as execFileSync3 } from "child_process";
 var GLANCE_LINES = 40;
-var SSH_OPTIONS2 = [
-  "-o",
-  "BatchMode=yes",
-  "-o",
-  "ControlMaster=no",
-  "-o",
-  "ControlPath=~/.ssh/control/%r@%h:%p",
-  "-o",
-  "ConnectTimeout=2"
-];
 function glance(store, agent, lines = GLANCE_LINES) {
   if (agent.host_id === loadIdentity()?.host_id) return tmux.capture(agent.pane, lines);
   const peer = store.peers().find((candidate) => candidate.host_id === agent.host_id);
@@ -572,7 +652,7 @@ function glance(store, agent, lines = GLANCE_LINES) {
     return execFileSync3(
       "ssh",
       [
-        ...SSH_OPTIONS2,
+        ...SSH_OPTIONS,
         target,
         "tmux",
         "capture-pane",
@@ -824,6 +904,14 @@ function openStore() {
       const rows = database.prepare("SELECT * FROM events ORDER BY ts, host_id, seq").all();
       return rows.map(toEvent);
     },
+    latestForAgent(hostId, agentId) {
+      const row = database.prepare(
+        `SELECT * FROM events
+            WHERE host_id = ? AND agent_id = ?
+            ORDER BY seq DESC LIMIT 1`
+      ).get(hostId, agentId);
+      return row ? toEvent(row) : null;
+    },
     maxSeq(hostId) {
       return selectMaxSeq.get(hostId).seq;
     },
@@ -884,8 +972,8 @@ function openStore() {
 var manifest = createRequire(import.meta.url)("../package.json");
 var VERSION = manifest.version;
 export {
-  COLLECT_INTERVAL_MS,
   DEFAULT_DRIVER,
+  MAX_CONCURRENT_PEERS,
   SCHEMA_VERSION,
   STALENESS_MS,
   STORE_VERSION,