atris 3.78.0 → 3.80.0

package/lib/member-up.js

@@ -184,6 +184,7 @@ function createMemberUp(deps, options = {}) {
   const max = Math.max(1, Number(options.max) || DEFAULT_MAX);
   const dryRun = options.dryRun === true;
   const print = deps.print || ((line) => console.log(line));
+  const reportError = deps.reportError || ((context, error) => console.error(`[member-up] ${context}:`, error));
   // only: when set, these members are the whole team for this supervisor.
   // Nobody else is started or counted, and unknown names are skipped.
   const only = Array.isArray(options.only) && options.only.length
@@ -319,6 +320,7 @@ function createMemberUp(deps, options = {}) {
       out = deps.requeueDropped();
       requeueFailSaid = false;
     } catch (error) {
+      reportError('could not check github merge line', error);
       if (!requeueFailSaid) print(`could not check github's merge line: ${oneLine(error && error.message || error)}`);
       requeueFailSaid = true;
       return;
@@ -336,6 +338,7 @@ function createMemberUp(deps, options = {}) {
     try {
       out = deps.writeNextJob(name, { dryRun });
     } catch (error) {
+      reportError(`${name} could not write its own next job`, error);
       if (!s.idle) print(`${name} could not write its own next job: ${oneLine(error && error.message || error)}`);
       return null;
     }
@@ -384,7 +387,13 @@ function createMemberUp(deps, options = {}) {
       if (working() + (dryRun ? started.length : 0) >= max) break;
       const s = seat(name);
       if (now < s.nextCheckAt) continue;
-      const picked = deps.pickTask(name, taken);
+      let picked;
+      try {
+        picked = deps.pickTask(name, taken);
+      } catch (error) {
+        reportError(`${name} could not read the task queue`, error);
+        continue;
+      }
       let task = picked && hasRunnableCheck(picked) ? picked : null;
       if (!task) task = ownNextJob(name, s);
       if (!task) {
@@ -412,6 +421,7 @@ function createMemberUp(deps, options = {}) {
         record({ member: name, pid: child && child.pid ? child.pid : null });
       } catch (error) {
         running.delete(name);
+        reportError(`${name} failed to start`, error);
         print(`${name} failed to start: ${oneLine(error && error.message || error)}`);
         continue;
       }
@@ -419,7 +429,9 @@ function createMemberUp(deps, options = {}) {
     }
     try {
       otherKindHints(team, members.filter((m) => m && m.name && !m.parked && !m.human && !m.sleeping), taken);
-    } catch {}
+    } catch (error) {
+      reportError('could not inspect waiting tasks', error);
+    }
     if (!dryRun && now - lastSummaryAt >= SUMMARY_MS) {
       lastSummaryAt = now;
       print(summary());
@@ -486,16 +498,41 @@ function parseMemberUpArgs(args = []) {
   return out;
 }
 
+// Persistent watcher failures can recur every second. Record the first one,
+// then preserve repeat counts in bounded summaries rather than filling disk.
+function watchErrorReporter(report, now = Date.now, intervalMs = 60000) {
+  const failures = new Map();
+  function emitPending(entry) {
+    if (!entry.repeats) return;
+    report(entry.context, new Error(`${entry.repeats} repeated watcher errors: ${entry.message}`));
+    entry.repeats = 0;
+    entry.last = now();
+  }
+  return {
+    report(context, error) {
+      const message = String(error?.message || error);
+      const key = `${context}\0${error?.code || ''}\0${message}`;
+      let entry = failures.get(key);
+      if (!entry) { entry = { context, message, repeats: 0, last: now() }; failures.set(key, entry); report(context, error); return; }
+      entry.repeats += 1;
+      if (now() - entry.last >= intervalMs) emitPending(entry);
+    },
+    flush() { for (const entry of failures.values()) emitPending(entry); },
+  };
+}
+
 // Runs checks every minute until stop() is called. `sleep(ms)` must resolve
 // after ms or as soon as the returned stop fires.
 function runMemberUp(deps, options = {}) {
   // Begin following before any child starts, so its first step cannot be lost.
   let follower;
+  const watchErrors = watchErrorReporter(deps.reportError || ((where, err) => console.error(`[member-up] ${where}:`, err)), deps.now);
   const up = createMemberUp({ ...deps, flushProgress: () => follower?.scan() }, options);
   if (!options.dryRun) follower = followBoard({
     dir: options.progressDir,
     pollMs: options.progressPollMs,
     orderByTime: true,
+    onError: (context, error) => watchErrors.report(context, error),
     emit: (event) => up.progress(event),
   });
   let stopped = false;
@@ -506,7 +543,11 @@ function runMemberUp(deps, options = {}) {
   }));
   const done = (async () => {
     while (!stopped) {
-      up.check();
+      try {
+        up.check();
+      } catch (error) {
+        (deps.reportError || ((context, err) => console.error(`[member-up] ${context}:`, err)))('supervisor check failed', error);
+      }
       if (stopped) break;
       await sleep(options.checkMs || CHECK_MS);
     }
@@ -519,6 +560,7 @@ function runMemberUp(deps, options = {}) {
     stop() {
       stopped = true;
       follower?.stop();
+      watchErrors.flush();
       if (wake) wake();
     },
   };
@@ -540,4 +582,5 @@ module.exports = {
   withMaintenanceLock,
   parseMemberUpArgs,
   runMemberUp,
+  watchErrorReporter,
 };