@algosuite/vo-mcp 0.2.0-beta.57 → 0.2.0-beta.58

package/dist/runner-supervisor.js

@@ -2318,6 +2318,157 @@ function resolveSupervisorChildEntry({
   };
 }
 
+// src/runner/update-drain-gate.mjs
+var DEFAULT_DRAIN_CAP_MS = 45 * 60 * 1e3;
+var DEFAULT_DRAIN_CHECK_MS = 3e4;
+var MAX_DRAIN_CAP_MS = 6 * 60 * 60 * 1e3;
+var MIN_DRAIN_CHECK_MS = 1e3;
+var MAX_DRAIN_CHECK_MS = 5 * 60 * 1e3;
+var UPDATE_RESTART_ERROR_MESSAGE = "runner update restart after drain timeout";
+var UPDATE_RESTART_RESULT = "runner_update_restart";
+var sleepMs = (ms) => new Promise((resolve5) => setTimeout(resolve5, ms));
+function positiveNumber(raw) {
+  if (raw === void 0 || raw === null || String(raw).trim() === "") return null;
+  const value = Number(raw);
+  return Number.isFinite(value) && value >= 0 ? value : null;
+}
+function resolveDrainCapMs(env = {}) {
+  const ms = positiveNumber(env.VO_RUNNER_UPDATE_DRAIN_CAP_MS);
+  if (ms !== null) return Math.min(ms, MAX_DRAIN_CAP_MS);
+  const minutes = positiveNumber(env.VO_RUNNER_UPDATE_DRAIN_CAP_MIN);
+  if (minutes !== null) return Math.min(minutes * 6e4, MAX_DRAIN_CAP_MS);
+  return DEFAULT_DRAIN_CAP_MS;
+}
+function resolveDrainCheckMs(env = {}) {
+  const ms = positiveNumber(env.VO_RUNNER_UPDATE_DRAIN_CHECK_MS);
+  if (ms === null) return DEFAULT_DRAIN_CHECK_MS;
+  return Math.min(Math.max(ms, MIN_DRAIN_CHECK_MS), MAX_DRAIN_CHECK_MS);
+}
+function readActiveTasks(status, { childRunning = true } = {}) {
+  if (!childRunning) {
+    return { count: 0, ids: [], known: true, runnerId: null, runnerInstanceId: null };
+  }
+  if (!status || status.ok === false) {
+    return { count: 0, ids: [], known: false, runnerId: null, runnerInstanceId: null };
+  }
+  const ids = Array.isArray(status.activeTaskIds) ? status.activeTaskIds.map((id) => String(id)).filter(Boolean) : [];
+  const reported = Number(status.activeTasks);
+  const count = Number.isFinite(reported) && reported >= 0 ? reported : ids.length;
+  return {
+    count: Math.max(count, ids.length),
+    ids,
+    known: true,
+    runnerId: status.runnerId ? String(status.runnerId) : null,
+    runnerInstanceId: status.runnerInstanceId ? String(status.runnerInstanceId) : null
+  };
+}
+function decideDrainStep({ active, elapsedMs, capMs }) {
+  const busy = active.known ? active.count : "unknown";
+  if (active.known && active.count === 0) return { action: "proceed", busy };
+  if (elapsedMs >= capMs) return { action: "cap", busy };
+  return { action: "wait", busy };
+}
+function describeBusy(active) {
+  if (!active.known) return "status unreadable from a running child (assumed busy)";
+  const ids = active.ids.length > 0 ? ` [${active.ids.join(", ")}]` : " [ids unavailable]";
+  return `${active.count} active task(s)${ids}`;
+}
+async function markCappedTasks({ active, failInFlightTask, log }) {
+  if (active.ids.length === 0) {
+    if (active.count > 0 || !active.known) {
+      log(`update drain cap reached with ${describeBusy(active)} \u2014 cannot annotate tasks the runner did not name`);
+    }
+    return [];
+  }
+  const marked = [];
+  for (const taskId of active.ids) {
+    try {
+      await failInFlightTask(taskId, {
+        message: UPDATE_RESTART_ERROR_MESSAGE,
+        result: UPDATE_RESTART_RESULT,
+        runnerId: active.runnerId,
+        runnerInstanceId: active.runnerInstanceId
+      });
+      marked.push(taskId);
+    } catch (error) {
+      log(`could not mark task ${taskId} before update restart: ${error instanceof Error ? error.message : String(error)}`);
+    }
+  }
+  return marked;
+}
+async function awaitRunnerUpdateDrain({
+  readStatus,
+  isChildRunning = () => true,
+  failInFlightTask = async () => {
+  },
+  drainable = false,
+  env = {},
+  now = Date.now,
+  sleep: sleep2 = sleepMs,
+  log = () => {
+  }
+} = {}) {
+  const capMs = resolveDrainCapMs(env);
+  const checkMs = resolveDrainCheckMs(env);
+  const startedAt = now();
+  let checks = 0;
+  let lastReported = null;
+  const snapshot = async () => {
+    checks += 1;
+    const childRunning = Boolean(isChildRunning());
+    let status;
+    try {
+      status = await readStatus();
+    } catch {
+      status = null;
+    }
+    return readActiveTasks(status, { childRunning });
+  };
+  if (!drainable) {
+    const active = await snapshot();
+    if (active.known && active.count > 0) {
+      return {
+        proceed: false,
+        detail: `deferred safely: ${active.count} active task(s); retry when the runner is idle`,
+        capped: false,
+        waitedMs: 0,
+        checks,
+        markedTaskIds: []
+      };
+    }
+    return { proceed: true, detail: "runner idle", capped: false, waitedMs: 0, checks, markedTaskIds: [] };
+  }
+  for (; ; ) {
+    const active = await snapshot();
+    const elapsedMs = now() - startedAt;
+    const step = decideDrainStep({ active, elapsedMs, capMs });
+    if (step.action === "proceed") {
+      if (lastReported !== null) {
+        log(`update drain complete after ${Math.round(elapsedMs / 1e3)}s \u2014 runner idle, applying staged update restart`);
+      }
+      return { proceed: true, detail: "runner idle", capped: false, waitedMs: elapsedMs, checks, markedTaskIds: [] };
+    }
+    if (step.action === "cap") {
+      log(`update drain cap reached after ${Math.round(elapsedMs / 1e3)}s (cap ${Math.round(capMs / 1e3)}s) \u2014 ${describeBusy(active)}; marking task(s) before restart`);
+      const markedTaskIds = await markCappedTasks({ active, failInFlightTask, log });
+      return {
+        proceed: true,
+        capped: true,
+        detail: `update restart applied after drain cap; marked ${markedTaskIds.length} task(s) as "${UPDATE_RESTART_ERROR_MESSAGE}"`,
+        waitedMs: elapsedMs,
+        checks,
+        markedTaskIds
+      };
+    }
+    const fingerprint = `${step.busy}:${active.ids.join(",")}`;
+    if (fingerprint !== lastReported) {
+      lastReported = fingerprint;
+      log(`deferring staged update restart \u2014 ${describeBusy(active)}; re-checking every ${Math.round(checkMs / 1e3)}s until idle (cap ${Math.round(capMs / 1e3)}s)`);
+    }
+    await sleep2(Math.max(1, Math.min(checkMs, capMs - elapsedMs)));
+  }
+}
+
 // src/runner/supervisor-child-env.mjs
 import { hostname as systemHostname } from "node:os";
 
@@ -3408,22 +3559,24 @@ async function main() {
       }
       deferredControlRecoveryGeneration = null;
       handling = true;
-      const beforeStop = await localStatus();
-      if (beforeStop && Number(beforeStop.activeTasks || 0) > 0) {
-        await client.completeRunnerControl(action.actionId, {
-          runnerId,
-          ...operatorId ? { operatorId } : {},
-          ...supervisorControlIdentity,
-          status: "failed",
-          detail: `deferred safely: ${beforeStop.activeTasks} active task(s); retry when the runner is idle`
-        });
+      const controlIdentity = { runnerId, ...operatorId ? { operatorId } : {}, ...supervisorControlIdentity };
+      const bundledAction = action.kind === "update" || action.kind === "reinstall";
+      const drain = await awaitRunnerUpdateDrain({
+        readStatus: localStatus,
+        drainable: bundledAction,
+        env: process.env,
+        isChildRunning: () => supervisorChildIsRunning(child),
+        failInFlightTask: (taskId, patch) => client.postProgress(taskId, { status: "failed", message: patch.message, result: patch.result, ...patch.runnerId ? { runner_id: patch.runnerId } : {}, ...patch.runnerInstanceId ? { runner_instance_id: patch.runnerInstanceId } : {} }),
+        log: (message) => console.warn(`[vo-runner supervisor] ${message}`)
+      });
+      if (!drain.proceed) {
+        await client.completeRunnerControl(action.actionId, { ...controlIdentity, status: "failed", detail: drain.detail });
         handling = false;
         respawn();
         continue;
       }
       controlRecoveryFence.markBeforeStop(child);
       await stopChildExpectedly(child);
-      const bundledAction = action.kind === "update" || action.kind === "reinstall";
       const result = bundledAction ? stageAndActivateBundledUpdate({
         runtimeRoot,
         packageSpec: `@algosuite/vo-mcp@${action.desired_package_version}`,
@@ -3445,6 +3598,7 @@ async function main() {
         log: (message) => console.warn(`[vo-runner supervisor] ${message}`)
       });
       if (result.ok && result.handoff) {
+        if (drain.capped) console.warn(`[vo-runner supervisor] ${drain.detail}`);
         console.warn(`[vo-runner supervisor] activated ${result.active.version}; exiting for new-process attestation`);
         return;
       }
@@ -3456,9 +3610,7 @@ async function main() {
       const relaunchedHealthy = recoveryAuthorized ? await ensureHealthyChildAfterControl() : false;
       if (!relaunchedHealthy) result.ok = false;
       await client.completeRunnerControl(action.actionId, {
-        runnerId,
-        ...operatorId ? { operatorId } : {},
-        ...supervisorControlIdentity,
+        ...controlIdentity,
         status: result.ok ? "succeeded" : "failed",
         detail: result.ok ? result.detail ? `${result.detail}; runner ${packageVersion()} reconnected`.slice(0, 1e3) : `runner ${packageVersion()} reconnected` : `maintenance exited ${result.status}${result.detail ? `: ${result.detail}` : ""}${relaunchedHealthy ? "" : "; runner relaunch did not become healthy"}`
       });