@jeffusion/bungee-core 3.2.1 → 3.3.0

package/dist/master.js

@@ -16534,6 +16534,228 @@ var init_plugin_permissions = __esm(() => {
   };
 });
 
+// src/plugin-runtime-multi-worker-convergence.ts
+class PluginRuntimeMultiWorkerConvergenceTracker {
+  convergenceDeadlineMs;
+  workers = new Map;
+  targetGeneration = null;
+  startedAt = null;
+  deadlineAt = null;
+  constructor(convergenceDeadlineMs = 5000) {
+    this.convergenceDeadlineMs = convergenceDeadlineMs;
+  }
+  upsertWorker(workerId, pid = null) {
+    const existing = this.workers.get(workerId);
+    if (existing) {
+      existing.pid = pid;
+      if (existing.baseStatus === "stopped") {
+        existing.baseStatus = "starting";
+      }
+      existing.updatedAt = Date.now();
+      return;
+    }
+    this.workers.set(workerId, {
+      workerId,
+      pid,
+      baseStatus: "starting",
+      servingGeneration: null,
+      lastRequestedGeneration: null,
+      lastAppliedGeneration: null,
+      failureGeneration: null,
+      error: null,
+      updatedAt: Date.now()
+    });
+  }
+  recordWorkerReady(workerId, generation, pid = null) {
+    const worker = this.getOrCreateWorker(workerId, pid);
+    worker.pid = pid ?? worker.pid;
+    worker.baseStatus = "ready";
+    worker.servingGeneration = generation;
+    worker.lastAppliedGeneration = generation;
+    worker.error = null;
+    worker.updatedAt = Date.now();
+  }
+  beginGeneration(targetGeneration, workerIds) {
+    this.targetGeneration = targetGeneration;
+    this.startedAt = Date.now();
+    this.deadlineAt = this.startedAt + Math.max(0, this.convergenceDeadlineMs);
+    for (const workerId of workerIds ?? Array.from(this.workers.keys())) {
+      const worker = this.getOrCreateWorker(workerId);
+      worker.lastRequestedGeneration = targetGeneration;
+      worker.failureGeneration = worker.failureGeneration === targetGeneration ? null : worker.failureGeneration;
+      worker.error = worker.failureGeneration === targetGeneration ? null : worker.error;
+      if (worker.baseStatus !== "stopped") {
+        worker.baseStatus = "reconciling";
+      }
+      worker.updatedAt = Date.now();
+    }
+  }
+  recordWorkerSuccess(workerId, generation, servingGeneration = generation, pid = null) {
+    const worker = this.getOrCreateWorker(workerId, pid);
+    worker.pid = pid ?? worker.pid;
+    worker.baseStatus = "ready";
+    worker.lastAppliedGeneration = generation;
+    worker.servingGeneration = servingGeneration;
+    worker.failureGeneration = null;
+    worker.error = null;
+    worker.updatedAt = Date.now();
+  }
+  recordWorkerFailure(workerId, generation, error, servingGeneration, pid = null) {
+    const worker = this.getOrCreateWorker(workerId, pid);
+    worker.pid = pid ?? worker.pid;
+    worker.baseStatus = "failed";
+    worker.lastAppliedGeneration = generation;
+    worker.failureGeneration = generation;
+    worker.servingGeneration = servingGeneration;
+    worker.error = error;
+    worker.updatedAt = Date.now();
+  }
+  recordWorkerStopped(workerId) {
+    const worker = this.getOrCreateWorker(workerId);
+    worker.baseStatus = "stopped";
+    worker.updatedAt = Date.now();
+  }
+  getStatusReport(now = Date.now()) {
+    const targetGeneration = this.targetGeneration;
+    const deadlineExceeded = this.deadlineAt !== null && now > this.deadlineAt;
+    const workers = Array.from(this.workers.values()).sort((left, right) => left.workerId - right.workerId).map((worker) => {
+      const status2 = deriveWorkerStatus(worker, targetGeneration);
+      return {
+        workerId: worker.workerId,
+        pid: worker.pid,
+        status: status2,
+        servingGeneration: worker.servingGeneration,
+        lastRequestedGeneration: worker.lastRequestedGeneration,
+        lastAppliedGeneration: worker.lastAppliedGeneration,
+        failureGeneration: worker.failureGeneration,
+        error: worker.error,
+        updatedAt: worker.updatedAt ? new Date(worker.updatedAt).toISOString() : null
+      };
+    });
+    const failedWorkers = workers.filter((worker) => worker.status === "failed").map((worker) => worker.workerId);
+    const staleWorkers = workers.filter((worker) => worker.status === "stale" || worker.status === "failed" && targetGeneration !== null && worker.servingGeneration !== null && worker.servingGeneration < targetGeneration).map((worker) => worker.workerId);
+    const pendingWorkers = workers.filter((worker) => worker.status === "reconciling" || worker.status === "starting" || worker.status === "ready").map((worker) => worker.workerId);
+    const convergedWorkers = workers.filter((worker) => worker.status === "converged").map((worker) => worker.workerId);
+    let status = "idle";
+    let convergedGeneration = null;
+    if (targetGeneration !== null) {
+      if (failedWorkers.length > 0 || deadlineExceeded && (staleWorkers.length > 0 || pendingWorkers.length > 0)) {
+        status = "failed";
+      } else if (workers.length > 0 && convergedWorkers.length === workers.length) {
+        status = "converged";
+        convergedGeneration = targetGeneration;
+      } else {
+        status = "pending";
+      }
+    }
+    return {
+      targetGeneration,
+      convergedGeneration,
+      startedAt: this.startedAt ? new Date(this.startedAt).toISOString() : null,
+      deadlineAt: this.deadlineAt ? new Date(this.deadlineAt).toISOString() : null,
+      status,
+      failedWorkers,
+      staleWorkers,
+      pendingWorkers,
+      convergedWorkers,
+      workers
+    };
+  }
+  getOrCreateWorker(workerId, pid = null) {
+    const existing = this.workers.get(workerId);
+    if (existing) {
+      if (pid !== null) {
+        existing.pid = pid;
+      }
+      return existing;
+    }
+    const worker = {
+      workerId,
+      pid,
+      baseStatus: "starting",
+      servingGeneration: null,
+      lastRequestedGeneration: null,
+      lastAppliedGeneration: null,
+      failureGeneration: null,
+      error: null,
+      updatedAt: null
+    };
+    this.workers.set(workerId, worker);
+    return worker;
+  }
+}
+
+class PluginRuntimeMultiWorkerCoordinator {
+  tracker;
+  pollIntervalMs;
+  requestedGeneration = 0;
+  constructor(tracker = new PluginRuntimeMultiWorkerConvergenceTracker, pollIntervalMs = 50) {
+    this.tracker = tracker;
+    this.pollIntervalMs = pollIntervalMs;
+  }
+  noteWorkerReady(workerId, generation, pid = null) {
+    this.requestedGeneration = Math.max(this.requestedGeneration, generation ?? 0);
+    this.tracker.recordWorkerReady(workerId, generation, pid);
+  }
+  noteWorkerStarting(workerId, pid = null) {
+    this.tracker.upsertWorker(workerId, pid);
+  }
+  noteWorkerStopped(workerId) {
+    this.tracker.recordWorkerStopped(workerId);
+  }
+  noteWorkerReconcileSuccess(workerId, generation, servingGeneration = generation, pid = null) {
+    this.requestedGeneration = Math.max(this.requestedGeneration, generation);
+    this.tracker.recordWorkerSuccess(workerId, generation, servingGeneration, pid);
+  }
+  noteWorkerReconcileFailure(workerId, generation, error, servingGeneration, pid = null) {
+    this.requestedGeneration = Math.max(this.requestedGeneration, generation);
+    this.tracker.recordWorkerFailure(workerId, generation, error, servingGeneration, pid);
+  }
+  getStatusReport(now = Date.now()) {
+    return this.tracker.getStatusReport(now);
+  }
+  async dispatchReconcile(workers) {
+    const activeWorkers = workers.filter((worker) => worker.status === "ready");
+    const targetGeneration = this.requestedGeneration + 1;
+    this.requestedGeneration = targetGeneration;
+    this.tracker.beginGeneration(targetGeneration, activeWorkers.map((worker) => worker.workerId));
+    const command = {
+      command: "reconcile-plugin-runtime",
+      generation: targetGeneration
+    };
+    for (const worker of activeWorkers) {
+      worker.send?.(command);
+    }
+    if (activeWorkers.length === 0) {
+      return this.tracker.getStatusReport();
+    }
+    return await new Promise((resolve) => {
+      const poller = setInterval(() => {
+        const report = this.tracker.getStatusReport();
+        if (report.status === "converged" || report.status === "failed") {
+          clearInterval(poller);
+          resolve(report);
+        }
+      }, this.pollIntervalMs);
+    });
+  }
+}
+function deriveWorkerStatus(worker, targetGeneration) {
+  if (worker.baseStatus === "stopped" || worker.baseStatus === "failed") {
+    return worker.baseStatus;
+  }
+  if (targetGeneration === null) {
+    return worker.baseStatus;
+  }
+  if (worker.servingGeneration === targetGeneration) {
+    return "converged";
+  }
+  if (worker.servingGeneration !== null && worker.servingGeneration < targetGeneration) {
+    return "stale";
+  }
+  return worker.baseStatus;
+}
+
 // src/master.ts
 var exports_master = {};
 import path7 from "path";
@@ -16547,6 +16769,7 @@ class Master {
   isReloading = false;
   reloadTimeout = null;
   pluginStorageCleanupService = null;
+  pluginRuntimeConvergenceCoordinator = new PluginRuntimeMultiWorkerCoordinator;
   constructor() {
     this.loadAndStart();
     this.watchConfig();
@@ -16601,6 +16824,14 @@ class Master {
         await new Promise((resolve) => setTimeout(resolve, 100));
       }
     }
+    const startupConvergence = await this.reconcilePluginRuntimeAcrossWorkers();
+    logger.info({
+      targetGeneration: startupConvergence.targetGeneration,
+      convergedGeneration: startupConvergence.convergedGeneration,
+      status: startupConvergence.status,
+      failedWorkers: startupConvergence.failedWorkers,
+      staleWorkers: startupConvergence.staleWorkers
+    }, "Initial multi-worker plugin runtime convergence completed");
     logger.info("All workers have been started.");
   }
   forkWorker(workerId) {
@@ -16626,10 +16857,12 @@ class Master {
           process: workerProcess,
           workerId,
           status: "starting",
+          pluginRuntimeGeneration: null,
           exitPromise,
           exitResolve
         };
         this.workers.set(workerId, workerInfo);
+        this.pluginRuntimeConvergenceCoordinator.noteWorkerStarting(workerId, workerProcess.pid ?? null);
         const startupTimeout = setTimeout(() => {
           logger.error(`Worker #${workerId} (PID: ${workerProcess.pid}) failed to start within timeout`);
           workerProcess.kill();
@@ -16646,6 +16879,7 @@ class Master {
           }
           this.workers.delete(workerId);
           workerInfo.status = "stopped";
+          this.pluginRuntimeConvergenceCoordinator.noteWorkerStopped(workerId);
           if (exitResolve) {
             exitResolve();
           }
@@ -16655,12 +16889,30 @@ class Master {
         });
         workerProcess.on("message", (message) => {
           clearTimeout(startupTimeout);
-          if (message.status === "ready") {
+          if ("command" in message && message.command === "cluster-reconcile-plugin-runtime") {
+            this.handleClusterPluginRuntimeReconcileRequest(workerInfo, message);
+            return;
+          }
+          if (!("status" in message)) {
+            return;
+          }
+          const workerMessage = message;
+          if (workerMessage.status === "ready") {
             logger.info(`Worker #${workerId} (PID: ${workerProcess.pid}) reported ready.`);
             workerInfo.status = "ready";
+            workerInfo.pluginRuntimeGeneration = workerMessage.pluginRuntime.generation;
+            this.pluginRuntimeConvergenceCoordinator.noteWorkerReady(workerId, workerMessage.pluginRuntime.generation, workerMessage.pid);
             resolve(true);
-          } else if (message.status === "error") {
-            const errorMsg = message.error || "Unknown error";
+          } else if (workerMessage.status === "plugin-runtime-reconcile-complete") {
+            workerInfo.pluginRuntimeGeneration = workerMessage.servingGeneration;
+            this.pluginRuntimeConvergenceCoordinator.noteWorkerReconcileSuccess(workerId, workerMessage.generation, workerMessage.servingGeneration, workerMessage.pid);
+            this.logPluginRuntimeConvergenceStatus();
+          } else if (workerMessage.status === "plugin-runtime-reconcile-failed") {
+            workerInfo.pluginRuntimeGeneration = workerMessage.servingGeneration;
+            this.pluginRuntimeConvergenceCoordinator.noteWorkerReconcileFailure(workerId, workerMessage.generation, workerMessage.error, workerMessage.servingGeneration, workerMessage.pid);
+            this.logPluginRuntimeConvergenceStatus();
+          } else if (workerMessage.status === "error") {
+            const errorMsg = workerMessage.error || "Unknown error";
             logger.error(`Worker #${workerId} (PID: ${workerProcess.pid}) reported an error: ${errorMsg}`);
             workerProcess.kill();
             this.workers.delete(workerId);
@@ -16680,6 +16932,42 @@ class Master {
       }
     });
   }
+  async reconcilePluginRuntimeAcrossWorkers() {
+    return await this.pluginRuntimeConvergenceCoordinator.dispatchReconcile(Array.from(this.workers.values()).map((workerInfo) => ({
+      workerId: workerInfo.workerId,
+      status: workerInfo.status,
+      send: workerInfo.process.send?.bind(workerInfo.process)
+    })));
+  }
+  async handleClusterPluginRuntimeReconcileRequest(requester, message) {
+    try {
+      const convergence = await this.reconcilePluginRuntimeAcrossWorkers();
+      requester.process.send?.({
+        status: "cluster-plugin-runtime-reconcile-result",
+        requestId: message.requestId,
+        convergence
+      });
+    } catch (error) {
+      requester.process.send?.({
+        status: "cluster-plugin-runtime-reconcile-result",
+        requestId: message.requestId,
+        convergence: null,
+        error: error instanceof Error ? error.message : String(error)
+      });
+    }
+  }
+  logPluginRuntimeConvergenceStatus() {
+    const report = this.pluginRuntimeConvergenceCoordinator.getStatusReport();
+    logger.info({
+      targetGeneration: report.targetGeneration,
+      convergedGeneration: report.convergedGeneration,
+      status: report.status,
+      failedWorkers: report.failedWorkers,
+      staleWorkers: report.staleWorkers,
+      pendingWorkers: report.pendingWorkers,
+      workers: report.workers
+    }, "Plugin runtime multi-worker convergence status updated");
+  }
   watchConfig() {
     fs7.watch(CONFIG_PATH, (eventType) => {
       if (eventType === "change") {