@zq-silk/yui 0.6.2 → 0.6.3

package/dist/controller/controller.js

@@ -17,6 +17,7 @@ import { formatTaskRecordReference } from "../task/taskRecordReference.js";
 import { parseDurableJobAcknowledgeParams, parseDurableJobCancelParams, parseDurableJobRefParams, parseDurableJobStartParams } from "./jobControl.js";
 const DEFAULT_RECONCILIATION_INTERVAL_MS = reconciliationIntervalMilliseconds();
 const DEFAULT_SIGNAL_WINDOW_MS = 100;
+const DEFAULT_RUNTIME_OBSERVER_INTERVAL_MS = 1_000;
 const DEFAULT_DELIVERY_RETRY_MS = 250;
 const DEFAULT_DELIVERY_RETRY_LIMIT = 60;
 const DEFAULT_TASK_ORCHESTRATION_RETRY_LIMIT = 2;
@@ -39,7 +40,7 @@ const ZERO_DRAIN_METRICS = Object.freeze({
  * Runs one lean scheduler pass. Due native Turn completions are folded before
  * liveness, so a valid Hook boundary fences destructive process reconciliation.
  */
-export async function runControllerSchedulerPass(store, delivery, now, workspacePreparer, scope = { kind: "full" }, includeOperator = true, runtimeCleanupOutcomes = [], lifecycleHost, stallWindowMs = DEFAULT_STALL_WINDOW_MS, resourceSuppressionKeys = new Set(), maintenanceFence, onMaintenanceFenceDefer) {
+export async function runControllerSchedulerPass(store, delivery, now, workspacePreparer, scope = { kind: "full" }, includeOperator = true, runtimeCleanupOutcomes = [], lifecycleHost, stallWindowMs = DEFAULT_STALL_WINDOW_MS, maintenanceFence, onMaintenanceFenceDefer) {
     const compiledSelection = compileReconcileSelection(scope);
     const selection = includeOperator
         ? compiledSelection
@@ -108,7 +109,7 @@ export async function runControllerSchedulerPass(store, delivery, now, workspace
         const resourceEvidence = new Map();
         const failedRunRefs = await reconcileExitedRoleRuns(store, delivery, now, roleSelection, unsettledRunRefs, liveStatuses, resourceEvidence);
         await controlEventLoopTurn();
-        await reconcileStalledRoleRuns(store, delivery, now, roleSelection, stallWindowMs, liveStatuses, resourceEvidence, resourceSuppressionKeys);
+        await reconcileStalledRoleRuns(store, delivery, now, roleSelection, stallWindowMs, liveStatuses, resourceEvidence);
         await controlEventLoopTurn();
         await reconcileDormantRuntimeOwners(store, delivery, lifecycleHost, scope, now);
         const autoResolvedInputs = selection.full
@@ -690,9 +691,9 @@ export class FileTaskController {
     #deliveryRetryLimit;
     #taskOrchestrationRetryLimit;
     #stallWindowMs;
-    /** Narrow-port fallback; FileSchedulerStoreAdapter durably records these keys. */
-    #resourceSuppressionKeys = new Set();
     #runtimeEventProcessor;
+    #runtimeObserver;
+    #runtimeObserverIntervalMs;
     #lifecycleHost;
     #deliveryRetryAttempts = new Map();
     #deliveryRetryTimers = new Map();
@@ -700,6 +701,7 @@ export class FileTaskController {
     #passRetryAttempt = 0;
     #timer;
     #deadlineTimer;
+    #runtimeObserverTimer;
     #signalScheduler;
     #operatorSignalScheduler;
     #configuration;
@@ -711,6 +713,7 @@ export class FileTaskController {
     #jobSupervisor;
     #current;
     #operatorCurrent;
+    #runtimeObserverCurrent;
     #pendingFull = false;
     #pendingKeys = new Set();
     #operatorStartupRetryArmed = false;
@@ -734,6 +737,8 @@ export class FileTaskController {
         this.#taskOrchestrationRetryLimit = positiveInteger(options.taskOrchestrationRetryLimit, DEFAULT_TASK_ORCHESTRATION_RETRY_LIMIT, "Controller Task orchestration retry limit");
         this.#stallWindowMs = positiveInteger(options.stallWindowMs, DEFAULT_STALL_WINDOW_MS, "Controller Run stall window");
         this.#runtimeEventProcessor = options.runtimeEventProcessor;
+        this.#runtimeObserver = options.runtimeObserver;
+        this.#runtimeObserverIntervalMs = positiveInteger(options.runtimeObserverIntervalMs, DEFAULT_RUNTIME_OBSERVER_INTERVAL_MS, "Controller runtime observer interval");
         this.#lifecycleHost = options.lifecycleHost;
         this.#configuration = options.configuration;
         this.#resourceReaper = options.resourceReaper;
@@ -820,6 +825,7 @@ export class FileTaskController {
         if (this.#stopped)
             throw new Error("Controller runtime is stopped.");
         this.#requestBackgroundPump();
+        this.#requestRuntimeObservation();
         // Arm the Operator lane once for work that was already pending when the
         // Controller started. Subsequent main-lane passes signal it only when a
         // new Operator batch is durably queued; an unchanged pending batch must
@@ -829,6 +835,12 @@ export class FileTaskController {
             this.#requestBackgroundPump();
         }, this.#intervalMs);
         this.#timer.unref();
+        if (this.#runtimeObserver !== undefined) {
+            this.#runtimeObserverTimer = setInterval(() => {
+                this.#requestRuntimeObservation();
+            }, this.#runtimeObserverIntervalMs);
+            this.#runtimeObserverTimer.unref();
+        }
     }
     stop() {
         this.#stopped = true;
@@ -842,6 +854,10 @@ export class FileTaskController {
             clearInterval(this.#timer);
             this.#timer = undefined;
         }
+        if (this.#runtimeObserverTimer !== undefined) {
+            clearInterval(this.#runtimeObserverTimer);
+            this.#runtimeObserverTimer = undefined;
+        }
         for (const timer of this.#deliveryRetryTimers.values())
             clearTimeout(timer);
         this.#deliveryRetryTimers.clear();
@@ -855,7 +871,8 @@ export class FileTaskController {
         this.stop();
         await Promise.allSettled([
             this.#current ?? Promise.resolve(),
-            this.#operatorCurrent ?? Promise.resolve()
+            this.#operatorCurrent ?? Promise.resolve(),
+            this.#runtimeObserverCurrent ?? Promise.resolve()
         ]);
     }
     /** Adds one dirty Task key to the fixed-window wake queue and returns immediately. */
@@ -875,6 +892,21 @@ export class FileTaskController {
     pump() {
         return this.#requestPass({ kind: "full" });
     }
+    #requestRuntimeObservation() {
+        if (this.#stopped || this.#runtimeObserver === undefined
+            || this.#runtimeObserverCurrent !== undefined)
+            return;
+        const running = this.#runtimeObserver.sample(this.#now()).then((keys) => {
+            if (this.#stopped)
+                return;
+            for (const key of keys)
+                this.signal(key);
+        }).catch(this.#onError).finally(() => {
+            if (this.#runtimeObserverCurrent === running)
+                this.#runtimeObserverCurrent = undefined;
+        });
+        this.#runtimeObserverCurrent = running;
+    }
     armOperatorStartupRetry() {
         if (this.#stopped)
             return;
@@ -954,7 +986,7 @@ export class FileTaskController {
                     // terminal job's Leader wakeup is enqueued in the same pass that
                     // processes Leader wakeups.
                     this.#jobSupervisor?.reconcile(this.#now());
-                    result = await runControllerSchedulerPass(this.store, this.delivery, this.#now(), this.#workspacePreparer, scope, false, runtimeCleanupOutcomes, this.#lifecycleHost, this.#stallWindowMs, this.#resourceSuppressionKeys, this.#maintenanceFence, this.#onMaintenanceFenceDefer);
+                    result = await runControllerSchedulerPass(this.store, this.delivery, this.#now(), this.#workspacePreparer, scope, false, runtimeCleanupOutcomes, this.#lifecycleHost, this.#stallWindowMs, this.#maintenanceFence, this.#onMaintenanceFenceDefer);
                     const secondRuntimeDrain = await this.#drainRuntimeEvents();
                     pendingRuntimeDrain = ((secondRuntimeDrain?.remainingEventCount ?? 0) > 0
                         && ((firstRuntimeDrain?.acknowledgedEventIds.length ?? 0) > 0
@@ -1133,6 +1165,7 @@ export class FileTaskController {
     #scheduleDeliveryRetries(result) {
         const retry = new Map();
         const settled = new Set();
+        const writerBlocked = new Set();
         const resignal = new Set();
         for (const delivery of result.activeRunDeliveries) {
             const key = `role:${encodeURIComponent(delivery.taskId)}/${encodeURIComponent(delivery.roleName)}`;
@@ -1140,6 +1173,8 @@ export class FileTaskController {
                 settled.add(key);
                 resignal.add(key);
             }
+            else if (delivery.reason === "writer-attached")
+                writerBlocked.add(key);
             else if (delivery.reason === "not-ready"
                 || delivery.reason === "runtime-unavailable"
                 || delivery.reason === "delivery-uncertain") {
@@ -1155,7 +1190,10 @@ export class FileTaskController {
         }
         for (const wakeup of result.wakeups) {
             const key = `role:${encodeURIComponent(wakeup.taskId)}/leader`;
-            if (wakeup.reason === "not-ready"
+            if (wakeup.reason === "writer-attached") {
+                writerBlocked.add(key);
+            }
+            else if (wakeup.reason === "not-ready"
                 || wakeup.reason === "delivery-uncertain") {
                 retry.set(key, { identity: wakeup.runId ?? key });
             }
@@ -1183,6 +1221,8 @@ export class FileTaskController {
         }
         for (const key of settled)
             this.#clearDeliveryRetry(key);
+        for (const key of writerBlocked)
+            this.#clearDeliveryRetry(key);
         for (const key of resignal)
             this.signal(key);
         for (const [key, candidate] of retry) {