memeloop 0.2.8 → 0.2.9

package/dist/index.js

@@ -73,7 +73,7 @@ import {
   runAgentToolLoopTurn,
   saveUserPermissions,
   validatePluginManifest
-} from "./chunk-OD6U6VAR.js";
+} from "./chunk-GH5RDPLJ.js";
 import {
   AGENT_AGENT_LOOP_ID,
   AGENT_TOOL_LOOP_ID,
@@ -262,7 +262,7 @@ import {
 import {
   MemoryDeviceNetworkService,
   syncCloudDevices
-} from "./chunk-NVJCRIEK.js";
+} from "./chunk-34B7MUI6.js";
 import {
   ChatSyncEngine,
   CloudDeviceFetchClient,
@@ -275,6 +275,7 @@ import {
   DEVICE_CONNECTION_GRANT_MAX_TTL_MS,
   DEVICE_CONNECTION_GRANT_SIGNATURE_DOMAIN,
   DEVICE_GRANT_MAX_CLOCK_SKEW_MS,
+  DEVICE_ORCHESTRATION_FRAME_LIMITS,
   DEVICE_ORCHESTRATION_PROTOCOL,
   DEVICE_PAIRING_INVITE_MAX_CLOCK_SKEW_MS,
   DEVICE_PAIRING_INVITE_PROTOCOL,
@@ -330,7 +331,7 @@ import {
   reconcileCloudDeviceDirectory,
   versionVectorCoversRange,
   versionVectorKey
-} from "./chunk-DACM5AMS.js";
+} from "./chunk-7W4FMKMO.js";
 import {
   DEVICE_HEARTBEAT_DEFAULT_MAX_CLOCK_SKEW_MS,
   DEVICE_HEARTBEAT_LIMITS,
@@ -5870,10 +5871,54 @@ function createExternalOrchestrationController(store, options) {
   const now = options.now ?? (() => /* @__PURE__ */ new Date());
   const onError = options.onError ?? (() => {
   });
-  const active = /* @__PURE__ */ new Set();
+  const active = /* @__PURE__ */ new Map();
   const activeByDriver = /* @__PURE__ */ new Map();
   const watchAbort = new AbortController();
+  const stoppedError = new OrchestrationError({
+    code: "CANCELLED",
+    message: "external orchestration controller stopped",
+    retryable: false
+  });
+  let resolveStopped;
+  const stoppedBoundary = new Promise((resolve) => {
+    resolveStopped = resolve;
+  });
   let stopped = false;
+  let stopPromise;
+  function throwIfStopped() {
+    if (stopped) throw stoppedError;
+  }
+  async function whileRunning(operation) {
+    throwIfStopped();
+    const operationResult = Promise.resolve().then(operation).then(
+      (value) => ({ kind: "value", value }),
+      (error2) => ({
+        kind: "error",
+        error: error2 instanceof Error ? error2 : new Error(safeErrorMessageFromUnknown(error2))
+      })
+    );
+    const result = await Promise.race([operationResult, stoppedBoundary]);
+    if (result.kind === "stopped") throw stoppedError;
+    if (result.kind === "error") throw result.error;
+    return result.value;
+  }
+  function reportUnlessStopped(error2) {
+    if (!stopped && error2 !== stoppedError) onError(error2);
+  }
+  async function drainWithDeadline(tasks) {
+    let timeout;
+    try {
+      await Promise.race([
+        Promise.allSettled(tasks).then(() => {
+        }),
+        new Promise((resolve) => {
+          timeout = setTimeout(resolve, 1e3);
+        })
+      ]);
+    } finally {
+      if (timeout) clearTimeout(timeout);
+    }
+  }
   async function acquireDriverSlot(entry) {
     const limit = entry.capabilities.maxConcurrency;
     while (!stopped) {
@@ -5882,7 +5927,7 @@ function createExternalOrchestrationController(store, options) {
         activeByDriver.set(entry.name, current + 1);
         return true;
       }
-      await sleep(pollIntervalMs);
+      await whileRunning(() => sleep(pollIntervalMs));
     }
     return false;
   }
@@ -5912,7 +5957,7 @@ function createExternalOrchestrationController(store, options) {
   }
   async function ensureRun(workload) {
     const reference = runReferenceOf(workload);
-    if (await store.get(reference)) return reference;
+    if (await whileRunning(() => store.get(reference))) return reference;
     const manifest = createAgentRunManifest(reference.name, {
       workloadRef: {
         apiVersion: workload.apiVersion,
@@ -5924,7 +5969,7 @@ function createExternalOrchestrationController(store, options) {
     });
     manifest.metadata.namespace = workload.metadata.namespace;
     try {
-      await store.create(options.actor, manifest);
+      await whileRunning(() => store.create(options.actor, manifest));
     } catch (error2) {
       if (!(error2 instanceof OrchestrationError) || error2.code !== "CONFLICT") throw error2;
     }
@@ -5932,12 +5977,14 @@ function createExternalOrchestrationController(store, options) {
   }
   async function updateStatus(reference, patch) {
     for (let attempt = 0; attempt < statusWriteAttempts; attempt += 1) {
-      const current = await store.get(reference);
+      const current = await whileRunning(() => store.get(reference));
       if (!current) return;
       try {
-        await store.updateStatus(options.actor, reference, patch(current.status), {
-          resourceVersion: current.metadata.resourceVersion
-        });
+        await whileRunning(
+          () => store.updateStatus(options.actor, reference, patch(current.status), {
+            resourceVersion: current.metadata.resourceVersion
+          })
+        );
         return;
       } catch (error2) {
         if (error2 instanceof OrchestrationError && error2.code === "CONFLICT") continue;
@@ -5977,6 +6024,7 @@ function createExternalOrchestrationController(store, options) {
     return entry;
   }
   async function fail(resource, error2) {
+    throwIfStopped();
     const message = safeErrorMessageFromUnknown(error2, { fallback: "External orchestration failed" });
     if (resource.kind === AGENT_WORKLOAD_KIND) {
       await updateStatus(runReferenceOf(resource), () => ({
@@ -6049,6 +6097,7 @@ function createExternalOrchestrationController(store, options) {
     return terminal;
   }
   async function reconcileWorkload(resource) {
+    throwIfStopped();
     const entry = findDriver(resource);
     const runtime = resolveExternalWorkloadRuntime(
       resource,
@@ -6057,7 +6106,7 @@ function createExternalOrchestrationController(store, options) {
     resolveExternalWorkloadResources(resource, runtime);
     const reference = referenceOf(resource);
     const runReference = await ensureRun(resource);
-    const latest = await store.get(reference);
+    const latest = await whileRunning(() => store.get(reference));
     let externalId = latest?.status?.externalId ?? resource.status?.externalId;
     if (!externalId) {
       if (!options.authorizeWorkloadPlacement) {
@@ -6067,8 +6116,8 @@ function createExternalOrchestrationController(store, options) {
           retryable: false
         });
       }
-      const authorization = await options.authorizeWorkloadPlacement(resource, entry);
-      const health = await entry.driver.getHealth();
+      const authorization = await whileRunning(() => options.authorizeWorkloadPlacement(resource, entry, watchAbort.signal));
+      const health = await whileRunning(() => entry.driver.getHealth());
       if (!health.healthy) {
         throw new OrchestrationError({ code: "UNAVAILABLE", message: `external orchestrator '${entry.name}' is unhealthy`, retryable: true });
       }
@@ -6081,21 +6130,25 @@ function createExternalOrchestrationController(store, options) {
         placementPolicyDigest: authorization.policyDigest
       }));
       let scriptSource;
-      if (resource.spec.scriptReference) {
-        scriptSource = await options.resolveScriptSource?.(resource.spec.scriptReference);
+      const scriptReference = resource.spec.scriptReference;
+      if (scriptReference) {
+        scriptSource = await whileRunning(async () => options.resolveScriptSource?.(scriptReference));
         if (scriptSource === void 0) {
           throw new OrchestrationError({
             code: "NOT_FOUND",
-            message: `script artifact '${resource.spec.scriptReference}' is unavailable for external placement`,
+            message: `script artifact '${scriptReference}' is unavailable for external placement`,
             retryable: false
           });
         }
       }
-      const workerBootstrap = await options.createWorkerBootstrap?.(resource, runReference);
-      const placed = await entry.driver.placeWorkload(resource, options.actor, {
-        ...scriptSource !== void 0 ? { scriptSource } : {},
-        ...workerBootstrap !== void 0 ? { workerBootstrap } : {}
-      });
+      const workerBootstrap = await whileRunning(async () => options.createWorkerBootstrap?.(resource, runReference));
+      const placed = await whileRunning(
+        () => entry.driver.placeWorkload(resource, options.actor, {
+          ...scriptSource !== void 0 ? { scriptSource } : {},
+          ...workerBootstrap !== void 0 ? { workerBootstrap } : {},
+          signal: watchAbort.signal
+        })
+      );
       externalId = placed.externalId;
       await updateStatus(reference, (current) => ({
         ...current,
@@ -6109,12 +6162,13 @@ function createExternalOrchestrationController(store, options) {
       await updateStatus(runReference, () => ({ phase: "Running" }));
     }
     while (!stopped) {
-      const external = await entry.driver.getWorkloadStatus(externalId);
+      const external = await whileRunning(() => entry.driver.getWorkloadStatus(externalId));
       if (await reflectWorkloadStatus(resource, external)) return;
-      await sleep(pollIntervalMs);
+      await whileRunning(() => sleep(pollIntervalMs));
     }
   }
   async function reconcileToolOperation(resource) {
+    throwIfStopped();
     const entry = findDriver(resource);
     const runtimeImage = resource.metadata.annotations?.["memeloop.io/runtime-image"];
     const contract = assertExternalToolOperationContract(
@@ -6123,10 +6177,10 @@ function createExternalOrchestrationController(store, options) {
       runtimeImage
     );
     const reference = referenceOf(resource);
-    const latest = await store.get(reference);
+    const latest = await whileRunning(() => store.get(reference));
     let externalId = latest?.status?.externalId ?? resource.status?.externalId;
     if (!externalId) {
-      const health = await entry.driver.getHealth();
+      const health = await whileRunning(() => entry.driver.getHealth());
       if (!health.healthy) {
         throw new OrchestrationError({ code: "UNAVAILABLE", message: `external orchestrator '${entry.name}' is unhealthy`, retryable: true });
       }
@@ -6142,14 +6196,14 @@ function createExternalOrchestrationController(store, options) {
           retryable: false
         });
       }
-      const authorization = await options.authorizeToolOperation(resource);
+      const authorization = await whileRunning(() => options.authorizeToolOperation(resource, watchAbort.signal));
       if (authorization.approval) {
         await updateStatus(reference, (current) => ({
           ...current,
           approval: authorization.approval
         }));
       }
-      const placed = await entry.driver.executeToolOperation(resource, options.actor);
+      const placed = await whileRunning(() => entry.driver.executeToolOperation(resource, options.actor));
       externalId = placed.externalId;
       await updateStatus(reference, (current) => ({
         ...current,
@@ -6162,27 +6216,28 @@ function createExternalOrchestrationController(store, options) {
       }));
     }
     while (!stopped) {
-      const external = await entry.driver.getToolOperationStatus(externalId);
+      const external = await whileRunning(() => entry.driver.getToolOperationStatus(externalId));
       if (await reflectToolStatus(resource, external, contract)) return;
-      await sleep(pollIntervalMs);
+      await whileRunning(() => sleep(pollIntervalMs));
     }
   }
   function maybeStart(resource) {
+    if (stopped) return;
     const routed = resource;
     if (!routeOf(routed)) return;
     const phase = routed.status?.phase;
     if (phase === "Completed" || phase === "Failed" || phase === "Cancelled") return;
     if (active.has(routed.metadata.uid)) return;
-    active.add(routed.metadata.uid);
-    void (async () => {
+    const task = (async () => {
       let slot;
       try {
         const entry = findDriver(routed);
         if (!await acquireDriverSlot(entry)) return;
         slot = entry.name;
       } catch (error2) {
+        if (stopped || error2 === stoppedError) return;
         onError(error2);
-        await fail(routed, error2).catch(onError);
+        await fail(routed, error2).catch(reportUnlessStopped);
         return;
       }
       try {
@@ -6195,18 +6250,25 @@ function createExternalOrchestrationController(store, options) {
             }
             return;
           } catch (error2) {
+            if (stopped || error2 === stoppedError) return;
             onError(error2);
             if (!(error2 instanceof OrchestrationError) || !error2.retryable) {
-              await fail(routed, error2).catch(onError);
+              await fail(routed, error2).catch(reportUnlessStopped);
               return;
             }
-            await sleep(pollIntervalMs);
+            await whileRunning(() => sleep(pollIntervalMs));
           }
         }
       } finally {
         if (slot) releaseDriverSlot(slot);
       }
-    })().finally(() => active.delete(routed.metadata.uid));
+    })();
+    active.set(routed.metadata.uid, task);
+    void task.finally(() => {
+      if (active.get(routed.metadata.uid) === task) active.delete(routed.metadata.uid);
+    }).catch((error2) => {
+      if (!stopped && error2 !== stoppedError) onError(error2);
+    });
   }
   async function cancelDeleted(resource) {
     const driverName = resource.status?.assignedDriver ?? routeOf(resource);
@@ -6227,7 +6289,7 @@ function createExternalOrchestrationController(store, options) {
           { apiVersion, kind },
           { signal: watchAbort.signal }
         )[Symbol.asyncIterator]();
-        const existing = await store.list({ apiVersion, kind });
+        const existing = await whileRunning(() => store.list({ apiVersion, kind }));
         for (const resource of existing.items) maybeStart(resource);
         while (!stopped) {
           const next = await iterator.next();
@@ -6235,30 +6297,35 @@ function createExternalOrchestrationController(store, options) {
           const event = next.value;
           if (stopped) break;
           if (event.type === "DELETED") {
-            await cancelDeleted(event.resource).catch(onError);
+            await cancelDeleted(event.resource).catch(reportUnlessStopped);
           } else if (event.type === "ADDED" || event.type === "MODIFIED") {
             maybeStart(event.resource);
           }
         }
       } catch (error2) {
-        onError(error2);
+        if (!stopped && error2 !== stoppedError) onError(error2);
+      }
+      if (!stopped) {
+        await whileRunning(() => sleep(1e3)).catch((error2) => {
+          if (!stopped && error2 !== stoppedError) onError(error2);
+        });
       }
-      if (!stopped) await sleep(1e3);
     }
   }
   const watchers = [
     runKind(AGENT_WORKLOAD_API_VERSION, AGENT_WORKLOAD_KIND),
     runKind(TOOL_OPERATION_API_VERSION, TOOL_OPERATION_KIND)
   ];
-  for (const watcher of watchers) void watcher.catch(onError);
+  for (const watcher of watchers) void watcher.catch(reportUnlessStopped);
   return {
     async stop() {
-      stopped = true;
-      watchAbort.abort();
-      await Promise.race([
-        Promise.allSettled(watchers),
-        new Promise((resolve) => setTimeout(resolve, 50))
-      ]);
+      if (!stopPromise) {
+        stopped = true;
+        resolveStopped({ kind: "stopped" });
+        watchAbort.abort(stoppedError);
+        stopPromise = drainWithDeadline([...watchers, ...active.values()]);
+      }
+      await stopPromise;
     }
   };
 }
@@ -9818,15 +9885,64 @@ function createWorkloadExecutionController(store, driver, options) {
     });
   }
   const active = /* @__PURE__ */ new Map();
+  const executions = /* @__PURE__ */ new Map();
+  const cancellationRequested = /* @__PURE__ */ new Set();
+  const watchAbort = new AbortController();
+  const stoppedError = new OrchestrationError({
+    code: "CANCELLED",
+    message: "workload execution controller stopped",
+    retryable: false
+  });
+  let resolveStopped;
+  const stoppedBoundary = new Promise((resolve) => {
+    resolveStopped = resolve;
+  });
   let stopped = false;
+  let stopPromise;
+  function throwIfStopped() {
+    if (stopped) throw stoppedError;
+  }
+  async function whileRunning(operation) {
+    throwIfStopped();
+    const operationResult = Promise.resolve().then(operation).then(
+      (value) => ({ kind: "value", value }),
+      (error2) => ({
+        kind: "error",
+        error: error2 instanceof Error ? error2 : new Error(safeErrorMessageFromUnknown(error2))
+      })
+    );
+    const result = await Promise.race([operationResult, stoppedBoundary]);
+    if (result.kind === "stopped") throw stoppedError;
+    if (result.kind === "error") throw result.error;
+    return result.value;
+  }
+  function reportUnlessStopped(error2) {
+    if (!stopped && error2 !== stoppedError) onError(error2);
+  }
+  async function drainWithDeadline(tasks) {
+    let timeout;
+    try {
+      await Promise.race([
+        Promise.allSettled(tasks).then(() => {
+        }),
+        new Promise((resolve) => {
+          timeout = setTimeout(resolve, 1e3);
+        })
+      ]);
+    } finally {
+      if (timeout) clearTimeout(timeout);
+    }
+  }
   async function updateStatusWithRetry(reference, patch) {
     for (let attempt = 0; attempt < statusWriteAttempts; attempt += 1) {
-      const current = await store.get(reference);
+      const current = await whileRunning(() => store.get(reference));
       if (!current) return;
       try {
-        await store.updateStatus(options.actor, reference, patch(current.status), {
-          resourceVersion: current.metadata.resourceVersion
-        });
+        await whileRunning(
+          () => store.updateStatus(options.actor, reference, patch(current.status), {
+            resourceVersion: current.metadata.resourceVersion
+          })
+        );
         return;
       } catch (error2) {
         if (error2 instanceof OrchestrationError && error2.code === "CONFLICT") continue;
@@ -9861,22 +9977,26 @@ function createWorkloadExecutionController(store, driver, options) {
   }
   async function claimRuntimeExecution(reference) {
     for (let attempt = 0; attempt < statusWriteAttempts; attempt += 1) {
-      const current = await store.get(reference);
+      const current = await whileRunning(
+        () => store.get(reference)
+      );
       if (!current) return null;
       if (current.status?.runtimeExecutionClaim) return null;
       try {
-        return await store.updateStatus(
-          options.actor,
-          reference,
-          {
-            ...current.status,
-            phase: "Starting",
-            runtimeExecutionClaim: {
-              controllerInstanceId,
-              claimedAt: (/* @__PURE__ */ new Date()).toISOString()
-            }
-          },
-          { resourceVersion: current.metadata.resourceVersion }
+        return await whileRunning(
+          () => store.updateStatus(
+            options.actor,
+            reference,
+            {
+              ...current.status,
+              phase: "Starting",
+              runtimeExecutionClaim: {
+                controllerInstanceId,
+                claimedAt: (/* @__PURE__ */ new Date()).toISOString()
+              }
+            },
+            { resourceVersion: current.metadata.resourceVersion }
+          )
         );
       } catch (error2) {
         if (error2 instanceof OrchestrationError && error2.code === "CONFLICT") {
@@ -9901,8 +10021,10 @@ function createWorkloadExecutionController(store, driver, options) {
           retryable: false
         });
       }
-      const current = await store.get(
-        runReference
+      const current = await whileRunning(
+        () => store.get(
+          runReference
+        )
       );
       if (!current) {
         throw new OrchestrationError({
@@ -9914,12 +10036,14 @@ function createWorkloadExecutionController(store, driver, options) {
       if (!workload.spec.modelPolicy?.modelClass) return { run: current };
       const binding = current.status?.assignedModelEndpoint;
       if (binding) {
-        const endpoint = await store.get({
-          apiVersion: binding.apiVersion || MODEL_ENDPOINT_API_VERSION,
-          kind: binding.kind || MODEL_ENDPOINT_KIND,
-          name: binding.name,
-          namespace: binding.namespace
-        });
+        const endpoint = await whileRunning(
+          () => store.get({
+            apiVersion: binding.apiVersion || MODEL_ENDPOINT_API_VERSION,
+            kind: binding.kind || MODEL_ENDPOINT_KIND,
+            name: binding.name,
+            namespace: binding.namespace
+          })
+        );
         if (endpoint && endpoint.metadata.uid === binding.uid && endpoint.status?.healthy === true && Number.isFinite(Date.parse(endpoint.status.heartbeat ?? "")) && Date.now() - Date.parse(endpoint.status.heartbeat ?? "") <= modelEndpointHeartbeatTtlMs && endpoint.spec.modelClassRef.name === workload.spec.modelPolicy.modelClass) {
           return { run: current, endpoint };
         }
@@ -9931,7 +10055,7 @@ function createWorkloadExecutionController(store, driver, options) {
           retryable: true
         });
       }
-      await sleep(dependencyPollIntervalMs);
+      await whileRunning(() => sleep(dependencyPollIntervalMs));
     }
   }
   async function ensureNetworkAttachment(workload, runReference, runUid) {
@@ -9944,7 +10068,9 @@ function createWorkloadExecutionController(store, driver, options) {
       name: attachmentName,
       namespace: workload.metadata.namespace
     };
-    let attachment = await store.get(reference);
+    let attachment = await whileRunning(
+      () => store.get(reference)
+    );
     if (!attachment) {
       const manifest = createNetworkAttachmentManifest(attachmentName, {
         networkClassRef: {
@@ -9968,7 +10094,7 @@ function createWorkloadExecutionController(store, driver, options) {
         nodeId: options.nodeId
       });
       manifest.metadata.namespace = workload.metadata.namespace;
-      attachment = await store.create(options.actor, manifest);
+      attachment = await whileRunning(() => store.create(options.actor, manifest));
     }
     await updateStatusWithRetry(runReference, (current) => ({
       ...current,
@@ -9989,7 +10115,9 @@ function createWorkloadExecutionController(store, driver, options) {
           retryable: false
         });
       }
-      const current = await store.get(reference);
+      const current = await whileRunning(
+        () => store.get(reference)
+      );
       if (!current || current.metadata.uid !== attachment.metadata.uid) {
         throw new OrchestrationError({
           code: "NOT_FOUND",
@@ -10005,7 +10133,9 @@ function createWorkloadExecutionController(store, driver, options) {
         });
       }
       if (current.status?.phase === "Attached" && current.status.handle) {
-        const networkClass = await store.get(current.spec.networkClassRef);
+        const networkClass = await whileRunning(
+          () => store.get(current.spec.networkClassRef)
+        );
         if (networkClass && current.status.binding?.networkClassResourceVersion === networkClass.metadata.resourceVersion && current.status.assignedNode === options.nodeId && current.status.assignedDriver === networkClass.spec.driver) {
           return current;
         }
@@ -10017,14 +10147,16 @@ function createWorkloadExecutionController(store, driver, options) {
           retryable: true
         });
       }
-      await sleep(dependencyPollIntervalMs);
+      await whileRunning(() => sleep(dependencyPollIntervalMs));
     }
   }
   async function waitForVolumeBindings(workload, runReference) {
     const expected = workload.spec.storagePolicy?.volumes ?? [];
     if (expected.length === 0) {
-      const run = await store.get(
-        runReference
+      const run = await whileRunning(
+        () => store.get(
+          runReference
+        )
       );
       return { run };
     }
@@ -10037,8 +10169,10 @@ function createWorkloadExecutionController(store, driver, options) {
           retryable: false
         });
       }
-      const run = await store.get(
-        runReference
+      const run = await whileRunning(
+        () => store.get(
+          runReference
+        )
       );
       if (!run) {
         throw new OrchestrationError({
@@ -10065,7 +10199,7 @@ function createWorkloadExecutionController(store, driver, options) {
             retryable: false
           });
         }
-        return { run, mounts: await options.resolveVolumeMounts(workload, run) };
+        return { run, mounts: await whileRunning(() => options.resolveVolumeMounts(workload, run)) };
       }
       if (Date.now() >= deadline) {
         throw new OrchestrationError({
@@ -10074,12 +10208,14 @@ function createWorkloadExecutionController(store, driver, options) {
           retryable: true
         });
       }
-      await sleep(dependencyPollIntervalMs);
+      await whileRunning(() => sleep(dependencyPollIntervalMs));
     }
   }
   async function requestDependencyRelease(runReference) {
-    const run = await store.get(
-      runReference
+    const run = await whileRunning(
+      () => store.get(
+        runReference
+      )
     );
     if (!run) return;
     const requestedAt = (/* @__PURE__ */ new Date()).toISOString();
@@ -10106,6 +10242,7 @@ function createWorkloadExecutionController(store, driver, options) {
     }
   }
   async function execute(workload) {
+    throwIfStopped();
     const workloadReference_ = workloadReference(workload);
     const runName = `${workload.metadata.name}-run`;
     const runReference = {
@@ -10117,18 +10254,19 @@ function createWorkloadExecutionController(store, driver, options) {
     let runtimeClaimed = false;
     try {
       let scriptSource;
-      if (workload.spec.scriptReference) {
-        scriptSource = await options.resolveScriptSource?.(workload.spec.scriptReference);
+      const scriptReference = workload.spec.scriptReference;
+      if (scriptReference) {
+        scriptSource = await whileRunning(async () => options.resolveScriptSource?.(scriptReference));
         if (!scriptSource) {
           await updateStatusWithRetry(workloadReference_, (current) => ({
             ...current,
             phase: "Failed",
-            lastRunResult: `script artifact '${workload.spec.scriptReference}' unavailable`
+            lastRunResult: `script artifact '${scriptReference}' unavailable`
           }));
           return;
         }
       }
-      let run = await store.get(runReference);
+      let run = await whileRunning(() => store.get(runReference));
       if (!run) {
         const manifest = createAgentRunManifest(runName, {
           workloadRef: {
@@ -10141,15 +10279,19 @@ function createWorkloadExecutionController(store, driver, options) {
         });
         manifest.metadata.namespace = workload.metadata.namespace;
         try {
-          run = await store.create(
-            options.actor,
-            manifest
+          run = await whileRunning(
+            () => store.create(
+              options.actor,
+              manifest
+            )
           );
         } catch (error2) {
           if (!(error2 instanceof OrchestrationError) || error2.code !== "CONFLICT") {
             throw error2;
           }
-          run = await store.get(runReference);
+          run = await whileRunning(
+            () => store.get(runReference)
+          );
           if (!run) throw error2;
         }
       }
@@ -10169,8 +10311,10 @@ function createWorkloadExecutionController(store, driver, options) {
         waitForVolumeBindings(workload, runReference)
       ]);
       run = volumeDependencies.run;
-      run = await store.get(
-        runReference
+      run = await whileRunning(
+        () => store.get(
+          runReference
+        )
       ) ?? run;
       await updateStatusWithRetry(workloadReference_, (current) => ({
         ...current,
@@ -10181,21 +10325,27 @@ function createWorkloadExecutionController(store, driver, options) {
       if (!claimedRun) return;
       runtimeClaimed = true;
       run = claimedRun;
-      const handle = await driver.start({
-        workload,
-        run,
-        ...dependencies.endpoint ? { modelEndpoint: dependencies.endpoint } : {},
-        ...networkAttachment ? { networkAttachment } : {},
-        ...volumeDependencies.mounts ? { volumeMounts: volumeDependencies.mounts } : {},
-        scriptSource,
-        message: options.messageForWorkload?.(workload) ?? workload.metadata.name
-      });
+      const handle = await whileRunning(
+        () => driver.start({
+          workload,
+          run,
+          ...dependencies.endpoint ? { modelEndpoint: dependencies.endpoint } : {},
+          ...networkAttachment ? { networkAttachment } : {},
+          ...volumeDependencies.mounts ? { volumeMounts: volumeDependencies.mounts } : {},
+          scriptSource,
+          message: options.messageForWorkload?.(workload) ?? workload.metadata.name
+        })
+      );
       active.set(workload.metadata.uid, handle);
+      if (cancellationRequested.has(workload.metadata.uid)) {
+        await handle.cancel().catch(reportUnlessStopped);
+        return;
+      }
       await updateStatusWithRetry(runReference, (current) => ({
         ...current,
         phase: "Running"
       }));
-      const outcome = await handle.wait();
+      const outcome = await whileRunning(() => handle.wait());
       await requestDependencyRelease(runReference);
       await updateStatusWithRetry(runReference, (current) => ({
         ...current,
@@ -10209,7 +10359,8 @@ function createWorkloadExecutionController(store, driver, options) {
         lastRunResult: outcome.summary ?? outcome.error?.message
       }));
     } catch (error2) {
-      await requestDependencyRelease(runReference).catch(onError);
+      if (stopped || error2 === stoppedError) return;
+      await requestDependencyRelease(runReference).catch(reportUnlessStopped);
       const cause = safeErrorMessageFromUnknown(error2, { fallback: "Runtime execution failed" });
       const message = runtimeClaimed ? `UNKNOWN_EFFECT: runtime execution failed after the durable pre-effect claim; verify external state before retrying: ${cause}` : cause;
       await updateStatusWithRetry(runReference, (current) => ({
@@ -10219,19 +10370,20 @@ function createWorkloadExecutionController(store, driver, options) {
           summary: message,
           exitCode: 1
         }
-      })).catch(onError);
-      if (stopped) return;
+      })).catch(reportUnlessStopped);
       onError(error2);
       await updateStatusWithRetry(workloadReference_, (current) => ({
         ...current,
         phase: "Failed",
         lastRunResult: message
-      })).catch(onError);
+      })).catch(reportUnlessStopped);
     } finally {
       active.delete(workload.metadata.uid);
+      cancellationRequested.delete(workload.metadata.uid);
     }
   }
   async function recoverRunning(workload) {
+    throwIfStopped();
     const runReference = {
       apiVersion: AGENT_RUN_API_VERSION,
       kind: AGENT_RUN_KIND,
@@ -10239,8 +10391,10 @@ function createWorkloadExecutionController(store, driver, options) {
       namespace: workload.metadata.namespace
     };
     try {
-      const run = await store.get(
-        runReference
+      const run = await whileRunning(
+        () => store.get(
+          runReference
+        )
       );
       if (run) {
         try {
@@ -10298,12 +10452,14 @@ function createWorkloadExecutionController(store, driver, options) {
         })
       );
     } catch (error2) {
-      onError(error2);
+      if (!stopped && error2 !== stoppedError) onError(error2);
     } finally {
       active.delete(workload.metadata.uid);
+      cancellationRequested.delete(workload.metadata.uid);
     }
   }
   function maybeStart(resource) {
+    if (stopped) return;
     const workload = resource;
     const status = workload.status;
     if (!status || status.phase !== "Scheduling" && status.phase !== "Running") return;
@@ -10311,39 +10467,57 @@ function createWorkloadExecutionController(store, driver, options) {
     if (active.has(workload.metadata.uid)) return;
     active.set(workload.metadata.uid, { cancel: async () => {
     } });
-    if (status.phase === "Running") {
-      void recoverRunning(workload);
-    } else {
-      void execute(workload).catch(onError);
-    }
+    const execution = status.phase === "Running" ? recoverRunning(workload) : execute(workload);
+    executions.set(workload.metadata.uid, execution);
+    void execution.finally(() => {
+      if (executions.get(workload.metadata.uid) === execution) {
+        executions.delete(workload.metadata.uid);
+      }
+    }).catch((error2) => {
+      if (!stopped && error2 !== stoppedError) onError(error2);
+    });
   }
-  void (async () => {
+  const watcher = (async () => {
     while (!stopped) {
       try {
         for await (const event of store.watch(
           { apiVersion: AGENT_WORKLOAD_API_VERSION, kind: AGENT_WORKLOAD_KIND },
-          { sendInitialEvents: true }
+          { sendInitialEvents: true, signal: watchAbort.signal }
         )) {
           if (stopped) break;
           if (event.type === "ADDED" || event.type === "MODIFIED") {
             maybeStart(event.resource);
           } else if (event.type === "DELETED") {
             const workload = event.resource;
-            await active.get(workload.metadata.uid)?.cancel().catch(onError);
+            cancellationRequested.add(workload.metadata.uid);
+            await active.get(workload.metadata.uid)?.cancel().catch(reportUnlessStopped);
           }
         }
       } catch (error2) {
-        onError(error2);
+        if (!stopped && error2 !== stoppedError) onError(error2);
+      }
+      if (!stopped) {
+        await whileRunning(() => sleep(1e3)).catch((error2) => {
+          if (!stopped && error2 !== stoppedError) onError(error2);
+        });
       }
-      if (!stopped) await sleep(1e3);
     }
-  })().catch(onError);
+  })();
+  void watcher.catch((error2) => {
+    if (!stopped && error2 !== stoppedError) onError(error2);
+  });
   return {
     async stop() {
-      stopped = true;
-      for (const handle of active.values()) {
-        await handle.cancel().catch(() => void 0);
+      if (!stopPromise) {
+        stopped = true;
+        resolveStopped({ kind: "stopped" });
+        watchAbort.abort(stoppedError);
+        const cancellations = [...active.values()].map(async (handle) => {
+          await handle.cancel().catch(() => void 0);
+        });
+        stopPromise = drainWithDeadline([...cancellations, watcher, ...executions.values()]);
       }
+      await stopPromise;
     }
   };
 }
@@ -12095,6 +12269,7 @@ export {
   DEVICE_HEARTBEAT_DEFAULT_MAX_CLOCK_SKEW_MS,
   DEVICE_HEARTBEAT_LIMITS,
   DEVICE_HEARTBEAT_SIGNATURE_DOMAIN,
+  DEVICE_ORCHESTRATION_FRAME_LIMITS,
   DEVICE_ORCHESTRATION_PROTOCOL,
   DEVICE_PAIRING_INVITE_MAX_CLOCK_SKEW_MS,
   DEVICE_PAIRING_INVITE_PROTOCOL,