@osolmaz/pi-workflows 0.12.1 → 0.13.1

package/src/controllers/effects.ts

@@ -2,6 +2,7 @@ import { jsonFingerprint } from "./json.js";
 import type { ControllerStore } from "./store.js";
 import type {
   ControllerEffects,
+  ControllerQueueClaim,
   ControllerResource,
   EffectApplication,
   EffectDefinition,
@@ -16,6 +17,7 @@ export class ControllerEffectService implements ControllerEffects {
   constructor(
     private readonly store: ControllerStore,
     private readonly resource: ControllerResource,
+    private readonly claim: ControllerQueueClaim,
     private readonly signal: AbortSignal,
   ) {}
 
@@ -28,6 +30,7 @@ export class ControllerEffectService implements ControllerEffects {
     const reservation = this.store.reserveEffect({
       key: definition.key,
       resourceUid: this.resource.metadata.uid,
+      claim: this.claim,
       generation: this.resource.metadata.generation,
       kind: definition.kind,
       requestFingerprint,
@@ -67,6 +70,7 @@ export class ControllerEffectService implements ControllerEffects {
     const record = this.store.updateEffect({
       resourceUid: this.resource.metadata.uid,
       key,
+      claim: this.claim,
       state: observation.state === "applied" ? "applied" : "indeterminate",
       ...("externalRef" in observation && observation.externalRef !== undefined
         ? { externalRef: observation.externalRef }
@@ -88,6 +92,7 @@ export class ControllerEffectService implements ControllerEffects {
       const record = this.store.updateEffect({
         resourceUid: this.resource.metadata.uid,
         key: definition.key,
+        claim: this.claim,
         state: "indeterminate",
         error: message,
       });
@@ -97,6 +102,7 @@ export class ControllerEffectService implements ControllerEffects {
     const record = this.store.updateEffect({
       resourceUid: this.resource.metadata.uid,
       key: definition.key,
+      claim: this.claim,
       state: result.state,
       ...(result.state === "applied" && result.externalRef !== undefined
         ? { externalRef: result.externalRef }
@@ -111,6 +117,7 @@ export class ControllerEffectService implements ControllerEffects {
     this.store.recordEvent({
       controller: this.resource.metadata.controller,
       key: this.resource.metadata.key,
+      claim: this.claim,
       type,
       payload: { effectKey, ...extra },
     });

package/src/controllers/index.ts

@@ -27,12 +27,6 @@ export {
   type WorkflowRunQueueRecord,
 } from "./sqlite.js";
 export {
-  CONTROLLER_STORE_SCHEMA,
-  controllerProjectScope,
-  controllerStoreBaseDir,
-  controllerStorePath,
-  projectControllerStoreBaseDir,
-  projectControllerStorePath,
   type ControllerStore,
   type EffectReservation,
   type QueueItem,

package/src/controllers/manager.ts

@@ -44,6 +44,7 @@ export type ControllerManagerOptions = {
 
 export class ControllerManager {
   readonly store: ControllerStore;
+  private readonly ownerId = `controller-manager-${randomUUID()}`;
   private readonly definitions = new Map<string, AnyControllerDefinition>();
   private readonly workflowCoordinator: ControllerWorkflowCoordinator;
   private readonly maxConcurrent: number;
@@ -137,7 +138,12 @@ export class ControllerManager {
       resource.metadata.generation === 1 ? "resource_applied" : "resource_updated",
       { generation: resource.metadata.generation },
     );
-    return resource;
+    const current = this.store.getResource<TSpec, TStatus>({
+      controller: definition.name,
+      key,
+    });
+    if (current === undefined) throw new Error("Controller resource disappeared after its event");
+    return current;
   }
 
   putResourceByName(controller: string, key: string, spec: unknown): ControllerResource {
@@ -158,13 +164,17 @@ export class ControllerManager {
       resource.metadata.generation === 1 ? "resource_applied" : "resource_updated",
       { generation: resource.metadata.generation },
     );
-    return resource;
+    const current = this.store.getResource({ controller, key });
+    if (current === undefined) throw new Error("Controller resource disappeared after its event");
+    return current;
   }
 
   requestDeletion(ref: ControllerResourceRef): ControllerResource {
-    const resource = this.store.requestDeletion(ref, this.now().toISOString());
+    this.store.requestDeletion(ref, this.now().toISOString());
     this.recordEvent(ref, "deletion_requested");
-    return resource;
+    const current = this.store.getResource(ref);
+    if (current === undefined) throw new Error("Controller resource disappeared after its event");
+    return current;
   }
 
   enqueue(ref: ControllerResourceRef, afterMs = 0): void {
@@ -230,13 +240,20 @@ export class ControllerManager {
         await sleep(this.pollIntervalMs, signal);
         continue;
       }
-      await this.processClaim(claim);
+      try {
+        await this.processClaim(claim);
+      } catch {
+        if (signal.aborted) return;
+        // A stale claim means another owner now controls that resource. Keep
+        // this worker alive so it can reconcile other queued resources.
+      }
     }
   }
 
   private claimNext(): ControllerQueueClaim | undefined {
     return this.store.claimNext({
       controllers: this.eligibleControllers(),
+      ownerId: this.ownerId,
       leaseMs: this.leaseMs,
       now: this.now().toISOString(),
     });
@@ -303,14 +320,19 @@ export class ControllerManager {
       Math.max(100, Math.floor(this.leaseMs / 3)),
     );
     renew.unref?.();
-    this.recordEvent(ref, "reconcile_started", {
-      reconcileId,
-      generation: resource.metadata.generation,
-    });
+    this.recordEvent(
+      ref,
+      "reconcile_started",
+      {
+        reconcileId,
+        generation: resource.metadata.generation,
+      },
+      claim,
+    );
 
     try {
       const result = await raceWithAbort(
-        this.callReconciler(definition, resource, abort.signal),
+        this.callReconciler(definition, resource, claim, abort.signal),
         abort.signal,
       );
       if (abort.signal.aborted) {
@@ -330,12 +352,13 @@ export class ControllerManager {
   private async callReconciler(
     definition: AnyControllerDefinition,
     resource: ControllerResource,
+    claim: ControllerQueueClaim,
     signal: AbortSignal,
   ): Promise<ReconcileResult<unknown>> {
     const context: ReconcileContext<unknown> = {
       signal,
-      effects: new ControllerEffectService(this.store, resource, signal),
-      workflows: this.workflowCoordinator.forResource(resource, signal),
+      effects: new ControllerEffectService(this.store, resource, claim, signal),
+      workflows: this.workflowCoordinator.forResource(resource, claim, signal),
       ...createResultHelpers<unknown>(),
     };
     const result = await definition.reconcile(context, asTypedResource(resource));
@@ -361,34 +384,40 @@ export class ControllerManager {
     try {
       const updated = this.store.updateStatus({
         ref,
-        expectedResourceVersion: resource.metadata.resourceVersion,
+        expectedResourceVersion: claim.resourceVersion,
+        claim,
         status,
         ...(result.status?.finalizers !== undefined
           ? { finalizers: result.status.finalizers }
           : {}),
         now: now.toISOString(),
       });
-      this.recordEvent(ref, "reconcile_finished", {
-        reconcileId,
-        result: result.kind,
-        generation: resource.metadata.generation,
-        durationMs: Math.max(0, now.getTime() - startedAtMs),
-        ...(result.kind === "requeue" && result.afterMs !== undefined
-          ? { requeueAfterMs: result.afterMs }
-          : {}),
-      });
+      this.recordEvent(
+        ref,
+        "reconcile_finished",
+        {
+          reconcileId,
+          result: result.kind,
+          generation: resource.metadata.generation,
+          durationMs: Math.max(0, now.getTime() - startedAtMs),
+          ...(result.kind === "requeue" && result.afterMs !== undefined
+            ? { requeueAfterMs: result.afterMs }
+            : {}),
+        },
+        claim,
+      );
       if (
         result.kind === "settled" &&
         updated.metadata.deletionTimestamp !== undefined &&
         updated.metadata.finalizers.length === 0
       ) {
-        this.recordEvent(ref, "resource_deleted", { reconcileId });
-        this.store.deleteResource(ref, updated.metadata.resourceVersion);
+        this.recordEvent(ref, "resource_deleted", { reconcileId }, claim);
+        this.store.deleteResource(ref, claim.resourceVersion, claim);
         return;
       }
     } catch (error) {
       if (error instanceof ResourceConflictError) {
-        this.recordEvent(ref, "reconcile_conflict", { reconcileId });
+        this.recordEvent(ref, "reconcile_conflict", { reconcileId }, claim);
         this.store.requeueClaim(claim, { availableAt: now.toISOString() }, now.toISOString());
         return;
       }
@@ -416,12 +445,17 @@ export class ControllerManager {
     const now = this.now();
     const message = boundedError(error);
     const delay = this.backoffMs(claim.consecutiveErrors + 1);
-    this.recordEvent(ref, "reconcile_failed", {
-      reconcileId,
-      error: message,
-      durationMs: Math.max(0, now.getTime() - startedAtMs),
-      requeueAfterMs: delay,
-    });
+    this.recordEvent(
+      ref,
+      "reconcile_failed",
+      {
+        reconcileId,
+        error: message,
+        durationMs: Math.max(0, now.getTime() - startedAtMs),
+        requeueAfterMs: delay,
+      },
+      claim,
+    );
     this.store.requeueClaim(
       claim,
       {
@@ -444,10 +478,16 @@ export class ControllerManager {
     return this.store.listQueue().some((item) => Date.parse(item.availableAt) <= now);
   }
 
-  private recordEvent(ref: ControllerResourceRef, type: string, payload: JsonObject = {}): void {
+  private recordEvent(
+    ref: ControllerResourceRef,
+    type: string,
+    payload: JsonObject = {},
+    claim?: ControllerQueueClaim,
+  ): void {
     this.store.recordEvent({
       controller: ref.controller,
       key: ref.key,
+      ...(claim === undefined ? {} : { claim }),
       type,
       payload,
       now: this.now().toISOString(),