@cotal-ai/manager 0.42.0 → 0.44.0

package/dist/manager.js

@@ -372,6 +372,16 @@ export class Manager {
      * hear `not-found`, which a seat reads as "drop it": failure reported for work that committed.
      * The settled answer lives here for a bounded window instead, and the sweep drops it after,
      * which is also what keeps this map from being one entry per turn for the process lifetime.
+     *
+     * `ref` is the goal coordinates the durable terminal is published under. A late yield that
+     * arrives after pendingTurns is gone and before `settled` is remembered (or after a concurrent
+     * sweep dropped the settled field) still has to name that terminal, never `not-found`. The
+     * index entry is not that answer: it is cleared at commit, and the yield path never consults it.
+     *
+     * `leftoverSince` is when the pending latch dropped. An unsettled leftover ages off that stamp,
+     * never off `acceptance.deadlineAt`: a turn that expired after a long outage would otherwise be
+     * pruned on the first sweep tick, taking the GoalRef with it, and a late yield in the retry
+     * window would hear `not-found` again.
      */
     turnAcceptances = new Map();
     /** Every relayed turn awaiting its seat's yield, by goal id. The seat's `turn-pending` pull
@@ -1353,13 +1363,25 @@ export class Manager {
      */
     async deregisterServiceOnStop() {
         const iid = this.managerInstanceId;
-        const dereg = ({ recordsKv }) => deregisterServiceInstance(recordsKv, { endpoint: MANAGER_ENDPOINT, instanceId: iid });
+        const dereg = ({ recordsKv, authKv }) => deregisterServiceInstance(recordsKv, {
+            endpoint: MANAGER_ENDPOINT,
+            instanceId: iid,
+            observeGeneration: async () => {
+                const key = epgateKey(MANAGER_ENDPOINT, iid);
+                const entry = await authKv.get(key);
+                if (!entry || entry.operation !== "PUT")
+                    throw new Error(`no issuance gate at ${key}`);
+                return parseEndpointGate(entry.value, key).generation;
+            },
+        });
         try {
             const outcome = await (this.auth ? this.withEndpointServeExecutor(dereg) : this.withOpenServeConnection(dereg));
             if (outcome.removed)
                 console.error(`✓ deregistered manager instance ${iid} from the ${MANAGER_ENDPOINT} service registry (spec revision ${outcome.specRevision})`);
             else if (outcome.reason === "superseded")
                 console.error(`! manager instance ${iid} was not deregistered: its registration moved while this stop ran, so another incarnation owns it now - leaving it alone`);
+            else if (outcome.reason === "registration-in-flight")
+                console.error(`! manager instance ${iid} was not deregistered: a registration is still in flight (governance slot held at the live gate generation) - leaving it alone`);
             // `absent` is silent: there was nothing registered to remove, which is not news at shutdown.
         }
         catch (e) {
@@ -4431,8 +4453,9 @@ export class Manager {
      *  `gone` into `unestablishable`). Live / unknown / unestablishable / wrong-op-kind stay loud
      *  refusals. No TTL: request-ingress has no process-epoch fence, so a clock would either let a
      *  superseded serve credential keep consuming ingress or just rename the freeze. An open gate
-     *  is a no-op (the successor's own freeze comes next). */
-    async healFrozenRegistrationGate(authKv, instanceId, auth) {
+     *  is a no-op (the successor's own freeze comes next). A committed spec under that freeze is
+     *  finished at the committed registrationRevision; only a definite no-commit abort-reopens. */
+    async healFrozenRegistrationGate(authKv, instanceId, auth, recordsKv) {
         const key = epgateKey(MANAGER_ENDPOINT, instanceId);
         const entry = await authKv.get(key);
         if (!entry || entry.operation !== "PUT")
@@ -4449,6 +4472,7 @@ export class Manager {
                 probeHolder: makeManagerHolderLivenessProbe({ space: this.space, servers, auth, log }),
                 evict: makeManagerEndpointEvictor({ space: this.space, servers, auth, log }),
                 log,
+                recordsKv,
             });
         }
         catch (e) {
@@ -4464,7 +4488,7 @@ export class Manager {
             }
             throw e;
         }
-        console.error(`✓ boot self-heal: ${MANAGER_ENDPOINT}/${instanceId} registration gate reopened at generation ${report.reopenedAtGeneration} (processEpoch unchanged at ${report.before.processEpoch}; freeze-holder gone, sweepComplete=true). Continuing the normal takeover.`);
+        console.error(`✓ boot self-heal: ${MANAGER_ENDPOINT}/${instanceId} registration gate reopened at generation ${report.reopenedAtGeneration} (processEpoch ${report.after.processEpoch}, registrationRevision ${report.after.registrationRevision}; freeze-holder gone, sweepComplete=true). Continuing the normal takeover.`);
     }
     /** P2 item 1: register the manager as an ordinary v0.4 `service` endpoint and serve its typed
      *  command surface on the ep rails - since 1d the manager's ONLY control door. On an AUTH mesh
@@ -4596,8 +4620,9 @@ export class Manager {
             // when the freeze-holder is gone. Complete that SAME op (abort-reopen) on independent
             // holder-gone evidence BEFORE this incarnation freezes a new one. Auth only: the
             // CONNZ oracle rides delivery-admin, which an open mesh does not have.
-            if (auth)
-                await this.healFrozenRegistrationGate(authKv, iid, auth);
+            if (auth) {
+                await this.healFrozenRegistrationGate(authKv, iid, auth, recordsKv);
+            }
             // P2 item 3 (slice 3a): on an AUTH mesh a RE-registration (restart of the persisted instanceId)
             // must VERIFY-EVICT the superseded serve family BEFORE the epoch advances (§13.1 "old authority
             // dies before new authority is visible"). Inject the SCOPED delivery-admin evictor; the OPEN
@@ -5608,7 +5633,7 @@ export class Manager {
         this.pendingTurns.set(goalId, pending);
         this.ensureTurnSweep();
         const acceptance = { name: a.name, owner: t.owner, actor: t.actor, uid: t.lifecycleUid, goalId, fingerprint, deadlineAt, executor };
-        this.turnAcceptances.set(goalId, { acceptance });
+        this.turnAcceptances.set(goalId, { acceptance, ref });
         this.emitGoalProgress(ref, executor.epoch, { phase: "relayed" });
         return acceptance;
     }
@@ -5662,11 +5687,22 @@ export class Manager {
             // reporting failure for work the run already has. The answer the first reply carried is
             // served again instead, to the addressee it was addressed to and nobody else.
             const settled = this.turnAcceptances.get(goalId);
-            if (settled?.settled !== undefined) {
+            if (settled !== undefined) {
                 const a = settled.acceptance;
                 if (a.owner !== c.owner || a.actor !== c.actor || a.uid !== c.uid)
                     throw new EpEnvelopeError("permission-denied", `turn "${goalId}" was addressed to ${a.owner}.${a.actor}/${a.uid}; a yield is the addressee's own (SPEC 13.6)`);
-                return { goalId, state: settled.settled.state };
+                if (settled.settled !== undefined)
+                    return { goalId, state: settled.settled.state };
+                // The in-memory answer is not yet remembered (pending deleted before the terminal CAS,
+                // or a concurrent sweep dropped `settled` while the commit was still in flight). The
+                // durable terminal is the one the run already has; name it, never `not-found`.
+                if (settled.ref !== undefined) {
+                    const ended = await readGoalResult(gw.ctx, settled.ref);
+                    if (ended !== undefined) {
+                        this.rememberSettledTurn(goalId, ended.state, ended.ts);
+                        return { goalId, state: ended.state };
+                    }
+                }
             }
             throw new EpEnvelopeError("not-found", `no pending turn "${goalId}" on this manager`);
         }
@@ -5704,6 +5740,7 @@ export class Manager {
         this.emitGoalProgress(p.ref, epoch, { phase: "terminal", state: fact.state, ...(fact.data !== undefined ? { data: fact.data } : {}) });
         await clearGoalIndex(gw.ctx, p.ref);
         this.pendingTurns.delete(goalId);
+        this.markTurnLatchDropped(goalId, at);
         this.rememberSettledTurn(goalId, fact.state, at);
         this.maybeStopTurnSweep();
         return { goalId, state: fact.state };
@@ -5718,17 +5755,18 @@ export class Manager {
             return;
         if (!this.pendingTurns.delete(p.goalId))
             return;
+        this.markTurnLatchDropped(p.goalId);
         const epoch = this.serviceServe?.grant.epoch ?? 0;
         try {
             await this.assertGoalWriterEpochCurrent(epoch);
             const { fact } = await commitGoalResult(gw.ctx, { ref: p.ref, now: Date.now(), cause: "deny", denial: { kind: "hold-expired", token: p.holdToken }, data: { reason: "turn-deadline", ...(p.seatDiedAt !== undefined ? { agentDownAt: p.seatDiedAt } : {}), ...(p.handoffFrom !== undefined ? { handoffFrom: p.handoffFrom } : {}) }, committer: { instanceId: this.managerInstanceId, epoch } });
             this.emitGoalProgress(p.ref, epoch, { phase: "terminal", state: fact.state, ...(fact.data !== undefined ? { data: fact.data } : {}) });
-            await clearGoalIndex(gw.ctx, p.ref);
-            // The SAME order the yield path keeps: remember, then ask whether the sweep may stop. Asked
-            // first, the stop saw no pending turn and no settled answer, cleared the acceptance map, and
-            // the remember below found nothing to write to; a yield whose reply was lost then heard
-            // `not-found` for the last expired turn, which the seat reads as "drop it".
+            // Remember BEFORE the index clear: a late yield that arrives after the pending latch
+            // (the delete at the top of this function) and after this CAS has to name this terminal.
+            // Clearing first, then remembering, left a window where maybeStopTurnSweep saw no pending
+            // turn and no settled answer, wiped the acceptance, and rememberSettledTurn no-op'd.
             this.rememberSettledTurn(p.goalId, fact.state, Date.now());
+            await clearGoalIndex(gw.ctx, p.ref);
         }
         catch (e) {
             console.error(`! turn deadline terminal for ${p.goalId}: ${e.message}`);
@@ -5775,16 +5813,27 @@ export class Manager {
             return;
         entry.settled = { state, at };
     }
+    /** Stamp when the pending latch dropped, so an unsettled leftover ages off that instant rather
+     *  than the original deadline. Idempotent: the first drop is the one the retry window starts at. */
+    markTurnLatchDropped(goalId, at = Date.now()) {
+        const entry = this.turnAcceptances.get(goalId);
+        if (entry === undefined || entry.leftoverSince !== undefined)
+            return;
+        entry.leftoverSince = at;
+    }
     /** Stop the sweep only when it has nothing left to do. It drives elapsed turns to their deadline
      *  terminals AND drops settled answers once their retry window has passed, so a stop while the
      *  second is still owed would leave one entry per turn for the process lifetime. */
     maybeStopTurnSweep() {
         if (this.pendingTurns.size > 0)
             return;
-        for (const e of this.turnAcceptances.values())
-            if (e.settled !== undefined)
-                return;
-        this.turnAcceptances.clear();
+        // An acceptance with no `settled` yet is the in-flight deadline commit: pending was the
+        // latch and is already gone, the terminal CAS has not returned, and a concurrent sweep
+        // tick used to clear the map here, so rememberSettledTurn no-op'd and a late yield heard
+        // `not-found` for a deny the run already has. Prune is the only deleter; stop only when
+        // it has left nothing.
+        if (this.turnAcceptances.size > 0)
+            return;
         this.stopTurnSweep();
     }
     stopTurnSweep() {
@@ -5798,9 +5847,17 @@ export class Manager {
     async sweepTurnDeadlines() {
         const now = Date.now();
         // A settled answer is kept only as long as a lost reply could still be retried under it.
-        for (const [goalId, e] of [...this.turnAcceptances.entries()])
-            if (e.settled !== undefined && now - e.settled.at >= TURN_ANSWER_RETENTION_MS)
+        // An UNSETTLED acceptance whose pending latch is already gone is the in-flight (or failed)
+        // deadline commit: keep it for the same window so a late yield can still read the durable
+        // terminal, then drop it so a commit that never remembered cannot pin the sweep forever.
+        for (const [goalId, e] of [...this.turnAcceptances.entries()]) {
+            if (this.pendingTurns.has(goalId))
+                continue;
+            // Unsettled leftovers age off the latch-drop stamp, never `deadlineAt`.
+            const at = e.settled?.at ?? e.leftoverSince;
+            if (at !== undefined && now - at >= TURN_ANSWER_RETENTION_MS)
                 this.turnAcceptances.delete(goalId);
+        }
         // The prune above is the one event after which the sweep may have nothing left to do, and
         // the settle-time callers cannot see it: they run when an answer has just been remembered.
         this.maybeStopTurnSweep();
@@ -5881,6 +5938,7 @@ export class Manager {
                 fingerprint: spec.value.fingerprint, deadlineAt: p.deadlineAt,
                 executor: { lifecycleUid: this.managerInstanceId, epoch: p.holdEpoch },
             },
+            ref: p.ref,
         });
         // NO LIVENESS VERDICT AT BOOT. The agents map is empty here: seats are re-registered later,
         // by the resume path, so `not in the map` at this moment means "not yet re-registered", never