@wildo-ai/saas-backend-lib 1.1.2 → 1.1.3

package/dist/esm/flows-actors/a2a/services/a2a-task-cancellation.backend.service.js

@@ -24,6 +24,13 @@
  * While a pod owns a turn it refreshes a TTL'd lease key. That is what lets
  * `tasks/get` report an abandoned task honestly instead of leaving it `working`
  * forever after a pod dies — see the lease helpers below.
+ *
+ * An absent lease is NOT self-explanatory, though, and reading it as death is an inversion that
+ * costs real work: a pod that could not write the lease looks exactly like a pod that died. So
+ * every successful lease write also leaves a long-lived ESTABLISHMENT marker, and
+ * `corroborateAbandonment` is what turns "no lease" into either a verdict or an admission that
+ * nothing was learned. Only the sweep — the one consumer that makes a liveness answer DURABLE —
+ * asks that question.
  */
 var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
     var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
@@ -40,10 +47,26 @@ var __param = (this && this.__param) || function (paramIndex, decorator) {
 import { inject, injectable } from 'inversify';
 import { SAAS_SERVICE_TYPES } from '../../../inject/models/service-types.js';
 import { projectCaughtErrorLogFields } from '../../../errors/utils/error-log-level-classification.backend.utils.js';
+import { A2A_AbandonmentEvidence } from '../../../flows-actors/a2a/models/a2a.backend.schemas.js';
 /** Channel every replica subscribes to. App-prefixed so tenants of one Redis never cross. */
 const CANCEL_CHANNEL_SUFFIX = 'a2a:task:cancel';
 /** Lease key prefix — presence means "a live pod is working this task right now". */
 const LEASE_KEY_PREFIX = 'a2a:task:lease:';
+/**
+ * ESTABLISHMENT marker prefix — presence means "this task's liveness WAS observable at some point
+ * recently", which is a different fact from "it is alive now" and has a different lifetime.
+ *
+ * It is what makes an absent lease MEAN something (see {@link A2A_AbandonmentEvidence}). Written
+ * by the same code path and in the same failure domain as the lease itself, so it can only exist
+ * if a write actually succeeded — which is exactly the property being recorded.
+ */
+const LEASE_ESTABLISHED_KEY_PREFIX = 'a2a:task:leased-once:';
+/**
+ * The sweep's own note that it has already seen one unexplained absence for a task. Written by the
+ * SWEEP's pod, whose write capability `isLeaseMechanismWritable` proved moments earlier — so the
+ * medium is known-good at the point of use, unlike the lease itself.
+ */
+const ABANDONMENT_OBSERVATION_KEY_PREFIX = 'a2a:task:absence-seen:';
 /**
  * Lease TTL and refresh cadence. The refresh interval is deliberately well under the
  * TTL: a single missed refresh (GC pause, a slow Redis round-trip) must not make a
@@ -51,6 +74,26 @@ const LEASE_KEY_PREFIX = 'a2a:task:lease:';
  */
 export const A2A_TASK_LEASE_TTL_SECONDS = 30;
 export const A2A_TASK_LEASE_REFRESH_MS = 10_000;
+/**
+ * How long the establishment marker outlives its last refresh.
+ *
+ * Sized to comfortably span the sweep's grace window plus its hourly period, so a pod that dies
+ * mid-turn is still recognisable as HAVING held the task when the next sweep looks. It is a
+ * timing optimisation, never a correctness requirement: if it lapses, the reading degrades to
+ * {@link A2A_AbandonmentEvidence.FIRST_ABSENCE} and the row is repaired one run later instead of
+ * on the first — which is the direction this whole mechanism errs in anyway.
+ */
+export const A2A_TASK_LEASE_ESTABLISHED_TTL_SECONDS = 6 * 60 * 60;
+/**
+ * How long one unexplained absence stands as corroboration for the next run.
+ *
+ * MUST exceed the sweep's period or no observation ever survives to be corroborated and nothing
+ * is ever repaired through that path; generous beyond it so a skipped run (the fleet-wide abstain)
+ * does not reset the count. Nothing clears it on recovery, and it does not need clearing: a task
+ * whose pod recovered answers ALIVE on the lease long before this is consulted, and once its lease
+ * lapses for real the establishment marker gives the verdict directly.
+ */
+export const A2A_TASK_ABSENCE_OBSERVATION_TTL_SECONDS = 3 * 60 * 60;
 let A2ATaskCancellationBackendService = class A2ATaskCancellationBackendService {
     logger;
     redisService;
@@ -70,6 +113,12 @@ let A2ATaskCancellationBackendService = class A2ATaskCancellationBackendService
     leaseKey(taskId) {
         return `${LEASE_KEY_PREFIX}${taskId}`;
     }
+    leaseEstablishedKey(taskId) {
+        return `${LEASE_ESTABLISHED_KEY_PREFIX}${taskId}`;
+    }
+    absenceObservationKey(taskId) {
+        return `${ABANDONMENT_OBSERVATION_KEY_PREFIX}${taskId}`;
+    }
     /**
      * Start this pod's single shared subscriber. Idempotent — safe to call from every
      * turn, so no startup ordering is required.
@@ -150,48 +199,85 @@ let A2ATaskCancellationBackendService = class A2ATaskCancellationBackendService
         // a non-terminal task with no lease and is told FAILED by derive-on-read.
         const controller = new AbortController();
         this.localTurns.set(taskId, controller);
-        const client = this.redisService.getReadyClient();
-        if (client) {
-            let leaseEverWritten = false;
-            const refresh = async () => {
-                try {
-                    await this.redisService.set(this.leaseKey(taskId), '1', { ttl: A2A_TASK_LEASE_TTL_SECONDS });
-                    leaseEverWritten = true;
+        let leaseEverWritten = false;
+        let unwritableAlreadyReported = false;
+        const refresh = async () => {
+            try {
+                await this.redisService.set(this.leaseKey(taskId), '1', { ttl: A2A_TASK_LEASE_TTL_SECONDS });
+                // The ESTABLISHMENT marker rides the same write, in the same failure domain, and is what
+                // makes a LATER absence of the lease mean something: a reader that finds no lease but
+                // finds this knows a pod really did hold the task and stopped, rather than that this
+                // task's liveness was never observable at all. Written every refresh so a long turn keeps
+                // it alive; deliberately NOT written before the lease, so it can never claim an
+                // establishment that did not happen.
+                await this.redisService.set(this.leaseEstablishedKey(taskId), '1', { ttl: A2A_TASK_LEASE_ESTABLISHED_TTL_SECONDS });
+                leaseEverWritten = true;
+            }
+            catch (error) {
+                // A missed REFRESH is survivable — the TTL is several refreshes long, so the lease
+                // written earlier still answers for this turn.
+                //
+                // A lease that was NEVER successfully written is a different thing entirely (review
+                // finding A3-4), and it used to log identically. No other pod can distinguish "the
+                // write failed" from "the pod died": `tasks/get` on any other replica reports this
+                // healthy running turn as FAILED, and the sweep then makes that answer DURABLE. This
+                // is the write-side twin of the read-side hole G2-8 closed — the JSDoc claim that
+                // "only a genuinely dead pod stops refreshing" was false in exactly this case.
+                if (leaseEverWritten) {
+                    this.logger.warn('[A2ATaskCancel] Lease refresh failed', { taskId, ...projectCaughtErrorLogFields(error) });
+                    return;
                 }
-                catch (error) {
-                    // A missed REFRESH is survivable — the TTL is several refreshes long, so the lease
-                    // written earlier still answers for this turn.
-                    //
-                    // A lease that was NEVER successfully written is a different thing entirely (review
-                    // finding A3-4), and it used to log identically. No other pod can distinguish "the
-                    // write failed" from "the pod died": `tasks/get` on any other replica reports this
-                    // healthy running turn as FAILED, and the sweep then makes that answer DURABLE. This
-                    // is the write-side twin of the read-side hole G2-8 closed — the JSDoc claim that
-                    // "only a genuinely dead pod stops refreshing" was false in exactly this case.
-                    if (leaseEverWritten) {
-                        this.logger.warn('[A2ATaskCancel] Lease refresh failed', { taskId, ...projectCaughtErrorLogFields(error) });
-                    }
-                    else {
-                        this.logger.error('[A2ATaskCancel] Task lease has NEVER been written; other pods will read this live turn as abandoned', {
-                            taskId,
-                            consequence: 'tasks/get on another replica reports failed, and the sweep may record it durably',
-                            ...projectCaughtErrorLogFields(error),
-                        });
-                    }
+                // ONCE per turn, not once per heartbeat. The heartbeat now runs even with no Redis at all
+                // (see below), so an unconditional log here would emit a line every 10 seconds for every
+                // concurrent turn of a Redis-less deployment — which is how a channel that carries a real
+                // signal gets tuned out.
+                if (unwritableAlreadyReported)
+                    return;
+                unwritableAlreadyReported = true;
+                // The two causes are logged at DIFFERENT levels because one is a configuration and the
+                // other is a fault. No client on this pod is the supported single-replica / local-dev
+                // shape: `isTurnAlive` fails safe toward "alive" there and nothing reads liveness across
+                // pods, so it is a WARN. A client that is present and REJECTED the write is the dangerous
+                // asymmetry — this pod is partitioned from a liveness plane every other replica can still
+                // read — and it keeps the ERROR level it had.
+                if (!this.redisService.getReadyClient()) {
+                    this.logger.warn('[A2ATaskCancel] No Redis on this pod; this turn has no liveness lease and cannot be cancelled from another replica', {
+                        taskId,
+                        consequence: 'harmless on a single replica; on a fleet, other pods cannot observe this turn until Redis returns',
+                        ...projectCaughtErrorLogFields(error),
+                    });
+                    return;
                 }
-            };
-            await refresh();
-            // Clear any prior heartbeat for this id before replacing it. Unreachable today
-            // (task ids are freshly minted uuids), but an overwritten `setInterval` handle is
-            // unclearable — it would refresh a lease forever and keep a dead task looking alive.
-            const existing = this.leaseTimers.get(taskId);
-            if (existing)
-                clearInterval(existing);
-            const timer = setInterval(() => { void refresh(); }, A2A_TASK_LEASE_REFRESH_MS);
-            // Never hold the event loop open on a heartbeat — a draining pod must still exit.
-            timer.unref?.();
-            this.leaseTimers.set(taskId, timer);
-        }
+                this.logger.error('[A2ATaskCancel] Task lease has NEVER been written; other pods will read this live turn as abandoned', {
+                    taskId,
+                    consequence: 'tasks/get on another replica reports failed until a later refresh lands; the sweep now abstains rather than recording it durably',
+                    ...projectCaughtErrorLogFields(error),
+                });
+            }
+        };
+        await refresh();
+        // Clear any prior heartbeat for this id before replacing it. Unreachable today
+        // (task ids are freshly minted uuids), but an overwritten `setInterval` handle is
+        // unclearable — it would refresh a lease forever and keep a dead task looking alive.
+        const existing = this.leaseTimers.get(taskId);
+        if (existing)
+            clearInterval(existing);
+        // ── The heartbeat is installed UNCONDITIONALLY, and that is a fix, not a tidy-up ──────────
+        //
+        // This whole block used to sit behind `if (this.redisService.getReadyClient())`, so a turn
+        // that began while this pod's client was not ready got NO lease and NO retry — it stayed
+        // unobservable for its entire life even after Redis came back, and nothing was logged at all,
+        // because the failing write never even ran. That is the SILENT half of the inversion the
+        // never-written branch above documents, and it is strictly worse: a failed first write
+        // self-heals on the next 10-second refresh, an absent heartbeat never does.
+        //
+        // The cost in a genuinely Redis-less deployment is one unref'd timer per in-flight turn and a
+        // caught rejection every 10 seconds; the log is emitted once (above), and `redisService.set`
+        // builds its error without logging. That is cheap enough to buy automatic recovery.
+        const timer = setInterval(() => { void refresh(); }, A2A_TASK_LEASE_REFRESH_MS);
+        // Never hold the event loop open on a heartbeat — a draining pod must still exit.
+        timer.unref?.();
+        this.leaseTimers.set(taskId, timer);
         // Subscriber setup happens AFTER the lease exists (see the note above). Not awaited:
         // a slow Redis handshake must not delay the turn, and a cancel arriving before the
         // subscription is live is already handled — `requestCancel` aborts a locally-owned turn
@@ -204,6 +290,14 @@ let A2ATaskCancellationBackendService = class A2ATaskCancellationBackendService
      *
      * Dropping the lease EAGERLY matters — a terminal task with a live lease would look
      * like it is still executing to anything reading liveness.
+     *
+     * The ESTABLISHMENT marker is deliberately NOT dropped with it. The two record different kinds
+     * of fact: the lease says "a pod is working this NOW" and stops being true the instant the turn
+     * unwinds, while the marker says "this task's liveness was observable", which stays true
+     * afterwards. Keeping it also covers the one case that would otherwise regress — a turn that
+     * finished but whose terminal write did not land — since the row is then genuinely abandoned and
+     * the sweep can say so on its first pass instead of waiting for corroboration. It expires on its
+     * own TTL, so nothing accumulates.
      */
     async endTurn(taskId) {
         const timer = this.leaseTimers.get(taskId);
@@ -274,6 +368,68 @@ let A2ATaskCancellationBackendService = class A2ATaskCancellationBackendService
             return true;
         }
     }
+    /**
+     * WHY is this task's lease absent — and may that absence be turned into a durable verdict?
+     *
+     * Asked by the abandoned-task sweep, and only by it, AFTER {@link isTurnAlive} has already
+     * answered "no lease". `isTurnAlive` returns a boolean because every other consumer only needs
+     * to know whether to keep waiting; the sweep is the one caller that writes a TERMINAL status,
+     * so it is the one caller that must be able to tell a verdict from a failed enquiry.
+     *
+     * ## The case this closes, which no existing guard catches
+     *
+     * `isLeaseMechanismWritable` is evaluated on the SWEEP's pod. A per-pod failure — the executing
+     * pod's Redis connection is broken while the sweep's is fine — passes it, `isTurnAlive`
+     * truthfully reports no lease, and a healthy running turn is durably recorded FAILED. That
+     * inversion has no compensating pass; it is the reason this method exists.
+     *
+     * ## How the two are told apart
+     *
+     * 1. **An establishment marker** ({@link LEASE_ESTABLISHED_KEY_PREFIX}) is written beside every
+     *    successful lease write and outlives it by hours. Present ⇒ a pod demonstrably held this
+     *    task and stopped ⇒ `LEASE_LAPSED`, a verdict, on the very first pass. This is what keeps
+     *    ordinary abandonment repaired in the same hour it happens.
+     * 2. **Otherwise the reading is indeterminate**, and the answer is TIME rather than another key:
+     *    a live pod that merely failed one write rewrites its lease on the next 10-second heartbeat,
+     *    so an absence that is still there a full sweep period later is not a missed write. The
+     *    first pass records the observation and returns `FIRST_ABSENCE`; a later pass that still
+     *    finds no lease and still finds the observation returns `CORROBORATED_ABSENCE`.
+     * 3. **If the observation can neither be read nor written, nothing was learned** —
+     *    `UNDETERMINED`. It is NOT collapsed into `FIRST_ABSENCE`, because that one made progress
+     *    toward a verdict and this one did not; recording it as progress would let two enquiries
+     *    that both failed add up to a durable kill.
+     *
+     * Losing the medium fails toward the SAFE side in both directions: with the markers gone (a
+     * Redis restart) every absence is merely uncorroborated, so genuine abandonment is still
+     * repaired — one run later — while a healthy turn is not killed on a single reading.
+     */
+    async corroborateAbandonment(taskId) {
+        // No client at all is the one case where this method can say nothing whatsoever. The sweep's
+        // fleet-wide abstain already refuses to run in that state, so reaching here means the client
+        // died between that probe and this row — indeterminate, by construction.
+        if (!this.redisService.getReadyClient())
+            return A2A_AbandonmentEvidence.UNDETERMINED;
+        try {
+            if (await this.redisService.exists(this.leaseEstablishedKey(taskId)))
+                return A2A_AbandonmentEvidence.LEASE_LAPSED;
+            const observationKey = this.absenceObservationKey(taskId);
+            if (await this.redisService.exists(observationKey))
+                return A2A_AbandonmentEvidence.CORROBORATED_ABSENCE;
+            // Record THIS observation so a later run can corroborate it. The write is inside the same
+            // try as the reads on purpose: an observation that was not durably recorded has not been
+            // made, and reporting `FIRST_ABSENCE` for it would let the next run corroborate against
+            // nothing.
+            await this.redisService.set(observationKey, '1', { ttl: A2A_TASK_ABSENCE_OBSERVATION_TTL_SECONDS });
+            return A2A_AbandonmentEvidence.FIRST_ABSENCE;
+        }
+        catch (error) {
+            this.logger.warn('[A2ATaskCancel] Could not establish why a task lease is absent; the sweep will not act on this reading', {
+                taskId,
+                ...projectCaughtErrorLogFields(error),
+            });
+            return A2A_AbandonmentEvidence.UNDETERMINED;
+        }
+    }
     /**
      * Can the lease mechanism actually be WRITTEN right now?
      *

package/dist/esm/flows-actors/a2a/services/a2a-task-cancellation.backend.service.js.map

@@ -1 +1 @@
-{"version":3,"file":"a2a-task-cancellation.backend.service.js","sourceRoot":"","sources":["../../../../../../src/flows-actors/a2a/services/a2a-task-cancellation.backend.service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;;;;;;;;;;;;;AAEH,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAG/C,OAAO,EAAE,kBAAkB,EAAE,MAAM,gDAAgD,CAAC;AAGpF,OAAO,EAAE,2BAA2B,EAAE,MAAM,8EAA8E,CAAC;AAE3H,6FAA6F;AAC7F,MAAM,qBAAqB,GAAG,iBAAiB,CAAC;AAChD,qFAAqF;AACrF,MAAM,gBAAgB,GAAG,iBAAiB,CAAC;AAE3C;;;;GAIG;AACH,MAAM,CAAC,MAAM,0BAA0B,GAAG,EAAE,CAAC;AAC7C,MAAM,CAAC,MAAM,yBAAyB,GAAG,MAAM,CAAC;AAGzC,IAAM,iCAAiC,GAAvC,MAAM,iCAAiC;IAUU;IACM;IAV5D,+EAA+E;IAC9D,UAAU,GAAG,IAAI,GAAG,EAA2B,CAAC;IACjE,6DAA6D;IAC5C,WAAW,GAAG,IAAI,GAAG,EAA0B,CAAC;IAEzD,UAAU,GAA2B,IAAI,CAAC;IAC1C,gBAAgB,GAAG,KAAK,CAAC;IAEjC,YACsD,MAAqB,EACf,YAA0B;QADhC,WAAM,GAAN,MAAM,CAAe;QACf,iBAAY,GAAZ,YAAY,CAAc;IACnF,CAAC;IAEI,OAAO;QACb,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,YAAY,EAAE,GAAG,qBAAqB,EAAE,CAAC;IACvE,CAAC;IAEO,QAAQ,CAAC,MAAc;QAC7B,OAAO,GAAG,gBAAgB,GAAG,MAAM,EAAE,CAAC;IACxC,CAAC;IAED;;;;;;;;;;OAUG;IACK,KAAK,CAAC,gBAAgB;QAC5B,IAAI,IAAI,CAAC,gBAAgB;YAAE,OAAO;QAClC,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC;QAE7B,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,cAAc,EAAE,CAAC;QAClD,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,qFAAqF,CAAC,CAAC;YACzG,OAAO;QACT,CAAC;QAED,IAAI,UAAuC,CAAC;QAC5C,IAAI,CAAC;YACH,UAAU,GAAG,MAAM,CAAC,SAAS,EAAqB,CAAC;YACnD,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAc,EAAE,EAAE;gBACxC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,iDAAiD,EAAE,EAAE,GAAG,2BAA2B,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAClH,CAAC,CAAC,CAAC;YACH,4FAA4F;YAC5F,8FAA8F;YAC9F,6FAA6F;YAC7F,6FAA6F;YAC7F,2FAA2F;YAC3F,6EAA6E;YAC7E,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;gBACxB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,0FAA0F,CAAC,CAAC;gBAC7G,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;gBAC5B,IAAI,CAAC,gBAAgB,GAAG,KAAK,CAAC;YAChC,CAAC,CAAC,CAAC;YACH,MAAM,UAAU,CAAC,OAAO,EAAE,CAAC;YAC3B,MAAM,UAAU,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC,OAAe,EAAE,EAAE,CAAC,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC;YACjG,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC/B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,iFAAiF;YACjF,yEAAyE;YACzE,sFAAsF;YACtF,oFAAoF;YACpF,IAAI,UAAU,EAAE,CAAC;gBACf,IAAI,CAAC;oBAAC,MAAM,UAAU,CAAC,OAAO,EAAE,CAAC;gBAAC,CAAC;gBAAC,MAAM,CAAC,CAAC,uCAAuC,CAAC,CAAC;YACvF,CAAC;YACD,IAAI,CAAC,gBAAgB,GAAG,KAAK,CAAC,CAAC,8BAA8B;YAC7D,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,2DAA2D,EAAE,EAAE,GAAG,2BAA2B,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC5H,CAAC;IACH,CAAC;IAED;;;;OAIG;IACK,iBAAiB,CAAC,MAAc;QACtC,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC/C,IAAI,CAAC,UAAU;YAAE,OAAO;QACxB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,gEAAgE,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;QAC/F,UAAU,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC;IAED;;;OAGG;IACI,KAAK,CAAC,SAAS,CAAC,MAAc;QACnC,oFAAoF;QACpF,wFAAwF;QACxF,qFAAqF;QACrF,yFAAyF;QACzF,0EAA0E;QAC1E,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;QAExC,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,cAAc,EAAE,CAAC;QAClD,IAAI,MAAM,EAAE,CAAC;YACX,IAAI,gBAAgB,GAAG,KAAK,CAAC;YAC7B,MAAM,OAAO,GAAG,KAAK,IAAmB,EAAE;gBACxC,IAAI,CAAC;oBACH,MAAM,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,0BAA0B,EAAE,CAAC,CAAC;oBAC7F,gBAAgB,GAAG,IAAI,CAAC;gBAC1B,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,mFAAmF;oBACnF,+CAA+C;oBAC/C,EAAE;oBACF,oFAAoF;oBACpF,mFAAmF;oBACnF,mFAAmF;oBACnF,qFAAqF;oBACrF,kFAAkF;oBAClF,+EAA+E;oBAC/E,IAAI,gBAAgB,EAAE,CAAC;wBACrB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,sCAAsC,EAAE,EAAE,MAAM,EAAE,GAAG,2BAA2B,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;oBAC9G,CAAC;yBAAM,CAAC;wBACN,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,qGAAqG,EAAE;4BACvH,MAAM;4BACN,WAAW,EAAE,kFAAkF;4BAC/F,GAAG,2BAA2B,CAAC,KAAK,CAAC;yBACtC,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;YACH,CAAC,CAAC;YACF,MAAM,OAAO,EAAE,CAAC;YAChB,+EAA+E;YAC/E,kFAAkF;YAClF,qFAAqF;YACrF,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YAC9C,IAAI,QAAQ;gBAAE,aAAa,CAAC,QAAQ,CAAC,CAAC;YAEtC,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,KAAK,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,yBAAyB,CAAC,CAAC;YAChF,kFAAkF;YAClF,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC;YAChB,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QACtC,CAAC;QAED,qFAAqF;QACrF,mFAAmF;QACnF,wFAAwF;QACxF,kCAAkC;QAClC,KAAK,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAE7B,OAAO,UAAU,CAAC,MAAM,CAAC;IAC3B,CAAC;IAED;;;;;OAKG;IACI,KAAK,CAAC,OAAO,CAAC,MAAc;QACjC,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC3C,IAAI,KAAK,EAAE,CAAC;YACV,aAAa,CAAC,KAAK,CAAC,CAAC;YACrB,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAClC,CAAC;QACD,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC/B,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;QACrD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,0DAA0D;YAC1D,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,6CAA6C,EAAE,EAAE,MAAM,EAAE,GAAG,2BAA2B,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACrH,CAAC;IACH,CAAC;IAED;;;;;;;;OAQG;IACI,KAAK,CAAC,aAAa,CAAC,MAAc;QACvC,mFAAmF;QACnF,sEAAsE;QACtE,MAAM,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC1C,IAAI,KAAK;YAAE,KAAK,CAAC,KAAK,EAAE,CAAC;QAEzB,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,cAAc,EAAE,CAAC;QAClD,IAAI,CAAC,MAAM;YAAE,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC;QAEnC,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,MAAM,CAAC,CAAC;YAC7C,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,8CAA8C,EAAE,EAAE,MAAM,EAAE,GAAG,2BAA2B,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YACrH,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC;QACxB,CAAC;IACH,CAAC;IAED;;;;;;;;;;OAUG;IACI,KAAK,CAAC,WAAW,CAAC,MAAc;QACrC,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC;YAAE,OAAO,IAAI,CAAC;QAC7C,wFAAwF;QACxF,0FAA0F;QAC1F,0FAA0F;QAC1F,oEAAoE;QACpE,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,cAAc,EAAE;YAAE,OAAO,IAAI,CAAC;QACrD,IAAI,CAAC;YACH,OAAO,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;QAC/D,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED;;;;;;;;;;;;;;;;OAgBG;IACI,KAAK,CAAC,wBAAwB;QACnC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,cAAc,EAAE;YAAE,OAAO,KAAK,CAAC;QACtD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,wBAAwB,CAAC,CAAC;QACzD,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,0BAA0B,EAAE,CAAC,CAAC;YAChF,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YACzD,MAAM,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;YACtD,OAAO,OAAO,CAAC;QACjB,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;CACF,CAAA;AA7PY,iCAAiC;IAD7C,UAAU,EAAE;IAWR,WAAA,MAAM,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAA;IACjC,WAAA,MAAM,CAAC,kBAAkB,CAAC,YAAY,CAAC,CAAA;;GAX/B,iCAAiC,CA6P7C","sourcesContent":["/**\n * Cross-pod A2A task cancellation.\n *\n * ## You do not need to find the executing pod\n *\n * With several backend replicas behind a load balancer, a `tasks/cancel` request\n * almost never lands on the pod that is running that task's LLM turn. The obvious\n * design — discover which pod owns it and route there — needs a pod registry, a\n * liveness view of it, and an internal call path, and it is wrong on every one of\n * those axes when a pod restarts mid-turn.\n *\n * Instead this BROADCASTS. Any pod publishes `{taskId}` to one Redis channel; every\n * pod runs a single shared subscriber; only the pod holding that `taskId` in its\n * in-process registry reacts. One connection per pod, no routing table, and a pod\n * that died simply does not answer — which is the correct behaviour, not an error to\n * handle.\n *\n * Rejected alternative: polling a Redis flag inside the token loop. That puts a GET on\n * the hottest path in the system (per N tokens, per concurrent turn) and bounds\n * cancellation latency by token cadence. Pub/sub costs nothing per token.\n *\n * ## Liveness is the same mechanism's other half\n *\n * While a pod owns a turn it refreshes a TTL'd lease key. That is what lets\n * `tasks/get` report an abandoned task honestly instead of leaving it `working`\n * forever after a pod dies — see the lease helpers below.\n */\n\nimport { inject, injectable } from 'inversify';\nimport type { RedisClientType } from 'redis';\n\nimport { SAAS_SERVICE_TYPES } from '~wildo-backend-lib/inject/models/service-types';\nimport type { LoggerService } from '~wildo-backend-lib/monitoring/logger/services/logger.service';\nimport type { RedisService } from '~wildo-backend-lib/databases/services/redis.service';\nimport { projectCaughtErrorLogFields } from '~wildo-backend-lib/errors/utils/error-log-level-classification.backend.utils';\n\n/** Channel every replica subscribes to. App-prefixed so tenants of one Redis never cross. */\nconst CANCEL_CHANNEL_SUFFIX = 'a2a:task:cancel';\n/** Lease key prefix — presence means \"a live pod is working this task right now\". */\nconst LEASE_KEY_PREFIX = 'a2a:task:lease:';\n\n/**\n * Lease TTL and refresh cadence. The refresh interval is deliberately well under the\n * TTL: a single missed refresh (GC pause, a slow Redis round-trip) must not make a\n * healthy turn look abandoned. Only a genuinely dead pod stops refreshing.\n */\nexport const A2A_TASK_LEASE_TTL_SECONDS = 30;\nexport const A2A_TASK_LEASE_REFRESH_MS = 10_000;\n\n@injectable()\nexport class A2ATaskCancellationBackendService {\n  /** Tasks this pod is currently executing → the controller that aborts them. */\n  private readonly localTurns = new Map<string, AbortController>();\n  /** Lease heartbeats keyed by task id, cleared on release. */\n  private readonly leaseTimers = new Map<string, NodeJS.Timeout>();\n\n  private subscriber: RedisClientType | null = null;\n  private subscribeStarted = false;\n\n  constructor(\n    @inject(SAAS_SERVICE_TYPES.Logger) private readonly logger: LoggerService,\n    @inject(SAAS_SERVICE_TYPES.RedisService) private readonly redisService: RedisService,\n  ) {}\n\n  private channel(): string {\n    return `${this.redisService.getAppPrefix()}${CANCEL_CHANNEL_SUFFIX}`;\n  }\n\n  private leaseKey(taskId: string): string {\n    return `${LEASE_KEY_PREFIX}${taskId}`;\n  }\n\n  /**\n   * Start this pod's single shared subscriber. Idempotent — safe to call from every\n   * turn, so no startup ordering is required.\n   *\n   * `node-redis` forbids commands on a subscribed connection, so the subscriber MUST be\n   * a duplicate rather than the shared client. Same pattern the WebSocket adapter uses.\n   *\n   * Fails OPEN by design: if Redis is unreachable the turn still runs — losing the\n   * ability to cancel is strictly better than refusing to answer at all. The failure is\n   * logged at ERROR because a persistent one silently disables cancellation fleet-wide.\n   */\n  private async ensureSubscriber(): Promise<void> {\n    if (this.subscribeStarted) return;\n    this.subscribeStarted = true;\n\n    const client = this.redisService.getReadyClient();\n    if (!client) {\n      this.logger.error('[A2ATaskCancel] Redis unavailable — cross-pod cancellation is DISABLED for this pod');\n      return;\n    }\n\n    let subscriber: RedisClientType | undefined;\n    try {\n      subscriber = client.duplicate() as RedisClientType;\n      subscriber.on('error', (error: unknown) => {\n        this.logger.error('[A2ATaskCancel] Cancel-channel subscriber error', { ...projectCaughtErrorLogFields(error) });\n      });\n      // REARM on a later death (review finding L-2). `subscribeStarted` was reset only inside the\n      // SETUP catch, so a subscriber that connected and then dropped — a Redis restart, a failover,\n      // an idle-timeout reap — logged from the handler above and was never rebuilt for the rest of\n      // the pod's lifetime. Cross-pod cancellation then silently stopped working on that pod while\n      // every other signal still looked healthy: the turns run, the leases refresh, and only the\n      // broadcast never arrives. Clearing the latch lets the next turn rebuild it.\n      subscriber.on('end', () => {\n        this.logger.warn('[A2ATaskCancel] Cancel-channel subscriber closed; re-arming so the next turn rebuilds it');\n        this.subscriber = undefined;\n        this.subscribeStarted = false;\n      });\n      await subscriber.connect();\n      await subscriber.subscribe(this.channel(), (message: string) => this.onCancelBroadcast(message));\n      this.subscriber = subscriber;\n    } catch (error) {\n      // DESTROY the half-built connection before retrying. `connect()` can succeed and\n      // `subscribe()` still throw, which leaves a live, unusable socket; since\n      // `subscribeStarted` resets to allow a retry, every later turn would open another one\n      // and leak a Redis connection per turn until the pod hit its file-descriptor limit.\n      if (subscriber) {\n        try { await subscriber.destroy(); } catch { /* already dead — nothing to reclaim */ }\n      }\n      this.subscribeStarted = false; // allow a later turn to retry\n      this.logger.error('[A2ATaskCancel] Failed to subscribe to the cancel channel', { ...projectCaughtErrorLogFields(error) });\n    }\n  }\n\n  /**\n   * Handle a broadcast. Every pod receives every cancel; only the owner acts, and a pod\n   * that does not own the task does nothing at all — that silence is the design, not a\n   * missed message.\n   */\n  private onCancelBroadcast(taskId: string): void {\n    const controller = this.localTurns.get(taskId);\n    if (!controller) return;\n    this.logger.info('[A2ATaskCancel] Aborting a locally-executing turn on broadcast', { taskId });\n    controller.abort();\n  }\n\n  /**\n   * Register a turn as executing on this pod: returns the signal to hand to the model\n   * call, and starts the liveness lease.\n   */\n  public async beginTurn(taskId: string): Promise<AbortSignal> {\n    // ORDER IS LOAD-BEARING. Register locally and write the lease BEFORE the subscriber\n    // setup, which on the first turn performs a full Redis duplicate + connect + subscribe.\n    // Awaiting that first would leave a window in which the task is materialised and the\n    // client — which asked for a non-blocking turn precisely so it could poll at once — sees\n    // a non-terminal task with no lease and is told FAILED by derive-on-read.\n    const controller = new AbortController();\n    this.localTurns.set(taskId, controller);\n\n    const client = this.redisService.getReadyClient();\n    if (client) {\n      let leaseEverWritten = false;\n      const refresh = async (): Promise<void> => {\n        try {\n          await this.redisService.set(this.leaseKey(taskId), '1', { ttl: A2A_TASK_LEASE_TTL_SECONDS });\n          leaseEverWritten = true;\n        } catch (error) {\n          // A missed REFRESH is survivable — the TTL is several refreshes long, so the lease\n          // written earlier still answers for this turn.\n          //\n          // A lease that was NEVER successfully written is a different thing entirely (review\n          // finding A3-4), and it used to log identically. No other pod can distinguish \"the\n          // write failed\" from \"the pod died\": `tasks/get` on any other replica reports this\n          // healthy running turn as FAILED, and the sweep then makes that answer DURABLE. This\n          // is the write-side twin of the read-side hole G2-8 closed — the JSDoc claim that\n          // \"only a genuinely dead pod stops refreshing\" was false in exactly this case.\n          if (leaseEverWritten) {\n            this.logger.warn('[A2ATaskCancel] Lease refresh failed', { taskId, ...projectCaughtErrorLogFields(error) });\n          } else {\n            this.logger.error('[A2ATaskCancel] Task lease has NEVER been written; other pods will read this live turn as abandoned', {\n              taskId,\n              consequence: 'tasks/get on another replica reports failed, and the sweep may record it durably',\n              ...projectCaughtErrorLogFields(error),\n            });\n          }\n        }\n      };\n      await refresh();\n      // Clear any prior heartbeat for this id before replacing it. Unreachable today\n      // (task ids are freshly minted uuids), but an overwritten `setInterval` handle is\n      // unclearable — it would refresh a lease forever and keep a dead task looking alive.\n      const existing = this.leaseTimers.get(taskId);\n      if (existing) clearInterval(existing);\n\n      const timer = setInterval(() => { void refresh(); }, A2A_TASK_LEASE_REFRESH_MS);\n      // Never hold the event loop open on a heartbeat — a draining pod must still exit.\n      timer.unref?.();\n      this.leaseTimers.set(taskId, timer);\n    }\n\n    // Subscriber setup happens AFTER the lease exists (see the note above). Not awaited:\n    // a slow Redis handshake must not delay the turn, and a cancel arriving before the\n    // subscription is live is already handled — `requestCancel` aborts a locally-owned turn\n    // directly, without a round-trip.\n    void this.ensureSubscriber();\n\n    return controller.signal;\n  }\n\n  /**\n   * Release a finished turn: stop the heartbeat, drop the lease, forget the controller.\n   *\n   * Dropping the lease EAGERLY matters — a terminal task with a live lease would look\n   * like it is still executing to anything reading liveness.\n   */\n  public async endTurn(taskId: string): Promise<void> {\n    const timer = this.leaseTimers.get(taskId);\n    if (timer) {\n      clearInterval(timer);\n      this.leaseTimers.delete(taskId);\n    }\n    this.localTurns.delete(taskId);\n    try {\n      await this.redisService.del(this.leaseKey(taskId));\n    } catch (error) {\n      // The TTL reaps it anyway; this only shortens the window.\n      this.logger.warn('[A2ATaskCancel] Failed to drop a task lease', { taskId, ...projectCaughtErrorLogFields(error) });\n    }\n  }\n\n  /**\n   * Ask the fleet to cancel a task.\n   *\n   * Returns whether the request was BROADCAST, never whether it was honoured: the owning\n   * pod may be gone, or the turn may already have finished. The caller must therefore\n   * treat the task record — not this return value — as the truth about cancellation,\n   * which is also what the A2A spec requires (`tasks/cancel` returns the task's updated\n   * state, not an acknowledgement).\n   */\n  public async requestCancel(taskId: string): Promise<boolean> {\n    // A turn owned by THIS pod is aborted directly — no round-trip, and it still works\n    // when Redis is down, which is the single-replica and local-dev case.\n    const local = this.localTurns.get(taskId);\n    if (local) local.abort();\n\n    const client = this.redisService.getReadyClient();\n    if (!client) return Boolean(local);\n\n    try {\n      await client.publish(this.channel(), taskId);\n      return true;\n    } catch (error) {\n      this.logger.error('[A2ATaskCancel] Failed to broadcast a cancel', { taskId, ...projectCaughtErrorLogFields(error) });\n      return Boolean(local);\n    }\n  }\n\n  /**\n   * Is some pod still working this task?\n   *\n   * This is the read half of the liveness design: a task whose stored status is\n   * non-terminal but whose lease has expired was abandoned by a pod that died, and must\n   * be reported as failed rather than left reading `working` forever.\n   *\n   * Fails SAFE toward \"alive\" when Redis is unreachable — reporting a healthy task as\n   * abandoned because the cache blipped would be a far worse lie than a brief stale\n   * `working`.\n   */\n  public async isTurnAlive(taskId: string): Promise<boolean> {\n    if (this.localTurns.has(taskId)) return true;\n    // Redis ABSENT is not evidence of death. `RedisService.exists` RETURNS FALSE when there\n    // is no client rather than throwing, so the catch below never fired for that case and the\n    // documented \"fails safe toward alive\" was untrue: with Redis down, every task on another\n    // pod would have been reported FAILED. Check the client explicitly.\n    if (!this.redisService.getReadyClient()) return true;\n    try {\n      return await this.redisService.exists(this.leaseKey(taskId));\n    } catch {\n      return true;\n    }\n  }\n\n  /**\n   * Can the lease mechanism actually be WRITTEN right now?\n   *\n   * Read by the abandoned-task sweep before it repairs anything, and only by it (review\n   * finding A3-4). The distinction that makes this necessary: `isTurnAlive` fails safe by\n   * answering \"alive\", so a degraded Redis costs a stale `working` — recoverable, and the next\n   * poll corrects it. The SWEEP is the one consumer that turns a liveness answer into a\n   * DURABLE terminal status, so it is the one that must refuse to act on a signal it cannot\n   * trust.\n   *\n   * A WRITE probe specifically, not a read: the dangerous failure mode is asymmetric — writes\n   * failing while reads succeed (a read-only replica, an OOM/quota condition). Under that\n   * failure every live turn's lease is missing while `exists` cheerfully reports it absent, so\n   * a read-based health check would pass and the sweep would fail every healthy task in the\n   * fleet. Probing the exact capability whose loss causes the false reading is the only check\n   * that catches it.\n   */\n  public async isLeaseMechanismWritable(): Promise<boolean> {\n    if (!this.redisService.getReadyClient()) return false;\n    const probeKey = this.leaseKey('__sweep-health-probe__');\n    try {\n      await this.redisService.set(probeKey, '1', { ttl: A2A_TASK_LEASE_TTL_SECONDS });\n      const present = await this.redisService.exists(probeKey);\n      await this.redisService.del(probeKey).catch(() => {});\n      return present;\n    } catch {\n      return false;\n    }\n  }\n}\n"]}
+{"version":3,"file":"a2a-task-cancellation.backend.service.js","sourceRoot":"","sources":["../../../../../../src/flows-actors/a2a/services/a2a-task-cancellation.backend.service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;;;;;;;;;;;;;AAEH,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAG/C,OAAO,EAAE,kBAAkB,EAAE,MAAM,gDAAgD,CAAC;AAGpF,OAAO,EAAE,2BAA2B,EAAE,MAAM,8EAA8E,CAAC;AAC3H,OAAO,EAAE,uBAAuB,EAAE,MAAM,gEAAgE,CAAC;AAEzG,6FAA6F;AAC7F,MAAM,qBAAqB,GAAG,iBAAiB,CAAC;AAChD,qFAAqF;AACrF,MAAM,gBAAgB,GAAG,iBAAiB,CAAC;AAC3C;;;;;;;GAOG;AACH,MAAM,4BAA4B,GAAG,uBAAuB,CAAC;AAC7D;;;;GAIG;AACH,MAAM,kCAAkC,GAAG,wBAAwB,CAAC;AAEpE;;;;GAIG;AACH,MAAM,CAAC,MAAM,0BAA0B,GAAG,EAAE,CAAC;AAC7C,MAAM,CAAC,MAAM,yBAAyB,GAAG,MAAM,CAAC;AAEhD;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,sCAAsC,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC;AAElE;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,wCAAwC,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC;AAG7D,IAAM,iCAAiC,GAAvC,MAAM,iCAAiC;IAUU;IACM;IAV5D,+EAA+E;IAC9D,UAAU,GAAG,IAAI,GAAG,EAA2B,CAAC;IACjE,6DAA6D;IAC5C,WAAW,GAAG,IAAI,GAAG,EAA0B,CAAC;IAEzD,UAAU,GAA2B,IAAI,CAAC;IAC1C,gBAAgB,GAAG,KAAK,CAAC;IAEjC,YACsD,MAAqB,EACf,YAA0B;QADhC,WAAM,GAAN,MAAM,CAAe;QACf,iBAAY,GAAZ,YAAY,CAAc;IACnF,CAAC;IAEI,OAAO;QACb,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,YAAY,EAAE,GAAG,qBAAqB,EAAE,CAAC;IACvE,CAAC;IAEO,QAAQ,CAAC,MAAc;QAC7B,OAAO,GAAG,gBAAgB,GAAG,MAAM,EAAE,CAAC;IACxC,CAAC;IAEO,mBAAmB,CAAC,MAAc;QACxC,OAAO,GAAG,4BAA4B,GAAG,MAAM,EAAE,CAAC;IACpD,CAAC;IAEO,qBAAqB,CAAC,MAAc;QAC1C,OAAO,GAAG,kCAAkC,GAAG,MAAM,EAAE,CAAC;IAC1D,CAAC;IAED;;;;;;;;;;OAUG;IACK,KAAK,CAAC,gBAAgB;QAC5B,IAAI,IAAI,CAAC,gBAAgB;YAAE,OAAO;QAClC,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC;QAE7B,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,cAAc,EAAE,CAAC;QAClD,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,qFAAqF,CAAC,CAAC;YACzG,OAAO;QACT,CAAC;QAED,IAAI,UAAuC,CAAC;QAC5C,IAAI,CAAC;YACH,UAAU,GAAG,MAAM,CAAC,SAAS,EAAqB,CAAC;YACnD,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAc,EAAE,EAAE;gBACxC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,iDAAiD,EAAE,EAAE,GAAG,2BAA2B,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAClH,CAAC,CAAC,CAAC;YACH,4FAA4F;YAC5F,8FAA8F;YAC9F,6FAA6F;YAC7F,6FAA6F;YAC7F,2FAA2F;YAC3F,6EAA6E;YAC7E,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;gBACxB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,0FAA0F,CAAC,CAAC;gBAC7G,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;gBAC5B,IAAI,CAAC,gBAAgB,GAAG,KAAK,CAAC;YAChC,CAAC,CAAC,CAAC;YACH,MAAM,UAAU,CAAC,OAAO,EAAE,CAAC;YAC3B,MAAM,UAAU,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC,OAAe,EAAE,EAAE,CAAC,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC;YACjG,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC/B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,iFAAiF;YACjF,yEAAyE;YACzE,sFAAsF;YACtF,oFAAoF;YACpF,IAAI,UAAU,EAAE,CAAC;gBACf,IAAI,CAAC;oBAAC,MAAM,UAAU,CAAC,OAAO,EAAE,CAAC;gBAAC,CAAC;gBAAC,MAAM,CAAC,CAAC,uCAAuC,CAAC,CAAC;YACvF,CAAC;YACD,IAAI,CAAC,gBAAgB,GAAG,KAAK,CAAC,CAAC,8BAA8B;YAC7D,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,2DAA2D,EAAE,EAAE,GAAG,2BAA2B,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC5H,CAAC;IACH,CAAC;IAED;;;;OAIG;IACK,iBAAiB,CAAC,MAAc;QACtC,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC/C,IAAI,CAAC,UAAU;YAAE,OAAO;QACxB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,gEAAgE,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;QAC/F,UAAU,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC;IAED;;;OAGG;IACI,KAAK,CAAC,SAAS,CAAC,MAAc;QACnC,oFAAoF;QACpF,wFAAwF;QACxF,qFAAqF;QACrF,yFAAyF;QACzF,0EAA0E;QAC1E,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;QAExC,IAAI,gBAAgB,GAAG,KAAK,CAAC;QAC7B,IAAI,yBAAyB,GAAG,KAAK,CAAC;QACtC,MAAM,OAAO,GAAG,KAAK,IAAmB,EAAE;YACxC,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,0BAA0B,EAAE,CAAC,CAAC;gBAC7F,yFAAyF;gBACzF,sFAAsF;gBACtF,qFAAqF;gBACrF,0FAA0F;gBAC1F,gFAAgF;gBAChF,qCAAqC;gBACrC,MAAM,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,sCAAsC,EAAE,CAAC,CAAC;gBACpH,gBAAgB,GAAG,IAAI,CAAC;YAC1B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,mFAAmF;gBACnF,+CAA+C;gBAC/C,EAAE;gBACF,oFAAoF;gBACpF,mFAAmF;gBACnF,mFAAmF;gBACnF,qFAAqF;gBACrF,kFAAkF;gBAClF,+EAA+E;gBAC/E,IAAI,gBAAgB,EAAE,CAAC;oBACrB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,sCAAsC,EAAE,EAAE,MAAM,EAAE,GAAG,2BAA2B,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;oBAC5G,OAAO;gBACT,CAAC;gBACD,0FAA0F;gBAC1F,yFAAyF;gBACzF,0FAA0F;gBAC1F,yBAAyB;gBACzB,IAAI,yBAAyB;oBAAE,OAAO;gBACtC,yBAAyB,GAAG,IAAI,CAAC;gBACjC,uFAAuF;gBACvF,sFAAsF;gBACtF,yFAAyF;gBACzF,0FAA0F;gBAC1F,0FAA0F;gBAC1F,8CAA8C;gBAC9C,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,cAAc,EAAE,EAAE,CAAC;oBACxC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,oHAAoH,EAAE;wBACrI,MAAM;wBACN,WAAW,EAAE,mGAAmG;wBAChH,GAAG,2BAA2B,CAAC,KAAK,CAAC;qBACtC,CAAC,CAAC;oBACH,OAAO;gBACT,CAAC;gBACD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,qGAAqG,EAAE;oBACvH,MAAM;oBACN,WAAW,EAAE,kIAAkI;oBAC/I,GAAG,2BAA2B,CAAC,KAAK,CAAC;iBACtC,CAAC,CAAC;YACL,CAAC;QACH,CAAC,CAAC;QACF,MAAM,OAAO,EAAE,CAAC;QAChB,+EAA+E;QAC/E,kFAAkF;QAClF,qFAAqF;QACrF,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC9C,IAAI,QAAQ;YAAE,aAAa,CAAC,QAAQ,CAAC,CAAC;QAEtC,6FAA6F;QAC7F,EAAE;QACF,2FAA2F;QAC3F,yFAAyF;QACzF,8FAA8F;QAC9F,yFAAyF;QACzF,uFAAuF;QACvF,4EAA4E;QAC5E,EAAE;QACF,8FAA8F;QAC9F,6FAA6F;QAC7F,oFAAoF;QACpF,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,KAAK,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,yBAAyB,CAAC,CAAC;QAChF,kFAAkF;QAClF,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC;QAChB,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QAEpC,qFAAqF;QACrF,mFAAmF;QACnF,wFAAwF;QACxF,kCAAkC;QAClC,KAAK,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAE7B,OAAO,UAAU,CAAC,MAAM,CAAC;IAC3B,CAAC;IAED;;;;;;;;;;;;;OAaG;IACI,KAAK,CAAC,OAAO,CAAC,MAAc;QACjC,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC3C,IAAI,KAAK,EAAE,CAAC;YACV,aAAa,CAAC,KAAK,CAAC,CAAC;YACrB,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAClC,CAAC;QACD,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC/B,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;QACrD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,0DAA0D;YAC1D,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,6CAA6C,EAAE,EAAE,MAAM,EAAE,GAAG,2BAA2B,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACrH,CAAC;IACH,CAAC;IAED;;;;;;;;OAQG;IACI,KAAK,CAAC,aAAa,CAAC,MAAc;QACvC,mFAAmF;QACnF,sEAAsE;QACtE,MAAM,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC1C,IAAI,KAAK;YAAE,KAAK,CAAC,KAAK,EAAE,CAAC;QAEzB,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,cAAc,EAAE,CAAC;QAClD,IAAI,CAAC,MAAM;YAAE,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC;QAEnC,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,MAAM,CAAC,CAAC;YAC7C,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,8CAA8C,EAAE,EAAE,MAAM,EAAE,GAAG,2BAA2B,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YACrH,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC;QACxB,CAAC;IACH,CAAC;IAED;;;;;;;;;;OAUG;IACI,KAAK,CAAC,WAAW,CAAC,MAAc;QACrC,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC;YAAE,OAAO,IAAI,CAAC;QAC7C,wFAAwF;QACxF,0FAA0F;QAC1F,0FAA0F;QAC1F,oEAAoE;QACpE,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,cAAc,EAAE;YAAE,OAAO,IAAI,CAAC;QACrD,IAAI,CAAC;YACH,OAAO,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;QAC/D,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAkCG;IACI,KAAK,CAAC,sBAAsB,CAAC,MAAc;QAChD,6FAA6F;QAC7F,6FAA6F;QAC7F,yEAAyE;QACzE,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,cAAc,EAAE;YAAE,OAAO,uBAAuB,CAAC,YAAY,CAAC;QAErF,IAAI,CAAC;YACH,IAAI,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC;gBAAE,OAAO,uBAAuB,CAAC,YAAY,CAAC;YAElH,MAAM,cAAc,GAAG,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,CAAC;YAC1D,IAAI,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,cAAc,CAAC;gBAAE,OAAO,uBAAuB,CAAC,oBAAoB,CAAC;YAExG,0FAA0F;YAC1F,yFAAyF;YACzF,wFAAwF;YACxF,WAAW;YACX,MAAM,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,cAAc,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,wCAAwC,EAAE,CAAC,CAAC;YACpG,OAAO,uBAAuB,CAAC,aAAa,CAAC;QAC/C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,wGAAwG,EAAE;gBACzH,MAAM;gBACN,GAAG,2BAA2B,CAAC,KAAK,CAAC;aACtC,CAAC,CAAC;YACH,OAAO,uBAAuB,CAAC,YAAY,CAAC;QAC9C,CAAC;IACH,CAAC;IAED;;;;;;;;;;;;;;;;OAgBG;IACI,KAAK,CAAC,wBAAwB;QACnC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,cAAc,EAAE;YAAE,OAAO,KAAK,CAAC;QACtD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,wBAAwB,CAAC,CAAC;QACzD,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,0BAA0B,EAAE,CAAC,CAAC;YAChF,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YACzD,MAAM,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;YACtD,OAAO,OAAO,CAAC;QACjB,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;CACF,CAAA;AAhXY,iCAAiC;IAD7C,UAAU,EAAE;IAWR,WAAA,MAAM,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAA;IACjC,WAAA,MAAM,CAAC,kBAAkB,CAAC,YAAY,CAAC,CAAA;;GAX/B,iCAAiC,CAgX7C","sourcesContent":["/**\n * Cross-pod A2A task cancellation.\n *\n * ## You do not need to find the executing pod\n *\n * With several backend replicas behind a load balancer, a `tasks/cancel` request\n * almost never lands on the pod that is running that task's LLM turn. The obvious\n * design — discover which pod owns it and route there — needs a pod registry, a\n * liveness view of it, and an internal call path, and it is wrong on every one of\n * those axes when a pod restarts mid-turn.\n *\n * Instead this BROADCASTS. Any pod publishes `{taskId}` to one Redis channel; every\n * pod runs a single shared subscriber; only the pod holding that `taskId` in its\n * in-process registry reacts. One connection per pod, no routing table, and a pod\n * that died simply does not answer — which is the correct behaviour, not an error to\n * handle.\n *\n * Rejected alternative: polling a Redis flag inside the token loop. That puts a GET on\n * the hottest path in the system (per N tokens, per concurrent turn) and bounds\n * cancellation latency by token cadence. Pub/sub costs nothing per token.\n *\n * ## Liveness is the same mechanism's other half\n *\n * While a pod owns a turn it refreshes a TTL'd lease key. That is what lets\n * `tasks/get` report an abandoned task honestly instead of leaving it `working`\n * forever after a pod dies — see the lease helpers below.\n *\n * An absent lease is NOT self-explanatory, though, and reading it as death is an inversion that\n * costs real work: a pod that could not write the lease looks exactly like a pod that died. So\n * every successful lease write also leaves a long-lived ESTABLISHMENT marker, and\n * `corroborateAbandonment` is what turns \"no lease\" into either a verdict or an admission that\n * nothing was learned. Only the sweep — the one consumer that makes a liveness answer DURABLE —\n * asks that question.\n */\n\nimport { inject, injectable } from 'inversify';\nimport type { RedisClientType } from 'redis';\n\nimport { SAAS_SERVICE_TYPES } from '~wildo-backend-lib/inject/models/service-types';\nimport type { LoggerService } from '~wildo-backend-lib/monitoring/logger/services/logger.service';\nimport type { RedisService } from '~wildo-backend-lib/databases/services/redis.service';\nimport { projectCaughtErrorLogFields } from '~wildo-backend-lib/errors/utils/error-log-level-classification.backend.utils';\nimport { A2A_AbandonmentEvidence } from '~wildo-backend-lib/flows-actors/a2a/models/a2a.backend.schemas';\n\n/** Channel every replica subscribes to. App-prefixed so tenants of one Redis never cross. */\nconst CANCEL_CHANNEL_SUFFIX = 'a2a:task:cancel';\n/** Lease key prefix — presence means \"a live pod is working this task right now\". */\nconst LEASE_KEY_PREFIX = 'a2a:task:lease:';\n/**\n * ESTABLISHMENT marker prefix — presence means \"this task's liveness WAS observable at some point\n * recently\", which is a different fact from \"it is alive now\" and has a different lifetime.\n *\n * It is what makes an absent lease MEAN something (see {@link A2A_AbandonmentEvidence}). Written\n * by the same code path and in the same failure domain as the lease itself, so it can only exist\n * if a write actually succeeded — which is exactly the property being recorded.\n */\nconst LEASE_ESTABLISHED_KEY_PREFIX = 'a2a:task:leased-once:';\n/**\n * The sweep's own note that it has already seen one unexplained absence for a task. Written by the\n * SWEEP's pod, whose write capability `isLeaseMechanismWritable` proved moments earlier — so the\n * medium is known-good at the point of use, unlike the lease itself.\n */\nconst ABANDONMENT_OBSERVATION_KEY_PREFIX = 'a2a:task:absence-seen:';\n\n/**\n * Lease TTL and refresh cadence. The refresh interval is deliberately well under the\n * TTL: a single missed refresh (GC pause, a slow Redis round-trip) must not make a\n * healthy turn look abandoned. Only a genuinely dead pod stops refreshing.\n */\nexport const A2A_TASK_LEASE_TTL_SECONDS = 30;\nexport const A2A_TASK_LEASE_REFRESH_MS = 10_000;\n\n/**\n * How long the establishment marker outlives its last refresh.\n *\n * Sized to comfortably span the sweep's grace window plus its hourly period, so a pod that dies\n * mid-turn is still recognisable as HAVING held the task when the next sweep looks. It is a\n * timing optimisation, never a correctness requirement: if it lapses, the reading degrades to\n * {@link A2A_AbandonmentEvidence.FIRST_ABSENCE} and the row is repaired one run later instead of\n * on the first — which is the direction this whole mechanism errs in anyway.\n */\nexport const A2A_TASK_LEASE_ESTABLISHED_TTL_SECONDS = 6 * 60 * 60;\n\n/**\n * How long one unexplained absence stands as corroboration for the next run.\n *\n * MUST exceed the sweep's period or no observation ever survives to be corroborated and nothing\n * is ever repaired through that path; generous beyond it so a skipped run (the fleet-wide abstain)\n * does not reset the count. Nothing clears it on recovery, and it does not need clearing: a task\n * whose pod recovered answers ALIVE on the lease long before this is consulted, and once its lease\n * lapses for real the establishment marker gives the verdict directly.\n */\nexport const A2A_TASK_ABSENCE_OBSERVATION_TTL_SECONDS = 3 * 60 * 60;\n\n@injectable()\nexport class A2ATaskCancellationBackendService {\n  /** Tasks this pod is currently executing → the controller that aborts them. */\n  private readonly localTurns = new Map<string, AbortController>();\n  /** Lease heartbeats keyed by task id, cleared on release. */\n  private readonly leaseTimers = new Map<string, NodeJS.Timeout>();\n\n  private subscriber: RedisClientType | null = null;\n  private subscribeStarted = false;\n\n  constructor(\n    @inject(SAAS_SERVICE_TYPES.Logger) private readonly logger: LoggerService,\n    @inject(SAAS_SERVICE_TYPES.RedisService) private readonly redisService: RedisService,\n  ) {}\n\n  private channel(): string {\n    return `${this.redisService.getAppPrefix()}${CANCEL_CHANNEL_SUFFIX}`;\n  }\n\n  private leaseKey(taskId: string): string {\n    return `${LEASE_KEY_PREFIX}${taskId}`;\n  }\n\n  private leaseEstablishedKey(taskId: string): string {\n    return `${LEASE_ESTABLISHED_KEY_PREFIX}${taskId}`;\n  }\n\n  private absenceObservationKey(taskId: string): string {\n    return `${ABANDONMENT_OBSERVATION_KEY_PREFIX}${taskId}`;\n  }\n\n  /**\n   * Start this pod's single shared subscriber. Idempotent — safe to call from every\n   * turn, so no startup ordering is required.\n   *\n   * `node-redis` forbids commands on a subscribed connection, so the subscriber MUST be\n   * a duplicate rather than the shared client. Same pattern the WebSocket adapter uses.\n   *\n   * Fails OPEN by design: if Redis is unreachable the turn still runs — losing the\n   * ability to cancel is strictly better than refusing to answer at all. The failure is\n   * logged at ERROR because a persistent one silently disables cancellation fleet-wide.\n   */\n  private async ensureSubscriber(): Promise<void> {\n    if (this.subscribeStarted) return;\n    this.subscribeStarted = true;\n\n    const client = this.redisService.getReadyClient();\n    if (!client) {\n      this.logger.error('[A2ATaskCancel] Redis unavailable — cross-pod cancellation is DISABLED for this pod');\n      return;\n    }\n\n    let subscriber: RedisClientType | undefined;\n    try {\n      subscriber = client.duplicate() as RedisClientType;\n      subscriber.on('error', (error: unknown) => {\n        this.logger.error('[A2ATaskCancel] Cancel-channel subscriber error', { ...projectCaughtErrorLogFields(error) });\n      });\n      // REARM on a later death (review finding L-2). `subscribeStarted` was reset only inside the\n      // SETUP catch, so a subscriber that connected and then dropped — a Redis restart, a failover,\n      // an idle-timeout reap — logged from the handler above and was never rebuilt for the rest of\n      // the pod's lifetime. Cross-pod cancellation then silently stopped working on that pod while\n      // every other signal still looked healthy: the turns run, the leases refresh, and only the\n      // broadcast never arrives. Clearing the latch lets the next turn rebuild it.\n      subscriber.on('end', () => {\n        this.logger.warn('[A2ATaskCancel] Cancel-channel subscriber closed; re-arming so the next turn rebuilds it');\n        this.subscriber = undefined;\n        this.subscribeStarted = false;\n      });\n      await subscriber.connect();\n      await subscriber.subscribe(this.channel(), (message: string) => this.onCancelBroadcast(message));\n      this.subscriber = subscriber;\n    } catch (error) {\n      // DESTROY the half-built connection before retrying. `connect()` can succeed and\n      // `subscribe()` still throw, which leaves a live, unusable socket; since\n      // `subscribeStarted` resets to allow a retry, every later turn would open another one\n      // and leak a Redis connection per turn until the pod hit its file-descriptor limit.\n      if (subscriber) {\n        try { await subscriber.destroy(); } catch { /* already dead — nothing to reclaim */ }\n      }\n      this.subscribeStarted = false; // allow a later turn to retry\n      this.logger.error('[A2ATaskCancel] Failed to subscribe to the cancel channel', { ...projectCaughtErrorLogFields(error) });\n    }\n  }\n\n  /**\n   * Handle a broadcast. Every pod receives every cancel; only the owner acts, and a pod\n   * that does not own the task does nothing at all — that silence is the design, not a\n   * missed message.\n   */\n  private onCancelBroadcast(taskId: string): void {\n    const controller = this.localTurns.get(taskId);\n    if (!controller) return;\n    this.logger.info('[A2ATaskCancel] Aborting a locally-executing turn on broadcast', { taskId });\n    controller.abort();\n  }\n\n  /**\n   * Register a turn as executing on this pod: returns the signal to hand to the model\n   * call, and starts the liveness lease.\n   */\n  public async beginTurn(taskId: string): Promise<AbortSignal> {\n    // ORDER IS LOAD-BEARING. Register locally and write the lease BEFORE the subscriber\n    // setup, which on the first turn performs a full Redis duplicate + connect + subscribe.\n    // Awaiting that first would leave a window in which the task is materialised and the\n    // client — which asked for a non-blocking turn precisely so it could poll at once — sees\n    // a non-terminal task with no lease and is told FAILED by derive-on-read.\n    const controller = new AbortController();\n    this.localTurns.set(taskId, controller);\n\n    let leaseEverWritten = false;\n    let unwritableAlreadyReported = false;\n    const refresh = async (): Promise<void> => {\n      try {\n        await this.redisService.set(this.leaseKey(taskId), '1', { ttl: A2A_TASK_LEASE_TTL_SECONDS });\n        // The ESTABLISHMENT marker rides the same write, in the same failure domain, and is what\n        // makes a LATER absence of the lease mean something: a reader that finds no lease but\n        // finds this knows a pod really did hold the task and stopped, rather than that this\n        // task's liveness was never observable at all. Written every refresh so a long turn keeps\n        // it alive; deliberately NOT written before the lease, so it can never claim an\n        // establishment that did not happen.\n        await this.redisService.set(this.leaseEstablishedKey(taskId), '1', { ttl: A2A_TASK_LEASE_ESTABLISHED_TTL_SECONDS });\n        leaseEverWritten = true;\n      } catch (error) {\n        // A missed REFRESH is survivable — the TTL is several refreshes long, so the lease\n        // written earlier still answers for this turn.\n        //\n        // A lease that was NEVER successfully written is a different thing entirely (review\n        // finding A3-4), and it used to log identically. No other pod can distinguish \"the\n        // write failed\" from \"the pod died\": `tasks/get` on any other replica reports this\n        // healthy running turn as FAILED, and the sweep then makes that answer DURABLE. This\n        // is the write-side twin of the read-side hole G2-8 closed — the JSDoc claim that\n        // \"only a genuinely dead pod stops refreshing\" was false in exactly this case.\n        if (leaseEverWritten) {\n          this.logger.warn('[A2ATaskCancel] Lease refresh failed', { taskId, ...projectCaughtErrorLogFields(error) });\n          return;\n        }\n        // ONCE per turn, not once per heartbeat. The heartbeat now runs even with no Redis at all\n        // (see below), so an unconditional log here would emit a line every 10 seconds for every\n        // concurrent turn of a Redis-less deployment — which is how a channel that carries a real\n        // signal gets tuned out.\n        if (unwritableAlreadyReported) return;\n        unwritableAlreadyReported = true;\n        // The two causes are logged at DIFFERENT levels because one is a configuration and the\n        // other is a fault. No client on this pod is the supported single-replica / local-dev\n        // shape: `isTurnAlive` fails safe toward \"alive\" there and nothing reads liveness across\n        // pods, so it is a WARN. A client that is present and REJECTED the write is the dangerous\n        // asymmetry — this pod is partitioned from a liveness plane every other replica can still\n        // read — and it keeps the ERROR level it had.\n        if (!this.redisService.getReadyClient()) {\n          this.logger.warn('[A2ATaskCancel] No Redis on this pod; this turn has no liveness lease and cannot be cancelled from another replica', {\n            taskId,\n            consequence: 'harmless on a single replica; on a fleet, other pods cannot observe this turn until Redis returns',\n            ...projectCaughtErrorLogFields(error),\n          });\n          return;\n        }\n        this.logger.error('[A2ATaskCancel] Task lease has NEVER been written; other pods will read this live turn as abandoned', {\n          taskId,\n          consequence: 'tasks/get on another replica reports failed until a later refresh lands; the sweep now abstains rather than recording it durably',\n          ...projectCaughtErrorLogFields(error),\n        });\n      }\n    };\n    await refresh();\n    // Clear any prior heartbeat for this id before replacing it. Unreachable today\n    // (task ids are freshly minted uuids), but an overwritten `setInterval` handle is\n    // unclearable — it would refresh a lease forever and keep a dead task looking alive.\n    const existing = this.leaseTimers.get(taskId);\n    if (existing) clearInterval(existing);\n\n    // ── The heartbeat is installed UNCONDITIONALLY, and that is a fix, not a tidy-up ──────────\n    //\n    // This whole block used to sit behind `if (this.redisService.getReadyClient())`, so a turn\n    // that began while this pod's client was not ready got NO lease and NO retry — it stayed\n    // unobservable for its entire life even after Redis came back, and nothing was logged at all,\n    // because the failing write never even ran. That is the SILENT half of the inversion the\n    // never-written branch above documents, and it is strictly worse: a failed first write\n    // self-heals on the next 10-second refresh, an absent heartbeat never does.\n    //\n    // The cost in a genuinely Redis-less deployment is one unref'd timer per in-flight turn and a\n    // caught rejection every 10 seconds; the log is emitted once (above), and `redisService.set`\n    // builds its error without logging. That is cheap enough to buy automatic recovery.\n    const timer = setInterval(() => { void refresh(); }, A2A_TASK_LEASE_REFRESH_MS);\n    // Never hold the event loop open on a heartbeat — a draining pod must still exit.\n    timer.unref?.();\n    this.leaseTimers.set(taskId, timer);\n\n    // Subscriber setup happens AFTER the lease exists (see the note above). Not awaited:\n    // a slow Redis handshake must not delay the turn, and a cancel arriving before the\n    // subscription is live is already handled — `requestCancel` aborts a locally-owned turn\n    // directly, without a round-trip.\n    void this.ensureSubscriber();\n\n    return controller.signal;\n  }\n\n  /**\n   * Release a finished turn: stop the heartbeat, drop the lease, forget the controller.\n   *\n   * Dropping the lease EAGERLY matters — a terminal task with a live lease would look\n   * like it is still executing to anything reading liveness.\n   *\n   * The ESTABLISHMENT marker is deliberately NOT dropped with it. The two record different kinds\n   * of fact: the lease says \"a pod is working this NOW\" and stops being true the instant the turn\n   * unwinds, while the marker says \"this task's liveness was observable\", which stays true\n   * afterwards. Keeping it also covers the one case that would otherwise regress — a turn that\n   * finished but whose terminal write did not land — since the row is then genuinely abandoned and\n   * the sweep can say so on its first pass instead of waiting for corroboration. It expires on its\n   * own TTL, so nothing accumulates.\n   */\n  public async endTurn(taskId: string): Promise<void> {\n    const timer = this.leaseTimers.get(taskId);\n    if (timer) {\n      clearInterval(timer);\n      this.leaseTimers.delete(taskId);\n    }\n    this.localTurns.delete(taskId);\n    try {\n      await this.redisService.del(this.leaseKey(taskId));\n    } catch (error) {\n      // The TTL reaps it anyway; this only shortens the window.\n      this.logger.warn('[A2ATaskCancel] Failed to drop a task lease', { taskId, ...projectCaughtErrorLogFields(error) });\n    }\n  }\n\n  /**\n   * Ask the fleet to cancel a task.\n   *\n   * Returns whether the request was BROADCAST, never whether it was honoured: the owning\n   * pod may be gone, or the turn may already have finished. The caller must therefore\n   * treat the task record — not this return value — as the truth about cancellation,\n   * which is also what the A2A spec requires (`tasks/cancel` returns the task's updated\n   * state, not an acknowledgement).\n   */\n  public async requestCancel(taskId: string): Promise<boolean> {\n    // A turn owned by THIS pod is aborted directly — no round-trip, and it still works\n    // when Redis is down, which is the single-replica and local-dev case.\n    const local = this.localTurns.get(taskId);\n    if (local) local.abort();\n\n    const client = this.redisService.getReadyClient();\n    if (!client) return Boolean(local);\n\n    try {\n      await client.publish(this.channel(), taskId);\n      return true;\n    } catch (error) {\n      this.logger.error('[A2ATaskCancel] Failed to broadcast a cancel', { taskId, ...projectCaughtErrorLogFields(error) });\n      return Boolean(local);\n    }\n  }\n\n  /**\n   * Is some pod still working this task?\n   *\n   * This is the read half of the liveness design: a task whose stored status is\n   * non-terminal but whose lease has expired was abandoned by a pod that died, and must\n   * be reported as failed rather than left reading `working` forever.\n   *\n   * Fails SAFE toward \"alive\" when Redis is unreachable — reporting a healthy task as\n   * abandoned because the cache blipped would be a far worse lie than a brief stale\n   * `working`.\n   */\n  public async isTurnAlive(taskId: string): Promise<boolean> {\n    if (this.localTurns.has(taskId)) return true;\n    // Redis ABSENT is not evidence of death. `RedisService.exists` RETURNS FALSE when there\n    // is no client rather than throwing, so the catch below never fired for that case and the\n    // documented \"fails safe toward alive\" was untrue: with Redis down, every task on another\n    // pod would have been reported FAILED. Check the client explicitly.\n    if (!this.redisService.getReadyClient()) return true;\n    try {\n      return await this.redisService.exists(this.leaseKey(taskId));\n    } catch {\n      return true;\n    }\n  }\n\n  /**\n   * WHY is this task's lease absent — and may that absence be turned into a durable verdict?\n   *\n   * Asked by the abandoned-task sweep, and only by it, AFTER {@link isTurnAlive} has already\n   * answered \"no lease\". `isTurnAlive` returns a boolean because every other consumer only needs\n   * to know whether to keep waiting; the sweep is the one caller that writes a TERMINAL status,\n   * so it is the one caller that must be able to tell a verdict from a failed enquiry.\n   *\n   * ## The case this closes, which no existing guard catches\n   *\n   * `isLeaseMechanismWritable` is evaluated on the SWEEP's pod. A per-pod failure — the executing\n   * pod's Redis connection is broken while the sweep's is fine — passes it, `isTurnAlive`\n   * truthfully reports no lease, and a healthy running turn is durably recorded FAILED. That\n   * inversion has no compensating pass; it is the reason this method exists.\n   *\n   * ## How the two are told apart\n   *\n   * 1. **An establishment marker** ({@link LEASE_ESTABLISHED_KEY_PREFIX}) is written beside every\n   *    successful lease write and outlives it by hours. Present ⇒ a pod demonstrably held this\n   *    task and stopped ⇒ `LEASE_LAPSED`, a verdict, on the very first pass. This is what keeps\n   *    ordinary abandonment repaired in the same hour it happens.\n   * 2. **Otherwise the reading is indeterminate**, and the answer is TIME rather than another key:\n   *    a live pod that merely failed one write rewrites its lease on the next 10-second heartbeat,\n   *    so an absence that is still there a full sweep period later is not a missed write. The\n   *    first pass records the observation and returns `FIRST_ABSENCE`; a later pass that still\n   *    finds no lease and still finds the observation returns `CORROBORATED_ABSENCE`.\n   * 3. **If the observation can neither be read nor written, nothing was learned** —\n   *    `UNDETERMINED`. It is NOT collapsed into `FIRST_ABSENCE`, because that one made progress\n   *    toward a verdict and this one did not; recording it as progress would let two enquiries\n   *    that both failed add up to a durable kill.\n   *\n   * Losing the medium fails toward the SAFE side in both directions: with the markers gone (a\n   * Redis restart) every absence is merely uncorroborated, so genuine abandonment is still\n   * repaired — one run later — while a healthy turn is not killed on a single reading.\n   */\n  public async corroborateAbandonment(taskId: string): Promise<A2A_AbandonmentEvidence> {\n    // No client at all is the one case where this method can say nothing whatsoever. The sweep's\n    // fleet-wide abstain already refuses to run in that state, so reaching here means the client\n    // died between that probe and this row — indeterminate, by construction.\n    if (!this.redisService.getReadyClient()) return A2A_AbandonmentEvidence.UNDETERMINED;\n\n    try {\n      if (await this.redisService.exists(this.leaseEstablishedKey(taskId))) return A2A_AbandonmentEvidence.LEASE_LAPSED;\n\n      const observationKey = this.absenceObservationKey(taskId);\n      if (await this.redisService.exists(observationKey)) return A2A_AbandonmentEvidence.CORROBORATED_ABSENCE;\n\n      // Record THIS observation so a later run can corroborate it. The write is inside the same\n      // try as the reads on purpose: an observation that was not durably recorded has not been\n      // made, and reporting `FIRST_ABSENCE` for it would let the next run corroborate against\n      // nothing.\n      await this.redisService.set(observationKey, '1', { ttl: A2A_TASK_ABSENCE_OBSERVATION_TTL_SECONDS });\n      return A2A_AbandonmentEvidence.FIRST_ABSENCE;\n    } catch (error) {\n      this.logger.warn('[A2ATaskCancel] Could not establish why a task lease is absent; the sweep will not act on this reading', {\n        taskId,\n        ...projectCaughtErrorLogFields(error),\n      });\n      return A2A_AbandonmentEvidence.UNDETERMINED;\n    }\n  }\n\n  /**\n   * Can the lease mechanism actually be WRITTEN right now?\n   *\n   * Read by the abandoned-task sweep before it repairs anything, and only by it (review\n   * finding A3-4). The distinction that makes this necessary: `isTurnAlive` fails safe by\n   * answering \"alive\", so a degraded Redis costs a stale `working` — recoverable, and the next\n   * poll corrects it. The SWEEP is the one consumer that turns a liveness answer into a\n   * DURABLE terminal status, so it is the one that must refuse to act on a signal it cannot\n   * trust.\n   *\n   * A WRITE probe specifically, not a read: the dangerous failure mode is asymmetric — writes\n   * failing while reads succeed (a read-only replica, an OOM/quota condition). Under that\n   * failure every live turn's lease is missing while `exists` cheerfully reports it absent, so\n   * a read-based health check would pass and the sweep would fail every healthy task in the\n   * fleet. Probing the exact capability whose loss causes the false reading is the only check\n   * that catches it.\n   */\n  public async isLeaseMechanismWritable(): Promise<boolean> {\n    if (!this.redisService.getReadyClient()) return false;\n    const probeKey = this.leaseKey('__sweep-health-probe__');\n    try {\n      await this.redisService.set(probeKey, '1', { ttl: A2A_TASK_LEASE_TTL_SECONDS });\n      const present = await this.redisService.exists(probeKey);\n      await this.redisService.del(probeKey).catch(() => {});\n      return present;\n    } catch {\n      return false;\n    }\n  }\n}\n"]}

package/dist/esm/flows-actors/a2a/services/a2a-task.backend.service.d.ts

@@ -234,8 +234,14 @@ export declare class A2ATaskBackendService {
      * Best-effort by the same reasoning as {@link materialiseTask}: the work itself has
      * already happened, so failing to record its outcome must not convert a successful
      * turn into an error the client sees.
+     *
+     * `reclaimAbandonedVerdict` is the ONE case in which a terminal status may be moved — see the
+     * block that uses it. It is opt-in per call site rather than a widened compare-and-swap set
+     * precisely so `tasks/cancel` can never reach it.
      */
-    advanceTask(owner: ConversationalOwnerScope, taskId: string, status: FlowsActors_Status): Promise<A2A_TaskTransitionOutcome>;
+    advanceTask(owner: ConversationalOwnerScope, taskId: string, status: FlowsActors_Status, options?: {
+        reclaimAbandonedVerdict?: boolean;
+    }): Promise<A2A_TaskTransitionOutcome>;
     /**
      * Read one task as the protocol object — backs `tasks/get`.
      *

package/dist/esm/flows-actors/a2a/services/a2a-task.backend.service.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"a2a-task.backend.service.d.ts","sourceRoot":"","sources":["../../../../../../src/flows-actors/a2a/services/a2a-task.backend.service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6CG;AAKH,OAAO,EACL,gBAAgB,EAEhB,kBAAkB,EAClB,KAAK,aAAa,EACnB,MAAM,uBAAuB,CAAC;AAG/B,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,8DAA8D,CAAC;AAClG,OAAO,KAAK,EAAE,0BAA0B,EAAE,MAAM,iFAAiF,CAAC;AAClI,OAAO,KAAK,EAAE,wBAAwB,EAAE,qCAAqC,EAAE,MAAM,mFAAmF,CAAC;AAEzK,OAAO,KAAK,EAAE,iCAAiC,EAAE,MAAM,yCAAyC,CAAC;AACjG,OAAO,KAAK,EAAE,+BAA+B,EAAE,MAAM,uCAAuC,CAAC;AAC7F,OAAO,EAOL,KAAK,gBAAgB,EACtB,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAGL,KAAK,gCAAgC,EACtC,MAAM,iDAAiD,CAAC;AA4CzD;;;;;;;GAOG;AACH,oBAAY,yBAAyB;IACnC,+EAA+E;IAC/E,QAAQ,aAAa;IACrB;;;;OAIG;IACH,OAAO,YAAY;IACnB;;;;OAIG;IACH,MAAM,WAAW;CAClB;AAED,2EAA2E;AAC3E,MAAM,WAAW,qBAAqB;IACpC,KAAK,EAAE,wBAAwB,CAAC;IAChC,mDAAmD;IACnD,SAAS,EAAE,MAAM,CAAC;IAClB,2EAA2E;IAC3E,KAAK,EAAE,MAAM,CAAC;IACd;;;;;OAKG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;;;OAMG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,kEAAkE;IAClE,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B;;;;;;;;;;OAUG;IACH,MAAM,CAAC,EAAE,kBAAkB,CAAC;CAC7B;AAED,qBACa,qBAAqB;IAEK,OAAO,CAAC,QAAQ,CAAC,MAAM;IACV,OAAO,CAAC,QAAQ,CAAC,YAAY;IAEtB,OAAO,CAAC,QAAQ,CAAC,YAAY;IAIlB,OAAO,CAAC,QAAQ,CAAC,uBAAuB;IACrD,OAAO,CAAC,QAAQ,CAAC,cAAc;gBARhC,MAAM,EAAE,aAAa,EACR,YAAY,EAAE,0BAA0B,EAEjC,YAAY,EAAE,iCAAiC,EAIpC,uBAAuB,EAAE,qCAAqC,EAC3E,cAAc,EAAE,+BAA+B;IAGvH;;;;;;;;;;;;;;;;;;;;;;OAsBG;YACW,0BAA0B;IA0BxC;;;;;;;OAOG;IACH;;;;;;;;;OASG;IACU,2BAA2B,CACtC,KAAK,EAAE,wBAAwB,EAC/B,MAAM,EAAE,MAAM,EACd,WAAW,CAAC,EAAE,MAAM,EACpB,mBAAmB,CAAC,EAAE,MAAM,GAC3B,OAAO,CAAC,gCAAgC,EAAE,GAAG,SAAS,CAAC;IAc1D;;;;;;;;;;;;;OAaG;IACU,8BAA8B,CACzC,KAAK,EAAE,wBAAwB,EAC/B,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,gCAAgC,EAAE,EAC3C,WAAW,CAAC,EAAE,MAAM,EACpB,mBAAmB,CAAC,EAAE,MAAM,GAC3B,OAAO,CAAC,OAAO,CAAC;IAiBnB;;;;;;;;;;;;;;;;;;OAkBG;IACU,SAAS,CACpB,KAAK,EAAE,wBAAwB,EAC/B,WAAW,EAAE,MAAM,GAAG,SAAS,EAC/B,OAAO,EAAE;QAAE,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,kBAAkB,CAAA;KAAE,EAC5D,UAAU,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,GAC7C,OAAO,CAAC;QAAE,KAAK,EAAE,gBAAgB,EAAE,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC;IA6B5D,OAAO,CAAC,gBAAgB;IAcxB;;;;;;;OAOG;IACI,uBAAuB,CAAC,KAAK,EAAE,wBAAwB,GAAG,gBAAgB,GAAG,SAAS;IAI7F;;;;;;;;OAQG;IACU,eAAe,CAAC,MAAM,EAAE,qBAAqB,GAAG,OAAO,CAAC,gBAAgB,GAAG,SAAS,CAAC;IA8ClG;;;;;;;OAOG;IACU,WAAW,CACtB,KAAK,EAAE,wBAAwB,EAC/B,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,kBAAkB,GACzB,OAAO,CAAC,yBAAyB,CAAC;IA4JrC;;;;;;;OAOG;IACU,QAAQ,CACnB,KAAK,EAAE,wBAAwB,EAC/B,MAAM,EAAE,MAAM,EACd,WAAW,CAAC,EAAE,MAAM,EACpB,mBAAmB,CAAC,EAAE,MAAM,GAC3B,OAAO,CAAC,gBAAgB,GAAG,SAAS,CAAC;IA2CxC,6DAA6D;IACtD,cAAc,CAAC,IAAI,EAAE,aAAa,GAAG,gBAAgB;CAmC7D"}
+{"version":3,"file":"a2a-task.backend.service.d.ts","sourceRoot":"","sources":["../../../../../../src/flows-actors/a2a/services/a2a-task.backend.service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6CG;AAKH,OAAO,EACL,gBAAgB,EAEhB,kBAAkB,EAClB,KAAK,aAAa,EACnB,MAAM,uBAAuB,CAAC;AAG/B,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,8DAA8D,CAAC;AAClG,OAAO,KAAK,EAAE,0BAA0B,EAAE,MAAM,iFAAiF,CAAC;AAClI,OAAO,KAAK,EAAE,wBAAwB,EAAE,qCAAqC,EAAE,MAAM,mFAAmF,CAAC;AAEzK,OAAO,KAAK,EAAE,iCAAiC,EAAE,MAAM,yCAAyC,CAAC;AACjG,OAAO,KAAK,EAAE,+BAA+B,EAAE,MAAM,uCAAuC,CAAC;AAC7F,OAAO,EAOL,KAAK,gBAAgB,EACtB,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAGL,KAAK,gCAAgC,EACtC,MAAM,iDAAiD,CAAC;AA4CzD;;;;;;;GAOG;AACH,oBAAY,yBAAyB;IACnC,+EAA+E;IAC/E,QAAQ,aAAa;IACrB;;;;OAIG;IACH,OAAO,YAAY;IACnB;;;;OAIG;IACH,MAAM,WAAW;CAClB;AAED,2EAA2E;AAC3E,MAAM,WAAW,qBAAqB;IACpC,KAAK,EAAE,wBAAwB,CAAC;IAChC,mDAAmD;IACnD,SAAS,EAAE,MAAM,CAAC;IAClB,2EAA2E;IAC3E,KAAK,EAAE,MAAM,CAAC;IACd;;;;;OAKG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;;;OAMG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,kEAAkE;IAClE,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B;;;;;;;;;;OAUG;IACH,MAAM,CAAC,EAAE,kBAAkB,CAAC;CAC7B;AAED,qBACa,qBAAqB;IAEK,OAAO,CAAC,QAAQ,CAAC,MAAM;IACV,OAAO,CAAC,QAAQ,CAAC,YAAY;IAEtB,OAAO,CAAC,QAAQ,CAAC,YAAY;IAIlB,OAAO,CAAC,QAAQ,CAAC,uBAAuB;IACrD,OAAO,CAAC,QAAQ,CAAC,cAAc;gBARhC,MAAM,EAAE,aAAa,EACR,YAAY,EAAE,0BAA0B,EAEjC,YAAY,EAAE,iCAAiC,EAIpC,uBAAuB,EAAE,qCAAqC,EAC3E,cAAc,EAAE,+BAA+B;IAGvH;;;;;;;;;;;;;;;;;;;;;;OAsBG;YACW,0BAA0B;IA0BxC;;;;;;;OAOG;IACH;;;;;;;;;OASG;IACU,2BAA2B,CACtC,KAAK,EAAE,wBAAwB,EAC/B,MAAM,EAAE,MAAM,EACd,WAAW,CAAC,EAAE,MAAM,EACpB,mBAAmB,CAAC,EAAE,MAAM,GAC3B,OAAO,CAAC,gCAAgC,EAAE,GAAG,SAAS,CAAC;IAc1D;;;;;;;;;;;;;OAaG;IACU,8BAA8B,CACzC,KAAK,EAAE,wBAAwB,EAC/B,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,gCAAgC,EAAE,EAC3C,WAAW,CAAC,EAAE,MAAM,EACpB,mBAAmB,CAAC,EAAE,MAAM,GAC3B,OAAO,CAAC,OAAO,CAAC;IAiBnB;;;;;;;;;;;;;;;;;;OAkBG;IACU,SAAS,CACpB,KAAK,EAAE,wBAAwB,EAC/B,WAAW,EAAE,MAAM,GAAG,SAAS,EAC/B,OAAO,EAAE;QAAE,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,kBAAkB,CAAA;KAAE,EAC5D,UAAU,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,GAC7C,OAAO,CAAC;QAAE,KAAK,EAAE,gBAAgB,EAAE,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC;IA6B5D,OAAO,CAAC,gBAAgB;IAcxB;;;;;;;OAOG;IACI,uBAAuB,CAAC,KAAK,EAAE,wBAAwB,GAAG,gBAAgB,GAAG,SAAS;IAI7F;;;;;;;;OAQG;IACU,eAAe,CAAC,MAAM,EAAE,qBAAqB,GAAG,OAAO,CAAC,gBAAgB,GAAG,SAAS,CAAC;IA8ClG;;;;;;;;;;;OAWG;IACU,WAAW,CACtB,KAAK,EAAE,wBAAwB,EAC/B,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,kBAAkB,EAC1B,OAAO,CAAC,EAAE;QAAE,uBAAuB,CAAC,EAAE,OAAO,CAAA;KAAE,GAC9C,OAAO,CAAC,yBAAyB,CAAC;IAsNrC;;;;;;;OAOG;IACU,QAAQ,CACnB,KAAK,EAAE,wBAAwB,EAC/B,MAAM,EAAE,MAAM,EACd,WAAW,CAAC,EAAE,MAAM,EACpB,mBAAmB,CAAC,EAAE,MAAM,GAC3B,OAAO,CAAC,gBAAgB,GAAG,SAAS,CAAC;IA2CxC,6DAA6D;IACtD,cAAc,CAAC,IAAI,EAAE,aAAa,GAAG,gBAAgB;CAmC7D"}

package/dist/esm/flows-actors/a2a/services/a2a-task.backend.service.js

@@ -349,8 +349,12 @@ let A2ATaskBackendService = class A2ATaskBackendService {
      * Best-effort by the same reasoning as {@link materialiseTask}: the work itself has
      * already happened, so failing to record its outcome must not convert a successful
      * turn into an error the client sees.
+     *
+     * `reclaimAbandonedVerdict` is the ONE case in which a terminal status may be moved — see the
+     * block that uses it. It is opt-in per call site rather than a widened compare-and-swap set
+     * precisely so `tasks/cancel` can never reach it.
      */
-    async advanceTask(owner, taskId, status) {
+    async advanceTask(owner, taskId, status, options) {
         // Returns whether the transition ACTUALLY happened (review finding G5-4). It used to return
         // `void`, which discarded the compare-and-swap's answer — so `tasks/cancel` could have its
         // write correctly refused by the CAS and still reply HTTP 200 with a `completed` task. That is
@@ -428,6 +432,54 @@ let A2ATaskBackendService = class A2ATaskBackendService {
             if (!updated) {
                 const current = await this.systemAccess.readAsSystem(resolved.resourceType, { ...resolved.ownerFields, taskId }, { scopeContext: resolved.scopeContext });
                 if (current) {
+                    // ── RECLAIMING a task the abandoned-task sweep killed underneath a live turn ─────────
+                    //
+                    // Until this existed there was NO way back from a wrong sweep verdict. The sweep can
+                    // only ever conclude "abandoned" from the ABSENCE of a liveness lease, and a pod whose
+                    // Redis writes are failing produces exactly that absence while running the turn
+                    // perfectly — so the sweep records FAILED, the client is told FAILED, and when the turn
+                    // then finishes for real its own outcome write is refused by the compare-and-swap and
+                    // the answer is discarded. A derived verdict outranking an observed fact is backwards.
+                    //
+                    // The turn is the AUTHORITATIVE party here: it ran the work. So its own outcome write —
+                    // and only that one — may move the task off a FAILED it did not write.
+                    //
+                    // Why that is safe, enumerated rather than assumed. Exactly THREE sites write FAILED to
+                    // a task row: this service's two callers in `runTurnAndAdvanceTask` (the `turn.kind ===
+                    // 'error'` arm and its `catch`, which are mutually exclusive and run at most once per
+                    // turn) and the sweep's `updateAsSystem`. So a turn whose own outcome write finds FAILED
+                    // stored did not put it there — the sweep did. Everything else is untouched:
+                    //   • CANCELLED is NOT in the reclaim filter, so a human's stop can never be overwritten
+                    //     — that is the one terminal state whose whole point is that it is a decision.
+                    //   • COMPLETED is not either, so a finished result stays final.
+                    //   • `tasks/cancel` never passes the flag, so no client-facing verb can reach this.
+                    // If a fourth writer of FAILED ever appears, it belongs in this list or the reasoning
+                    // above stops holding.
+                    if (options?.reclaimAbandonedVerdict && current.status === FlowsActors_Status.FAILED && status !== FlowsActors_Status.FAILED) {
+                        const reclaimed = await this.systemAccess.updateAsSystem(resolved.resourceType, 
+                        // Filtered on FAILED, not merely attempted: that is what makes the log below a
+                        // statement of fact rather than an inference, and it keeps this a compare-and-swap
+                        // so a genuine CANCELLED landing in between still wins.
+                        { ...resolved.ownerFields, taskId, status: FlowsActors_Status.FAILED }, { status, ...(isTerminalA2ATaskStatus(status) ? { finishedAt: new Date() } : {}) }, { scopeContext: resolved.scopeContext });
+                        if (reclaimed) {
+                            // WARN, and loudly: a reclaim means the sweep declared a LIVE turn dead. The row is
+                            // now right, but the fleet has a pod that cannot write its liveness leases, and this
+                            // is the only place that fact surfaces where an operator will see it — the owning
+                            // pod's own error lands on the pod nobody is watching.
+                            this.logger.warn('[A2ATask] Reclaimed a task the abandoned-task sweep had failed; the turn was alive after all', {
+                                taskId, reclaimedTo: status,
+                                consequence: 'the sweep acted on a missing liveness lease for a running turn — check that every pod can write Redis',
+                            });
+                            void this.pushDispatcher
+                                .dispatchTaskUpdate(owner, this.toOutboundTask({ ...reclaimed, status }), reclaimed.pushNotificationConfigs, reclaimed.resourceServerInstanceRef, reclaimed.machineCredentialId)
+                                .catch((error) => {
+                                this.logger.warn('[A2ATask] Push dispatch rejected despite its no-throw contract', {
+                                    taskId, ...projectCaughtErrorLogFields(error),
+                                });
+                            });
+                            return A2A_TaskTransitionOutcome.ADVANCED;
+                        }
+                    }
                     this.logger.info('[A2ATask] Task already reached a terminal state; the later transition was refused', {
                         taskId, attemptedStatus: status, storedStatus: current.status,
                     });