@tokenfactory/acc-runner 0.40.27 → 0.40.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/attachments/allowlist.d.ts +16 -0
- package/dist/attachments/allowlist.d.ts.map +1 -0
- package/dist/attachments/allowlist.js +34 -0
- package/dist/attachments/allowlist.js.map +1 -0
- package/dist/attachments/content-block.d.ts +16 -0
- package/dist/attachments/content-block.d.ts.map +1 -0
- package/dist/attachments/content-block.js +38 -0
- package/dist/attachments/content-block.js.map +1 -0
- package/dist/attachments/fetch-attachment.d.ts +24 -0
- package/dist/attachments/fetch-attachment.d.ts.map +1 -0
- package/dist/attachments/fetch-attachment.js +55 -0
- package/dist/attachments/fetch-attachment.js.map +1 -0
- package/dist/attachments/index.d.ts +21 -0
- package/dist/attachments/index.d.ts.map +1 -0
- package/dist/attachments/index.js +20 -0
- package/dist/attachments/index.js.map +1 -0
- package/dist/attachments/types.d.ts +39 -0
- package/dist/attachments/types.d.ts.map +1 -0
- package/dist/attachments/types.js +9 -0
- package/dist/attachments/types.js.map +1 -0
- package/dist/chat-cancel/cancel-controller.d.ts +77 -0
- package/dist/chat-cancel/cancel-controller.d.ts.map +1 -0
- package/dist/chat-cancel/cancel-controller.js +148 -0
- package/dist/chat-cancel/cancel-controller.js.map +1 -0
- package/dist/chat-cancel/cancel-source.d.ts +37 -0
- package/dist/chat-cancel/cancel-source.d.ts.map +1 -0
- package/dist/chat-cancel/cancel-source.js +26 -0
- package/dist/chat-cancel/cancel-source.js.map +1 -0
- package/dist/chat-cancel/child-terminator.d.ts +37 -0
- package/dist/chat-cancel/child-terminator.d.ts.map +1 -0
- package/dist/chat-cancel/child-terminator.js +81 -0
- package/dist/chat-cancel/child-terminator.js.map +1 -0
- package/dist/chat-cancel/index.d.ts +21 -0
- package/dist/chat-cancel/index.d.ts.map +1 -0
- package/dist/chat-cancel/index.js +20 -0
- package/dist/chat-cancel/index.js.map +1 -0
- package/dist/chat-cancel/types.d.ts +69 -0
- package/dist/chat-cancel/types.d.ts.map +1 -0
- package/dist/chat-cancel/types.js +7 -0
- package/dist/chat-cancel/types.js.map +1 -0
- package/dist/claim-deadman/deadman.d.ts +152 -0
- package/dist/claim-deadman/deadman.d.ts.map +1 -0
- package/dist/claim-deadman/deadman.js +176 -0
- package/dist/claim-deadman/deadman.js.map +1 -0
- package/dist/claim-deadman/heartbeat-signal.d.ts +48 -0
- package/dist/claim-deadman/heartbeat-signal.d.ts.map +1 -0
- package/dist/claim-deadman/heartbeat-signal.js +31 -0
- package/dist/claim-deadman/heartbeat-signal.js.map +1 -0
- package/dist/claim-deadman/index.d.ts +19 -0
- package/dist/claim-deadman/index.d.ts.map +1 -0
- package/dist/claim-deadman/index.js +17 -0
- package/dist/claim-deadman/index.js.map +1 -0
- package/dist/watch.d.ts +74 -0
- package/dist/watch.d.ts.map +1 -1
- package/dist/watch.js +289 -1
- package/dist/watch.js.map +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* CLAIM-DEADMAN: the runner-side reader that turns "what
|
|
3
|
+
* the server knows about my pending dispatches" into a `DispatchSignal` for the
|
|
4
|
+
* ClaimDeadman.
|
|
5
|
+
*
|
|
6
|
+
* DESIGN / SCOPE NOTE. The task's north star is "the heartbeat RESPONSE carries
|
|
7
|
+
* a pending-dispatch count (server already knows) so the runner learns it is
|
|
8
|
+
* being offered work even with a dead WS." The canonical wiring is to extend the
|
|
9
|
+
* `acc.heartbeat_runner` RPC to return `{ pending_dispatch, suspect }`. That RPC
|
|
10
|
+
* is a published, frozen contract and changing it needs a forward-only migration
|
|
11
|
+
* (`supabase/migrations/*`) — OUT OF SCOPE for this task and a Track A follow-up
|
|
12
|
+
* (see docs/acc/CLAIM_DEADMAN.md). Until then the runner obtains the SAME fact
|
|
13
|
+
* over the SAME HTTP path the heartbeat uses, by counting its assigned+queued
|
|
14
|
+
* tasks via the existing `list_assigned_queued_tasks` RPC (the very rows the
|
|
15
|
+
* server would put in the heartbeat response). The `DispatchSignal` contract the
|
|
16
|
+
* deadman consumes is identical either way, so the migration is a drop-in swap
|
|
17
|
+
* of the source, not a change to the deadman.
|
|
18
|
+
*/
|
|
19
|
+
import type { DispatchSignal } from "./deadman.js";
|
|
20
|
+
/**
|
|
21
|
+
* Minimal client surface this reader needs — satisfied by RunnerSupabaseClient
|
|
22
|
+
* (supabase-js `rpc` returns a thenable PostgrestFilterBuilder, hence PromiseLike
|
|
23
|
+
* and a structural `{data, error}` resolve shape rather than a concrete Promise).
|
|
24
|
+
*/
|
|
25
|
+
export interface DispatchSignalClient {
|
|
26
|
+
rpc(name: string, params?: Record<string, unknown>): PromiseLike<{
|
|
27
|
+
data: unknown;
|
|
28
|
+
error: unknown;
|
|
29
|
+
}>;
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* The pending-dispatch fields as they would arrive on the heartbeat response
|
|
33
|
+
* once `heartbeat_runner` is extended server-side. Shared shape so the eventual
|
|
34
|
+
* migration wiring and today's derived read produce the identical signal.
|
|
35
|
+
*/
|
|
36
|
+
export interface HeartbeatDeadmanFields {
|
|
37
|
+
pending_dispatch: number;
|
|
38
|
+
suspect: boolean;
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Read the pending-dispatch signal for `runnerId`. Returns pendingDispatch=0 on
|
|
42
|
+
* any error (fail-safe: a read blip must not fabricate a wedge). `suspect` is
|
|
43
|
+
* carried through from a heartbeat-supplied value when available; the derived
|
|
44
|
+
* read alone cannot know the dispatcher's consecutive-undelivered count, so it
|
|
45
|
+
* defaults to false until the RPC extension lands.
|
|
46
|
+
*/
|
|
47
|
+
export declare function readDispatchSignal(supabase: DispatchSignalClient, runnerId: string, heartbeat?: HeartbeatDeadmanFields | null): Promise<DispatchSignal>;
|
|
48
|
+
//# sourceMappingURL=heartbeat-signal.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"heartbeat-signal.d.ts","sourceRoot":"","sources":["../../src/claim-deadman/heartbeat-signal.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AACH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAEnD;;;;GAIG;AACH,MAAM,WAAW,oBAAoB;IACnC,GAAG,CACD,IAAI,EAAE,MAAM,EACZ,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAC/B,WAAW,CAAC;QAAE,IAAI,EAAE,OAAO,CAAC;QAAC,KAAK,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;CACnD;AAED;;;;GAIG;AACH,MAAM,WAAW,sBAAsB;IACrC,gBAAgB,EAAE,MAAM,CAAC;IACzB,OAAO,EAAE,OAAO,CAAC;CAClB;AAED;;;;;;GAMG;AACH,wBAAsB,kBAAkB,CACtC,QAAQ,EAAE,oBAAoB,EAC9B,QAAQ,EAAE,MAAM,EAChB,SAAS,CAAC,EAAE,sBAAsB,GAAG,IAAI,GACxC,OAAO,CAAC,cAAc,CAAC,CAuBzB"}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Read the pending-dispatch signal for `runnerId`. Returns pendingDispatch=0 on
|
|
3
|
+
* any error (fail-safe: a read blip must not fabricate a wedge). `suspect` is
|
|
4
|
+
* carried through from a heartbeat-supplied value when available; the derived
|
|
5
|
+
* read alone cannot know the dispatcher's consecutive-undelivered count, so it
|
|
6
|
+
* defaults to false until the RPC extension lands.
|
|
7
|
+
*/
|
|
8
|
+
export async function readDispatchSignal(supabase, runnerId, heartbeat) {
|
|
9
|
+
// Prefer the authoritative heartbeat-carried count when the server supplied it.
|
|
10
|
+
if (heartbeat && typeof heartbeat.pending_dispatch === "number") {
|
|
11
|
+
return {
|
|
12
|
+
pendingDispatch: Math.max(0, Math.floor(heartbeat.pending_dispatch)),
|
|
13
|
+
suspect: Boolean(heartbeat.suspect),
|
|
14
|
+
};
|
|
15
|
+
}
|
|
16
|
+
try {
|
|
17
|
+
const { data, error } = await supabase.rpc("list_assigned_queued_tasks", {
|
|
18
|
+
p_runner_id: runnerId,
|
|
19
|
+
p_since: "1970-01-01T00:00:00.000Z",
|
|
20
|
+
});
|
|
21
|
+
if (error)
|
|
22
|
+
return { pendingDispatch: 0, suspect: false };
|
|
23
|
+
const rows = Array.isArray(data) ? data : [];
|
|
24
|
+
const pending = rows.filter((r) => !!r && typeof r.id === "string").length;
|
|
25
|
+
return { pendingDispatch: pending, suspect: false };
|
|
26
|
+
}
|
|
27
|
+
catch {
|
|
28
|
+
return { pendingDispatch: 0, suspect: false };
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
//# sourceMappingURL=heartbeat-signal.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"heartbeat-signal.js","sourceRoot":"","sources":["../../src/claim-deadman/heartbeat-signal.ts"],"names":[],"mappings":"AA0CA;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,QAA8B,EAC9B,QAAgB,EAChB,SAAyC;IAEzC,gFAAgF;IAChF,IAAI,SAAS,IAAI,OAAO,SAAS,CAAC,gBAAgB,KAAK,QAAQ,EAAE,CAAC;QAChE,OAAO;YACL,eAAe,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,gBAAgB,CAAC,CAAC;YACpE,OAAO,EAAE,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC;SACpC,CAAC;IACJ,CAAC;IACD,IAAI,CAAC;QACH,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,QAAQ,CAAC,GAAG,CAAC,4BAA4B,EAAE;YACvE,WAAW,EAAE,QAAQ;YACrB,OAAO,EAAE,0BAA0B;SACpC,CAAC,CAAC;QACH,IAAI,KAAK;YAAE,OAAO,EAAE,eAAe,EAAE,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;QACzD,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7C,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CACzB,CAAC,CAAC,EAAuB,EAAE,CACzB,CAAC,CAAC,CAAC,IAAI,OAAQ,CAAsB,CAAC,EAAE,KAAK,QAAQ,CACxD,CAAC,MAAM,CAAC;QACT,OAAO,EAAE,eAAe,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;IACtD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,eAAe,EAAE,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;IAChD,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* CLAIM-DEADMAN public surface.
|
|
3
|
+
*
|
|
4
|
+
* The dead-man switch that self-heals a wedged claim channel: heartbeat-alive
|
|
5
|
+
* but claim-dead runners force a full Realtime-client rebuild, then fall back to
|
|
6
|
+
* degraded DB-poll claiming — with `runner.claim_channel_wedged` /
|
|
7
|
+
* `..._recovered` audit events. See ./deadman.ts for the state machine and
|
|
8
|
+
* docs/acc/CLAIM_DEADMAN.md for the wiring + rollout.
|
|
9
|
+
*/
|
|
10
|
+
export { ClaimDeadman, DEFAULT_WEDGE_THRESHOLD_MS, DEFAULT_REBUILD_GRACE_MS, DEFAULT_MAX_REBUILDS, DEFAULT_FALLBACK_POLL_MS, DEFAULT_DEADMAN_CHECK_MS, } from "./deadman.js";
|
|
11
|
+
export type { DeadmanActionKind, DeadmanDecision, DispatchSignal, ClaimDeadmanOptions, } from "./deadman.js";
|
|
12
|
+
export { readDispatchSignal, } from "./heartbeat-signal.js";
|
|
13
|
+
export type { DispatchSignalClient, HeartbeatDeadmanFields, } from "./heartbeat-signal.js";
|
|
14
|
+
/** Audit verbs emitted by the watch loop when the deadman escalates / recovers. */
|
|
15
|
+
export declare const CLAIM_CHANNEL_WEDGED_VERB = "runner.claim_channel_wedged";
|
|
16
|
+
export declare const CLAIM_CHANNEL_RECOVERED_VERB = "runner.claim_channel_recovered";
|
|
17
|
+
export declare const CLAIM_CHANNEL_REBUILD_VERB = "runner.claim_channel_rebuild";
|
|
18
|
+
export declare const CLAIM_CHANNEL_FALLBACK_VERB = "runner.claim_channel_fallback";
|
|
19
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/claim-deadman/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,OAAO,EACL,YAAY,EACZ,0BAA0B,EAC1B,wBAAwB,EACxB,oBAAoB,EACpB,wBAAwB,EACxB,wBAAwB,GACzB,MAAM,cAAc,CAAC;AACtB,YAAY,EACV,iBAAiB,EACjB,eAAe,EACf,cAAc,EACd,mBAAmB,GACpB,MAAM,cAAc,CAAC;AACtB,OAAO,EACL,kBAAkB,GACnB,MAAM,uBAAuB,CAAC;AAC/B,YAAY,EACV,oBAAoB,EACpB,sBAAsB,GACvB,MAAM,uBAAuB,CAAC;AAE/B,mFAAmF;AACnF,eAAO,MAAM,yBAAyB,gCAAgC,CAAC;AACvE,eAAO,MAAM,4BAA4B,mCAAmC,CAAC;AAC7E,eAAO,MAAM,0BAA0B,iCAAiC,CAAC;AACzE,eAAO,MAAM,2BAA2B,kCAAkC,CAAC"}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* CLAIM-DEADMAN public surface.
|
|
3
|
+
*
|
|
4
|
+
* The dead-man switch that self-heals a wedged claim channel: heartbeat-alive
|
|
5
|
+
* but claim-dead runners force a full Realtime-client rebuild, then fall back to
|
|
6
|
+
* degraded DB-poll claiming — with `runner.claim_channel_wedged` /
|
|
7
|
+
* `..._recovered` audit events. See ./deadman.ts for the state machine and
|
|
8
|
+
* docs/acc/CLAIM_DEADMAN.md for the wiring + rollout.
|
|
9
|
+
*/
|
|
10
|
+
export { ClaimDeadman, DEFAULT_WEDGE_THRESHOLD_MS, DEFAULT_REBUILD_GRACE_MS, DEFAULT_MAX_REBUILDS, DEFAULT_FALLBACK_POLL_MS, DEFAULT_DEADMAN_CHECK_MS, } from "./deadman.js";
|
|
11
|
+
export { readDispatchSignal, } from "./heartbeat-signal.js";
|
|
12
|
+
/** Audit verbs emitted by the watch loop when the deadman escalates / recovers. */
|
|
13
|
+
export const CLAIM_CHANNEL_WEDGED_VERB = "runner.claim_channel_wedged";
|
|
14
|
+
export const CLAIM_CHANNEL_RECOVERED_VERB = "runner.claim_channel_recovered";
|
|
15
|
+
export const CLAIM_CHANNEL_REBUILD_VERB = "runner.claim_channel_rebuild";
|
|
16
|
+
export const CLAIM_CHANNEL_FALLBACK_VERB = "runner.claim_channel_fallback";
|
|
17
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/claim-deadman/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,OAAO,EACL,YAAY,EACZ,0BAA0B,EAC1B,wBAAwB,EACxB,oBAAoB,EACpB,wBAAwB,EACxB,wBAAwB,GACzB,MAAM,cAAc,CAAC;AAOtB,OAAO,EACL,kBAAkB,GACnB,MAAM,uBAAuB,CAAC;AAM/B,mFAAmF;AACnF,MAAM,CAAC,MAAM,yBAAyB,GAAG,6BAA6B,CAAC;AACvE,MAAM,CAAC,MAAM,4BAA4B,GAAG,gCAAgC,CAAC;AAC7E,MAAM,CAAC,MAAM,0BAA0B,GAAG,8BAA8B,CAAC;AACzE,MAAM,CAAC,MAAM,2BAA2B,GAAG,+BAA+B,CAAC"}
|
package/dist/watch.d.ts
CHANGED
|
@@ -11,6 +11,7 @@ import { GitAuthController, type GitAuthProbe } from "./git-auth/index.js";
|
|
|
11
11
|
import { ResumeController, type ResumeControllerDeps } from "./capacity/resume-controller.js";
|
|
12
12
|
import { AccountProbeScheduler } from "./engines/account-probe.js";
|
|
13
13
|
import { type FleetChatServingHandle } from "./watch-chat/wire.js";
|
|
14
|
+
import { ClaimDeadman, type HeartbeatDeadmanFields } from "./claim-deadman/index.js";
|
|
14
15
|
import type { EngineAccountIdentity } from "./engines/types.js";
|
|
15
16
|
import type { StoredSession } from "./types.js";
|
|
16
17
|
import { RefreshReuseDetectedError, classifyHeartbeatError, nextHeartbeatDelayMs, type HeartbeatErrorKind } from "./runtime/auth-lifecycle.js";
|
|
@@ -87,6 +88,37 @@ export interface WatchOptions {
|
|
|
87
88
|
* hook drives it deterministically. Knob: `claim_backstop_interval_ms`.
|
|
88
89
|
*/
|
|
89
90
|
claimBackstopIntervalMs?: number;
|
|
91
|
+
/**
|
|
92
|
+
* CLAIM-DEADMAN: wedge threshold (ms). Work is pending
|
|
93
|
+
* and no claim has landed for this long → the claim channel is wedged; escalate
|
|
94
|
+
* to a full Realtime-client rebuild, then DB-poll fallback. Defaults to 120000
|
|
95
|
+
* (120s, > the 90s claim watchdog so the cheaper resubscribe fires first).
|
|
96
|
+
* Tests set a tiny value to observe the escalation ladder without real waits.
|
|
97
|
+
*/
|
|
98
|
+
claimDeadmanWedgeMs?: number;
|
|
99
|
+
/**
|
|
100
|
+
* CLAIM-DEADMAN: deadman evaluation cadence (ms). How often the watch loop
|
|
101
|
+
* reads its pending-dispatch and runs the deadman tick. Defaults to 30000.
|
|
102
|
+
* Tests set a large value so only the `claimDeadmanOnce()` hook drives it.
|
|
103
|
+
*/
|
|
104
|
+
claimDeadmanCheckMs?: number;
|
|
105
|
+
/**
|
|
106
|
+
* CLAIM-DEADMAN: minimum gap between escalation steps (ms) so a rebuild gets
|
|
107
|
+
* time to take effect before the next step. Defaults to 15000.
|
|
108
|
+
*/
|
|
109
|
+
claimDeadmanRebuildGraceMs?: number;
|
|
110
|
+
/**
|
|
111
|
+
* CLAIM-DEADMAN: full Realtime-client rebuilds attempted before degrading to
|
|
112
|
+
* DB-poll fallback claiming. Defaults to 2.
|
|
113
|
+
*/
|
|
114
|
+
claimDeadmanMaxRebuilds?: number;
|
|
115
|
+
/**
|
|
116
|
+
* CLAIM-DEADMAN: degraded DB-poll fallback cadence (ms). Once the rebuild
|
|
117
|
+
* budget is spent the runner claims by polling the claim path directly at this
|
|
118
|
+
* interval — degraded but functional — until the backlog drains. Defaults to
|
|
119
|
+
* 15000. Tests set a tiny value to observe fallback claiming without real waits.
|
|
120
|
+
*/
|
|
121
|
+
claimDeadmanFallbackPollMs?: number;
|
|
90
122
|
/** Returns a controller for tests; production wiring uses the default. */
|
|
91
123
|
taskRunnerFactory?: typeof runTask;
|
|
92
124
|
/** v0.6.2: review handler injection for tests. Defaults to runReview. */
|
|
@@ -246,6 +278,12 @@ export interface WatchHandle {
|
|
|
246
278
|
* from the backstop timer.
|
|
247
279
|
*/
|
|
248
280
|
claimBackstopOnce(): Promise<void>;
|
|
281
|
+
/**
|
|
282
|
+
* CLAIM-DEADMAN test hook: forces one deadman evaluation
|
|
283
|
+
* (read pending-dispatch → tick → rebuild/fallback/none) and resolves when it
|
|
284
|
+
* finishes. Production drives this from the deadman timer.
|
|
285
|
+
*/
|
|
286
|
+
claimDeadmanOnce(): Promise<void>;
|
|
249
287
|
/**
|
|
250
288
|
* GIT-AUTH-ALERT test hook: forces one git-credential health probe and
|
|
251
289
|
* resolves when it finishes (including any degraded/recovered emit). Resolves
|
|
@@ -303,6 +341,25 @@ interface WatchState {
|
|
|
303
341
|
claimBackstopTimer: NodeJS.Timeout;
|
|
304
342
|
claimBackstopIntervalMs: number;
|
|
305
343
|
backstopScanning: boolean;
|
|
344
|
+
/**
|
|
345
|
+
* CLAIM-DEADMAN: dead-man switch for a wedged claim
|
|
346
|
+
* channel. `claimDeadman` is the pure state machine; `claimDeadmanTimer` drives
|
|
347
|
+
* its evaluation every `claimDeadmanCheckMs`; `claimDeadmanChecking` is the
|
|
348
|
+
* single-flight guard. `deadmanRebuilding` guards the full Realtime-client
|
|
349
|
+
* rebuild. `deadmanFallbackTimer` runs the degraded DB-poll claim loop at
|
|
350
|
+
* `deadmanFallbackPollMs` once the rebuild budget is spent (null when not
|
|
351
|
+
* degraded). `lastHeartbeatDeadmanFields` caches the pending-dispatch/suspect
|
|
352
|
+
* the heartbeat response carried (null until the server extension lands, in
|
|
353
|
+
* which case the deadman falls back to the assigned-queued read).
|
|
354
|
+
*/
|
|
355
|
+
claimDeadman: ClaimDeadman;
|
|
356
|
+
claimDeadmanTimer: NodeJS.Timeout;
|
|
357
|
+
claimDeadmanCheckMs: number;
|
|
358
|
+
claimDeadmanChecking: boolean;
|
|
359
|
+
deadmanRebuilding: boolean;
|
|
360
|
+
deadmanFallbackTimer: NodeJS.Timeout | null;
|
|
361
|
+
deadmanFallbackPollMs: number;
|
|
362
|
+
lastHeartbeatDeadmanFields: HeartbeatDeadmanFields | null;
|
|
306
363
|
/**
|
|
307
364
|
* GIT-AUTH-ALERT: the git-credential health state machine + its timer.
|
|
308
365
|
* `gitAuthController` is null when the probe loop is disabled (no base repo
|
|
@@ -519,6 +576,23 @@ interface WatchState {
|
|
|
519
576
|
}
|
|
520
577
|
export declare function resubscribeChannel(state: WatchState, factory: typeof runTask): Promise<void>;
|
|
521
578
|
export declare function reassertRunningClaims(state: WatchState): Promise<void>;
|
|
579
|
+
/**
|
|
580
|
+
* CLAIM-DEADMAN: parse the pending-dispatch fields off a `heartbeat_runner`
|
|
581
|
+
* response. Tolerant of a scalar row, a single-row array, or null (today's
|
|
582
|
+
* void return) — returns null unless a numeric `pending_dispatch` is present, so
|
|
583
|
+
* a pre-extension server leaves the deadman on its assigned-queued read.
|
|
584
|
+
*/
|
|
585
|
+
export declare function parseHeartbeatDeadmanFields(data: unknown): HeartbeatDeadmanFields | null;
|
|
586
|
+
/**
|
|
587
|
+
* CLAIM-DEADMAN: tear down + FULLY REBUILD the Realtime client — not just a
|
|
588
|
+
* resubscribe. The v0.68 resubscribe / v0.73 watchdog both remove+re-add a
|
|
589
|
+
* channel on the SAME Supabase client, so a wedged underlying socket / Realtime
|
|
590
|
+
* instance survives them (the incident's failure mode). This discards the whole
|
|
591
|
+
* client and builds a fresh one around the same JWT + realtime config (mirrors
|
|
592
|
+
* doRefresh/doFailover's client rebuild, minus the token change), then recovers
|
|
593
|
+
* anything the dead channel missed. Single-flight via `deadmanRebuilding`.
|
|
594
|
+
*/
|
|
595
|
+
export declare function rebuildRealtimeClient(state: WatchState, factory: typeof runTask, attempt: number): Promise<void>;
|
|
522
596
|
export declare function watchCommand(options?: WatchOptions): Promise<WatchHandle>;
|
|
523
597
|
/**
|
|
524
598
|
* PA-11C-SIGTERM-REQUEUE: graceful shutdown handler for SIGINT/SIGTERM.
|
package/dist/watch.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"watch.d.ts","sourceRoot":"","sources":["../src/watch.ts"],"names":[],"mappings":"AA4BA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAC7D,OAAO,EAML,KAAK,iBAAiB,EACtB,KAAK,YAAY,EAGlB,MAAM,aAAa,CAAC;AAIrB,OAAO,EAAE,uBAAuB,EAAE,0BAA0B,EAAE,MAAM,aAAa,CAAC;AAClF,OAAO,EAAoB,KAAK,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAG3E,OAAO,EAAsB,KAAK,oBAAoB,EAAE,MAAM,eAAe,CAAC;AAE9E,OAAO,EAAE,OAAO,EAAE,KAAK,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AACnE,OAAO,EAEL,KAAK,gBAAgB,EACrB,KAAK,eAAe,EACrB,MAAM,uBAAuB,CAAC;AAK/B,OAAO,EAIL,KAAK,aAAa,EACnB,MAAM,wBAAwB,CAAC;AAKhC,OAAO,EAIL,KAAK,iBAAiB,EACtB,KAAK,mBAAmB,EACzB,MAAM,2BAA2B,CAAC;AAEnC,OAAO,EACL,iBAAiB,EAIjB,KAAK,YAAY,EAClB,MAAM,qBAAqB,CAAC;AAI7B,OAAO,EAAE,gBAAgB,EAAE,KAAK,oBAAoB,EAAE,MAAM,iCAAiC,CAAC;AAK9F,OAAO,EAAE,qBAAqB,EAAmB,MAAM,4BAA4B,CAAC;AAKpF,OAAO,EAEL,KAAK,sBAAsB,EAC5B,MAAM,sBAAsB,CAAC;
|
|
1
|
+
{"version":3,"file":"watch.d.ts","sourceRoot":"","sources":["../src/watch.ts"],"names":[],"mappings":"AA4BA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAC7D,OAAO,EAML,KAAK,iBAAiB,EACtB,KAAK,YAAY,EAGlB,MAAM,aAAa,CAAC;AAIrB,OAAO,EAAE,uBAAuB,EAAE,0BAA0B,EAAE,MAAM,aAAa,CAAC;AAClF,OAAO,EAAoB,KAAK,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAG3E,OAAO,EAAsB,KAAK,oBAAoB,EAAE,MAAM,eAAe,CAAC;AAE9E,OAAO,EAAE,OAAO,EAAE,KAAK,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AACnE,OAAO,EAEL,KAAK,gBAAgB,EACrB,KAAK,eAAe,EACrB,MAAM,uBAAuB,CAAC;AAK/B,OAAO,EAIL,KAAK,aAAa,EACnB,MAAM,wBAAwB,CAAC;AAKhC,OAAO,EAIL,KAAK,iBAAiB,EACtB,KAAK,mBAAmB,EACzB,MAAM,2BAA2B,CAAC;AAEnC,OAAO,EACL,iBAAiB,EAIjB,KAAK,YAAY,EAClB,MAAM,qBAAqB,CAAC;AAI7B,OAAO,EAAE,gBAAgB,EAAE,KAAK,oBAAoB,EAAE,MAAM,iCAAiC,CAAC;AAK9F,OAAO,EAAE,qBAAqB,EAAmB,MAAM,4BAA4B,CAAC;AAKpF,OAAO,EAEL,KAAK,sBAAsB,EAC5B,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EACL,YAAY,EAQZ,KAAK,sBAAsB,EAC5B,MAAM,0BAA0B,CAAC;AAElC,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAGhE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAYhD,OAAO,EACL,yBAAyB,EACzB,sBAAsB,EAEtB,oBAAoB,EAEpB,KAAK,kBAAkB,EACxB,MAAM,6BAA6B,CAAC;AAErC,OAAO,EACL,yBAAyB,EACzB,sBAAsB,EACtB,oBAAoB,EACpB,KAAK,kBAAkB,GACxB,CAAC;AAoGF;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,qBAAqB,CAAC,IAAI,EAAE;IAC1C,gBAAgB,EAAE,MAAM,CAAC;IACzB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB,GAAG,MAAM,CAMT;AAED,MAAM,WAAW,YAAY;IAC3B,0DAA0D;IAC1D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,yDAAyD;IACzD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;;;;;OAOG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B;;;;;;;;;OASG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B;;;;;;;OAOG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B;;;;;OAKG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B;;;;;;;;OAQG;IACH,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC;;;;;;OAMG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B;;;OAGG;IACH,0BAA0B,CAAC,EAAE,MAAM,CAAC;IACpC;;;OAGG;IACH,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC;;;;;OAKG;IACH,0BAA0B,CAAC,EAAE,MAAM,CAAC;IACpC,0EAA0E;IAC1E,iBAAiB,CAAC,EAAE,OAAO,OAAO,CAAC;IACnC,yEAAyE;IACzE,eAAe,CAAC,EAAE,CAChB,UAAU,EAAE,gBAAgB,EAC5B,IAAI,EAAE;QAAE,QAAQ,EAAE,oBAAoB,CAAA;KAAE,KACrC,OAAO,CAAC,eAAe,CAAC,CAAC;IAC9B;;;;;;;;OAQG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B;;;;;OAKG;IACH,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,+DAA+D;IAC/D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,uEAAuE;IACvE,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;IAC9B;;;;;OAKG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B;;;;;OAKG;IACH,WAAW,CAAC,EAAE,CAAC,KAAK,EAAE;QAAE,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,KAAK,OAAO,CAAC,iBAAiB,CAAC,CAAC;IACnF;;;;;OAKG;IACH,aAAa,CAAC,EAAE,CAAC,KAAK,EAAE;QACtB,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAC3B,OAAO,EAAE,OAAO,CAAC;KAClB,KAAK,OAAO,CAAC,mBAAmB,CAAC,CAAC;IACnC;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;;;OAKG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;;;OAKG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B;;;;;;OAMG;IACH,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC;;;;;OAKG;IACH,0BAA0B,CAAC,EAAE,MAAM,CAAC;IACpC;;;;;OAKG;IACH,YAAY,CAAC,EAAE,MAAM,OAAO,CAAC,qBAAqB,CAAC,CAAC;IACpD;;;;;;OAMG;IACH,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B;;;OAGG;IACH,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC;;;OAGG;IACH,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC;;;;;OAKG;IACH,YAAY,CAAC,EAAE,YAAY,CAAC;CAC7B;AAED;;;;GAIG;AACH,wBAAgB,gBAAgB,CAC9B,GAAG,GAAE,MAAM,CAAC,UAAwB,GACnC,MAAM,GAAG,IAAI,CAWf;AAED,MAAM,WAAW,WAAW;IAC1B,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACtB,wEAAwE;IACxE,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1B;;;OAGG;IACH,aAAa,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/B;;;;OAIG;IACH,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAChC;;;;OAIG;IACH,iBAAiB,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACnC;;;;OAIG;IACH,iBAAiB,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACnC;;;;OAIG;IACH,gBAAgB,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAClC;;;;;OAKG;IACH,gBAAgB,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;IACrC,2EAA2E;IAC3E,YAAY,IAAI,MAAM,CAAC;CACxB;AAED,KAAK,eAAe,GAAG,CACrB,UAAU,EAAE,gBAAgB,EAC5B,IAAI,EAAE;IAAE,QAAQ,EAAE,oBAAoB,CAAA;CAAE,KACrC,OAAO,CAAC,eAAe,CAAC,CAAC;AAE9B,UAAU,UAAU;IAClB,GAAG,EAAE,YAAY,CAAC;IAClB,OAAO,EAAE,aAAa,CAAC;IACvB,uEAAuE;IACvE,aAAa,EAAE,aAAa,CAAC;IAC7B,QAAQ,EAAE,oBAAoB,CAAC;IAC/B,OAAO,EAAE,eAAe,CAAC;IACzB,cAAc,EAAE,MAAM,CAAC,OAAO,CAAC;IAC/B,YAAY,EAAE,MAAM,CAAC,OAAO,CAAC;IAC7B,SAAS,EAAE,MAAM,CAAC,OAAO,CAAC;IAC1B,iFAAiF;IACjF,cAAc,EAAE,MAAM,CAAC,OAAO,CAAC;IAC/B,sFAAsF;IACtF,mBAAmB,EAAE,MAAM,CAAC;IAC5B,+EAA+E;IAC/E,aAAa,EAAE,OAAO,CAAC;IACvB,6EAA6E;IAC7E,eAAe,EAAE,MAAM,CAAC,OAAO,CAAC;IAChC,kFAAkF;IAClF,oBAAoB,EAAE,MAAM,CAAC;IAC7B,+EAA+E;IAC/E,cAAc,EAAE,OAAO,CAAC;IACxB;;;;;;;OAOG;IACH,kBAAkB,EAAE,MAAM,CAAC,OAAO,CAAC;IACnC,oBAAoB,EAAE,MAAM,CAAC;IAC7B,kBAAkB,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACxC,qBAAqB,EAAE,OAAO,CAAC;IAC/B;;;;;;OAMG;IACH,kBAAkB,EAAE,MAAM,CAAC,OAAO,CAAC;IACnC,uBAAuB,EAAE,MAAM,CAAC;IAChC,gBAAgB,EAAE,OAAO,CAAC;IAC1B;;;;;;;;;;OAUG;IACH,YAAY,EAAE,YAAY,CAAC;IAC3B,iBAAiB,EAAE,MAAM,CAAC,OAAO,CAAC;IAClC,mBAAmB,EAAE,MAAM,CAAC;IAC5B,oBAAoB,EAAE,OAAO,CAAC;IAC9B,iBAAiB,EAAE,OAAO,CAAC;IAC3B,oBAAoB,EAAE,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC;IAC5C,qBAAqB,EAAE,MAAM,CAAC;IAC9B,0BAA0B,EAAE,sBAAsB,GAAG,IAAI,CAAC;IAC1D;;;;;;;OAOG;IACH,iBAAiB,EAAE,iBAAiB,GAAG,IAAI,CAAC;IAC5C,YAAY,EAAE,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC;IACpC,sBAAsB,EAAE,MAAM,CAAC;IAC/B,eAAe,EAAE,OAAO,CAAC;IACzB,aAAa,EAAE,OAAO,CAAC;IACvB;;;;;;;OAOG;IACH,aAAa,EAAE,OAAO,CAAC;IACvB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,gBAAgB,EAAE,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC;IACxC;;;;;OAKG;IACH,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,iBAAiB,CAAC,CAAC;IACxC;;;;;OAKG;IACH,gBAAgB,EAAE,MAAM,CAAC;IACzB;;;;OAIG;IACH,mBAAmB,EAAE,MAAM,CAAC;IAC5B;;;;;;OAMG;IACH,eAAe,EAAE,MAAM,CAAC;IACxB,uBAAuB,EAAE,MAAM,CAAC;IAChC,oFAAoF;IACpF,sBAAsB,EAAE,OAAO,CAAC;IAChC;;;;;OAKG;IACH,iBAAiB,EAAE,iBAAiB,CAAC;IACrC;;;;;;OAMG;IACH,wBAAwB,EAAE,OAAO,CAAC;IAClC;;;;;;;OAOG;IACH,mBAAmB,EAAE,OAAO,CAAC;IAC7B,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,4EAA4E;IAC5E,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAClB,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,EAAE,OAAO,CAAC;IACjB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,UAAU,EAAE,OAAO,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC;IACpC,2EAA2E;IAC3E,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,OAAO,CAAC;IACjB;;;;;OAKG;IACH,WAAW,EAAE,gBAAgB,EAAE,CAAC;IAChC,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IACzB,aAAa,EAAE,OAAO,CAAC;IACvB,SAAS,EAAE,eAAe,CAAC;IAC3B;;;;;OAKG;IACH,WAAW,EAAE,OAAO,CAAC;IACrB;;;;;;;OAOG;IACH,gBAAgB,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAC9B;;;;;OAKG;IACH,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC;IACjC,2DAA2D;IAC3D,WAAW,EAAE,OAAO,CAAC;IACrB;;;;;;OAMG;IACH,gBAAgB,EAAE,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC;IACxC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,aAAa,EAAE,CAAC,KAAK,EAAE;QAAE,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,KAAK,OAAO,CAAC,iBAAiB,CAAC,CAAC;IACpF,aAAa,EAAE,OAAO,CAAC;IACvB;;;;OAIG;IACH,iBAAiB,EAAE,OAAO,OAAO,CAAC;IAClC;;;;;;OAMG;IACH,cAAc,EAAE,OAAO,CAAC;IACxB,uEAAuE;IACvE,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B;;;;;;OAMG;IACH,qBAAqB,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC;;;;;OAKG;IACH,mBAAmB,EAAE,MAAM,CAAC;IAC5B;;;;;;;OAOG;IACH,gBAAgB,EAAE,gBAAgB,CAAC;IACnC;;;;;OAKG;IACH,YAAY,EAAE,qBAAqB,GAAG,IAAI,CAAC;IAC3C;;;;OAIG;IACH,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC;;;;;OAKG;IACH,aAAa,EAAE,OAAO,CAAC;IACvB,sEAAsE;IACtE,iBAAiB,EAAE,MAAM,CAAC;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB;;;OAGG;IACH,aAAa,EAAE,aAAa,GAAG,IAAI,CAAC;IACpC;;;;OAIG;IACH,SAAS,EAAE,sBAAsB,GAAG,IAAI,CAAC;CAC1C;AA8ID,wBAAsB,kBAAkB,CACtC,KAAK,EAAE,UAAU,EACjB,OAAO,EAAE,OAAO,OAAO,GACtB,OAAO,CAAC,IAAI,CAAC,CAoBf;AAMD,wBAAsB,qBAAqB,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAqB5E;AA0VD;;;;;GAKG;AACH,wBAAgB,2BAA2B,CACzC,IAAI,EAAE,OAAO,GACZ,sBAAsB,GAAG,IAAI,CAS/B;AAsCD;;;;;;;;GAQG;AACH,wBAAsB,qBAAqB,CACzC,KAAK,EAAE,UAAU,EACjB,OAAO,EAAE,OAAO,OAAO,EACvB,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,IAAI,CAAC,CAsDf;AAuUD,wBAAsB,YAAY,CAAC,OAAO,GAAE,YAAiB,GAAG,OAAO,CAAC,WAAW,CAAC,CAmwBnF;AAwRD;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAsB,gBAAgB,CACpC,KAAK,EAAE,IAAI,CAAC,UAAU,EAAE,SAAS,GAAG,UAAU,CAAC,EAC/C,IAAI,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,EACzB,MAAM,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,EAC9B,MAAM,EAAE,QAAQ,GAAG,SAAS,GAC3B,OAAO,CAAC,IAAI,CAAC,CAwBf;AAkSD;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAgB,yBAAyB,CACvC,KAAK,EAAE,UAAU,EACjB,OAAO,EAAE,OAAO,OAAO,GACtB,IAAI,CACL,oBAAoB,EACpB,gBAAgB,GAAG,YAAY,GAAG,kBAAkB,GAAG,eAAe,GAAG,OAAO,CACjF,CA6GA"}
|
package/dist/watch.js
CHANGED
|
@@ -55,6 +55,7 @@ import { getChatEngine } from "./engines/registry.js";
|
|
|
55
55
|
// a `watch` runner — the runner serves its bound user's chat turns behind their
|
|
56
56
|
// companion. Glue lives in watch-chat/wire.ts to keep this file the task loop.
|
|
57
57
|
import { startFleetChatServing, } from "./watch-chat/wire.js";
|
|
58
|
+
import { ClaimDeadman, readDispatchSignal, DEFAULT_DEADMAN_CHECK_MS, DEFAULT_FALLBACK_POLL_MS, CLAIM_CHANNEL_WEDGED_VERB, CLAIM_CHANNEL_RECOVERED_VERB, CLAIM_CHANNEL_REBUILD_VERB, CLAIM_CHANNEL_FALLBACK_VERB, } from "./claim-deadman/index.js";
|
|
58
59
|
import { unknownAccountIdentity } from "./engines/account-identity.js";
|
|
59
60
|
import { checkVersion, compareSemver } from "./version-check.js";
|
|
60
61
|
import { PACKAGE_VERSION } from "./pkg-version.js";
|
|
@@ -120,6 +121,22 @@ const DEFAULT_CLAIM_WATCHDOG_CHECK_MS = 30_000;
|
|
|
120
121
|
// regressions of the same shape. Default 60s — prompt recovery without a third
|
|
121
122
|
// frequent DB poll racing the watchdog. Knob: `claim_backstop_interval_ms`.
|
|
122
123
|
const DEFAULT_CLAIM_BACKSTOP_INTERVAL_MS = 60_000;
|
|
124
|
+
// CLAIM-DEADMAN: dead-man switch for a WEDGED claim channel
|
|
125
|
+
// that the 90s watchdog resubscribe can't clear. Live incident 2026-07-22: a
|
|
126
|
+
// runner heartbeated fine (hb_age 0min) but its Realtime claim subscription was
|
|
127
|
+
// dead AND the watchdog resubscribe had itself stopped firing — 12 queued / 0
|
|
128
|
+
// running with two online runners, cleared only by a manual restart. The deadman
|
|
129
|
+
// tracks pending-dispatch (work the server is offering) vs the last processed
|
|
130
|
+
// claim; when work is pending and no claim lands for CLAIM_DEADMAN_WEDGE_MS
|
|
131
|
+
// (default 120s, > the 90s watchdog so the cheap resubscribe gets first crack)
|
|
132
|
+
// it (1) TEARS DOWN + FULLY REBUILDS the Realtime client up to
|
|
133
|
+
// CLAIM_DEADMAN_MAX_REBUILDS times, then (2) falls back to degraded DB-poll
|
|
134
|
+
// claiming every CLAIM_DEADMAN_FALLBACK_POLL_MS. CLAIM_DEADMAN_CHECK_MS is the
|
|
135
|
+
// evaluation cadence. Tests override all of these for speed. See
|
|
136
|
+
// packages/acc-runner/src/claim-deadman/deadman.ts + docs/acc/CLAIM_DEADMAN.md.
|
|
137
|
+
const DEFAULT_CLAIM_DEADMAN_WEDGE_MS = 120_000;
|
|
138
|
+
const DEFAULT_CLAIM_DEADMAN_REBUILD_GRACE_MS = 15_000;
|
|
139
|
+
const DEFAULT_CLAIM_DEADMAN_MAX_REBUILDS = 2;
|
|
123
140
|
// GIT-AUTH-ALERT: how often the runner probes its base clone's git credential
|
|
124
141
|
// (a masked `ls-remote origin` dry-run). The same 60s cadence detects the first
|
|
125
142
|
// failure AND polls for recovery — while degraded, task-lane claiming is paused
|
|
@@ -208,6 +225,13 @@ function enqueue(state, taskId, factory) {
|
|
|
208
225
|
state.seen.add(taskId);
|
|
209
226
|
state.queue.push(taskId);
|
|
210
227
|
touchActivity(state);
|
|
228
|
+
// CLAIM-DEADMAN: a claim was processed via SOME path
|
|
229
|
+
// (realtime / poll / scan / watchdog / the deadman's own DB-poll fallback).
|
|
230
|
+
// Reset the deadman's staleness clock so a runner that IS claiming — even
|
|
231
|
+
// slowly, even degraded — is never treated as wedged. Recovery itself is
|
|
232
|
+
// declared by the offered backlog draining (observeDispatchSignal), so this
|
|
233
|
+
// never prematurely clears a wedge episode.
|
|
234
|
+
state.claimDeadman?.observeClaimProcessed();
|
|
211
235
|
void pump(state, factory);
|
|
212
236
|
return true;
|
|
213
237
|
}
|
|
@@ -656,6 +680,230 @@ async function claimBackstopScan(state, factory) {
|
|
|
656
680
|
state.backstopScanning = false;
|
|
657
681
|
}
|
|
658
682
|
}
|
|
683
|
+
/**
|
|
684
|
+
* CLAIM-DEADMAN: a runner may have free capacity but be
|
|
685
|
+
* deliberately not claiming (quarantined, capacity-paused, review-claim-paused,
|
|
686
|
+
* or already at its effective task-claim ceiling). In those states pending work
|
|
687
|
+
* SHOULD sit queued, so it must NOT be read as a wedge. Mirrors the
|
|
688
|
+
* claimBackstopScan guard. Returns true only when the runner both wants to and
|
|
689
|
+
* can claim another task.
|
|
690
|
+
*/
|
|
691
|
+
function deadmanCanClaim(state) {
|
|
692
|
+
if (state.quarantined || state.pausedCapacity || state.capacityClaimPaused) {
|
|
693
|
+
return false;
|
|
694
|
+
}
|
|
695
|
+
const limit = computeTaskClaimLimit({
|
|
696
|
+
concurrencyLimit: state.concurrencyLimit,
|
|
697
|
+
reservedReviewSlots: state.reservedReviewSlots,
|
|
698
|
+
capacityProbe: state.capacityProbe,
|
|
699
|
+
claimPaused: state.capacityClaimPaused,
|
|
700
|
+
});
|
|
701
|
+
return state.running.size < limit;
|
|
702
|
+
}
|
|
703
|
+
/**
|
|
704
|
+
* CLAIM-DEADMAN: parse the pending-dispatch fields off a `heartbeat_runner`
|
|
705
|
+
* response. Tolerant of a scalar row, a single-row array, or null (today's
|
|
706
|
+
* void return) — returns null unless a numeric `pending_dispatch` is present, so
|
|
707
|
+
* a pre-extension server leaves the deadman on its assigned-queued read.
|
|
708
|
+
*/
|
|
709
|
+
export function parseHeartbeatDeadmanFields(data) {
|
|
710
|
+
const row = Array.isArray(data) ? data[0] : data;
|
|
711
|
+
if (!row || typeof row !== "object")
|
|
712
|
+
return null;
|
|
713
|
+
const r = row;
|
|
714
|
+
if (typeof r.pending_dispatch !== "number")
|
|
715
|
+
return null;
|
|
716
|
+
return {
|
|
717
|
+
pending_dispatch: Math.max(0, Math.floor(r.pending_dispatch)),
|
|
718
|
+
suspect: Boolean(r.suspect),
|
|
719
|
+
};
|
|
720
|
+
}
|
|
721
|
+
/**
|
|
722
|
+
* CLAIM-DEADMAN: emit the audit event + operator-facing alert for a claim-
|
|
723
|
+
* channel state change. `runner.claim_channel_wedged` / `..._recovered` are the
|
|
724
|
+
* UI alert breadcrumbs; the payload NAMES the runner (id + label) so the derived
|
|
725
|
+
* notifications read model can surface "runner X's claim channel is wedged"
|
|
726
|
+
* without a join. Best-effort by contract — a logging failure never disturbs the
|
|
727
|
+
* watch loop. Also mirrors to stderr so a locally-tailed runner shows it.
|
|
728
|
+
*/
|
|
729
|
+
function emitDeadmanEvent(state, verb, payload, humanLine) {
|
|
730
|
+
process.stderr.write(`[acc-runner] ${humanLine}\n`);
|
|
731
|
+
void (async () => {
|
|
732
|
+
try {
|
|
733
|
+
await state.supabase.rpc("log_activity", {
|
|
734
|
+
p_verb: verb,
|
|
735
|
+
p_target_id: state.session.runner_id,
|
|
736
|
+
p_payload: {
|
|
737
|
+
// NAMES the runner so the derived notifications read model can surface
|
|
738
|
+
// "runner <id>'s claim channel is wedged" without a join (the UI
|
|
739
|
+
// resolves the display name from acc.runners by this id).
|
|
740
|
+
runner_id: state.session.runner_id,
|
|
741
|
+
version: PACKAGE_VERSION,
|
|
742
|
+
...payload,
|
|
743
|
+
},
|
|
744
|
+
p_target_type: "runner",
|
|
745
|
+
});
|
|
746
|
+
}
|
|
747
|
+
catch {
|
|
748
|
+
/* best-effort breadcrumb */
|
|
749
|
+
}
|
|
750
|
+
})();
|
|
751
|
+
}
|
|
752
|
+
/**
|
|
753
|
+
* CLAIM-DEADMAN: tear down + FULLY REBUILD the Realtime client — not just a
|
|
754
|
+
* resubscribe. The v0.68 resubscribe / v0.73 watchdog both remove+re-add a
|
|
755
|
+
* channel on the SAME Supabase client, so a wedged underlying socket / Realtime
|
|
756
|
+
* instance survives them (the incident's failure mode). This discards the whole
|
|
757
|
+
* client and builds a fresh one around the same JWT + realtime config (mirrors
|
|
758
|
+
* doRefresh/doFailover's client rebuild, minus the token change), then recovers
|
|
759
|
+
* anything the dead channel missed. Single-flight via `deadmanRebuilding`.
|
|
760
|
+
*/
|
|
761
|
+
export async function rebuildRealtimeClient(state, factory, attempt) {
|
|
762
|
+
if (state.stopped || state.deadmanRebuilding)
|
|
763
|
+
return;
|
|
764
|
+
state.deadmanRebuilding = true;
|
|
765
|
+
try {
|
|
766
|
+
// Tear down the old channel + socket as fully as the client allows.
|
|
767
|
+
try {
|
|
768
|
+
await state.supabase.removeChannel(state.channel);
|
|
769
|
+
}
|
|
770
|
+
catch {
|
|
771
|
+
/* channel may already be dead */
|
|
772
|
+
}
|
|
773
|
+
try {
|
|
774
|
+
await state.supabase.removeAllChannels?.();
|
|
775
|
+
}
|
|
776
|
+
catch {
|
|
777
|
+
/* best-effort */
|
|
778
|
+
}
|
|
779
|
+
try {
|
|
780
|
+
// any-allowed: `.realtime` is not on the narrowed RunnerSupabaseClient type
|
|
781
|
+
// but supabase-js exposes a RealtimeClient with disconnect() here — this is
|
|
782
|
+
// the "fully rebuild, not just resubscribe" teardown of the live socket.
|
|
783
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
784
|
+
state.supabase.realtime?.disconnect?.();
|
|
785
|
+
}
|
|
786
|
+
catch {
|
|
787
|
+
/* best-effort */
|
|
788
|
+
}
|
|
789
|
+
// Fresh client around the current (still-valid) credential.
|
|
790
|
+
state.supabase = createRunnerClient(state.cfg, state.session.access_token, state.session.realtime_config);
|
|
791
|
+
subscribeChannel(state, factory);
|
|
792
|
+
// FLEET-SERVE (wire): hot-swap the chat lane onto the fresh client so chat
|
|
793
|
+
// serving survives the rebuild (mirrors doRefresh/doFailover).
|
|
794
|
+
await state.fleetChat?.refreshClient(state.supabase, state.session.access_token);
|
|
795
|
+
emitDeadmanEvent(state, CLAIM_CHANNEL_REBUILD_VERB, { attempt }, `claim-deadman: rebuilding Realtime client (attempt ${attempt}) — ` +
|
|
796
|
+
`claim channel wedged with a live heartbeat`);
|
|
797
|
+
// Recover anything the wedged channel missed. The seen/seenReviews guards
|
|
798
|
+
// dedupe against a late re-delivery on the fresh channel.
|
|
799
|
+
void reassertRunningClaims(state);
|
|
800
|
+
void pollOnce(state, factory);
|
|
801
|
+
void claimScan(state, factory, "periodic");
|
|
802
|
+
void reviewScan(state, "periodic");
|
|
803
|
+
}
|
|
804
|
+
catch (err) {
|
|
805
|
+
process.stderr.write(`[acc-runner] claim-deadman rebuild failed: ${err.message}\n`);
|
|
806
|
+
}
|
|
807
|
+
finally {
|
|
808
|
+
state.deadmanRebuilding = false;
|
|
809
|
+
}
|
|
810
|
+
}
|
|
811
|
+
/**
|
|
812
|
+
* CLAIM-DEADMAN: the degraded DB-poll fallback loop. Once the rebuild budget is
|
|
813
|
+
* spent the runner claims by polling the claim path directly — a full epoch-
|
|
814
|
+
* watermark claim scan plus the `seen`-reconcile backstop — every
|
|
815
|
+
* `deadmanFallbackPollMs`. Degraded (no realtime latency) but FUNCTIONAL: tasks
|
|
816
|
+
* get claimed with zero human action while the WS stays dead. Idempotent —
|
|
817
|
+
* starting an already-running loop is a no-op.
|
|
818
|
+
*/
|
|
819
|
+
function startDeadmanFallback(state, factory) {
|
|
820
|
+
if (state.stopped || state.deadmanFallbackTimer)
|
|
821
|
+
return;
|
|
822
|
+
emitDeadmanEvent(state, CLAIM_CHANNEL_FALLBACK_VERB, { poll_ms: state.deadmanFallbackPollMs }, `claim-deadman: Realtime rebuild budget spent — falling back to DB-poll ` +
|
|
823
|
+
`claiming every ${state.deadmanFallbackPollMs}ms (degraded but functional)`);
|
|
824
|
+
const tick = () => {
|
|
825
|
+
void (async () => {
|
|
826
|
+
if (state.stopped)
|
|
827
|
+
return;
|
|
828
|
+
await claimScan(state, factory, "periodic");
|
|
829
|
+
await claimBackstopScan(state, factory);
|
|
830
|
+
})();
|
|
831
|
+
};
|
|
832
|
+
tick(); // claim immediately, don't wait a full interval
|
|
833
|
+
state.deadmanFallbackTimer = setInterval(tick, state.deadmanFallbackPollMs);
|
|
834
|
+
state.deadmanFallbackTimer.unref?.();
|
|
835
|
+
}
|
|
836
|
+
/** CLAIM-DEADMAN: stop the degraded DB-poll fallback loop (on recovery / stop). */
|
|
837
|
+
function stopDeadmanFallback(state) {
|
|
838
|
+
if (state.deadmanFallbackTimer) {
|
|
839
|
+
clearInterval(state.deadmanFallbackTimer);
|
|
840
|
+
state.deadmanFallbackTimer = null;
|
|
841
|
+
}
|
|
842
|
+
}
|
|
843
|
+
/**
|
|
844
|
+
* CLAIM-DEADMAN: the dead-man evaluation. Reads how much
|
|
845
|
+
* work the server is offering this runner (pending-dispatch — from the heartbeat
|
|
846
|
+
* response when the server supplies it, else the assigned-queued read over the
|
|
847
|
+
* same HTTP path that survives a dead WS), feeds it + the last-claim clock to the
|
|
848
|
+
* pure ClaimDeadman, and acts on its decision:
|
|
849
|
+
*
|
|
850
|
+
* - recovered (offered backlog drained after a wedge episode) → stop the
|
|
851
|
+
* fallback loop + emit `runner.claim_channel_recovered`.
|
|
852
|
+
* - rebuild(N) → full Realtime-client rebuild (rebuildRealtimeClient).
|
|
853
|
+
* - fallback → engage/keep the degraded DB-poll claim loop.
|
|
854
|
+
*
|
|
855
|
+
* A heartbeat failure short-circuits to observeHeartbeatFailure (a network/auth
|
|
856
|
+
* outage is a different failure with its own handling — not a wedge). Bounded
|
|
857
|
+
* and side-effect-safe: never throws out of the loop. Single-flight via
|
|
858
|
+
* `claimDeadmanChecking`.
|
|
859
|
+
*/
|
|
860
|
+
async function claimDeadmanScan(state, factory) {
|
|
861
|
+
if (state.stopped || state.claimDeadmanChecking)
|
|
862
|
+
return;
|
|
863
|
+
state.claimDeadmanChecking = true;
|
|
864
|
+
try {
|
|
865
|
+
if (state.heartbeatFailures > 0) {
|
|
866
|
+
state.claimDeadman.observeHeartbeatFailure();
|
|
867
|
+
return;
|
|
868
|
+
}
|
|
869
|
+
// Only pending work the runner WOULD claim counts as offered — a runner that
|
|
870
|
+
// is deliberately not claiming (paused/quarantined/at-capacity) is not wedged.
|
|
871
|
+
let signal;
|
|
872
|
+
if (deadmanCanClaim(state)) {
|
|
873
|
+
signal = await readDispatchSignal(state.supabase, state.session.runner_id, state.lastHeartbeatDeadmanFields);
|
|
874
|
+
}
|
|
875
|
+
else {
|
|
876
|
+
signal = { pendingDispatch: 0, suspect: false };
|
|
877
|
+
}
|
|
878
|
+
const { recovered } = state.claimDeadman.observeDispatchSignal(signal);
|
|
879
|
+
if (recovered) {
|
|
880
|
+
stopDeadmanFallback(state);
|
|
881
|
+
emitDeadmanEvent(state, CLAIM_CHANNEL_RECOVERED_VERB, {}, `claim-deadman: claim channel recovered — offered backlog drained, ` +
|
|
882
|
+
`runner is claiming normally again`);
|
|
883
|
+
}
|
|
884
|
+
const decision = state.claimDeadman.tick();
|
|
885
|
+
if (decision.wedged) {
|
|
886
|
+
emitDeadmanEvent(state, CLAIM_CHANNEL_WEDGED_VERB, {
|
|
887
|
+
pending_dispatch: decision.pendingDispatch,
|
|
888
|
+
ms_since_claim: decision.msSinceClaim,
|
|
889
|
+
}, `claim-deadman: claim channel WEDGED — ${decision.pendingDispatch} ` +
|
|
890
|
+
`task(s) offered but none claimed for ${Math.round(decision.msSinceClaim / 1000)}s ` +
|
|
891
|
+
`with a live heartbeat; self-healing`);
|
|
892
|
+
}
|
|
893
|
+
if (decision.action === "rebuild") {
|
|
894
|
+
await rebuildRealtimeClient(state, factory, decision.rebuildAttempt ?? 1);
|
|
895
|
+
}
|
|
896
|
+
else if (decision.action === "fallback") {
|
|
897
|
+
startDeadmanFallback(state, factory);
|
|
898
|
+
}
|
|
899
|
+
}
|
|
900
|
+
catch (err) {
|
|
901
|
+
process.stderr.write(`[acc-runner] claim deadman scan failed: ${err.message}\n`);
|
|
902
|
+
}
|
|
903
|
+
finally {
|
|
904
|
+
state.claimDeadmanChecking = false;
|
|
905
|
+
}
|
|
906
|
+
}
|
|
659
907
|
/**
|
|
660
908
|
* v0.65 (T-65-2): full review scan — the review twin of claimScan. Lists EVERY
|
|
661
909
|
* pending review assigned to this runner with an epoch watermark (`1970-…Z`)
|
|
@@ -923,6 +1171,7 @@ export async function watchCommand(options = {}) {
|
|
|
923
1171
|
reviewScanOnce: async () => { },
|
|
924
1172
|
claimWatchdogOnce: async () => { },
|
|
925
1173
|
claimBackstopOnce: async () => { },
|
|
1174
|
+
claimDeadmanOnce: async () => { },
|
|
926
1175
|
gitAuthProbeOnce: async () => false,
|
|
927
1176
|
runningCount: () => 0,
|
|
928
1177
|
};
|
|
@@ -1105,6 +1354,20 @@ export async function watchCommand(options = {}) {
|
|
|
1105
1354
|
claimBackstopTimer: undefined,
|
|
1106
1355
|
claimBackstopIntervalMs: options.claimBackstopIntervalMs ?? DEFAULT_CLAIM_BACKSTOP_INTERVAL_MS,
|
|
1107
1356
|
backstopScanning: false,
|
|
1357
|
+
// CLAIM-DEADMAN: wedge dead-man switch. The pure state
|
|
1358
|
+
// machine carries its own clock (Date.now); tests inject tiny thresholds.
|
|
1359
|
+
claimDeadman: new ClaimDeadman({
|
|
1360
|
+
wedgeThresholdMs: options.claimDeadmanWedgeMs ?? DEFAULT_CLAIM_DEADMAN_WEDGE_MS,
|
|
1361
|
+
rebuildGraceMs: options.claimDeadmanRebuildGraceMs ?? DEFAULT_CLAIM_DEADMAN_REBUILD_GRACE_MS,
|
|
1362
|
+
maxRebuilds: options.claimDeadmanMaxRebuilds ?? DEFAULT_CLAIM_DEADMAN_MAX_REBUILDS,
|
|
1363
|
+
}),
|
|
1364
|
+
claimDeadmanTimer: undefined,
|
|
1365
|
+
claimDeadmanCheckMs: options.claimDeadmanCheckMs ?? DEFAULT_DEADMAN_CHECK_MS,
|
|
1366
|
+
claimDeadmanChecking: false,
|
|
1367
|
+
deadmanRebuilding: false,
|
|
1368
|
+
deadmanFallbackTimer: null,
|
|
1369
|
+
deadmanFallbackPollMs: options.claimDeadmanFallbackPollMs ?? DEFAULT_FALLBACK_POLL_MS,
|
|
1370
|
+
lastHeartbeatDeadmanFields: null,
|
|
1108
1371
|
// GIT-AUTH-ALERT: controller wired below once `state` exists (its emit
|
|
1109
1372
|
// callbacks close over `state`); timer armed in the timer-setup block.
|
|
1110
1373
|
gitAuthController: null,
|
|
@@ -1263,6 +1526,18 @@ export async function watchCommand(options = {}) {
|
|
|
1263
1526
|
state.claimBackstopTimer = setInterval(() => {
|
|
1264
1527
|
void claimBackstopScan(state, taskRunnerFactory);
|
|
1265
1528
|
}, state.claimBackstopIntervalMs);
|
|
1529
|
+
// CLAIM-DEADMAN: the wedge dead-man switch. Escalates
|
|
1530
|
+
// BEYOND the 90s watchdog resubscribe (which only re-adds a channel on the
|
|
1531
|
+
// same client and, per the incident, can itself stop firing): tracks pending-
|
|
1532
|
+
// dispatch vs the last processed claim, and when work is offered but nothing
|
|
1533
|
+
// is claimed for the wedge threshold it FULLY REBUILDS the Realtime client
|
|
1534
|
+
// (twice) then degrades to DB-poll claiming — self-healing the "heartbeat-
|
|
1535
|
+
// alive but claim-dead" runner with zero human action. Independent timer +
|
|
1536
|
+
// single-flight inside claimDeadmanScan().
|
|
1537
|
+
state.claimDeadmanTimer = setInterval(() => {
|
|
1538
|
+
void claimDeadmanScan(state, taskRunnerFactory);
|
|
1539
|
+
}, state.claimDeadmanCheckMs);
|
|
1540
|
+
state.claimDeadmanTimer.unref?.();
|
|
1266
1541
|
// GIT-AUTH-ALERT: periodic git-credential health probe. Armed only when the
|
|
1267
1542
|
// controller is wired (a base clone is resolvable). The same cadence detects
|
|
1268
1543
|
// the first auth failure AND polls for recovery while degraded. Single-flight
|
|
@@ -1368,6 +1643,8 @@ export async function watchCommand(options = {}) {
|
|
|
1368
1643
|
clearInterval(state.reviewScanTimer); // v0.65 (T-65-2)
|
|
1369
1644
|
clearInterval(state.claimWatchdogTimer); // v0.73 (T-73-2)
|
|
1370
1645
|
clearInterval(state.claimBackstopTimer); // FIX-D
|
|
1646
|
+
clearInterval(state.claimDeadmanTimer); // CLAIM-DEADMAN
|
|
1647
|
+
stopDeadmanFallback(state); // CLAIM-DEADMAN
|
|
1371
1648
|
if (state.gitAuthTimer)
|
|
1372
1649
|
clearInterval(state.gitAuthTimer); // GIT-AUTH-ALERT
|
|
1373
1650
|
if (state.resubscribeTimer)
|
|
@@ -1480,6 +1757,7 @@ export async function watchCommand(options = {}) {
|
|
|
1480
1757
|
reviewScanOnce: () => reviewScan(state, "periodic"), // v0.65 (T-65-2)
|
|
1481
1758
|
claimWatchdogOnce: () => claimWatchdogScan(state, taskRunnerFactory), // v0.73 (T-73-2)
|
|
1482
1759
|
claimBackstopOnce: () => claimBackstopScan(state, taskRunnerFactory), // FIX-D
|
|
1760
|
+
claimDeadmanOnce: () => claimDeadmanScan(state, taskRunnerFactory), // CLAIM-DEADMAN
|
|
1483
1761
|
gitAuthProbeOnce: () => gitAuthProbeScan(state), // GIT-AUTH-ALERT
|
|
1484
1762
|
runningCount: () => state.running.size,
|
|
1485
1763
|
};
|
|
@@ -1572,12 +1850,20 @@ function scheduleHeartbeat(state, taskRunnerFactory, baseMs) {
|
|
|
1572
1850
|
const delay = nextHeartbeatDelayMs(state.heartbeatFailures, baseMs);
|
|
1573
1851
|
state.heartbeatTimer = setTimeout(() => {
|
|
1574
1852
|
void (async () => {
|
|
1575
|
-
const { error } = await state.supabase.rpc("heartbeat_runner", {
|
|
1853
|
+
const { data, error } = await state.supabase.rpc("heartbeat_runner", {
|
|
1576
1854
|
p_id: state.session.runner_id,
|
|
1577
1855
|
p_version: PACKAGE_VERSION,
|
|
1578
1856
|
});
|
|
1579
1857
|
if (!error) {
|
|
1580
1858
|
state.heartbeatFailures = 0;
|
|
1859
|
+
// CLAIM-DEADMAN: capture the pending-dispatch count
|
|
1860
|
+
// + suspect flag when the heartbeat RESPONSE carries them, so the runner
|
|
1861
|
+
// learns it is being offered work even with a dead WS. Forward-compatible:
|
|
1862
|
+
// today `heartbeat_runner` returns void and this is a no-op (the deadman
|
|
1863
|
+
// reads the assigned-queued count over HTTP instead). Once the RPC is
|
|
1864
|
+
// extended (Track A migration) this is the authoritative signal — the
|
|
1865
|
+
// deadman prefers it in readDispatchSignal.
|
|
1866
|
+
state.lastHeartbeatDeadmanFields = parseHeartbeatDeadmanFields(data);
|
|
1581
1867
|
// RUNCFG-1: pull the operator's server-set concurrency and apply it to
|
|
1582
1868
|
// the live ceiling within this heartbeat. Runs only on a healthy
|
|
1583
1869
|
// heartbeat (a network/auth failure would fail this RPC too). Awaited so
|
|
@@ -1611,6 +1897,8 @@ function scheduleHeartbeat(state, taskRunnerFactory, baseMs) {
|
|
|
1611
1897
|
clearInterval(state.reviewScanTimer); // v0.65 (T-65-2)
|
|
1612
1898
|
clearInterval(state.claimWatchdogTimer); // v0.73 (T-73-2)
|
|
1613
1899
|
clearInterval(state.claimBackstopTimer); // FIX-D
|
|
1900
|
+
clearInterval(state.claimDeadmanTimer); // CLAIM-DEADMAN
|
|
1901
|
+
stopDeadmanFallback(state); // CLAIM-DEADMAN
|
|
1614
1902
|
if (state.resubscribeTimer)
|
|
1615
1903
|
clearTimeout(state.resubscribeTimer); // v0.68 (T-68-1)
|
|
1616
1904
|
process.exit(1);
|