@adhdev/daemon-core 0.9.82-rc.536 → 0.9.82-rc.538
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chat/source-resolver.d.ts +21 -0
- package/dist/cli-adapters/cli-state-engine.d.ts +13 -0
- package/dist/index.js +344 -17
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +344 -17
- package/dist/index.mjs.map +1 -1
- package/dist/mesh/mesh-completion-synthesis.d.ts +14 -0
- package/dist/mesh/mesh-work-queue.d.ts +22 -0
- package/package.json +3 -3
- package/src/chat/source-resolver.ts +0 -0
- package/src/cli-adapters/cli-state-engine.ts +51 -1
- package/src/cli-adapters/provider-cli-adapter.ts +7 -0
- package/src/commands/chat-commands-read.ts +103 -7
- package/src/mesh/mesh-completion-synthesis.ts +85 -0
- package/src/mesh/mesh-event-forwarding.ts +96 -1
- package/src/mesh/mesh-queue-assignment.ts +7 -3
- package/src/mesh/mesh-reconcile-loop.ts +53 -2
- package/src/mesh/mesh-work-queue.ts +74 -4
- package/src/providers/sdk/v1/builders/cli/detect-status.ts +108 -1
- package/src/providers/sdk/v1/builders/cli/parse-approval.ts +33 -1
|
@@ -2,3 +2,17 @@ import type { DaemonComponents } from '../boot/daemon-lifecycle.js';
|
|
|
2
2
|
import type { LocalMeshEntry } from '../repo-mesh-types.js';
|
|
3
3
|
export declare function reconcileUnterminatedDirectDispatches(components: DaemonComponents, mesh: LocalMeshEntry, selfIds: string[], localDaemonId: string | undefined): Promise<void>;
|
|
4
4
|
export declare function autoPruneStaleDirectDispatches(components: DaemonComponents, mesh: LocalMeshEntry, selfIds: string[], localDaemonId: string | undefined, minAgeMs: number): Promise<void>;
|
|
5
|
+
export declare function pollAssignedTaskTerminalEvidence(components: DaemonComponents, mesh: {
|
|
6
|
+
id: string;
|
|
7
|
+
nodes?: Array<{
|
|
8
|
+
id: string;
|
|
9
|
+
daemonId?: string;
|
|
10
|
+
workspace?: string;
|
|
11
|
+
}>;
|
|
12
|
+
}, row: {
|
|
13
|
+
id: string;
|
|
14
|
+
assignedSessionId?: string;
|
|
15
|
+
assignedNodeId?: string;
|
|
16
|
+
assignedProviderType?: string;
|
|
17
|
+
dispatchTimestamp?: string;
|
|
18
|
+
}): Promise<'completed' | null>;
|
|
@@ -374,6 +374,28 @@ export declare function requeueTask(meshId: string, taskId: string, opts?: {
|
|
|
374
374
|
/** Per-task retry cap override. Falls back to mesh policy maxTaskRetries (default 1). */
|
|
375
375
|
maxRetries?: number;
|
|
376
376
|
} & MeshQueueMutationOptions): MeshWorkQueueEntry | null;
|
|
377
|
+
/**
|
|
378
|
+
* TASK-PROMPT-REDRIVE-AFTER-COMPLETE: the reclaim reasons the assigned-stranded watchdog
|
|
379
|
+
* uses when it RE-DRIVES a delivered-but-not-terminal task (returns it to 'pending' so the
|
|
380
|
+
* SAME prompt is re-dispatched). These are distinct from `assigned_stranded_dispatch_unconfirmed`
|
|
381
|
+
* (a dispatch that was NEVER handed off — nothing ran, so a late completion is impossible).
|
|
382
|
+
*
|
|
383
|
+
* A re-drive assumes the worker never finished. But for an autoLaunch/worktree worker the
|
|
384
|
+
* turn-lifecycle events (agent:generating_started/completed) do NOT reliably reach the
|
|
385
|
+
* coordinator ledger, so the deadline can elapse and re-drive fire while the worker's genuine
|
|
386
|
+
* completion is merely LATE (observed live: it lands 0.9s–98s AFTER the reclaim). The late
|
|
387
|
+
* completion must then SUPERSEDE the re-drive rather than be dropped — the completion handler's
|
|
388
|
+
* flip-miss safety net checks a row reclaimed for one of these reasons within
|
|
389
|
+
* {@link REDRIVE_SUPERSEDE_WINDOW_MS} of its `requeuedAt`.
|
|
390
|
+
*/
|
|
391
|
+
export declare const REDRIVE_RECLAIM_REASONS: ReadonlySet<string>;
|
|
392
|
+
/**
|
|
393
|
+
* How long after a re-drive reclaim's `requeuedAt` a late completion still supersedes the
|
|
394
|
+
* re-dispatch. Comfortably covers the observed 0.9s–98s completion-vs-reclaim race with margin,
|
|
395
|
+
* while staying far short of the time it would take a genuinely fresh re-dispatched turn to
|
|
396
|
+
* produce its OWN completion — so a real second turn is never mistaken for the superseded one.
|
|
397
|
+
*/
|
|
398
|
+
export declare const REDRIVE_SUPERSEDE_WINDOW_MS: number;
|
|
377
399
|
/**
|
|
378
400
|
* Bug B: reclaim a task stuck in 'assigned' because its dispatch was never confirmed.
|
|
379
401
|
*
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@adhdev/daemon-core",
|
|
3
|
-
"version": "0.9.82-rc.
|
|
3
|
+
"version": "0.9.82-rc.538",
|
|
4
4
|
"description": "ADHDev daemon core — CDP, IDE detection, providers, command execution",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -47,8 +47,8 @@
|
|
|
47
47
|
"author": "vilmire",
|
|
48
48
|
"license": "AGPL-3.0-or-later",
|
|
49
49
|
"dependencies": {
|
|
50
|
-
"@adhdev/mesh-shared": "0.9.82-rc.
|
|
51
|
-
"@adhdev/session-host-core": "0.9.82-rc.
|
|
50
|
+
"@adhdev/mesh-shared": "0.9.82-rc.538",
|
|
51
|
+
"@adhdev/session-host-core": "0.9.82-rc.538",
|
|
52
52
|
"@agentclientprotocol/sdk": "^0.16.1",
|
|
53
53
|
"ajv": "^8.20.0",
|
|
54
54
|
"ajv-formats": "^3.0.1",
|
|
Binary file
|
|
@@ -175,6 +175,7 @@ export class CliStateEngine {
|
|
|
175
175
|
* paint and made the engine type "1" repeatedly into the prompt.
|
|
176
176
|
*/
|
|
177
177
|
modalLostAt = 0;
|
|
178
|
+
private modalLostRecheckTimer: NodeJS.Timeout | null = null;
|
|
178
179
|
private approvalExitTimeout: NodeJS.Timeout | null = null;
|
|
179
180
|
|
|
180
181
|
// ── Response tracking ────────────────────────────
|
|
@@ -243,6 +244,12 @@ export class CliStateEngine {
|
|
|
243
244
|
setStatus(status: CliSessionStatus['status'], trigger?: string): void {
|
|
244
245
|
const prev = this.currentStatus;
|
|
245
246
|
if (prev === status) return;
|
|
247
|
+
// Leaving waiting_approval — cancel any pending modal-lost recheck; it
|
|
248
|
+
// only exists to wake a quiescent PTY still pinned to waiting_approval.
|
|
249
|
+
if (prev === 'waiting_approval' && this.modalLostRecheckTimer) {
|
|
250
|
+
clearTimeout(this.modalLostRecheckTimer);
|
|
251
|
+
this.modalLostRecheckTimer = null;
|
|
252
|
+
}
|
|
246
253
|
this.currentStatus = status;
|
|
247
254
|
this.statusHistory.push({ status, at: Date.now(), trigger });
|
|
248
255
|
if (this.statusHistory.length > 50) this.statusHistory.shift();
|
|
@@ -456,6 +463,7 @@ export class CliStateEngine {
|
|
|
456
463
|
if (this.settleTimer) { clearTimeout(this.settleTimer); this.settleTimer = null; }
|
|
457
464
|
if (this.idleTimeout) { clearTimeout(this.idleTimeout); this.idleTimeout = null; }
|
|
458
465
|
if (this.approvalExitTimeout) { clearTimeout(this.approvalExitTimeout); this.approvalExitTimeout = null; }
|
|
466
|
+
if (this.modalLostRecheckTimer) { clearTimeout(this.modalLostRecheckTimer); this.modalLostRecheckTimer = null; }
|
|
459
467
|
if (this.finishRetryTimer) { clearTimeout(this.finishRetryTimer); this.finishRetryTimer = null; }
|
|
460
468
|
if (this.pendingScriptStatusTimer) { clearTimeout(this.pendingScriptStatusTimer); this.pendingScriptStatusTimer = null; }
|
|
461
469
|
if (this.providerErrorRetryTimer) { clearTimeout(this.providerErrorRetryTimer); this.providerErrorRetryTimer = null; }
|
|
@@ -822,7 +830,23 @@ export class CliStateEngine {
|
|
|
822
830
|
// signature each time, which typed the approval key
|
|
823
831
|
// ("1") into the prompt repeatedly. Wait for the modal to
|
|
824
832
|
// stay gone for `approvalCooldown` before clearing.
|
|
825
|
-
|
|
833
|
+
//
|
|
834
|
+
// (fix: kimi approve-resolve-stuck) The recovery used to require
|
|
835
|
+
// `this.activeModal` to be non-null. But a provider can be pinned
|
|
836
|
+
// to `waiting_approval` with `activeModal === null`: kimi's
|
|
837
|
+
// questionPattern (`run.*command`) false-positive-matches the
|
|
838
|
+
// user's echoed prompt ("✨ Run the shell command: …"), so
|
|
839
|
+
// detectStatus reports `waiting_approval` from the question cue
|
|
840
|
+
// alone while parseApproval extracts zero buttons (→ null modal,
|
|
841
|
+
// never captured). After the real approval resolves and kimi goes
|
|
842
|
+
// quiet, no further PTY output arrives, so this is the LAST
|
|
843
|
+
// settled evaluation — with the old `&& this.activeModal` guard it
|
|
844
|
+
// bare-returned and the FSM latched `waiting_approval` forever
|
|
845
|
+
// (the approval appears never to resolve). Recover whenever we are
|
|
846
|
+
// pinned to waiting_approval with no actionable modal, captured or
|
|
847
|
+
// not, and arm a re-check so a quiescent PTY still gets one more
|
|
848
|
+
// evaluation pass to reach the hysteresis deadline.
|
|
849
|
+
if (this.currentStatus === 'waiting_approval') {
|
|
826
850
|
const lostAt = this.modalLostAt || Date.now();
|
|
827
851
|
if (!this.modalLostAt) this.modalLostAt = lostAt;
|
|
828
852
|
if (Date.now() - lostAt >= this.timeouts.approvalCooldown) {
|
|
@@ -830,6 +854,12 @@ export class CliStateEngine {
|
|
|
830
854
|
this.modalLostAt = 0;
|
|
831
855
|
this.setStatus('generating', 'approval_lost_modal');
|
|
832
856
|
this.callbacks.onStatusChange();
|
|
857
|
+
} else {
|
|
858
|
+
// Not yet past the hysteresis window. The PTY may be
|
|
859
|
+
// quiescent (kimi emits nothing once idle), so schedule an
|
|
860
|
+
// explicit re-evaluation — otherwise no settle tick ever
|
|
861
|
+
// fires again and the recovery above is never reached.
|
|
862
|
+
this.armModalLostRecheck();
|
|
833
863
|
}
|
|
834
864
|
}
|
|
835
865
|
return;
|
|
@@ -1151,6 +1181,26 @@ export class CliStateEngine {
|
|
|
1151
1181
|
|
|
1152
1182
|
// ─── Helpers ────────────────────────────────────────────────────────────
|
|
1153
1183
|
|
|
1184
|
+
/**
|
|
1185
|
+
* Schedule one more settled evaluation while pinned to `waiting_approval`
|
|
1186
|
+
* with no actionable modal. The settled FSM normally only re-runs on new
|
|
1187
|
+
* PTY output; a provider whose modal cue lingers in a form detectStatus
|
|
1188
|
+
* still matches (e.g. kimi's questionPattern hitting the user echo) but
|
|
1189
|
+
* whose PTY has gone quiet would never get another evaluation, latching
|
|
1190
|
+
* `waiting_approval` forever. This timer guarantees the modal-lost recovery
|
|
1191
|
+
* in `applyWaitingApproval` is reached even against a silent PTY. It is a
|
|
1192
|
+
* no-op once the FSM leaves `waiting_approval` (the re-evaluation itself
|
|
1193
|
+
* takes the recovery branch and clears the state).
|
|
1194
|
+
*/
|
|
1195
|
+
private armModalLostRecheck(): void {
|
|
1196
|
+
if (this.modalLostRecheckTimer) return;
|
|
1197
|
+
this.modalLostRecheckTimer = setTimeout(() => {
|
|
1198
|
+
this.modalLostRecheckTimer = null;
|
|
1199
|
+
if (this.currentStatus !== 'waiting_approval') return;
|
|
1200
|
+
this.evaluateSettled(this.transport.getSnapshot());
|
|
1201
|
+
}, this.timeouts.approvalCooldown);
|
|
1202
|
+
}
|
|
1203
|
+
|
|
1154
1204
|
private armApprovalExitTimeout(): void {
|
|
1155
1205
|
if (this.approvalExitTimeout) clearTimeout(this.approvalExitTimeout);
|
|
1156
1206
|
this.approvalExitTimeout = setTimeout(() => {
|
|
@@ -356,6 +356,13 @@ export class ProviderCliAdapter implements CliAdapter {
|
|
|
356
356
|
const currentScreenLooksIdle = /(?:^|\n|\r)\s*[❯›>]\s*(?:Try\s+["“][^\n\r"”]+["”])?\s*(?:\n|\r|$)/.test(screenText)
|
|
357
357
|
&& !activeScreenPattern.test(screenText);
|
|
358
358
|
if (staleSnapshotLooksActive && currentScreenLooksIdle) return screenText;
|
|
359
|
+
// When the current frame already resolves to a settled/idle prompt via the
|
|
360
|
+
// provider's own detector, do NOT graft an older snapshot onto it: the
|
|
361
|
+
// older frame can carry a dead modal box (e.g. cursor-agent's leftover
|
|
362
|
+
// "Workspace Trust Required" rows) that re-fires `waiting_approval` at the
|
|
363
|
+
// appended tail and wedges the turn in `generating`. A truly idle current
|
|
364
|
+
// screen needs no historical supplement.
|
|
365
|
+
if (this.runDetectStatus(screenText) === 'idle') return screenText;
|
|
359
366
|
if (currentSnapshot.length >= lastSnapshot.length) return screenText;
|
|
360
367
|
// Terminal screen reads can miss a just-rendered completed Hermes box while
|
|
361
368
|
// the normalized snapshot captured during output still has it. Feed both
|
|
@@ -745,8 +745,90 @@ function decideCliReadChatSource(args: {
|
|
|
745
745
|
const supportsNative = supportsCliNativeTranscript(args.providerType, args.provider);
|
|
746
746
|
const observation = buildObservationForCli(args, supportsNative);
|
|
747
747
|
const sessionKey = chatSourceSessionKey(args.providerType, args.sessionId);
|
|
748
|
+
|
|
749
|
+
// STICKY-NATIVE: once a session's transcript is bound to native-history via
|
|
750
|
+
// a TRUSTED EXACT provider-session identity (the on-disk uuid == this
|
|
751
|
+
// session's own transcript file, not a recency/mtime guess), a single read
|
|
752
|
+
// that momentarily fails to reproduce the full native slice is a TRANSIENT
|
|
753
|
+
// gap, NOT evidence native regressed or vanished. Concretely, cursor-agent
|
|
754
|
+
// rewrites its JSONL tail mid-turn — the file is briefly empty, or exposes a
|
|
755
|
+
// SHRUNK slice (old assistant row dropped while the new turn is streamed) —
|
|
756
|
+
// so a per-read observation sees either `native_unavailable/empty` or a
|
|
757
|
+
// `native_regressed_shrunk`. Either one flips a NativeLocked machine to
|
|
758
|
+
// `pty-parser` for that read (and, after enough misses, permanently to
|
|
759
|
+
// PtyOnly). That is exactly the reported symptom: cursor chat provenance
|
|
760
|
+
// "flips back and forth between PTY and native-history".
|
|
761
|
+
//
|
|
762
|
+
// Fix: for a session that (a) resolved via a trusted EXACT identity and
|
|
763
|
+
// (b) is already committed to native (NativeLocked/Recovering), speculatively
|
|
764
|
+
// observe, and if the machine WOULD flip to pty-parser, roll the machine
|
|
765
|
+
// state back and HOLD native-history as authoritative for this read. Exact
|
|
766
|
+
// identity means the read is provably against THIS session's own transcript
|
|
767
|
+
// file, so a transient gap on it can only be a mid-write artifact — never a
|
|
768
|
+
// genuine session switch or data loss (cursor's file is cumulative). A
|
|
769
|
+
// workspace-heuristic (non-exact) read is never eligible, so a real session
|
|
770
|
+
// switch still flips normally.
|
|
771
|
+
const priorSnapshot = CHAT_SOURCE_REGISTRY.snapshotRecord(sessionKey);
|
|
772
|
+
const priorState = priorSnapshot?.state ?? CHAT_SOURCE_REGISTRY.getState(sessionKey);
|
|
773
|
+
const eligibleForStickyHold = args.trustedExactNativeIdentity === true
|
|
774
|
+
&& (priorState.name === 'NativeLocked' || priorState.name === 'Recovering');
|
|
775
|
+
|
|
748
776
|
let decision = CHAT_SOURCE_REGISTRY.observe(sessionKey, observation);
|
|
749
777
|
|
|
778
|
+
if (eligibleForStickyHold && decision.selected === 'pty-parser') {
|
|
779
|
+
// The machine flipped a native-committed, exact-identity session to PTY
|
|
780
|
+
// on this read. Treat it as a transient native gap: undo the observe and
|
|
781
|
+
// report native-history held. We surface whatever native rows this read
|
|
782
|
+
// DID map (empty on a true gap, a shrunk slice on a mid-write rewrite);
|
|
783
|
+
// the streaming/tail delivery layer already suppresses an empty tail from
|
|
784
|
+
// clobbering the last real tail, so holding native with a thin slice is
|
|
785
|
+
// strictly safer than emitting PTY under a native-authority session.
|
|
786
|
+
CHAT_SOURCE_REGISTRY.restoreRecord(sessionKey, priorSnapshot);
|
|
787
|
+
const heldNativeMessages: ChatMessage[] = observation.kind === 'native_present'
|
|
788
|
+
? extractNativeMessagesFromResult(args.providerType, args.nativeHistoryResult)
|
|
789
|
+
: [];
|
|
790
|
+
const messageSource = buildCliMessageSourceProvenance({
|
|
791
|
+
selected: 'native-history',
|
|
792
|
+
provider: args.providerType,
|
|
793
|
+
nativeHandle: typeof args.nativeHistoryResult?.providerSessionId === 'string'
|
|
794
|
+
? args.nativeHistoryResult.providerSessionId
|
|
795
|
+
: undefined,
|
|
796
|
+
sessionWorkspace: args.sessionWorkspace,
|
|
797
|
+
intendedWorkspace: args.intendedWorkspace,
|
|
798
|
+
transcriptWorkspace: undefined,
|
|
799
|
+
fallbackReason: 'native_history_transient_gap_held',
|
|
800
|
+
nativeSource: 'provider-native',
|
|
801
|
+
sourcePath: typeof args.nativeHistoryResult?.sourcePath === 'string' ? args.nativeHistoryResult.sourcePath : undefined,
|
|
802
|
+
sourceMtimeMs: typeof args.nativeHistoryResult?.sourceMtimeMs === 'number' ? args.nativeHistoryResult.sourceMtimeMs : undefined,
|
|
803
|
+
nativeHistoryCoverage: undefined,
|
|
804
|
+
partialReason: undefined,
|
|
805
|
+
unavailableReason: observation.kind === 'native_unavailable' ? observation.reason : undefined,
|
|
806
|
+
nativeMessages: heldNativeMessages,
|
|
807
|
+
ptyMessages: args.ptyMessages,
|
|
808
|
+
returnedMessages: heldNativeMessages,
|
|
809
|
+
safeMapping: args.safeMapping,
|
|
810
|
+
freshEnough: true,
|
|
811
|
+
ptyStatusApprovalOnly: true,
|
|
812
|
+
});
|
|
813
|
+
return {
|
|
814
|
+
decision: {
|
|
815
|
+
selected: 'native-history',
|
|
816
|
+
nextState: priorState,
|
|
817
|
+
transition: {
|
|
818
|
+
fromState: priorState.name,
|
|
819
|
+
toState: priorState.name,
|
|
820
|
+
event: 'NoOp',
|
|
821
|
+
cause: decision.transition.cause,
|
|
822
|
+
at: Date.now(),
|
|
823
|
+
},
|
|
824
|
+
lockState: { locked: priorState.name === 'NativeLocked' },
|
|
825
|
+
},
|
|
826
|
+
messageSource,
|
|
827
|
+
nativeMessages: heldNativeMessages,
|
|
828
|
+
nativeSelected: true,
|
|
829
|
+
};
|
|
830
|
+
}
|
|
831
|
+
|
|
750
832
|
// A restored runtime can briefly expose different native slices while the
|
|
751
833
|
// provider transcript settles (for example, startup/system rows may be
|
|
752
834
|
// filtered after the first read). The source machine correctly treats a
|
|
@@ -1335,13 +1417,27 @@ function readCliProviderNativeHistory(agentStr: string, args: {
|
|
|
1335
1417
|
const pinnedProviderSessionId = typeof args.pinnedProviderSessionId === 'string'
|
|
1336
1418
|
? args.pinnedProviderSessionId.trim()
|
|
1337
1419
|
: '';
|
|
1338
|
-
// Pin reuse (PRIMARY): a
|
|
1339
|
-
//
|
|
1340
|
-
//
|
|
1341
|
-
//
|
|
1342
|
-
//
|
|
1343
|
-
//
|
|
1344
|
-
|
|
1420
|
+
// Pin reuse (PRIMARY): a read with no caller-supplied historySessionId can
|
|
1421
|
+
// still resolve to the session it was last bound to. Read THAT session
|
|
1422
|
+
// directly by threading the pin through as historySessionId — same code path
|
|
1423
|
+
// an explicit session read takes — instead of relying on the live spawn/cwd/
|
|
1424
|
+
// mtime heuristic. Never overrides a caller-supplied historySessionId; only
|
|
1425
|
+
// kicks in when there is none.
|
|
1426
|
+
//
|
|
1427
|
+
// The pin is preferred EVEN FOR A LIVE SESSION (canBindFromLiveSession).
|
|
1428
|
+
// The pin is keyed on this session's own mesh id (getBoundProviderSessionIdPin
|
|
1429
|
+
// (targetSessionId)) and holds the provider-native uuid proven in a prior
|
|
1430
|
+
// read, so it can only ever resolve to THIS session's own transcript — it
|
|
1431
|
+
// cannot alias a concurrent session sharing the cwd. Bypassing the pin while
|
|
1432
|
+
// a session is live (the old behaviour) forced the FIRST read of every new
|
|
1433
|
+
// turn back onto the spawn/mtime heuristic: cursor's native tail is briefly
|
|
1434
|
+
// stale (previous turn) versus the just-echoed PTY user line, so the
|
|
1435
|
+
// workspace-overlap safe-mapping gate fails and the read flips to PTY
|
|
1436
|
+
// (native_history_not_safely_mapped) before native re-locks. Preferring the
|
|
1437
|
+
// pin makes that first read an EXACT-identity lookup (trustedExactNativeIdentity
|
|
1438
|
+
// = true), which reads the correct cumulative file AND lets the STICKY-NATIVE
|
|
1439
|
+
// hold cover the turn boundary. This is the pin-bypass class fix.
|
|
1440
|
+
const effectiveHistorySessionId = args.historySessionId || pinnedProviderSessionId || '';
|
|
1345
1441
|
// Last-resort workspace-latest (b): only when nothing above resolved a
|
|
1346
1442
|
// session id AND no pin exists AND the caller opted in with a workspace.
|
|
1347
1443
|
// Strictly behind pin reuse — pinnedProviderSessionId being set disables it.
|
|
@@ -414,3 +414,88 @@ export async function autoPruneStaleDirectDispatches(
|
|
|
414
414
|
LOG.info('MeshReconcile', `Auto-pruned ${result.prunedCount} orphaned direct dispatch record(s) for mesh ${mesh.id}`);
|
|
415
415
|
}
|
|
416
416
|
}
|
|
417
|
+
|
|
418
|
+
// TASK-PROMPT-REDRIVE-AFTER-COMPLETE (Fix A-i). Single-shot transcript poll for a CLAIM-PATH
|
|
419
|
+
// (queue-assigned) row that PHASE 2.5's long delivered-no-turn deadline is about to RE-DRIVE.
|
|
420
|
+
//
|
|
421
|
+
// PHASE 4 (reconcileUnterminatedDirectDispatches) recovers a lost completion by re-reading the
|
|
422
|
+
// worker transcript, but it is bound to DIRECT-dispatch ledger rows — it never touches a
|
|
423
|
+
// claim-path queue row. So the F3 long-deadline reclaim reaches its 15-min deadline with an empty
|
|
424
|
+
// ledger for an autoLaunch/worktree worker whose generating_started/completed events never
|
|
425
|
+
// propagated, and re-drives a task the worker actually FINISHED (the owner's symptom). This poll
|
|
426
|
+
// gives that reclaim the SAME transcript evidence PHASE 4 uses, for the queue row: if the worker
|
|
427
|
+
// session is idle with a final assistant summary dated at/after this task's dispatch, the task is
|
|
428
|
+
// done — return its terminal outcome so the caller short-circuits to that status instead of
|
|
429
|
+
// reclaiming. Unlike PHASE 4 there is no acked-hold/grace machinery: the caller only invokes this
|
|
430
|
+
// AFTER the full 15-min deadline, so a single idle-with-final-assistant read is decisive.
|
|
431
|
+
//
|
|
432
|
+
// Conservative: a non-idle read, a read failure, no final summary, or a summary provably BEFORE
|
|
433
|
+
// dispatch all yield null (fall through to the caller's normal reclaim decision) — the poll can
|
|
434
|
+
// only PREVENT a wrong re-drive, never invent a completion.
|
|
435
|
+
export async function pollAssignedTaskTerminalEvidence(
|
|
436
|
+
components: DaemonComponents,
|
|
437
|
+
mesh: { id: string; nodes?: Array<{ id: string; daemonId?: string; workspace?: string }> },
|
|
438
|
+
row: { id: string; assignedSessionId?: string; assignedNodeId?: string; assignedProviderType?: string; dispatchTimestamp?: string },
|
|
439
|
+
): Promise<'completed' | null> {
|
|
440
|
+
const sessionId = readNonEmptyString(row.assignedSessionId);
|
|
441
|
+
const nodeId = readNonEmptyString(row.assignedNodeId);
|
|
442
|
+
if (!sessionId || !nodeId) return null; // no worker to read
|
|
443
|
+
|
|
444
|
+
const node = (mesh.nodes ?? []).find(n => n.id === nodeId);
|
|
445
|
+
const nodeDaemonId = readNonEmptyString(node?.daemonId);
|
|
446
|
+
const localDaemonId = readNonEmptyString((components as { statusInstanceId?: string }).statusInstanceId);
|
|
447
|
+
const isLocalNode = !nodeDaemonId
|
|
448
|
+
|| daemonIdsEquivalent(nodeDaemonId, localDaemonId)
|
|
449
|
+
|| !!components.instanceManager.getInstance(sessionId);
|
|
450
|
+
|
|
451
|
+
const providerType = readNonEmptyString(row.assignedProviderType);
|
|
452
|
+
const readArgs: Record<string, unknown> = {
|
|
453
|
+
sessionId,
|
|
454
|
+
targetSessionId: sessionId,
|
|
455
|
+
tailLimit: 10,
|
|
456
|
+
...(node?.workspace ? { workspace: node.workspace } : {}),
|
|
457
|
+
...(providerType ? { agentType: providerType, providerType } : {}),
|
|
458
|
+
};
|
|
459
|
+
|
|
460
|
+
let payload: Record<string, unknown> | null = null;
|
|
461
|
+
try {
|
|
462
|
+
if (isLocalNode) {
|
|
463
|
+
const result = await components.commandHandler?.handle('read_chat', readArgs);
|
|
464
|
+
if (result && (result as { success?: boolean }).success === false) return null;
|
|
465
|
+
payload = unwrapReadChatPayload(result);
|
|
466
|
+
} else if (components.dispatchMeshCommand) {
|
|
467
|
+
const result = await components.dispatchMeshCommand(nodeDaemonId, 'read_chat', readArgs);
|
|
468
|
+
payload = unwrapReadChatPayload(result);
|
|
469
|
+
if (payload && (payload as { success?: boolean }).success === false) return null;
|
|
470
|
+
} else {
|
|
471
|
+
return null; // remote node, no P2P transport — can't read this tick
|
|
472
|
+
}
|
|
473
|
+
} catch {
|
|
474
|
+
return null; // transport error / session gone → inconclusive, let the caller decide
|
|
475
|
+
}
|
|
476
|
+
if (!payload) return null;
|
|
477
|
+
|
|
478
|
+
// Only a settled-idle session is a turn-end; a generating/waiting session is mid-turn and
|
|
479
|
+
// must NEVER be short-circuited to completed.
|
|
480
|
+
if (readChatPayloadStatus(payload) !== 'idle') return null;
|
|
481
|
+
|
|
482
|
+
const messages = Array.isArray(payload.messages) ? payload.messages as ChatMessage[] : [];
|
|
483
|
+
const evidence = extractFinalAssistantSummaryEvidence(messages);
|
|
484
|
+
if (!evidence.finalSummary) return null; // idle but no assistant result yet → not a turn-end
|
|
485
|
+
|
|
486
|
+
// Stale-summary guard (same bar as PHASE 4): a reused session's transcript tail may hold a
|
|
487
|
+
// PRIOR task's summary. Require the final assistant message to be dated at/after THIS task's
|
|
488
|
+
// dispatch. When either timestamp is unusable, do NOT short-circuit — fall through so we never
|
|
489
|
+
// synthesize a completion off a possibly-stale tail (the reclaim path is the safe default).
|
|
490
|
+
const dispatchedAtMs = Date.parse(readNonEmptyString(row.dispatchTimestamp));
|
|
491
|
+
const transcriptAtMs = Date.parse(evidence.transcriptMessageAt ?? '');
|
|
492
|
+
if (!(Number.isFinite(dispatchedAtMs) && Number.isFinite(transcriptAtMs) && transcriptAtMs >= dispatchedAtMs)) {
|
|
493
|
+
return null;
|
|
494
|
+
}
|
|
495
|
+
|
|
496
|
+
// Idle + a final assistant message dated after dispatch = the worker finished this turn. We
|
|
497
|
+
// cannot distinguish a self-reported failure from the plain transcript tail here (that lives
|
|
498
|
+
// in buildTaskCompletionEvidence's structured-result path), and the alternative — re-driving a
|
|
499
|
+
// finished worker — is strictly worse, so a proven turn-end short-circuits to 'completed'.
|
|
500
|
+
return 'completed';
|
|
501
|
+
}
|
|
@@ -4,7 +4,8 @@ import { getMesh, getMeshByRepo, listMeshes } from '../config/mesh-config.js';
|
|
|
4
4
|
import { LOG } from '../logging/logger.js';
|
|
5
5
|
import { appendLedgerEntry, buildTaskCompletionEvidence, getSessionRecoveryContext, isIntentionalCleanupStopEntry, readLedgerEntries } from './mesh-ledger.js';
|
|
6
6
|
import type { SessionRecoveryContext } from './mesh-ledger.js';
|
|
7
|
-
import { updateSessionTaskStatus, enqueueTask, updateDirectDispatchStatus, cleanupTerminalDirectDispatches, getActiveDirectDispatches, hasPendingDependents, getQueue } from './mesh-work-queue.js';
|
|
7
|
+
import { updateSessionTaskStatus, updateTaskStatus, enqueueTask, updateDirectDispatchStatus, cleanupTerminalDirectDispatches, getActiveDirectDispatches, hasPendingDependents, getQueue, REDRIVE_RECLAIM_REASONS, REDRIVE_SUPERSEDE_WINDOW_MS } from './mesh-work-queue.js';
|
|
8
|
+
import type { MeshWorkQueueEntry } from './mesh-work-queue.js';
|
|
8
9
|
import { markSessionDeliveriesTerminal, updateSessionDeliveryStatus, consumeSessionDelivery } from './mesh-delivery-policy.js';
|
|
9
10
|
import { MeshRuntimeStore, pruneMeshRuntimeRetention } from './mesh-runtime-store.js';
|
|
10
11
|
import { maybeInjectIdleActiveMissionReminder } from './mesh-idle-reminder.js';
|
|
@@ -810,6 +811,93 @@ function stopStaleMeshWorker(
|
|
|
810
811
|
}
|
|
811
812
|
}
|
|
812
813
|
|
|
814
|
+
/**
|
|
815
|
+
* TASK-PROMPT-REDRIVE-AFTER-COMPLETE (Fix C): a genuine completion echoed a taskId whose queue
|
|
816
|
+
* row is NOT 'assigned' anymore — because the assigned-stranded watchdog RE-DROVE it (long
|
|
817
|
+
* delivered-no-turn deadline / unknown-grace) while this very completion was still in flight.
|
|
818
|
+
* For an autoLaunch/worktree worker the turn-lifecycle events don't reliably reach the
|
|
819
|
+
* coordinator ledger, so the deadline fires before the completion propagates (observed live at
|
|
820
|
+
* 0.9s–98s late). The re-drive re-injects the SAME prompt into the already-finished worker (the
|
|
821
|
+
* owner's symptom). This late completion PROVES the original turn finished, so it SUPERSEDES the
|
|
822
|
+
* re-drive: flip the row terminal and stop any duplicate re-dispatch.
|
|
823
|
+
*
|
|
824
|
+
* Bounded to a row reclaimed for a RE-DRIVE reason within {@link REDRIVE_SUPERSEDE_WINDOW_MS} of
|
|
825
|
+
* its `requeuedAt` — outside that window a `pending` row is an unrelated retry (or a fresh
|
|
826
|
+
* re-dispatched turn whose own completion this is not), and is left untouched. Returns true when
|
|
827
|
+
* it superseded the re-drive (row flipped terminal), false to fall through to normal handling.
|
|
828
|
+
*/
|
|
829
|
+
function supersedeRedriveReclaimForLateCompletion(
|
|
830
|
+
components: DaemonComponents,
|
|
831
|
+
meshId: string,
|
|
832
|
+
row: MeshWorkQueueEntry,
|
|
833
|
+
completingSessionId: string,
|
|
834
|
+
outcome: 'completed' | 'failed',
|
|
835
|
+
args: { nodeId?: string; event: string; metadataEvent: Record<string, unknown> },
|
|
836
|
+
): boolean {
|
|
837
|
+
// Only a re-drive reclaim is superseded. A row reclaimed as never-delivered
|
|
838
|
+
// (assigned_stranded_dispatch_unconfirmed) never ran, so a completion for it is not a
|
|
839
|
+
// late race — leave it to the normal path. A row already terminal is a no-op.
|
|
840
|
+
if (!row.requeueReason || !REDRIVE_RECLAIM_REASONS.has(row.requeueReason)) return false;
|
|
841
|
+
if (row.status === 'completed' || row.status === 'failed' || row.status === 'cancelled') return false;
|
|
842
|
+
const requeuedAtMs = Date.parse(row.requeuedAt ?? '');
|
|
843
|
+
if (!Number.isFinite(requeuedAtMs)) return false;
|
|
844
|
+
if (Date.now() - requeuedAtMs > REDRIVE_SUPERSEDE_WINDOW_MS) return false;
|
|
845
|
+
|
|
846
|
+
// The re-drive may have already re-dispatched the SAME prompt onto a FRESH session (row is
|
|
847
|
+
// 'assigned' again to a different session). That duplicate is executing work the original
|
|
848
|
+
// worker already finished — stop it so the prompt is not run twice. (When the row is still
|
|
849
|
+
// 'pending' there is no duplicate yet; flipping it terminal below prevents PHASE 3 from ever
|
|
850
|
+
// re-dispatching it.)
|
|
851
|
+
const reDispatchedSessionId = row.assignedSessionId;
|
|
852
|
+
if (
|
|
853
|
+
row.status === 'assigned'
|
|
854
|
+
&& reDispatchedSessionId
|
|
855
|
+
&& !sessionIdsEquivalent(reDispatchedSessionId, completingSessionId)
|
|
856
|
+
) {
|
|
857
|
+
stopStaleMeshWorker(components, {
|
|
858
|
+
meshId,
|
|
859
|
+
sessionId: reDispatchedSessionId,
|
|
860
|
+
nodeId: row.assignedNodeId,
|
|
861
|
+
providerType: row.assignedProviderType,
|
|
862
|
+
});
|
|
863
|
+
}
|
|
864
|
+
|
|
865
|
+
// Flip the row terminal by its exact id (immune to the cleared session ownership) and record
|
|
866
|
+
// the terminal ledger evidence, so the reconcile watchdog's terminal-ledger branch also sees
|
|
867
|
+
// it and no further reclaim fires.
|
|
868
|
+
endTaskDispatchInFlight(meshId, row.id);
|
|
869
|
+
updateTaskStatus(meshId, row.id, outcome === 'completed' ? 'completed' : 'failed');
|
|
870
|
+
if (!findTerminalLedgerEvidenceForTask({ meshId, taskId: row.id })) {
|
|
871
|
+
try {
|
|
872
|
+
appendLedgerEntry(meshId, {
|
|
873
|
+
kind: outcome === 'completed' ? 'task_completed' : 'task_failed',
|
|
874
|
+
sessionId: completingSessionId,
|
|
875
|
+
nodeId: readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId) || undefined,
|
|
876
|
+
providerType: readNonEmptyString(args.metadataEvent.providerType) || undefined,
|
|
877
|
+
payload: {
|
|
878
|
+
taskId: row.id,
|
|
879
|
+
event: args.event,
|
|
880
|
+
source: 'redrive_late_completion_supersede',
|
|
881
|
+
reclaimReason: row.requeueReason,
|
|
882
|
+
reclaimAgeMs: Date.now() - requeuedAtMs,
|
|
883
|
+
finalSummary: readNonEmptyString(args.metadataEvent.finalSummary) || undefined,
|
|
884
|
+
},
|
|
885
|
+
});
|
|
886
|
+
} catch { /* best-effort ledger write */ }
|
|
887
|
+
}
|
|
888
|
+
LOG.warn('MeshQueue', `Late completion superseded re-drive for task ${row.id} on mesh ${meshId} `
|
|
889
|
+
+ `(reclaimed '${row.requeueReason}' ${Math.round((Date.now() - requeuedAtMs) / 1000)}s ago; completing session ${completingSessionId}) `
|
|
890
|
+
+ `→ flipped ${outcome}${(row.status === 'assigned' && reDispatchedSessionId && !sessionIdsEquivalent(reDispatchedSessionId, completingSessionId)) ? `, stopped duplicate re-dispatch on ${reDispatchedSessionId}` : ''}`);
|
|
891
|
+
traceMeshEventDrop('redrive_late_completion_supersede', {
|
|
892
|
+
taskId: row.id,
|
|
893
|
+
sessionId: completingSessionId,
|
|
894
|
+
nodeId: row.assignedNodeId ?? args.nodeId,
|
|
895
|
+
meshId,
|
|
896
|
+
event: args.event,
|
|
897
|
+
}, `${row.requeueReason} ${Math.round((Date.now() - requeuedAtMs) / 1000)}s → ${outcome}`);
|
|
898
|
+
return true;
|
|
899
|
+
}
|
|
900
|
+
|
|
813
901
|
function injectMeshSystemMessage(components: DaemonComponents, args: {
|
|
814
902
|
meshId: string;
|
|
815
903
|
sourceInstanceId?: string;
|
|
@@ -1022,6 +1110,13 @@ function injectMeshSystemMessage(components: DaemonComponents, args: {
|
|
|
1022
1110
|
},
|
|
1023
1111
|
});
|
|
1024
1112
|
}
|
|
1113
|
+
} else if (strandedRow && supersedeRedriveReclaimForLateCompletion(components, args.meshId, strandedRow, sessionId, outcome, args)) {
|
|
1114
|
+
// TASK-PROMPT-REDRIVE-AFTER-COMPLETE (Fix C, late-completion supersede): the row
|
|
1115
|
+
// is NOT 'assigned' — it was re-driven (reclaimed → 'pending', or already
|
|
1116
|
+
// re-dispatched to a fresh session) because the long delivered-no-turn deadline
|
|
1117
|
+
// fired before this genuine completion propagated. The completion proves the
|
|
1118
|
+
// ORIGINAL worker finished, so it SUPERSEDES the re-drive: the helper flipped the
|
|
1119
|
+
// row terminal and stopped any duplicate re-dispatch (handled inside).
|
|
1025
1120
|
}
|
|
1026
1121
|
} catch { /* best-effort safety net — never fail the completion path */ }
|
|
1027
1122
|
}
|
|
@@ -1967,10 +1967,14 @@ async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, mesh
|
|
|
1967
1967
|
// a launch that can never claim. Mirrors claimNextQueueTask's convergence gate.
|
|
1968
1968
|
if (task.taskMode === 'convergence' && node?.isLocalWorktree === true) return false;
|
|
1969
1969
|
// Skip nodes that can never satisfy requiredTags regardless of which provider
|
|
1970
|
-
//
|
|
1971
|
-
//
|
|
1970
|
+
// is selected. A node satisfies tags if at least one provider it can launch
|
|
1971
|
+
// would produce matching capability tags. Enumerate providers from the node's
|
|
1972
|
+
// capability slots (the single source of truth — a provider that lives only in
|
|
1973
|
+
// slots, e.g. cursor-cli, is otherwise invisible to providerPriority-keyed
|
|
1974
|
+
// enumeration), falling back to the legacy providerPriority.
|
|
1972
1975
|
if (task.requiredTags?.length) {
|
|
1973
|
-
const
|
|
1976
|
+
const slotProviders = resolveNodeCapabilitySlots(node).map(s => s.provider).filter(Boolean);
|
|
1977
|
+
const priorities = slotProviders.length ? slotProviders : normalizeProviderPriority(node?.policy);
|
|
1974
1978
|
const providerCandidates = priorities.length ? priorities : [undefined as unknown as string];
|
|
1975
1979
|
return providerCandidates.some(p =>
|
|
1976
1980
|
nodeSatisfiesRequiredTags(task.requiredTags, buildMeshNodeCapabilityTags(node, p))
|
|
@@ -79,6 +79,7 @@ import { pullRemoteNodeQueues } from './mesh-remote-event-pull.js';
|
|
|
79
79
|
import {
|
|
80
80
|
reconcileUnterminatedDirectDispatches,
|
|
81
81
|
autoPruneStaleDirectDispatches,
|
|
82
|
+
pollAssignedTaskTerminalEvidence,
|
|
82
83
|
} from './mesh-completion-synthesis.js';
|
|
83
84
|
|
|
84
85
|
// Re-export the extracted public API so existing importers (mesh-events.ts barrel;
|
|
@@ -737,7 +738,12 @@ export function __resetReclaimUnknownStreakForTests(): void {
|
|
|
737
738
|
// never reclaimed here. And the deadline is generous so a slow-but-live dispatch still in
|
|
738
739
|
// its normal confirm window is never reclaimed early. Reclaimed rows return to 'pending'
|
|
739
740
|
// with ownership cleared, so the PHASE 3 trigger below re-dispatches them this same tick.
|
|
740
|
-
function recoverStrandedAssignedDispatches(
|
|
741
|
+
async function recoverStrandedAssignedDispatches(
|
|
742
|
+
components: DaemonComponents,
|
|
743
|
+
mesh: { id: string; nodes?: Array<{ id: string; daemonId?: string; workspace?: string }> },
|
|
744
|
+
store: MeshRuntimeStore,
|
|
745
|
+
): Promise<void> {
|
|
746
|
+
const meshId = mesh.id;
|
|
741
747
|
const assigned = getQueue(meshId, { status: ['assigned'] });
|
|
742
748
|
if (!assigned.length) return;
|
|
743
749
|
const nowMs = Date.now();
|
|
@@ -910,6 +916,51 @@ function recoverStrandedAssignedDispatches(components: DaemonComponents, meshId:
|
|
|
910
916
|
}
|
|
911
917
|
reclaimReason = 'reclaim_after_unknown_grace';
|
|
912
918
|
}
|
|
919
|
+
// TASK-PROMPT-REDRIVE-AFTER-COMPLETE (Fix A-i): before re-driving, poll the worker
|
|
920
|
+
// transcript for terminal evidence — the SAME check PHASE 4 does for direct dispatches,
|
|
921
|
+
// now for this claim-path queue row. An autoLaunch/worktree worker's
|
|
922
|
+
// generating_started/completed events don't reliably reach the coordinator ledger, so
|
|
923
|
+
// the ledger check above (findTerminalLedgerEvidenceForTask) can be empty at the 15-min
|
|
924
|
+
// deadline for a task the worker actually FINISHED — and re-driving then re-injects the
|
|
925
|
+
// same prompt into the already-idle worker (the owner's symptom). If the worker is idle
|
|
926
|
+
// with a final assistant summary dated after dispatch, the task is done: flip it
|
|
927
|
+
// 'completed' instead of reclaiming. Conservative by construction (mid-turn / no
|
|
928
|
+
// summary / stale summary / unreadable → null → fall through to the reclaim below), so
|
|
929
|
+
// this can only PREVENT a wrong re-drive, never invent a completion. Runs only at the
|
|
930
|
+
// deadline (rare), so the extra read is not a hot-path cost.
|
|
931
|
+
const terminalEvidence = await pollAssignedTaskTerminalEvidence(components, mesh, row);
|
|
932
|
+
if (terminalEvidence) {
|
|
933
|
+
deliveredNoTurnUnknownStreak.delete(streakKey);
|
|
934
|
+
// updateTaskStatus ends the single-flight dispatch window on any transition off
|
|
935
|
+
// 'assigned', so a later requeue/re-dispatch is never blocked by a stale mark.
|
|
936
|
+
updateTaskStatus(meshId, row.id, terminalEvidence);
|
|
937
|
+
if (!findTerminalLedgerEvidenceForTask({ meshId, taskId: row.id })) {
|
|
938
|
+
try {
|
|
939
|
+
appendLedgerEntry(meshId, {
|
|
940
|
+
kind: terminalEvidence === 'completed' ? 'task_completed' : 'task_failed',
|
|
941
|
+
nodeId: row.assignedNodeId,
|
|
942
|
+
sessionId: row.assignedSessionId,
|
|
943
|
+
providerType: row.assignedProviderType,
|
|
944
|
+
payload: {
|
|
945
|
+
taskId: row.id,
|
|
946
|
+
event: 'agent:generating_completed',
|
|
947
|
+
source: 'redrive_deadline_transcript_evidence',
|
|
948
|
+
},
|
|
949
|
+
});
|
|
950
|
+
} catch { /* best-effort ledger write */ }
|
|
951
|
+
}
|
|
952
|
+
LOG.warn('MeshReconcile', `Skipped delivered-no-turn re-drive for task ${row.id} on mesh ${meshId} `
|
|
953
|
+
+ `(node=${row.assignedNodeId ?? '?'} session=${row.assignedSessionId ?? '?'}): worker transcript is idle with a `
|
|
954
|
+
+ `final assistant message after dispatch — the completion event was lost/late, task is ${terminalEvidence}, NOT re-driving`);
|
|
955
|
+
traceMeshEventDrop('redrive_deadline_transcript_completed', {
|
|
956
|
+
taskId: row.id,
|
|
957
|
+
sessionId: row.assignedSessionId,
|
|
958
|
+
nodeId: row.assignedNodeId,
|
|
959
|
+
meshId,
|
|
960
|
+
event: 'agent:generating_completed',
|
|
961
|
+
}, `${reclaimReason} → transcript ${terminalEvidence}`);
|
|
962
|
+
continue;
|
|
963
|
+
}
|
|
913
964
|
const reclaimedLost = reclaimStrandedAssignedTask(meshId, row.id, {
|
|
914
965
|
reason: reclaimReason,
|
|
915
966
|
ageMs: nowMs - dispatchedAtMs,
|
|
@@ -1108,7 +1159,7 @@ export async function runMeshReconcileTick(components: DaemonComponents): Promis
|
|
|
1108
1159
|
const selfIds = resolveCoordinatorSelfIds(mesh, drainDaemonIds);
|
|
1109
1160
|
if (!daemonHostsMesh(mesh, selfIds)) continue;
|
|
1110
1161
|
try {
|
|
1111
|
-
recoverStrandedAssignedDispatches(components, mesh
|
|
1162
|
+
await recoverStrandedAssignedDispatches(components, mesh, store);
|
|
1112
1163
|
} catch (e: any) {
|
|
1113
1164
|
LOG.warn('MeshReconcile', `Assigned-stranded watchdog failed for mesh ${mesh.id}: ${e?.message || e}`);
|
|
1114
1165
|
}
|