@adhdev/daemon-core 0.9.82-rc.454 → 0.9.82-rc.456
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/router-refine.d.ts +137 -0
- package/dist/commands/router-worktree-cleanup.d.ts +133 -0
- package/dist/commands/router.d.ts +103 -151
- package/dist/index.js +3043 -2882
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +3043 -2882
- package/dist/index.mjs.map +1 -1
- package/dist/mesh/mesh-event-classify.d.ts +2 -0
- package/dist/mesh/worktree-bootstrap-config.d.ts +20 -0
- package/dist/providers/approval-utils.d.ts +25 -0
- package/dist/providers/cli-provider-instance.d.ts +26 -1
- package/dist/providers/manual-attendance.d.ts +16 -0
- package/dist/providers/provider-instance.d.ts +8 -1
- package/package.json +3 -3
- package/src/commands/handler.ts +6 -3
- package/src/commands/med-family/cli-agent.ts +26 -23
- package/src/commands/router-refine.ts +1698 -0
- package/src/commands/router-worktree-cleanup.ts +870 -0
- package/src/commands/router.ts +59 -2452
- package/src/mesh/mesh-active-work.ts +11 -2
- package/src/mesh/mesh-event-classify.ts +22 -0
- package/src/mesh/mesh-event-forwarding.ts +36 -0
- package/src/mesh/mesh-queue-assignment.ts +38 -11
- package/src/mesh/mesh-reconcile-loop.ts +174 -4
- package/src/mesh/mesh-runtime-store.ts +27 -13
- package/src/mesh/worktree-bootstrap-config.ts +19 -0
- package/src/providers/approval-utils.ts +42 -0
- package/src/providers/cli-provider-instance.ts +87 -6
- package/src/providers/manual-attendance.ts +20 -0
- package/src/providers/provider-instance.ts +6 -1
- package/src/providers/sdk/v1/builders/cli/parse-approval.ts +13 -2
|
@@ -223,9 +223,18 @@ function buildLedgerDirectDispatchRecord(
|
|
|
223
223
|
ctx: { terminals: MeshLedgerEntry[]; nodes: any[] | undefined; now: number },
|
|
224
224
|
): { record: MeshActiveWorkRecord; terminalRow: boolean } {
|
|
225
225
|
const taskId = directDispatchTaskId(dispatch);
|
|
226
|
-
const
|
|
226
|
+
const matching = ctx.terminals
|
|
227
227
|
.filter(entry => new Date(entry.timestamp).getTime() >= new Date(dispatch.timestamp).getTime())
|
|
228
|
-
.
|
|
228
|
+
.filter(entry => terminalMatchesDispatch(entry, dispatch, taskId));
|
|
229
|
+
// APPROVAL-Q1-REALTIME (stale level state): prefer a REAL terminal (task_completed /
|
|
230
|
+
// task_failed) over an earlier task_approval_needed for the same dispatch. An approval
|
|
231
|
+
// that was subsequently resolved — the worker went on to complete or fail — must NOT keep
|
|
232
|
+
// the node pinned to awaiting_approval, which would falsely tell the coordinator (via
|
|
233
|
+
// mesh_status/read_chat) the worker is still blocked (the UX inversion this fix avoids).
|
|
234
|
+
// Among real terminals the earliest still wins (unchanged); approval-needed is selected
|
|
235
|
+
// only when no real terminal followed it. `terminals` is sorted ascending, so `.find`
|
|
236
|
+
// returns the earliest real terminal.
|
|
237
|
+
const terminal = matching.find(entry => entry.kind !== 'task_approval_needed') || matching[0];
|
|
229
238
|
const terminalStatus = terminal ? statusFromTerminal(terminal) : undefined;
|
|
230
239
|
const live = sessionStatusFromNodes(ctx.nodes, dispatch.nodeId, dispatch.sessionId);
|
|
231
240
|
const status = terminalStatus || live.status || 'assigned';
|
|
@@ -49,3 +49,25 @@ export const MESH_FORCE_INJECT_EVENTS: ReadonlySet<string> = new Set([
|
|
|
49
49
|
export function shouldForceInjectMeshEvent(eventName: unknown): boolean {
|
|
50
50
|
return typeof eventName === 'string' && MESH_FORCE_INJECT_EVENTS.has(eventName);
|
|
51
51
|
}
|
|
52
|
+
|
|
53
|
+
// APPROVAL-Q1-REALTIME. Approval-kind coordinator events: a worker is blocked on an
|
|
54
|
+
// approval prompt and needs the coordinator to act (mesh_approve). Approval is treated
|
|
55
|
+
// differently from a completion in the reconcile loop's no-idle hold, and the reason is
|
|
56
|
+
// WHERE each event's authoritative state lives:
|
|
57
|
+
// - A completion's payload (finalSummary / worker result) exists ONLY in the pending
|
|
58
|
+
// event, so a drain-without-inject loses it forever → it MUST ride the idle-edge hold
|
|
59
|
+
// until it can land in the coordinator as a real turn.
|
|
60
|
+
// - An approval's authoritative state is recorded at LEVEL in the ledger the moment the
|
|
61
|
+
// event is processed (task_approval_needed → mesh_status awaiting_approval, see
|
|
62
|
+
// onMeshCoordinatorEventForwarded + mesh-active-work). The pending approval event is
|
|
63
|
+
// therefore only a real-time NUDGE, not the source of truth: it can be delivered to a
|
|
64
|
+
// busy coordinator's inbox (and dropped) without data loss, because the level state
|
|
65
|
+
// re-derives it. That is why approval is exempt from the idle-edge hold completions
|
|
66
|
+
// require, and why a stale/resolved approval nudge can simply be dropped.
|
|
67
|
+
export const MESH_APPROVAL_EVENTS: ReadonlySet<string> = new Set([
|
|
68
|
+
'agent:waiting_approval',
|
|
69
|
+
]);
|
|
70
|
+
|
|
71
|
+
export function isMeshApprovalEvent(eventName: unknown): boolean {
|
|
72
|
+
return typeof eventName === 'string' && MESH_APPROVAL_EVENTS.has(eventName);
|
|
73
|
+
}
|
|
@@ -18,10 +18,12 @@ import { resolveDelegatedWorkerAutoApprove } from '../repo-mesh-types.js';
|
|
|
18
18
|
import { meshNodeIdMatches, daemonIdsEquivalent, expandDaemonIdForms, sessionIdsEquivalent, type MeshNodeIdentified } from '@adhdev/mesh-shared';
|
|
19
19
|
import {
|
|
20
20
|
findRecentTerminalLedgerEvidence,
|
|
21
|
+
findTerminalLedgerEvidenceForTask,
|
|
21
22
|
hasDispatchAfterTerminal,
|
|
22
23
|
hasUnterminalDirectDispatchLedgerEntry,
|
|
23
24
|
buildNoProgressCompletionReconciliation,
|
|
24
25
|
} from './mesh-events-stale.js';
|
|
26
|
+
import { endTaskDispatchInFlight } from './mesh-task-inflight.js';
|
|
25
27
|
import {
|
|
26
28
|
buildMeshSystemMessage,
|
|
27
29
|
readNonEmptyString,
|
|
@@ -833,6 +835,40 @@ function injectMeshSystemMessage(components: DaemonComponents, args: {
|
|
|
833
835
|
updateDirectDispatchStatus(args.meshId, sessionId, outcome, eventTaskId);
|
|
834
836
|
}
|
|
835
837
|
markSessionDeliveriesTerminal(args.meshId, sessionId, outcome);
|
|
838
|
+
// COMPLETION-PROPAGATION F2 (double safety net): the flip found no matching assigned
|
|
839
|
+
// row (task === null) but the completion echoed a taskId AND a queue row for that id is
|
|
840
|
+
// STILL 'assigned'. This is the stranded flip-miss case F1's equivalence match is meant
|
|
841
|
+
// to reconcile — NOT a direct dispatch (which legitimately has no queue row and is
|
|
842
|
+
// covered by updateDirectDispatchStatus above). Record a terminal ledger entry keyed by
|
|
843
|
+
// the echoed taskId and release the single-flight lock, so the reconcile PHASE 2.5
|
|
844
|
+
// terminal-ledger branch (findTerminalLedgerEvidenceForTask by row.id) has a taskId-based
|
|
845
|
+
// path to flip the stranded row terminal even if the direct SQL flip could not resolve
|
|
846
|
+
// it, and a subsequent reclaim/requeue is not blocked by a stale in-flight mark. Gated
|
|
847
|
+
// to GENUINE terminals (a weak / false-idle completion is left for the transcript
|
|
848
|
+
// reconcile, matching the leaveDirectDispatchActive philosophy above).
|
|
849
|
+
const genuineTerminal = outcome === 'failed' || !isWeakCompletionEvidence(args.metadataEvent);
|
|
850
|
+
if (!task && eventTaskId && genuineTerminal) {
|
|
851
|
+
try {
|
|
852
|
+
const strandedRow = MeshRuntimeStore.getInstance().findQueueEntryById(args.meshId, eventTaskId);
|
|
853
|
+
if (strandedRow && strandedRow.status === 'assigned') {
|
|
854
|
+
endTaskDispatchInFlight(args.meshId, eventTaskId);
|
|
855
|
+
if (!findTerminalLedgerEvidenceForTask({ meshId: args.meshId, taskId: eventTaskId })) {
|
|
856
|
+
appendLedgerEntry(args.meshId, {
|
|
857
|
+
kind: outcome === 'completed' ? 'task_completed' : 'task_failed',
|
|
858
|
+
sessionId,
|
|
859
|
+
nodeId: readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId) || undefined,
|
|
860
|
+
providerType: readNonEmptyString(args.metadataEvent.providerType) || undefined,
|
|
861
|
+
payload: {
|
|
862
|
+
taskId: eventTaskId,
|
|
863
|
+
event: args.event,
|
|
864
|
+
source: 'flip_miss_safety_net',
|
|
865
|
+
finalSummary: readNonEmptyString(args.metadataEvent.finalSummary) || undefined,
|
|
866
|
+
},
|
|
867
|
+
});
|
|
868
|
+
}
|
|
869
|
+
}
|
|
870
|
+
} catch { /* best-effort safety net — never fail the completion path */ }
|
|
871
|
+
}
|
|
836
872
|
setImmediate(() => cleanupTerminalDirectDispatches());
|
|
837
873
|
return task ? { id: task.id } : null;
|
|
838
874
|
}
|
|
@@ -19,7 +19,7 @@ import { normalizeMeshNodeId, meshNodeIdMatches, daemonIdsEquivalent, canonicalD
|
|
|
19
19
|
import { findTerminalLedgerEvidenceForTask, hasUnterminalDirectDispatchLedgerEntry } from './mesh-events-stale.js';
|
|
20
20
|
import { readNonEmptyString } from './mesh-events-utils.js';
|
|
21
21
|
import { queuePendingMeshCoordinatorEvent, retractPendingDispatchBlockedEvent } from './mesh-events-pending.js';
|
|
22
|
-
import { isWorktreeBootstrapStaleRunning } from './worktree-bootstrap-config.js';
|
|
22
|
+
import { isWorktreeBootstrapStaleRunning, shouldDeferDispatchForBootstrap } from './worktree-bootstrap-config.js';
|
|
23
23
|
import { isWithinCloneBootstrapGrace } from './mesh-clone-grace.js';
|
|
24
24
|
import { beginTaskDispatchInFlight, endTaskDispatchInFlight } from './mesh-task-inflight.js';
|
|
25
25
|
|
|
@@ -318,13 +318,32 @@ export function tryAssignQueueTask(
|
|
|
318
318
|
// 3-form normalizer the defer guard uses), never a raw === — canon-identity regression guard.
|
|
319
319
|
// Conservative: any non-'running' status (idle/complete/failed/absent/unknown) does NOT gate,
|
|
320
320
|
// so a base node and a fully-bootstrapped worktree keep prior behavior exactly.
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
321
|
+
// COMPLETION-PROPAGATION F7 (C2 SSOT): resolve the node's bootstrap status from the router's
|
|
322
|
+
// synchronous inline cache FIRST — the authoritative source markWorktreeBootstrapTerminalState
|
|
323
|
+
// stamps synchronously — falling back to the merged claim view only when the inline node carries
|
|
324
|
+
// no bootstrap status. getMeshWithCache takes a config-REGISTERED node verbatim from local
|
|
325
|
+
// config, whose bootstrap status lags the inline stamp (the detached async persist chain), so a
|
|
326
|
+
// config-registered worktree node could read a stale 'running' here and defer a claim whose
|
|
327
|
+
// bootstrap is already complete. Reading the inline node removes that stale-'running' defer,
|
|
328
|
+
// symmetric with the remote dispatch guard (cli-agent.ts F6). Conservative: only override with
|
|
329
|
+
// the inline node when it actually carries a status (an incomplete inline entry never masks a
|
|
330
|
+
// genuine config 'running').
|
|
331
|
+
const inlineBootstrapNode = (() => {
|
|
332
|
+
try {
|
|
333
|
+
const inlineMesh = components.router?.getCachedInlineMesh?.(meshId);
|
|
334
|
+
const inlineNode = Array.isArray(inlineMesh?.nodes)
|
|
335
|
+
? inlineMesh.nodes.find((n: any) => meshNodeIdMatches(n, nodeId))
|
|
336
|
+
: undefined;
|
|
337
|
+
return readNonEmptyString(inlineNode?.worktreeBootstrap?.status) ? inlineNode : undefined;
|
|
338
|
+
} catch { return undefined; }
|
|
339
|
+
})();
|
|
340
|
+
const bootstrapGateNode = inlineBootstrapNode ?? node;
|
|
341
|
+
if ((bootstrapGateNode as { worktreeBootstrap?: { status?: string } } | undefined)?.worktreeBootstrap?.status === 'running') {
|
|
342
|
+
// Fix (3) safety net + F7: shouldDeferDispatchForBootstrap returns false when the 'running'
|
|
343
|
+
// state is stale (older than the backstop AND git-clean) — treat that as silently complete
|
|
344
|
+
// and allow the claim; otherwise defer (leave the task pending) so the claim re-fires once
|
|
345
|
+
// bootstrap reaches a terminal state and never dispatches into a half-built worktree.
|
|
346
|
+
if (!shouldDeferDispatchForBootstrap(bootstrapGateNode as any)) {
|
|
328
347
|
LOG.warn('MeshQueue', `Worktree node ${nodeId} (${sessionId}) bootstrap stuck 'running' beyond the stale backstop and its worktree is git-clean — treating bootstrap as silently complete and allowing the claim (the terminal-state stamp likely never reached this daemon's mesh view)`);
|
|
329
348
|
} else {
|
|
330
349
|
LOG.info('MeshQueue', `Gating queue claim for worktree node ${nodeId} (${sessionId}): worktree bootstrap still running — task left pending; claim re-fires once bootstrap reaches a terminal state (guards against dispatching into a half-built worktree → empty session)`);
|
|
@@ -521,9 +540,17 @@ export function tryAssignQueueTask(
|
|
|
521
540
|
launchedByCoordinator: true,
|
|
522
541
|
autoApprove: resolveDelegatedWorkerAutoApprove(mesh?.policy, node?.policy),
|
|
523
542
|
...(localDaemonId ? { meshCoordinatorDaemonId: localDaemonId } : {}),
|
|
524
|
-
// (
|
|
525
|
-
//
|
|
526
|
-
|
|
543
|
+
// COMPLETION-PROPAGATION F5: (re)stamp the coordinator SESSION anchor from THIS
|
|
544
|
+
// task's sourceCoordinatorSessionId with PRIORITY — a manually-launched (or reused)
|
|
545
|
+
// session may already carry a stale anchor from mesh_launch_session or a prior task,
|
|
546
|
+
// and a stale session anchor makes the completion unicast to the wrong/absent
|
|
547
|
+
// coordinator session (targetCoordinatorSessionId), stranding it. When this task
|
|
548
|
+
// carries a source, overwrite; when it carries NONE, CLEAR the anchor to undefined
|
|
549
|
+
// (updateSettings merges, so an explicit undefined overrides) so the completion
|
|
550
|
+
// cannot be misrouted by a stale unicast anchor and instead BROADCASTS — the real
|
|
551
|
+
// coordinator (which drains its own pending queue) then picks it up. Daemon-level
|
|
552
|
+
// routing (meshCoordinatorDaemonId above) is unaffected.
|
|
553
|
+
meshCoordinatorSessionId: localSourceCoordinatorSessionId || undefined,
|
|
527
554
|
});
|
|
528
555
|
}
|
|
529
556
|
} catch { /* best-effort — dispatch still proceeds */ }
|
|
@@ -50,6 +50,7 @@ import type { PendingMeshCoordinatorEvent } from './mesh-events-pending.js';
|
|
|
50
50
|
import { appendLedgerEntry } from './mesh-ledger.js';
|
|
51
51
|
import { MeshRuntimeStore } from './mesh-runtime-store.js';
|
|
52
52
|
import { handleMeshForwardEvent, shouldForceInjectMeshEvent, triggerMeshQueue, resolveForwardEventMeshId } from './mesh-events-coordinator.js';
|
|
53
|
+
import { isMeshApprovalEvent, MESH_APPROVAL_EVENTS } from './mesh-event-classify.js';
|
|
53
54
|
import {
|
|
54
55
|
peekUnresolvedDelegateForwards,
|
|
55
56
|
ackUnresolvedDelegateForward,
|
|
@@ -59,7 +60,9 @@ import { readNonEmptyString, readMeshCompletionSummary, buildMeshSystemMessage }
|
|
|
59
60
|
import { traceMeshEventStage, traceMeshEventDrop } from './mesh-event-trace.js';
|
|
60
61
|
import { expandDaemonIdForms, daemonIdsEquivalent, sessionIdsEquivalent } from '@adhdev/mesh-shared';
|
|
61
62
|
import { getActiveDirectDispatches, getQueue, reclaimStrandedAssignedTask, updateTaskStatus } from './mesh-work-queue.js';
|
|
63
|
+
import { isSessionActivelyGenerating } from './mesh-queue-assignment.js';
|
|
62
64
|
import { readLedgerEntries } from './mesh-ledger.js';
|
|
65
|
+
import type { MeshLedgerEntry } from './mesh-ledger.js';
|
|
63
66
|
import { pruneStaleDirectDispatches } from './mesh-active-work.js';
|
|
64
67
|
import { findTerminalLedgerEvidenceForTask, reconcileDirectDispatchCompletionFromTranscript } from './mesh-events-stale.js';
|
|
65
68
|
import { extractFinalAssistantSummaryEvidence } from '../providers/chat-message-normalization.js';
|
|
@@ -569,6 +572,7 @@ export function shouldHoldPendingDrainForBusyLocalCoordinator(
|
|
|
569
572
|
function injectPendingIntoCoordinator(
|
|
570
573
|
coordinator: LiveCoordinator['instance'],
|
|
571
574
|
pending: PendingMeshCoordinatorEvent,
|
|
575
|
+
opts?: { forceOverride?: boolean },
|
|
572
576
|
): void {
|
|
573
577
|
if (!coordinator) return;
|
|
574
578
|
// NOTIF-DROP-SYNTH-NO-MESSAGE (defence-in-depth): a queued event with no coordinatorMessage
|
|
@@ -594,7 +598,11 @@ function injectPendingIntoCoordinator(
|
|
|
594
598
|
if (!coordinatorMessage) return; // builder produced nothing — nothing to surface
|
|
595
599
|
LOG.warn('MeshReconcile', `Lazily synthesized missing coordinatorMessage for ${pending.event} (mesh ${pending.meshId}) at inject time — a queued terminal event arrived message-less`);
|
|
596
600
|
}
|
|
597
|
-
|
|
601
|
+
// forceOverride lets the APPROVAL-Q1-REALTIME nudge path deliver into a busy
|
|
602
|
+
// coordinator WITHOUT a raw PTY force-write (force-inject-into-generating stays
|
|
603
|
+
// intentionally removed): a non-force send_message enters the adapter's
|
|
604
|
+
// pendingOutboundQueue and is surfaced at the coordinator's next turn boundary.
|
|
605
|
+
const force = opts?.forceOverride ?? shouldForceInjectMeshEvent(pending.event);
|
|
598
606
|
// EVTTRACE: event surfaced to the coordinator (injected into its live CLI session).
|
|
599
607
|
// This is the terminal happy-path stage. Observation only.
|
|
600
608
|
traceMeshEventStage('surfaced', {
|
|
@@ -775,6 +783,112 @@ function drainAndInjectIntoTargets(
|
|
|
775
783
|
return pendingEvents.length;
|
|
776
784
|
}
|
|
777
785
|
|
|
786
|
+
// APPROVAL-Q1-REALTIME stale guard. An approval nudge is RESOLVED once a real terminal
|
|
787
|
+
// ledger entry (task_completed / task_failed) for the same node/session landed at or
|
|
788
|
+
// after the nudge was queued — the worker either finished or died, so it is no longer
|
|
789
|
+
// waiting on that approval. Delivering the nudge then would falsely tell the coordinator
|
|
790
|
+
// the worker is still blocked (the exact UX inversion this fix must avoid), so a resolved
|
|
791
|
+
// nudge is dropped rather than delivered. Ledger-based so the check is daemon-local and
|
|
792
|
+
// deterministic (no dependence on a possibly-remote worker instance's live state).
|
|
793
|
+
function isApprovalNudgeResolved(meshId: string, pending: PendingMeshCoordinatorEvent): boolean {
|
|
794
|
+
const metadataEvent = (pending.metadataEvent && typeof pending.metadataEvent === 'object')
|
|
795
|
+
? pending.metadataEvent as Record<string, unknown>
|
|
796
|
+
: {};
|
|
797
|
+
const nodeId = readNonEmptyString(pending.nodeId) || readNonEmptyString(metadataEvent.meshNodeId);
|
|
798
|
+
const sessionId = readNonEmptyString(metadataEvent.targetSessionId) || readNonEmptyString(metadataEvent.sessionId);
|
|
799
|
+
if (!nodeId && !sessionId) return false; // nothing to correlate a terminal against
|
|
800
|
+
const queuedAt = typeof pending.queuedAt === 'number' && Number.isFinite(pending.queuedAt) ? pending.queuedAt : 0;
|
|
801
|
+
let entries: MeshLedgerEntry[];
|
|
802
|
+
try {
|
|
803
|
+
entries = readLedgerEntries(meshId);
|
|
804
|
+
} catch {
|
|
805
|
+
return false; // best-effort — a read failure never blocks delivery
|
|
806
|
+
}
|
|
807
|
+
return entries.some(e => {
|
|
808
|
+
if (e.kind !== 'task_completed' && e.kind !== 'task_failed') return false;
|
|
809
|
+
if (queuedAt > 0) {
|
|
810
|
+
const t = new Date(e.timestamp).getTime();
|
|
811
|
+
if (Number.isFinite(t) && t < queuedAt) return false; // terminal predates the nudge
|
|
812
|
+
}
|
|
813
|
+
const nodeMatch = !!nodeId && !!e.nodeId && daemonIdsEquivalent(e.nodeId, nodeId);
|
|
814
|
+
const sessionMatch = !!sessionId && !!e.sessionId && sessionIdsEquivalent(e.sessionId, sessionId);
|
|
815
|
+
return nodeMatch || sessionMatch;
|
|
816
|
+
});
|
|
817
|
+
}
|
|
818
|
+
|
|
819
|
+
// APPROVAL-Q1-REALTIME. Deliver queued approval nudges to a mesh's coordinators every
|
|
820
|
+
// reconcile tick, EVEN when the only coordinators are busy (generating / modal-parked)
|
|
821
|
+
// and there is no idle drain target. This is the crux of the fix: a completion rides the
|
|
822
|
+
// idle-edge hold below (its payload lives only in the pending event), but an approval is
|
|
823
|
+
// LEVEL-backed (task_approval_needed ledger → mesh_status awaiting_approval) so it must
|
|
824
|
+
// NOT wait for an idle edge — during orchestration a coordinator can stay `generating`
|
|
825
|
+
// awaiting the very worker that is blocked on the approval, so the idle edge (the flush
|
|
826
|
+
// point) may never come, and the coordinator's mesh_approve arrives only after a human
|
|
827
|
+
// resolves it ('Not in approval state'). We drain ONLY approval events (leaving every
|
|
828
|
+
// other event for the unchanged hold), drop any already-resolved (stale) nudge, and
|
|
829
|
+
// deliver the rest into each coordinator's inbox WITHOUT a raw PTY force-write (non-force
|
|
830
|
+
// send_message → adapter pendingOutboundQueue → surfaced at the coordinator's next turn
|
|
831
|
+
// boundary). Dropping the pending copy after delivery is safe and prevents re-nudging
|
|
832
|
+
// every 4s — the level ledger state remains the durable, re-derivable source of truth.
|
|
833
|
+
// Returns the number of nudges delivered (0 when none were queued/deliverable).
|
|
834
|
+
function drainAndDeliverApprovalNudges(
|
|
835
|
+
meshId: string,
|
|
836
|
+
drainDaemonIds: string[],
|
|
837
|
+
localDaemonId: string | undefined,
|
|
838
|
+
meshCoordinators: LiveCoordinator[],
|
|
839
|
+
): number {
|
|
840
|
+
// O(1) guard: only touch the queue when an approval event is actually present.
|
|
841
|
+
let peeked: readonly PendingMeshCoordinatorEvent[];
|
|
842
|
+
try {
|
|
843
|
+
peeked = getPendingMeshCoordinatorEvents(meshId, drainDaemonIds.length > 0 ? drainDaemonIds : undefined);
|
|
844
|
+
} catch {
|
|
845
|
+
return 0;
|
|
846
|
+
}
|
|
847
|
+
if (!peeked.some(e => isMeshApprovalEvent(e.event))) return 0;
|
|
848
|
+
|
|
849
|
+
let drained: PendingMeshCoordinatorEvent[];
|
|
850
|
+
try {
|
|
851
|
+
drained = drainPendingMeshCoordinatorEvents(
|
|
852
|
+
meshId,
|
|
853
|
+
drainDaemonIds.length > 0 ? drainDaemonIds : localDaemonId,
|
|
854
|
+
{ onlyEvents: MESH_APPROVAL_EVENTS },
|
|
855
|
+
);
|
|
856
|
+
} catch (e: any) {
|
|
857
|
+
LOG.warn('MeshReconcile', `Approval-nudge drain failed for mesh ${meshId}: ${e?.message || e}`);
|
|
858
|
+
return 0;
|
|
859
|
+
}
|
|
860
|
+
|
|
861
|
+
let delivered = 0;
|
|
862
|
+
for (const pending of drained) {
|
|
863
|
+
if (isApprovalNudgeResolved(meshId, pending)) {
|
|
864
|
+
// Stale: already resolved. Drop without delivery — re-surfacing it would
|
|
865
|
+
// mislead the coordinator into believing the worker is still awaiting approval.
|
|
866
|
+
traceMeshEventDrop('approval_nudge_stale_resolved', {
|
|
867
|
+
taskId: readNonEmptyString((pending.metadataEvent as Record<string, unknown>)?.taskId),
|
|
868
|
+
sessionId: readNonEmptyString((pending.metadataEvent as Record<string, unknown>)?.targetSessionId) ?? pending.targetCoordinatorSessionId,
|
|
869
|
+
nodeId: pending.nodeId,
|
|
870
|
+
meshId,
|
|
871
|
+
event: pending.event,
|
|
872
|
+
}, 'approval already resolved (terminal ledger entry present)');
|
|
873
|
+
LOG.info('MeshReconcile', `Dropped stale approval nudge for mesh ${meshId} (${pending.nodeLabel}) — approval already resolved`);
|
|
874
|
+
continue;
|
|
875
|
+
}
|
|
876
|
+
// Strict session routing (multi-coordinator): deliver only to the originating
|
|
877
|
+
// coordinator session when the nudge names one; otherwise broadcast to every
|
|
878
|
+
// coordinator for this mesh. Absent a live matching coordinator we drop the nudge —
|
|
879
|
+
// the level state (awaiting_approval) still surfaces via mesh_status, so nothing is lost.
|
|
880
|
+
const wantSession = readNonEmptyString(pending.targetCoordinatorSessionId);
|
|
881
|
+
const targets = wantSession
|
|
882
|
+
? meshCoordinators.filter(c => sessionIdsEquivalent(c.sessionId, wantSession))
|
|
883
|
+
: meshCoordinators;
|
|
884
|
+
if (targets.length === 0) continue;
|
|
885
|
+
for (const c of targets) injectPendingIntoCoordinator(c.instance, pending, { forceOverride: false });
|
|
886
|
+
delivered++;
|
|
887
|
+
LOG.info('MeshReconcile', `Delivered approval nudge (level) for mesh ${meshId} (${pending.nodeLabel}) → ${targets.length} coordinator(s) without waiting for an idle edge`);
|
|
888
|
+
}
|
|
889
|
+
return delivered;
|
|
890
|
+
}
|
|
891
|
+
|
|
778
892
|
// One reconcile tick. Two independent phases:
|
|
779
893
|
//
|
|
780
894
|
// PHASE 1 — Remote queue pull (the fix for remote worktree completions never
|
|
@@ -802,6 +916,16 @@ function drainAndInjectIntoTargets(
|
|
|
802
916
|
// lost to a daemon restart between claim and confirm).
|
|
803
917
|
const ASSIGNED_STRANDED_DEADLINE_MS = 5 * 60_000;
|
|
804
918
|
|
|
919
|
+
// COMPLETION-PROPAGATION F3: how long a row may sit 'assigned' with a CONFIRMED delivery
|
|
920
|
+
// (delivered/acked) but no terminal completion before the watchdog reclaims it as a
|
|
921
|
+
// delivered-but-lost completion. Distinct from — and deliberately larger than —
|
|
922
|
+
// ASSIGNED_STRANDED_DEADLINE_MS: a confirmed-delivered dispatch was genuinely handed to a
|
|
923
|
+
// worker, so the deadline must comfortably exceed any realistic single worker turn (a large
|
|
924
|
+
// generation) before we treat the missing completion as lost and re-open the task. Paired with
|
|
925
|
+
// the non-generating + no-terminal-ledger guards below so a worker still mid-turn is never
|
|
926
|
+
// reclaimed out from under itself.
|
|
927
|
+
const DELIVERED_NO_TURN_DEADLINE_MS = 15 * 60_000;
|
|
928
|
+
|
|
805
929
|
// PHASE 2.5 — assigned-stranded dispatch watchdog (Bug B). claimNextTask atomically
|
|
806
930
|
// flips a row to 'assigned' BEFORE the fire-and-forget dispatch runs. If that dispatch
|
|
807
931
|
// neither rejects (→ no .catch requeue) nor is confirmed delivered — a relay that hangs
|
|
@@ -814,7 +938,7 @@ const ASSIGNED_STRANDED_DEADLINE_MS = 5 * 60_000;
|
|
|
814
938
|
// never reclaimed here. And the deadline is generous so a slow-but-live dispatch still in
|
|
815
939
|
// its normal confirm window is never reclaimed early. Reclaimed rows return to 'pending'
|
|
816
940
|
// with ownership cleared, so the PHASE 3 trigger below re-dispatches them this same tick.
|
|
817
|
-
function recoverStrandedAssignedDispatches(meshId: string, store: MeshRuntimeStore): void {
|
|
941
|
+
function recoverStrandedAssignedDispatches(components: DaemonComponents, meshId: string, store: MeshRuntimeStore): void {
|
|
818
942
|
const assigned = getQueue(meshId, { status: ['assigned'] });
|
|
819
943
|
if (!assigned.length) return;
|
|
820
944
|
const nowMs = Date.now();
|
|
@@ -839,7 +963,37 @@ function recoverStrandedAssignedDispatches(meshId: string, store: MeshRuntimeSto
|
|
|
839
963
|
}, terminal.kind);
|
|
840
964
|
continue;
|
|
841
965
|
}
|
|
842
|
-
if (store.taskHasConfirmedDelivery(meshId, row.id))
|
|
966
|
+
if (store.taskHasConfirmedDelivery(meshId, row.id)) {
|
|
967
|
+
// COMPLETION-PROPAGATION F3 (delivered-but-lost completion): the dispatch WAS
|
|
968
|
+
// confirmed handed to a worker (delivered/acked) but no terminal completion ever
|
|
969
|
+
// landed and none is in the ledger (checked just above). Normally this is PHASE 4's
|
|
970
|
+
// job, but PHASE 4 only covers direct-dispatch rows / a live re-read; a claim-path
|
|
971
|
+
// queue row whose completion event was lost (the manual-launch flip-miss signature)
|
|
972
|
+
// sits 'assigned' forever. Reclaim it — but ONLY once the session is idle/dead (its
|
|
973
|
+
// live local instance is not actively generating; a remote/absent instance reports
|
|
974
|
+
// non-generating too) AND a generous delivered-no-turn deadline has elapsed, so a
|
|
975
|
+
// worker genuinely mid-turn is never torn off its task. reclaimStrandedAssignedTask
|
|
976
|
+
// ends the single-flight window (F4), so a subsequent re-dispatch/requeue is unblocked.
|
|
977
|
+
if (nowMs - dispatchedAtMs < DELIVERED_NO_TURN_DEADLINE_MS) continue; // still within turn budget
|
|
978
|
+
if (row.assignedSessionId && isSessionActivelyGenerating(components, row.assignedSessionId)) continue; // worker still working
|
|
979
|
+
const reclaimedLost = reclaimStrandedAssignedTask(meshId, row.id, {
|
|
980
|
+
reason: 'delivered_no_turn_deadline',
|
|
981
|
+
ageMs: nowMs - dispatchedAtMs,
|
|
982
|
+
});
|
|
983
|
+
if (reclaimedLost) {
|
|
984
|
+
LOG.warn('MeshReconcile', `Reclaimed delivered-but-lost task ${row.id} on mesh ${meshId} `
|
|
985
|
+
+ `(node=${row.assignedNodeId ?? '?'} session=${row.assignedSessionId ?? '?'}, delivered but no `
|
|
986
|
+
+ `completion in ${Math.round((nowMs - dispatchedAtMs) / 1000)}s, session non-generating → ${reclaimedLost.status})`);
|
|
987
|
+
traceMeshEventDrop('assigned_stranded_delivered_no_turn', {
|
|
988
|
+
taskId: row.id,
|
|
989
|
+
sessionId: row.assignedSessionId,
|
|
990
|
+
nodeId: row.assignedNodeId,
|
|
991
|
+
meshId,
|
|
992
|
+
event: 'agent:generating_completed',
|
|
993
|
+
}, `delivered ${Math.round((nowMs - dispatchedAtMs) / 1000)}s → ${reclaimedLost.status}`);
|
|
994
|
+
}
|
|
995
|
+
continue;
|
|
996
|
+
}
|
|
843
997
|
const reclaimed = reclaimStrandedAssignedTask(meshId, row.id, {
|
|
844
998
|
reason: 'assigned_stranded_dispatch_unconfirmed',
|
|
845
999
|
ageMs: nowMs - dispatchedAtMs,
|
|
@@ -914,7 +1068,7 @@ export async function runMeshReconcileTick(components: DaemonComponents): Promis
|
|
|
914
1068
|
const selfIds = resolveCoordinatorSelfIds(mesh, drainDaemonIds);
|
|
915
1069
|
if (!daemonHostsMesh(mesh, selfIds)) continue;
|
|
916
1070
|
try {
|
|
917
|
-
recoverStrandedAssignedDispatches(mesh.id, store);
|
|
1071
|
+
recoverStrandedAssignedDispatches(components, mesh.id, store);
|
|
918
1072
|
} catch (e: any) {
|
|
919
1073
|
LOG.warn('MeshReconcile', `Assigned-stranded watchdog failed for mesh ${mesh.id}: ${e?.message || e}`);
|
|
920
1074
|
}
|
|
@@ -1075,6 +1229,22 @@ export async function runMeshReconcileTick(components: DaemonComponents): Promis
|
|
|
1075
1229
|
// worker summary is recoverable even if the coordinator never returns or the pending
|
|
1076
1230
|
// file is later trimmed.
|
|
1077
1231
|
if (targetCoordinators.length === 0) {
|
|
1232
|
+
// ── APPROVAL-Q1-REALTIME: level-deliver approval nudges BEFORE the hold ──
|
|
1233
|
+
// Approval events are LEVEL-backed (task_approval_needed ledger →
|
|
1234
|
+
// mesh_status awaiting_approval), so they must not be edge-held like a
|
|
1235
|
+
// completion (whose payload lives only in the pending event). Drain and
|
|
1236
|
+
// deliver them to the busy coordinator's inbox (non-force, next-turn-boundary)
|
|
1237
|
+
// this tick, dropping any already-resolved (stale) nudge — and leave ONLY the
|
|
1238
|
+
// completion/other events in the queue for the existing hold semantics below
|
|
1239
|
+
// (their behaviour is unchanged: shouldForceInjectMeshEvent no longer sees the
|
|
1240
|
+
// approval rows because this drained them). MUST run first so the modal-park
|
|
1241
|
+
// orphan-escape and the generating-hold audit only ever see non-approval events.
|
|
1242
|
+
drainAndDeliverApprovalNudges(meshId, drainDaemonIds, localDaemonId, meshCoordinators);
|
|
1243
|
+
// If approval nudges were the only queued events, nothing remains to hold — skip
|
|
1244
|
+
// the hold branches (and their "holding pending event(s)" log) entirely.
|
|
1245
|
+
if (store) {
|
|
1246
|
+
try { if (store.pendingEventCount(meshId) === 0) continue; } catch { /* fall through */ }
|
|
1247
|
+
}
|
|
1078
1248
|
if (modalParkedCoordinators.length > 0) {
|
|
1079
1249
|
// ── orphan escape (MUST precede the blanket modal-park hold) ──────────
|
|
1080
1250
|
// A modal-parked coordinator with no idle/generating sibling otherwise
|
|
@@ -883,25 +883,39 @@ export class MeshRuntimeStore {
|
|
|
883
883
|
): MeshWorkQueueEntry | null {
|
|
884
884
|
this.ensureLegacyQueueMigrated(meshId);
|
|
885
885
|
|
|
886
|
-
//
|
|
886
|
+
// WRITE/READ PREDICATE SYMMETRY (COMPLETION-PROPAGATION F1): the claim path writes
|
|
887
|
+
// assigned_session_id RAW (claimNextTask), and the sibling gates that decide whether a
|
|
888
|
+
// session already holds work (sessionHasActiveAssignment) and which pending row a
|
|
889
|
+
// session may claim (targetMatches) compare it through sessionIdsEquivalent — the
|
|
890
|
+
// canonical single-form predicate that TRIMS both sides. A raw SQL `assigned_session_id
|
|
891
|
+
// = ?` here is asymmetric with that write/sibling predicate: a completion whose
|
|
892
|
+
// resolveEventSessionId-reinterpreted sessionId is equivalent-but-not-byte-identical to
|
|
893
|
+
// the stored column (e.g. a whitespace/serialization skew from a manually-launched
|
|
894
|
+
// session) silently fetched zero rows and stranded the finished task as `assigned`
|
|
895
|
+
// forever (the mesh-work-queue :1251 "N assigned row(s) exist" warning is that exact
|
|
896
|
+
// signature). Fetch every `assigned` row for the mesh and filter session membership in
|
|
897
|
+
// JS with sessionIdsEquivalent, mirroring the node-id IN(...)+JS-revalidate pattern the
|
|
898
|
+
// claim SELECT uses (claimNextTask :720-736 / targetMatches :797-811).
|
|
899
|
+
const allRows = this.db.prepare(
|
|
900
|
+
`SELECT payload FROM mesh_queue WHERE mesh_id = ? AND status = 'assigned'`
|
|
901
|
+
).all(meshId) as Array<{ payload: string }>;
|
|
902
|
+
const sessionEntries = allRows
|
|
903
|
+
.map(r => { try { return JSON.parse(r.payload) as MeshWorkQueueEntry; } catch { return null; } })
|
|
904
|
+
.filter((e): e is MeshWorkQueueEntry => e !== null)
|
|
905
|
+
.filter(e => sessionIdsEquivalent(e.assignedSessionId, sessionId));
|
|
906
|
+
|
|
907
|
+
// 1. Exact taskId match — robust against clock skew and stale rows. Scoped to the
|
|
908
|
+
// session-equivalent set (as the raw `AND assigned_session_id = ? AND id = ?` was),
|
|
909
|
+
// now via the trimming equivalence predicate.
|
|
887
910
|
if (taskId) {
|
|
888
|
-
const
|
|
889
|
-
|
|
890
|
-
).get(meshId, sessionId, taskId) as { payload: string } | undefined;
|
|
891
|
-
if (row) return JSON.parse(row.payload) as MeshWorkQueueEntry;
|
|
911
|
+
const byId = sessionEntries.find(e => e.id === taskId);
|
|
912
|
+
if (byId) return byId;
|
|
892
913
|
// Fall through to session-based matching if the id didn't line up
|
|
893
914
|
// (e.g. event carried a stale/foreign taskId).
|
|
894
915
|
}
|
|
895
916
|
|
|
896
917
|
// 2. Session-based match WITHOUT the mutable updated_at filter.
|
|
897
|
-
const
|
|
898
|
-
`SELECT payload FROM mesh_queue WHERE mesh_id = ? AND assigned_session_id = ? AND status = 'assigned'`
|
|
899
|
-
).all(meshId, sessionId) as Array<{ payload: string }>;
|
|
900
|
-
if (rows.length === 0) return null;
|
|
901
|
-
|
|
902
|
-
const entries = rows
|
|
903
|
-
.map(r => { try { return JSON.parse(r.payload) as MeshWorkQueueEntry; } catch { return null; } })
|
|
904
|
-
.filter((e): e is MeshWorkQueueEntry => e !== null);
|
|
918
|
+
const entries = sessionEntries;
|
|
905
919
|
if (entries.length === 0) return null;
|
|
906
920
|
if (entries.length === 1) return entries[0];
|
|
907
921
|
|
|
@@ -154,6 +154,25 @@ export function isWorktreeBootstrapStaleRunning(
|
|
|
154
154
|
}
|
|
155
155
|
}
|
|
156
156
|
|
|
157
|
+
/**
|
|
158
|
+
* COMPLETION-PROPAGATION F7 (C2): the single shared consume-ready / bootstrap-pending defer
|
|
159
|
+
* predicate. A task must NOT be injected into a worktree node whose bootstrap is still 'running'
|
|
160
|
+
* — the provider is not yet ready to consume input, so the inject lands in the input buffer and
|
|
161
|
+
* is silently swallowed (empty session). Both the remote dispatch guard (the router agent_command
|
|
162
|
+
* handler) and the local queue-claim gate (tryAssignQueueTask) route through THIS predicate so
|
|
163
|
+
* they agree on exactly when to defer. Returns true = defer. The stale-'running' backstop
|
|
164
|
+
* (isWorktreeBootstrapStaleRunning) is honored here too: a 'running' state far older than any real
|
|
165
|
+
* bootstrap whose worktree is git-clean is treated as silently complete (do NOT defer), so a node
|
|
166
|
+
* whose terminal stamp never reached this daemon is not stranded forever.
|
|
167
|
+
*/
|
|
168
|
+
export function shouldDeferDispatchForBootstrap(
|
|
169
|
+
node: { worktreeBootstrap?: { status?: string; startedAt?: string; updatedAt?: string; completedAt?: string }; workspace?: string } | undefined,
|
|
170
|
+
nowMs: number = Date.now(),
|
|
171
|
+
): boolean {
|
|
172
|
+
if (node?.worktreeBootstrap?.status !== 'running') return false;
|
|
173
|
+
return !isWorktreeBootstrapStaleRunning(node, nowMs);
|
|
174
|
+
}
|
|
175
|
+
|
|
157
176
|
export interface WorktreeBootstrapConfigLoadResult {
|
|
158
177
|
config?: RepoMeshWorktreeBootstrapConfig;
|
|
159
178
|
source: string;
|
|
@@ -41,6 +41,48 @@ export function hasNegativeApprovalOption(buttons: string[] | null | undefined):
|
|
|
41
41
|
return (buttons || []).some((button) => isNegativeApprovalLabel(String(button || '')));
|
|
42
42
|
}
|
|
43
43
|
|
|
44
|
+
/**
|
|
45
|
+
* True when a button reliably identifies a tool-CONSENT modal on its own — a
|
|
46
|
+
* scoped permission-grant affirmative such as:
|
|
47
|
+
* - "Yes, allow all edits in tmp/ during this session"
|
|
48
|
+
* - "Yes, and don't ask again for example.com"
|
|
49
|
+
* - "Yes, allow reading from etc/ from this project"
|
|
50
|
+
* - "Always allow"
|
|
51
|
+
*
|
|
52
|
+
* These options only ever appear in a genuine approval/permission prompt; a
|
|
53
|
+
* /model or /mode picker ("1. Default 2. Opus 3. Sonnet") never offers a
|
|
54
|
+
* "grant this scope" choice. They therefore serve as a SECOND reliable
|
|
55
|
+
* structural anchor alongside {@link hasNegativeApprovalOption}.
|
|
56
|
+
*
|
|
57
|
+
* Why this exists (tall-diff fallback, #137): when a Write/Edit diff is tall,
|
|
58
|
+
* the trailing decline option ("3. No") can scroll off the bottom of the
|
|
59
|
+
* captured PTY frame, leaving only "1. Yes" + "2. Yes, allow … this session".
|
|
60
|
+
* hasNegativeApprovalOption then reads false and the auto-approve gate bails —
|
|
61
|
+
* a delegated worker sits forever on a modal it could safely have approved. The
|
|
62
|
+
* grant-scope affirmative lets the gate recognize the consent modal WITHOUT
|
|
63
|
+
* seeing the off-frame decline. The gate still selects the plain "Yes"
|
|
64
|
+
* (allow-once) via pickApprovalButton, never the broader grant, and the settle
|
|
65
|
+
* gate still requires a stable modal — so a half-rendered frame never fires.
|
|
66
|
+
* Kept deliberately narrow so no picker/confirm modal can trip it.
|
|
67
|
+
*/
|
|
68
|
+
export function hasReliableApprovalAffirmative(buttons: string[] | null | undefined): boolean {
|
|
69
|
+
return (buttons || []).some((button) => {
|
|
70
|
+
const label = normalizeApprovalLabel(String(button || ''));
|
|
71
|
+
if (!label) return false;
|
|
72
|
+
// "always allow …" as a standalone grant option.
|
|
73
|
+
if (/^always allow\b/.test(label)) return true;
|
|
74
|
+
// "yes, …" scoped grants: allow / always allow / don't ask again / etc.
|
|
75
|
+
// (normalizeApprovalLabel strips the apostrophe, so "don't" → "don t").
|
|
76
|
+
if (/^yes\b/.test(label)) {
|
|
77
|
+
if (/\ballow\b/.test(label)) return true;
|
|
78
|
+
if (/\bask again\b/.test(label)) return true;
|
|
79
|
+
if (/\bduring this session\b/.test(label)) return true;
|
|
80
|
+
if (/\bfrom this project\b/.test(label)) return true;
|
|
81
|
+
}
|
|
82
|
+
return false;
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
|
|
44
86
|
export function getApprovalPositiveHints(provider?: Pick<ProviderModule, 'approvalPositiveHints'> | null): string[] {
|
|
45
87
|
const customHints = Array.isArray(provider?.approvalPositiveHints)
|
|
46
88
|
? provider.approvalPositiveHints
|