@adhdev/daemon-core 1.0.18-rc.8 → 1.0.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli-adapters/provider-cli-shared.d.ts +33 -0
- package/dist/commands/cli-manager.d.ts +9 -0
- package/dist/commands/upgrade-helper.d.ts +1 -0
- package/dist/commands/windows-atomic-upgrade.d.ts +5 -0
- package/dist/config/mesh-json-config.d.ts +62 -0
- package/dist/index.d.ts +4 -2
- package/dist/index.js +2309 -689
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +2277 -666
- package/dist/index.mjs.map +1 -1
- package/dist/mesh/mesh-active-work.d.ts +1 -1
- package/dist/mesh/mesh-ledger.d.ts +1 -1
- package/dist/mesh/mesh-node-identity.d.ts +23 -0
- package/dist/mesh/mesh-refine-gates.d.ts +89 -0
- package/dist/mesh/mesh-remote-event-pull.d.ts +3 -0
- package/dist/mesh/mesh-work-queue.d.ts +24 -1
- package/dist/providers/auto-approve-modes.d.ts +14 -0
- package/dist/providers/chat-message-normalization.d.ts +22 -0
- package/dist/providers/cli-provider-instance-types.d.ts +2 -0
- package/dist/providers/cli-provider-instance.d.ts +76 -0
- package/dist/providers/contracts.d.ts +17 -0
- package/dist/providers/provider-schema.d.ts +1 -0
- package/dist/providers/sdk/v1/builders/cli/detect-status.d.ts +14 -0
- package/dist/providers/types/interactive-prompt.d.ts +18 -0
- package/dist/repo-mesh-types.d.ts +38 -6
- package/dist/shared-types.d.ts +4 -2
- package/package.json +3 -3
- package/src/cli-adapters/cli-state-engine.ts +17 -1
- package/src/cli-adapters/provider-cli-adapter.ts +2 -6
- package/src/cli-adapters/provider-cli-shared.ts +41 -0
- package/src/commands/cli-manager.ts +72 -8
- package/src/commands/high-family/mesh-events.ts +13 -2
- package/src/commands/med-family/mesh-crud.ts +155 -0
- package/src/commands/med-family/mesh-queue.ts +74 -1
- package/src/commands/router-refine.ts +71 -4
- package/src/commands/router.ts +8 -0
- package/src/commands/upgrade-helper.ts +78 -15
- package/src/commands/windows-atomic-upgrade.ts +146 -38
- package/src/config/mesh-json-config.ts +103 -0
- package/src/index.ts +21 -1
- package/src/mesh/coordinator-prompt.ts +2 -1
- package/src/mesh/mesh-active-work.ts +11 -1
- package/src/mesh/mesh-completion-synthesis.ts +12 -1
- package/src/mesh/mesh-event-classify.ts +19 -0
- package/src/mesh/mesh-event-forwarding.ts +64 -6
- package/src/mesh/mesh-events-utils.ts +42 -0
- package/src/mesh/mesh-ledger.ts +5 -0
- package/src/mesh/mesh-node-identity.ts +49 -0
- package/src/mesh/mesh-queue-assignment.ts +210 -11
- package/src/mesh/mesh-reconcile-loop.ts +257 -3
- package/src/mesh/mesh-refine-gates.ts +300 -0
- package/src/mesh/mesh-remote-event-pull.ts +68 -47
- package/src/mesh/mesh-work-queue.ts +85 -2
- package/src/providers/auto-approve-modes.ts +97 -0
- package/src/providers/chat-message-normalization.ts +53 -0
- package/src/providers/cli-provider-instance-types.ts +28 -0
- package/src/providers/cli-provider-instance.ts +394 -28
- package/src/providers/contracts.ts +25 -1
- package/src/providers/provider-schema.ts +83 -0
- package/src/providers/sdk/v1/builders/cli/detect-status.ts +23 -0
- package/src/providers/sdk/v1/builders/cli/parse-approval.ts +20 -0
- package/src/providers/sdk/v1/schemas/cli/provider.schema.json +44 -0
- package/src/providers/types/interactive-prompt.ts +77 -0
- package/src/repo-mesh-types.ts +112 -12
- package/src/shared-types.ts +9 -1
- package/src/status/reporter.ts +1 -0
- package/src/status/snapshot.ts +2 -0
|
@@ -26,7 +26,7 @@ import { getActiveDirectDispatches, getQueue } from './mesh-work-queue.js';
|
|
|
26
26
|
import { readLedgerEntries } from './mesh-ledger.js';
|
|
27
27
|
import { pruneStaleDirectDispatches } from './mesh-active-work.js';
|
|
28
28
|
import { reconcileDirectDispatchCompletionFromTranscript } from './mesh-events-stale.js';
|
|
29
|
-
import { extractFinalAssistantSummaryEvidence } from '../providers/chat-message-normalization.js';
|
|
29
|
+
import { extractFinalAssistantSummaryEvidence, hasTrailingToolActivityAfterFinalAssistant } from '../providers/chat-message-normalization.js';
|
|
30
30
|
import type { ChatMessage } from '../types.js';
|
|
31
31
|
import {
|
|
32
32
|
getMeshV2BackstopCounters,
|
|
@@ -483,6 +483,17 @@ export async function pollAssignedTaskTerminalEvidence(
|
|
|
483
483
|
const evidence = extractFinalAssistantSummaryEvidence(messages);
|
|
484
484
|
if (!evidence.finalSummary) return null; // idle but no assistant result yet → not a turn-end
|
|
485
485
|
|
|
486
|
+
// EARLY-IDLE-COMPLETION-FALSE-POSITIVE (poll defense-in-depth): a momentary idle read
|
|
487
|
+
// (startup-grace, or the sliver between an assistant preamble and the tool it fires) can
|
|
488
|
+
// show an assistant "Let me explore…" bubble FOLLOWED by trailing Read/Grep tool_use — a
|
|
489
|
+
// turn that is still executing, not a turn-end. selectFinalAssistantTurnEndMessage skips
|
|
490
|
+
// those trailing tool bubbles and would promote the preamble, so guard here: if a
|
|
491
|
+
// tool/terminal activity bubble trails the final assistant message, the worker is mid-turn
|
|
492
|
+
// — refuse the completion (fall through to the caller's reclaim/grace path). A genuinely
|
|
493
|
+
// finished pure-PTY worker ends on its final assistant with no trailing tool activity, so
|
|
494
|
+
// the kimi rescue is preserved.
|
|
495
|
+
if (hasTrailingToolActivityAfterFinalAssistant(messages)) return null;
|
|
496
|
+
|
|
486
497
|
// Stale-summary guard (same bar as PHASE 4): a reused session's transcript tail may hold a
|
|
487
498
|
// PRIOR task's summary. Require the final assistant message to be dated at/after THIS task's
|
|
488
499
|
// dispatch. When either timestamp is unusable, do NOT short-circuit — fall through so we never
|
|
@@ -8,6 +8,11 @@ const MESH_COORDINATOR_EVENTS = new Set([
|
|
|
8
8
|
'agent:generating_started',
|
|
9
9
|
'agent:generating_completed',
|
|
10
10
|
'agent:waiting_approval',
|
|
11
|
+
// A worker parked on an AskUserQuestion multi-choice prompt. DISTINCT from
|
|
12
|
+
// agent:waiting_approval (a yes/no tool-consent modal): a question is answered
|
|
13
|
+
// with mesh_answer_question, never mesh_approve (mission f1d25e11). Carries the
|
|
14
|
+
// full InteractivePrompt payload so the coordinator can render + answer.
|
|
15
|
+
'agent:waiting_choice',
|
|
11
16
|
'agent:stopped',
|
|
12
17
|
'agent:ready',
|
|
13
18
|
'monitor:no_progress',
|
|
@@ -26,6 +31,7 @@ const MESH_COORDINATOR_EVENTS = new Set([
|
|
|
26
31
|
export const EVENT_TO_LEDGER_KIND: Record<string, MeshLedgerKind> = {
|
|
27
32
|
'agent:generating_completed': 'task_completed',
|
|
28
33
|
'agent:waiting_approval': 'task_approval_needed',
|
|
34
|
+
'agent:waiting_choice': 'task_question_pending',
|
|
29
35
|
'agent:stopped': 'task_failed',
|
|
30
36
|
'monitor:no_progress': 'task_stalled',
|
|
31
37
|
};
|
|
@@ -45,6 +51,11 @@ export const MESH_FORCE_INJECT_EVENTS: ReadonlySet<string> = new Set([
|
|
|
45
51
|
'agent:generating_completed',
|
|
46
52
|
'agent:stopped',
|
|
47
53
|
'agent:waiting_approval',
|
|
54
|
+
// A worker's question (waiting_choice) is real-time and the coordinator may be
|
|
55
|
+
// generating when it arrives; force-inject it like approval so the busy coordinator
|
|
56
|
+
// learns it must answer (mesh_answer_question) rather than the nudge sitting in the
|
|
57
|
+
// adapter's outbound queue until the coordinator idles on its own (mission f1d25e11).
|
|
58
|
+
'agent:waiting_choice',
|
|
48
59
|
'refine:completed',
|
|
49
60
|
'refine:failed',
|
|
50
61
|
'worktree_bootstrap_complete',
|
|
@@ -71,6 +82,14 @@ export function shouldForceInjectMeshEvent(eventName: unknown): boolean {
|
|
|
71
82
|
// require, and why a stale/resolved approval nudge can simply be dropped.
|
|
72
83
|
export const MESH_APPROVAL_EVENTS: ReadonlySet<string> = new Set([
|
|
73
84
|
'agent:waiting_approval',
|
|
85
|
+
// agent:waiting_choice (a worker's AskUserQuestion) shares the approval class's
|
|
86
|
+
// real-time-nudge / level-backed contract: its authoritative state is recorded at
|
|
87
|
+
// LEVEL the moment it is processed (task_question_pending → mesh_status
|
|
88
|
+
// awaiting_choice), so the pending event is only a NUDGE and can be delivered to a
|
|
89
|
+
// busy coordinator's inbox (and dropped) without data loss — the level state
|
|
90
|
+
// re-derives it. It is therefore exempt from the idle-edge hold completions require,
|
|
91
|
+
// exactly like an approval nudge (mission f1d25e11).
|
|
92
|
+
'agent:waiting_choice',
|
|
74
93
|
]);
|
|
75
94
|
|
|
76
95
|
export function isMeshApprovalEvent(eventName: unknown): boolean {
|
|
@@ -16,7 +16,8 @@ import { resolveMeshHostStatus } from './mesh-host-ownership.js';
|
|
|
16
16
|
import { enqueueUnresolvedDelegateForward, nudgeUnresolvedForwardRetry } from './mesh-unresolved-forward-outbox.js';
|
|
17
17
|
import { traceMeshEventStage, traceMeshEventDrop } from './mesh-event-trace.js';
|
|
18
18
|
import { getLastDisplayMessage } from '../status/snapshot.js';
|
|
19
|
-
import {
|
|
19
|
+
import { delegatedWorkerAutoApproveSettings } from '../repo-mesh-types.js';
|
|
20
|
+
import { loadRepoMeshJsonConfig } from '../config/mesh-json-config.js';
|
|
20
21
|
import { meshNodeIdMatches, daemonIdsEquivalent, expandDaemonIdForms, sessionIdsEquivalent, withStatusProbeMarker, type MeshNodeIdentified } from '@adhdev/mesh-shared';
|
|
21
22
|
import {
|
|
22
23
|
findRecentTerminalLedgerEvidence,
|
|
@@ -1071,10 +1072,44 @@ function injectMeshSystemMessage(components: DaemonComponents, args: {
|
|
|
1071
1072
|
// C2: prefer an exact taskId match when the completion event carries one —
|
|
1072
1073
|
// it's immune to coordinator↔worker clock skew that can hide the assigned row.
|
|
1073
1074
|
const eventTaskId = readNonEmptyString(args.metadataEvent.taskId) || undefined;
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1075
|
+
// WEAK-QUEUE-TENTATIVE (early-completed safety net, symmetric with the direct-dispatch
|
|
1076
|
+
// `tentativeIfDirect` guard below): a `completed` outcome whose evidence is WEAK — a
|
|
1077
|
+
// false-idle / missing_final_assistant emit (the CANON-C decoupled-immediate path that
|
|
1078
|
+
// stamps evidenceLevel=insufficient) — must NOT hard-flip a matched QUEUE row to
|
|
1079
|
+
// terminal. Until this net, only DIRECT dispatches were kept tentative on weak evidence;
|
|
1080
|
+
// a queue-claimed task's session flipped its row to 'completed' unconditionally, so a
|
|
1081
|
+
// worker that dropped to idle ~13s in with no final assistant closed the task early even
|
|
1082
|
+
// though it never produced an answer. Leaving the row 'assigned' hands it to the reconcile
|
|
1083
|
+
// loop's proven net (PHASE 4 records the GENUINE completion once the transcript lands via
|
|
1084
|
+
// findTerminalLedgerEvidenceForTask; PHASE 2.5 reclaims a truly-stranded row back to
|
|
1085
|
+
// 'pending' for re-dispatch, bounded by MAX_STRANDED_RECLAIMS) — never a permanent wedge.
|
|
1086
|
+
//
|
|
1087
|
+
// CRUCIAL SCOPE: this covers only the PREMATURE decoupled-immediate emit
|
|
1088
|
+
// (emittedAfterFinalizationTimeout !== true). A weak completion that already waited out the
|
|
1089
|
+
// full 30s COMPLETED_FINALIZATION_MAX_WAIT_MS window (emittedAfterFinalizationTimeout=true)
|
|
1090
|
+
// is a GENUINE — if answerless — terminal: the worker exhausted its finalization wait and
|
|
1091
|
+
// is done, so it flips 'completed' as before (a tool-only turn that never produced an
|
|
1092
|
+
// assistant bubble must still close its task). A `failed` outcome or a completion with
|
|
1093
|
+
// genuine evidence flips terminal as before too.
|
|
1094
|
+
const completionDiagnostic = args.metadataEvent.completionDiagnostic && typeof args.metadataEvent.completionDiagnostic === 'object'
|
|
1095
|
+
? args.metadataEvent.completionDiagnostic as Record<string, unknown>
|
|
1096
|
+
: undefined;
|
|
1097
|
+
const emittedAfterFinalizationTimeout = completionDiagnostic?.emittedAfterFinalizationTimeout === true;
|
|
1098
|
+
const weakCompleted = outcome === 'completed'
|
|
1099
|
+
&& isWeakCompletionEvidence(args.metadataEvent)
|
|
1100
|
+
&& !emittedAfterFinalizationTimeout;
|
|
1101
|
+
const task = weakCompleted
|
|
1102
|
+
? updateSessionTaskStatus(args.meshId, sessionId, 'assigned', {
|
|
1103
|
+
occurredAt: occurredAtMs != null ? new Date(occurredAtMs).toISOString() : undefined,
|
|
1104
|
+
taskId: eventTaskId,
|
|
1105
|
+
})
|
|
1106
|
+
: updateSessionTaskStatus(args.meshId, sessionId, outcome, {
|
|
1107
|
+
occurredAt: occurredAtMs != null ? new Date(occurredAtMs).toISOString() : undefined,
|
|
1108
|
+
taskId: eventTaskId,
|
|
1109
|
+
});
|
|
1110
|
+
if (weakCompleted && task) {
|
|
1111
|
+
LOG.info('MeshQueue', `Weak completion (${readNonEmptyString(args.metadataEvent.evidenceLevel) || 'missing_final_assistant'}) kept queue task ${task.id} tentative (session ${sessionId}); reconcile owns the genuine terminal`);
|
|
1112
|
+
}
|
|
1078
1113
|
// Fix A (early-terminal prevention): a false-idle completion (no confirmed final
|
|
1079
1114
|
// assistant) for a DIRECT dispatch — i.e. no work-queue row matched — must not flip the
|
|
1080
1115
|
// dispatch row terminal. Leaving it active lets the reconcile loop (PHASE 4) re-read the
|
|
@@ -1580,7 +1615,22 @@ function injectMeshSystemMessage(components: DaemonComponents, args: {
|
|
|
1580
1615
|
spawnedSessionVisibility: mesh?.policy?.spawnedSessionVisibility || 'hidden',
|
|
1581
1616
|
// Coordinator-dispatched recovery relaunch: same auto-approve
|
|
1582
1617
|
// policy as the primary worker launch path.
|
|
1583
|
-
|
|
1618
|
+
...delegatedWorkerAutoApproveSettings(
|
|
1619
|
+
mesh?.policy,
|
|
1620
|
+
node?.policy,
|
|
1621
|
+
components.providerLoader?.getMeta(recoveryContext.failedProviderType),
|
|
1622
|
+
// Recovery relaunch: same repo-declared requested mode as the
|
|
1623
|
+
// primary path. node.workspace missing → null → provider default.
|
|
1624
|
+
(() => {
|
|
1625
|
+
const ws = typeof node?.workspace === 'string' && node.workspace.trim() ? node.workspace.trim() : '';
|
|
1626
|
+
if (!ws) return null;
|
|
1627
|
+
try {
|
|
1628
|
+
const r = loadRepoMeshJsonConfig(ws);
|
|
1629
|
+
return r.sourceType === 'repo_file' && r.config ? r.config : null;
|
|
1630
|
+
} catch { return null; }
|
|
1631
|
+
})(),
|
|
1632
|
+
recoveryContext.failedProviderType,
|
|
1633
|
+
),
|
|
1584
1634
|
launchedByCoordinator: true,
|
|
1585
1635
|
}
|
|
1586
1636
|
}).catch((e: any) => LOG.error('MeshRecovery', `Failed to auto-relaunch session for ${node.id}: ${e?.message}`));
|
|
@@ -1754,6 +1804,14 @@ export function buildRelayMetadataEvent(payload: Record<string, unknown>): Recor
|
|
|
1754
1804
|
retryOfJobId: readNonEmptyString(payload.retryOfJobId),
|
|
1755
1805
|
...(relayModalMessage ? { modalMessage: relayModalMessage } : {}),
|
|
1756
1806
|
...(relayModalButtons && relayModalButtons.length > 0 ? { modalButtons: relayModalButtons } : {}),
|
|
1807
|
+
// agent:waiting_choice (mission f1d25e11): carry the FULL structured question
|
|
1808
|
+
// payload across the machine boundary so a REMOTE worker's AskUserQuestion reaches
|
|
1809
|
+
// the coordinator with every question + option intact — the coordinator renders
|
|
1810
|
+
// these and answers with mesh_answer_question. The local in-process forward path
|
|
1811
|
+
// preserves the whole event for free; this mirrors the fields for the remote relay.
|
|
1812
|
+
...(payload.interactivePrompt && typeof payload.interactivePrompt === 'object' && !Array.isArray(payload.interactivePrompt) ? { interactivePrompt: payload.interactivePrompt } : {}),
|
|
1813
|
+
...(readNonEmptyString(payload.promptId) ? { promptId: readNonEmptyString(payload.promptId) } : {}),
|
|
1814
|
+
...(payload.multiSelect === true ? { multiSelect: true } : {}),
|
|
1757
1815
|
...(payload.result && typeof payload.result === 'object' && !Array.isArray(payload.result) ? { result: payload.result } : {}),
|
|
1758
1816
|
...(payload.completionDiagnostic && typeof payload.completionDiagnostic === 'object' && !Array.isArray(payload.completionDiagnostic) ? { completionDiagnostic: payload.completionDiagnostic } : {}),
|
|
1759
1817
|
...(payload.workerResult && typeof payload.workerResult === 'object' && !Array.isArray(payload.workerResult) ? { workerResult: payload.workerResult } : {}),
|
|
@@ -330,6 +330,48 @@ export function buildMeshSystemMessage(args: {
|
|
|
330
330
|
if (args.event === 'agent:waiting_approval') {
|
|
331
331
|
return `[System] ${args.nodeLabel} is waiting for approval to proceed${metadata}. You may use mesh_read_chat and mesh_approve to handle it.`;
|
|
332
332
|
}
|
|
333
|
+
if (args.event === 'agent:waiting_choice') {
|
|
334
|
+
// A multi-choice QUESTION (AskUserQuestion) — NOT an approval. It is answered with
|
|
335
|
+
// mesh_answer_question (never mesh_approve). Surface the question text + choices
|
|
336
|
+
// inline so the coordinator can decide without a mesh_read_chat round-trip, and
|
|
337
|
+
// name the correct tool + promptId (mission f1d25e11).
|
|
338
|
+
const prompt = args.metadataEvent.interactivePrompt as
|
|
339
|
+
| { promptId?: unknown; questions?: Array<Record<string, unknown>> }
|
|
340
|
+
| undefined;
|
|
341
|
+
const promptId = readNonEmptyString(args.metadataEvent.promptId)
|
|
342
|
+
|| (prompt && readNonEmptyString(prompt.promptId));
|
|
343
|
+
const lines: string[] = [
|
|
344
|
+
`[System] ${args.nodeLabel} is asking a question and is waiting for your answer${metadata}.`,
|
|
345
|
+
];
|
|
346
|
+
const questions = Array.isArray(prompt?.questions) ? prompt!.questions! : [];
|
|
347
|
+
if (questions.length > 0) {
|
|
348
|
+
for (const q of questions) {
|
|
349
|
+
const header = readNonEmptyString(q.header);
|
|
350
|
+
const question = readNonEmptyString(q.question);
|
|
351
|
+
const multiSelect = q.multiSelect === true;
|
|
352
|
+
if (question) {
|
|
353
|
+
lines.push(`\n**${header ? `${header}: ` : ''}${question}**${multiSelect ? ' (select one or more)' : ''}`);
|
|
354
|
+
}
|
|
355
|
+
const options = Array.isArray(q.options) ? q.options : [];
|
|
356
|
+
options.forEach((opt, i) => {
|
|
357
|
+
const record = (opt && typeof opt === 'object') ? opt as Record<string, unknown> : {};
|
|
358
|
+
const label = readNonEmptyString(record.label);
|
|
359
|
+
if (!label) return;
|
|
360
|
+
const description = readNonEmptyString(record.description);
|
|
361
|
+
lines.push(` ${i + 1}. ${label}${description ? ` — ${description}` : ''}`);
|
|
362
|
+
});
|
|
363
|
+
}
|
|
364
|
+
} else {
|
|
365
|
+
const modalMessage = readNonEmptyString(args.metadataEvent.modalMessage);
|
|
366
|
+
if (modalMessage) lines.push(`\n${modalMessage}`);
|
|
367
|
+
}
|
|
368
|
+
lines.push(
|
|
369
|
+
`\nAnswer with mesh_answer_question(node_id, session_id${promptId ? `, promptId: "${promptId}"` : ''}, answers). ` +
|
|
370
|
+
`Do NOT use mesh_approve — that only resolves yes/no consent modals, not a question. ` +
|
|
371
|
+
`Use mesh_read_chat once if you need the full context first.`,
|
|
372
|
+
);
|
|
373
|
+
return lines.join('\n');
|
|
374
|
+
}
|
|
333
375
|
if (args.event === 'agent:stopped') {
|
|
334
376
|
const rc = args.recoveryContext;
|
|
335
377
|
if (rc && rc.consecutiveNodeFailures > 0) {
|
package/src/mesh/mesh-ledger.ts
CHANGED
|
@@ -33,6 +33,11 @@ export type MeshLedgerKind =
|
|
|
33
33
|
| 'task_failed'
|
|
34
34
|
| 'task_stalled'
|
|
35
35
|
| 'task_approval_needed'
|
|
36
|
+
// A worker is parked on an AskUserQuestion multi-choice prompt (waiting_choice) —
|
|
37
|
+
// distinct from task_approval_needed (a yes/no tool-consent modal). The coordinator
|
|
38
|
+
// answers a question with mesh_answer_question, never mesh_approve (mission f1d25e11).
|
|
39
|
+
// payload carries the full InteractivePrompt (promptId + questions + options).
|
|
40
|
+
| 'task_question_pending'
|
|
36
41
|
| 'p2p_dispatch_failed'
|
|
37
42
|
| 'session_launched'
|
|
38
43
|
| 'session_auto_launch'
|
|
@@ -907,6 +907,55 @@ export function isMeshNodeHealthLaunchable(node: any): boolean {
|
|
|
907
907
|
return health === 'online' || health === 'unknown';
|
|
908
908
|
}
|
|
909
909
|
|
|
910
|
+
/** Resolve the git telemetry object a node carries, in the same precedence
|
|
911
|
+
* resolveEffectiveMeshNodeHealth uses: a fresh `node.git`, else the cached inline
|
|
912
|
+
* status git (`node.cachedStatus.git`). Returns an empty record when neither is
|
|
913
|
+
* present (no telemetry). */
|
|
914
|
+
function resolveEffectiveNodeGit(node: any): Record<string, any> {
|
|
915
|
+
const git = readObjectRecord(node?.git);
|
|
916
|
+
if (Object.keys(git).length > 0) return git;
|
|
917
|
+
const cachedStatus = readObjectRecord(node?.cachedStatus);
|
|
918
|
+
return readObjectRecord(cachedStatus.git);
|
|
919
|
+
}
|
|
920
|
+
|
|
921
|
+
/**
|
|
922
|
+
* Launch FRESHNESS gate — distinct from the health gate above.
|
|
923
|
+
*
|
|
924
|
+
* `deriveMeshNodeHealthFromGit` reports a clean-tree node with `behind > 0` as
|
|
925
|
+
* 'online', so the health gate (isMeshNodeHealthLaunchable) happily lets a STALE
|
|
926
|
+
* node — one whose branch is N commits behind its upstream — win auto-launch fitness
|
|
927
|
+
* routing and run a fresh worker against out-of-date code. `behind` is deliberately
|
|
928
|
+
* NOT folded into deriveMeshNodeHealthFromGit because "behind" is not universally
|
|
929
|
+
* unhealthy (the MAGI planner and other callers share that resolver); it is only
|
|
930
|
+
* unhealthy for *spawning new work*. This gate encodes exactly that launch-time axis.
|
|
931
|
+
*
|
|
932
|
+
* Returns false (NOT fresh → skip / de-rank) only when git telemetry is PRESENT and
|
|
933
|
+
* proves staleness:
|
|
934
|
+
* - behind count exceeds `maxBehind` (default 0 — any behind blocks), OR
|
|
935
|
+
* - a submodule is out of sync (gitlink points off upstream — cannot be caught up
|
|
936
|
+
* by a simple worktree ff and would launch against a mismatched submodule).
|
|
937
|
+
*
|
|
938
|
+
* When telemetry is absent it returns true (fresh), preserving the online/unknown-pass
|
|
939
|
+
* philosophy: we never block on missing data, only on data that proves the node stale.
|
|
940
|
+
*/
|
|
941
|
+
export function isMeshNodeFreshEnoughToLaunch(node: any, opts?: { maxBehind?: number }): boolean {
|
|
942
|
+
const git = resolveEffectiveNodeGit(node);
|
|
943
|
+
// No git telemetry at all → cannot prove stale → do not block.
|
|
944
|
+
if (Object.keys(git).length === 0) return true;
|
|
945
|
+
// Not a git repo / no branch: leave this to the health gate (it returns 'degraded'
|
|
946
|
+
// and blocks there); freshness has nothing to add.
|
|
947
|
+
if (readBooleanValue(git.isGitRepo) === false) return true;
|
|
948
|
+
const submoduleDrift = getGitSubmoduleDriftState(git);
|
|
949
|
+
if (submoduleDrift.outOfSync) return false;
|
|
950
|
+
const behind = readNumberValue(git.behind);
|
|
951
|
+
// No behind datum reported → treat as fresh (don't infer staleness from absence).
|
|
952
|
+
if (behind === undefined) return true;
|
|
953
|
+
const maxBehind = Number.isFinite(opts?.maxBehind as number) && (opts?.maxBehind as number) >= 0
|
|
954
|
+
? Math.floor(opts!.maxBehind as number)
|
|
955
|
+
: 0;
|
|
956
|
+
return behind <= maxBehind;
|
|
957
|
+
}
|
|
958
|
+
|
|
910
959
|
function readMeshNodeLabel(status: Record<string, unknown>, node: any): string {
|
|
911
960
|
return readStringValue(status.nodeId, normalizeMeshNodeId(node)) ?? 'unknown';
|
|
912
961
|
}
|
|
@@ -6,20 +6,22 @@ import { getMesh } from '../config/mesh-config.js';
|
|
|
6
6
|
import { detectCLI } from '../detection/cli-detector.js';
|
|
7
7
|
import { LOG } from '../logging/logger.js';
|
|
8
8
|
import { appendLedgerEntry } from './mesh-ledger.js';
|
|
9
|
-
import { buildMeshNodeCapabilityTags, nodeSatisfiesRequiredTags, claimNextTask, updateTaskStatus, getQueue, recordTaskAutoLaunch, getActiveDirectDispatches, isTaskReadonly, taskDependenciesSatisfied, meshTaskNotBeforeReady, meshTaskPriorityRank } from './mesh-work-queue.js';
|
|
9
|
+
import { buildMeshNodeCapabilityTags, nodeSatisfiesRequiredTags, claimNextTask, updateTaskStatus, getQueue, recordTaskAutoLaunch, getActiveDirectDispatches, isTaskReadonly, taskDependenciesSatisfied, meshTaskNotBeforeReady, meshTaskPriorityRank, requeueTask } from './mesh-work-queue.js';
|
|
10
10
|
import type { MeshWorkQueueEntry } from './mesh-work-queue.js';
|
|
11
11
|
import { fastForwardMeshNode } from './mesh-fast-forward.js';
|
|
12
12
|
import { createSessionDelivery, updateSessionDeliveryStatus } from './mesh-delivery-policy.js';
|
|
13
13
|
import { MeshRuntimeStore } from './mesh-runtime-store.js';
|
|
14
14
|
import { traceMeshEventDrop } from './mesh-event-trace.js';
|
|
15
15
|
import { awaitWithWarmupDeadline, resolveWarmupDeadlineOpts } from './mesh-warmup-deadline.js';
|
|
16
|
-
import {
|
|
16
|
+
import { delegatedWorkerAutoApproveSettings, resolveProviderMaxParallel, resolveNodeSchedulingPriority, normalizeMeshSchedulingStrategy, resolveMaxParallelTasks, resolveMaxReadonlyParallelTasks, resolveCoordinatorIdlePushPolicy } from '../repo-mesh-types.js';
|
|
17
|
+
import { loadRepoMeshJsonConfig } from '../config/mesh-json-config.js';
|
|
18
|
+
import type { RepoMeshDeclarativeConfig } from '../config/mesh-json-config.js';
|
|
17
19
|
import type { RepoMeshSchedulingStrategy } from '../repo-mesh-types.js';
|
|
18
20
|
import { normalizeMeshNodeId, meshNodeIdMatches, daemonIdsEquivalent, canonicalDaemonId, expandDaemonIdForms, normalizeMeshWorkspaceForCompare, meshWorkspacesEquivalent, sessionIdsEquivalent, normalizeNodeCapabilitySlots, isMeshTaskDifficulty, withStatusProbeMarker, type MeshNodeIdentified, type NodeCapabilitySlot, type MeshTaskDifficulty } from '@adhdev/mesh-shared';
|
|
19
21
|
import { resolveNodeCapabilitySlots } from './mesh-node-slots.js';
|
|
20
22
|
import { findTerminalLedgerEvidenceForTask, hasUnterminalDirectDispatchLedgerEntry } from './mesh-events-stale.js';
|
|
21
23
|
import { readNonEmptyString } from './mesh-events-utils.js';
|
|
22
|
-
import { readMeshNodeDaemonId, isMeshNodeHealthLaunchable } from './mesh-node-identity.js';
|
|
24
|
+
import { readMeshNodeDaemonId, isMeshNodeHealthLaunchable, isMeshNodeFreshEnoughToLaunch } from './mesh-node-identity.js';
|
|
23
25
|
import { queuePendingMeshCoordinatorEvent, retractPendingDispatchBlockedEvent, drainPendingMeshCoordinatorEvents } from './mesh-events-pending.js';
|
|
24
26
|
import { isWorktreeBootstrapStaleRunning, shouldDeferDispatchForBootstrap } from './worktree-bootstrap-config.js';
|
|
25
27
|
import { isWithinCloneBootstrapGrace } from './mesh-clone-grace.js';
|
|
@@ -90,6 +92,25 @@ export function __resetIdleAutoFastForwardForTests(): void {
|
|
|
90
92
|
autoFastForwardWorkspaceLease.clear();
|
|
91
93
|
}
|
|
92
94
|
|
|
95
|
+
/**
|
|
96
|
+
* Load the repo-shared `.adhdev/mesh.json` for a node's workspace, tolerating a
|
|
97
|
+
* missing/invalid file (returns null → resolver falls back to provider-spec
|
|
98
|
+
* defaults, i.e. exactly the pre-providerDefaults behavior). Only the
|
|
99
|
+
* `providerDefaults` zone influences the delegated-worker MODE selection; it never
|
|
100
|
+
* touches the ENABLE decision. When a node carries no workspace path (should not
|
|
101
|
+
* happen for a launchable node, but be defensive), we skip the read entirely.
|
|
102
|
+
*/
|
|
103
|
+
function loadRepoConfigForNode(node: any): RepoMeshDeclarativeConfig | null {
|
|
104
|
+
const workspace = typeof node?.workspace === 'string' && node.workspace.trim() ? node.workspace.trim() : '';
|
|
105
|
+
if (!workspace) return null;
|
|
106
|
+
try {
|
|
107
|
+
const result = loadRepoMeshJsonConfig(workspace);
|
|
108
|
+
return result.sourceType === 'repo_file' && result.config ? result.config : null;
|
|
109
|
+
} catch {
|
|
110
|
+
return null;
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
93
114
|
export function getMeshWithCache(components: DaemonComponents, meshId: string): any | undefined {
|
|
94
115
|
const localMesh = getMesh(meshId);
|
|
95
116
|
const cachedMesh = components.router?.getCachedInlineMesh(meshId);
|
|
@@ -690,7 +711,7 @@ export function tryAssignQueueTask(
|
|
|
690
711
|
if (inst && typeof inst.updateSettings === 'function') {
|
|
691
712
|
// Adopting a (possibly manually-opened) local session as a worker: apply the
|
|
692
713
|
// delegated-worker auto-approve policy here too, so a session that was launched
|
|
693
|
-
// without
|
|
714
|
+
// without an auto-approve boolean/mode still resolves the delegated policy once dispatched
|
|
694
715
|
// to it (the "approval notification fires only for certain delegated sessions"
|
|
695
716
|
// case). updateSettings preserves runtime mesh keys; passing autoApprove keeps it.
|
|
696
717
|
//
|
|
@@ -704,7 +725,13 @@ export function tryAssignQueueTask(
|
|
|
704
725
|
meshNodeFor: meshId,
|
|
705
726
|
meshNodeId: nodeId,
|
|
706
727
|
launchedByCoordinator: true,
|
|
707
|
-
|
|
728
|
+
...delegatedWorkerAutoApproveSettings(
|
|
729
|
+
mesh?.policy,
|
|
730
|
+
node?.policy,
|
|
731
|
+
components.providerLoader?.getMeta(providerType),
|
|
732
|
+
loadRepoConfigForNode(node),
|
|
733
|
+
providerType,
|
|
734
|
+
),
|
|
708
735
|
...(localDaemonId ? { meshCoordinatorDaemonId: localDaemonId } : {}),
|
|
709
736
|
// COMPLETION-PROPAGATION F5: (re)stamp the coordinator SESSION anchor from THIS
|
|
710
737
|
// task's sourceCoordinatorSessionId with PRIORITY — a manually-launched (or reused)
|
|
@@ -931,6 +958,105 @@ function isTargetNodeTransientlyUnresolved(mesh: any, task: MeshWorkQueueEntry):
|
|
|
931
958
|
return isWithinCloneBootstrapGrace(targetNodeId);
|
|
932
959
|
}
|
|
933
960
|
|
|
961
|
+
// ---------------------------------------------------------------------------
|
|
962
|
+
// DEAD-TARGET-SELFHEAL: unpin a queue task hard-pinned to a session/node that has
|
|
963
|
+
// died (absent from the live mesh) so a live idle session can claim it, instead of
|
|
964
|
+
// leaving it stranded 'pending' forever behind the target_session_constraint skip.
|
|
965
|
+
// ---------------------------------------------------------------------------
|
|
966
|
+
|
|
967
|
+
// Conservative age gate before a pinned-but-dead target is unpinned. A target that is
|
|
968
|
+
// merely briefly unassigned or momentarily reconnecting must not be reclaimed on the
|
|
969
|
+
// tick it drops out of view; we require the task to have been idle (no updatedAt bump)
|
|
970
|
+
// for at least this window first. Sized to comfortably outlast a transient P2P blip /
|
|
971
|
+
// reconnect while staying well inside the reclaim cadence of the rest of the file
|
|
972
|
+
// (AUTO_LAUNCH_AWAIT_CLAIM_MS is 90s; the stranded-reclaim watchdog fires on similar
|
|
973
|
+
// scales), so a real reconnect wins the race and the self-heal only fires on a target
|
|
974
|
+
// that is genuinely gone.
|
|
975
|
+
const DEAD_TARGET_GRACE_MS = 60_000;
|
|
976
|
+
|
|
977
|
+
interface DeadTargetVerdict {
|
|
978
|
+
/** The pinned target is confirmed dead and past the grace window → safe to unpin. */
|
|
979
|
+
dead: boolean;
|
|
980
|
+
/** True when the target NODE itself is absent from the live mesh (clear targetNodeId too). */
|
|
981
|
+
nodeDead: boolean;
|
|
982
|
+
/** Short reason string for the ledger/requeue. */
|
|
983
|
+
reason: string;
|
|
984
|
+
}
|
|
985
|
+
|
|
986
|
+
/**
|
|
987
|
+
* Decide whether a task's hard target pin (targetSessionId and/or targetNodeId) points at
|
|
988
|
+
* something that has DIED — i.e. is absent from the live mesh snapshot — and has been so
|
|
989
|
+
* long enough (DEAD_TARGET_GRACE_MS since the task's last update) that unpinning is safe.
|
|
990
|
+
*
|
|
991
|
+
* Two definitive death signals, deliberately conservative to never race a reconnecting node:
|
|
992
|
+
*
|
|
993
|
+
* (1) NODE dead — the task pins a targetNodeId that matches NO node in the live mesh
|
|
994
|
+
* (the same `meshNodeIdMatches`-over-mesh.nodes signal the targetPinUnmatched relabel
|
|
995
|
+
* uses at the empty-candidate site). A pinned session on an absent node is unreachable
|
|
996
|
+
* regardless, so the session pin is dead too. `nodeDead` ⇒ clear targetNodeId as well.
|
|
997
|
+
* Excluded: a target that is only TRANSIENTLY unresolved (a freshly-cloned worktree
|
|
998
|
+
* still propagating / bootstrapping) — isTargetNodeTransientlyUnresolved gates it out.
|
|
999
|
+
*
|
|
1000
|
+
* (2) SESSION dead on a LIVE LOCAL node — the target node IS present and is THIS daemon's
|
|
1001
|
+
* node, but the pinned session is absent from the local instance manager
|
|
1002
|
+
* (resolveSessionBusyVerdict === 'UNKNOWN'). Local session visibility is complete, so
|
|
1003
|
+
* absence here is genuine death, not a busy/generating flip. We KEEP targetNodeId (only
|
|
1004
|
+
* the session died; the node is healthy and can host a replacement claim).
|
|
1005
|
+
*
|
|
1006
|
+
* A live REMOTE node whose session is not in our idle view is NOT treated as dead: absence
|
|
1007
|
+
* from the remote-idle mirror is explicitly UNKNOWN liveness (the session may be busy or its
|
|
1008
|
+
* agent:ready pull merely lost), so unpinning it could race healthy in-flight work. Returns
|
|
1009
|
+
* dead=false in that case, leaving the existing skip in place.
|
|
1010
|
+
*/
|
|
1011
|
+
function resolveDeadTargetVerdict(components: DaemonComponents, meshId: string, mesh: any, task: MeshWorkQueueEntry): DeadTargetVerdict {
|
|
1012
|
+
const NOT_DEAD: DeadTargetVerdict = { dead: false, nodeDead: false, reason: '' };
|
|
1013
|
+
const targetSessionId = readNonEmptyString(task.targetSessionId);
|
|
1014
|
+
const targetNodeId = readNonEmptyString(task.targetNodeId);
|
|
1015
|
+
if (!targetSessionId && !targetNodeId) return NOT_DEAD;
|
|
1016
|
+
|
|
1017
|
+
// Age gate: never reclaim a pin younger than the grace window (guards against a target
|
|
1018
|
+
// that has only just dropped out of view for a momentary reconnect).
|
|
1019
|
+
const lastUpdateMs = Date.parse(task.updatedAt || task.createdAt || '');
|
|
1020
|
+
if (Number.isFinite(lastUpdateMs) && Date.now() - lastUpdateMs < DEAD_TARGET_GRACE_MS) return NOT_DEAD;
|
|
1021
|
+
|
|
1022
|
+
const nodes: any[] = Array.isArray(mesh?.nodes) ? mesh.nodes : [];
|
|
1023
|
+
|
|
1024
|
+
// (1) NODE-dead — a pinned node absent from the live mesh, and NOT merely transiently
|
|
1025
|
+
// unresolved (a propagating/bootstrapping clone). This is a permanent routing miss.
|
|
1026
|
+
if (targetNodeId) {
|
|
1027
|
+
const nodePresent = nodes.some(n => meshNodeIdMatches(n, targetNodeId));
|
|
1028
|
+
if (!nodePresent) {
|
|
1029
|
+
if (isTargetNodeTransientlyUnresolved(mesh, task)) return NOT_DEAD;
|
|
1030
|
+
return { dead: true, nodeDead: true, reason: 'dead_target_node_absent' };
|
|
1031
|
+
}
|
|
1032
|
+
}
|
|
1033
|
+
|
|
1034
|
+
// (2) SESSION-dead on a LIVE LOCAL node. Only meaningful when a session is pinned.
|
|
1035
|
+
if (targetSessionId) {
|
|
1036
|
+
// Resolve the pinned target's node (if any) to decide whether we can trust local
|
|
1037
|
+
// absence. Without a targetNodeId, fall back to whichever live node hosts the session
|
|
1038
|
+
// is unknowable here; treat that as a LOCAL check only (a session id we cannot see
|
|
1039
|
+
// locally on a node we cannot resolve remotely stays UNKNOWN → not dead).
|
|
1040
|
+
const node = targetNodeId
|
|
1041
|
+
? nodes.find(n => meshNodeIdMatches(n, targetNodeId))
|
|
1042
|
+
: undefined;
|
|
1043
|
+
// A pinned session on a REMOTE live node: absence from our view is UNKNOWN, not death.
|
|
1044
|
+
// Only a LOCAL node (or no node pin at all — same-daemon assumption) lets us conclude
|
|
1045
|
+
// death from local instance-manager absence.
|
|
1046
|
+
const nodeIsLocal = node ? isLocalAutoLaunchNode(node) : true;
|
|
1047
|
+
if (nodeIsLocal) {
|
|
1048
|
+
const verdict = resolveSessionBusyVerdict(components, targetSessionId);
|
|
1049
|
+
if (verdict === 'UNKNOWN') {
|
|
1050
|
+
// Session absent from the complete local session view → genuinely gone.
|
|
1051
|
+
return { dead: true, nodeDead: false, reason: 'dead_target_session_absent' };
|
|
1052
|
+
}
|
|
1053
|
+
// GENERATING / IDLE_CONFIRMED → the session is alive (possibly busy). Never disturb.
|
|
1054
|
+
}
|
|
1055
|
+
}
|
|
1056
|
+
|
|
1057
|
+
return NOT_DEAD;
|
|
1058
|
+
}
|
|
1059
|
+
|
|
934
1060
|
/**
|
|
935
1061
|
* FALSE-BLOCKER-CLONE-QUEUE (stale-event clear): once a task whose actionable blocker we
|
|
936
1062
|
* previously paged either gets claimed or transitions to a self-resolving state, re-arm the
|
|
@@ -1595,8 +1721,20 @@ function liveSessionCountForNode(components: DaemonComponents, meshId: string, n
|
|
|
1595
1721
|
* free claimer, and is excluded — so a read-only auto-launch onto a busy-but-no-idle node
|
|
1596
1722
|
* is still allowed. A node with NO live mesh session at all (dead, or never launched) does
|
|
1597
1723
|
* not match, preserving the legitimate first-session spawn.
|
|
1724
|
+
*
|
|
1725
|
+
* PROVIDER-MATCH gate (DISPATCH-DEADLOCK-PROVIDER-MISMATCH): a live session only suppresses
|
|
1726
|
+
* this launch if it could ACTUALLY claim THIS task. A session whose providerType does not
|
|
1727
|
+
* satisfy the task's requiredTags (e.g. a claude-cli coordinator/worker session on a node,
|
|
1728
|
+
* while the pending task is required_tags: [provider=codex-cli]) is NOT a pending claimer for
|
|
1729
|
+
* this task — claimNextQueueTask's nodeSatisfiesRequiredTags gate would reject its claim. Left
|
|
1730
|
+
* unchecked, such a mismatched session made this gate return true for a task it can never claim,
|
|
1731
|
+
* so the required-provider worker never auto-launched AND no session could claim → nobody made
|
|
1732
|
+
* progress → permanent silent deadlock. Mirror the claim path: build the session's own
|
|
1733
|
+
* capability tags (its providerType pinned onto the node) and only count it as a pending claimer
|
|
1734
|
+
* when those tags satisfy task.requiredTags. `node` is passed in so the tag set reflects this
|
|
1735
|
+
* node's os/arch/worktree/converge context, matching claimNextQueueTask exactly.
|
|
1598
1736
|
*/
|
|
1599
|
-
function nodeHasLiveSessionPendingClaim(components: DaemonComponents, meshId: string, nodeId: string): boolean {
|
|
1737
|
+
function nodeHasLiveSessionPendingClaim(components: DaemonComponents, meshId: string, nodeId: string, task: MeshWorkQueueEntry, node: any): boolean {
|
|
1600
1738
|
// (A) AUTOLAUNCH-CLAIM-CHURN remote-awareness: a task whose auto-launch record targets this
|
|
1601
1739
|
// node and is still inside its await-claim window (base or backoff) already has a session on
|
|
1602
1740
|
// its way to claim — even when that session is REMOTE and thus invisible to the local
|
|
@@ -1635,7 +1773,20 @@ function nodeHasLiveSessionPendingClaim(components: DaemonComponents, meshId: st
|
|
|
1635
1773
|
if (isTerminalSessionStatus(status)) return false; // dead → no claimer here, allow launch
|
|
1636
1774
|
const sessionId = readNonEmptyString(state.instanceId);
|
|
1637
1775
|
if (sessionId && busySessionIds.has(sessionId)) return false; // busy with its own assigned task
|
|
1638
|
-
|
|
1776
|
+
// PROVIDER-MATCH gate (DISPATCH-DEADLOCK-PROVIDER-MISMATCH): only count this session as a
|
|
1777
|
+
// pending claimer if its own provider could satisfy THIS task's requiredTags. A session
|
|
1778
|
+
// whose providerType does not match (e.g. a claude-cli session while task requires
|
|
1779
|
+
// provider=codex-cli) can never claim this task via claimNextQueueTask's
|
|
1780
|
+
// nodeSatisfiesRequiredTags gate, so it must not suppress the required-provider launch —
|
|
1781
|
+
// otherwise the task deadlocks (mismatched session blocks launch, yet cannot claim). Mirror
|
|
1782
|
+
// the claim path: pin the session's providerType onto this node and check the tags.
|
|
1783
|
+
if (task.requiredTags?.length) {
|
|
1784
|
+
const sessionProviderType = state.type || readNonEmptyString(settings.providerType);
|
|
1785
|
+
if (sessionProviderType && !nodeSatisfiesRequiredTags(task.requiredTags, buildMeshNodeCapabilityTags(node, sessionProviderType))) {
|
|
1786
|
+
return false; // provider mismatch → this session can't claim this task; not a pending claimer
|
|
1787
|
+
}
|
|
1788
|
+
}
|
|
1789
|
+
return true; // live + unassigned + provider-capable → will claim the pending task itself
|
|
1639
1790
|
});
|
|
1640
1791
|
}
|
|
1641
1792
|
|
|
@@ -1888,6 +2039,11 @@ async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, mesh
|
|
|
1888
2039
|
}
|
|
1889
2040
|
if (!pending.length) return false;
|
|
1890
2041
|
|
|
2042
|
+
// Launch-freshness threshold: reuse the auto-fast-forward policy's maxBehind (default
|
|
2043
|
+
// 0 → any behind blocks) so the launch gate and the repair path agree on how far
|
|
2044
|
+
// behind is tolerable. Resolved once per pass and shared across every candidate node.
|
|
2045
|
+
const freshnessGate = { maxBehind: resolveAutoFastForwardPolicy(mesh).maxBehind };
|
|
2046
|
+
|
|
1891
2047
|
// Write cap + read-only cap resolved through the shared helpers from the
|
|
1892
2048
|
// MACHINE-LOCAL stored mesh policy (no repo-file overlay). These are the same
|
|
1893
2049
|
// resolvers the observability projection uses, so the enforced and exposed
|
|
@@ -1929,6 +2085,32 @@ async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, mesh
|
|
|
1929
2085
|
continue;
|
|
1930
2086
|
}
|
|
1931
2087
|
if (task.targetSessionId) {
|
|
2088
|
+
// DEAD-TARGET-SELFHEAL: before the unconditional target_session_constraint skip,
|
|
2089
|
+
// check whether the pinned session/node has DIED (absent from the live mesh). A
|
|
2090
|
+
// hard-pinned task whose target is gone can NEVER re-enter 'assigned' (the claim
|
|
2091
|
+
// gate refuses every non-matching session) and this skip fires forever with no
|
|
2092
|
+
// liveness check — the triple-walled stranded-pending defect. If the pin is
|
|
2093
|
+
// confirmed dead past the grace window, requeue it (clearing the dead session
|
|
2094
|
+
// pin, and the node pin too when the NODE itself is gone) so a live idle session
|
|
2095
|
+
// can claim it. requeueTask counts toward maxTaskRetries → bounded self-heal that
|
|
2096
|
+
// auto-fails past the cap (the desired terminal state, unblocking dependents).
|
|
2097
|
+
const deadTarget = resolveDeadTargetVerdict(components, meshId, mesh, task);
|
|
2098
|
+
if (deadTarget.dead) {
|
|
2099
|
+
const requeued = requeueTask(meshId, task.id, {
|
|
2100
|
+
reason: deadTarget.reason,
|
|
2101
|
+
clearTargetSession: true,
|
|
2102
|
+
// Keep the node pin if only the SESSION died on a still-live node; clear it
|
|
2103
|
+
// when the NODE itself is absent (nothing to pin to).
|
|
2104
|
+
clearTargetNode: deadTarget.nodeDead,
|
|
2105
|
+
});
|
|
2106
|
+
if (requeued) {
|
|
2107
|
+
LOG.warn('MeshQueue', `DEAD-TARGET-SELFHEAL: task ${task.id} (mesh ${meshId}) was pinned to a dead target (${deadTarget.reason}); requeued${deadTarget.nodeDead ? ' and unpinned node' : ''} (requeueCount=${requeued.requeueCount ?? '?'}, status=${requeued.status}).`);
|
|
2108
|
+
}
|
|
2109
|
+
// Keep the skip for THIS tick (the requeue already flipped the row to
|
|
2110
|
+
// pending/failed); a later tick assigns/launches the now-unpinned task.
|
|
2111
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'target_session_dead_requeued' });
|
|
2112
|
+
continue;
|
|
2113
|
+
}
|
|
1932
2114
|
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'target_session_constraint' });
|
|
1933
2115
|
continue;
|
|
1934
2116
|
}
|
|
@@ -2085,6 +2267,18 @@ async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, mesh
|
|
|
2085
2267
|
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'node_not_launch_ready', nodeId });
|
|
2086
2268
|
continue;
|
|
2087
2269
|
}
|
|
2270
|
+
// FRESHNESS gate (distinct from the health gate above): a clean-tree node that
|
|
2271
|
+
// is `behind` its upstream reads as 'online' and passes isLaunchableNode, so
|
|
2272
|
+
// without this it could win fitness routing and run a fresh worker against
|
|
2273
|
+
// stale code. Skip a node whose git telemetry proves it stale (behind >
|
|
2274
|
+
// maxBehind, or a submodule out of sync). Reuse the auto-fast-forward policy's
|
|
2275
|
+
// maxBehind threshold so "how far behind is tolerable" is configured in ONE
|
|
2276
|
+
// place. Telemetry-absent nodes pass (never block on missing data). The 4s
|
|
2277
|
+
// reconcile retries once the node's auto-ff repair path catches it up.
|
|
2278
|
+
if (!isMeshNodeFreshEnoughToLaunch(node, freshnessGate)) {
|
|
2279
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'node_stale_behind_upstream', nodeId });
|
|
2280
|
+
continue;
|
|
2281
|
+
}
|
|
2088
2282
|
const launchTarget = resolveAutoLaunchTarget(components, node);
|
|
2089
2283
|
if (launchTarget.mode === 'skip') {
|
|
2090
2284
|
// Remote node we can't reach (no transport / no coordinator daemonId).
|
|
@@ -2104,7 +2298,7 @@ async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, mesh
|
|
|
2104
2298
|
// busy-but-no-idle node is still allowed. Skip with a transient (non-actionable)
|
|
2105
2299
|
// reason so the coordinator is not paged; the 4s reconcile retries, and once the
|
|
2106
2300
|
// existing session goes terminal this gate clears and a legitimate launch proceeds.
|
|
2107
|
-
if (nodeHasLiveSessionPendingClaim(components, meshId, nodeId)) {
|
|
2301
|
+
if (nodeHasLiveSessionPendingClaim(components, meshId, nodeId, task, node)) {
|
|
2108
2302
|
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'node_has_live_session_pending_claim', nodeId });
|
|
2109
2303
|
continue;
|
|
2110
2304
|
}
|
|
@@ -2176,8 +2370,14 @@ async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, mesh
|
|
|
2176
2370
|
spawnedSessionVisibility: mesh?.policy?.spawnedSessionVisibility || 'hidden',
|
|
2177
2371
|
// Coordinator-dispatched worker: auto-approve unless mesh/node policy
|
|
2178
2372
|
// opts out (default true). Lands in settingsOverride and beats the
|
|
2179
|
-
// global per-provider-type
|
|
2180
|
-
|
|
2373
|
+
// global per-provider-type boolean/mode through explicit opposite-key clearing.
|
|
2374
|
+
...delegatedWorkerAutoApproveSettings(
|
|
2375
|
+
mesh?.policy,
|
|
2376
|
+
node?.policy,
|
|
2377
|
+
components.providerLoader?.getMeta(resolved.providerType),
|
|
2378
|
+
loadRepoConfigForNode(node),
|
|
2379
|
+
resolved.providerType,
|
|
2380
|
+
),
|
|
2181
2381
|
launchedByCoordinator: true,
|
|
2182
2382
|
autoLaunchedForQueueTaskId: task.id,
|
|
2183
2383
|
};
|
|
@@ -2842,4 +3042,3 @@ export function runIdleMaintenanceThenAssignQueue(components: DaemonComponents,
|
|
|
2842
3042
|
});
|
|
2843
3043
|
});
|
|
2844
3044
|
}
|
|
2845
|
-
|