@adhdev/daemon-core 0.9.82-rc.373 → 0.9.82-rc.375
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +372 -66
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +372 -66
- package/dist/index.mjs.map +1 -1
- package/dist/mesh/coordinator-prompt.d.ts +53 -0
- package/dist/mesh/mesh-events-stale.d.ts +12 -0
- package/dist/mesh/mesh-ledger.d.ts +1 -1
- package/dist/providers/chat-message-normalization.d.ts +1 -1
- package/dist/providers/cli-provider-instance.d.ts +3 -0
- package/package.json +2 -2
- package/src/commands/high-family/mesh-coordinator-launch.ts +67 -2
- package/src/commands/med-family/cli-agent.ts +25 -13
- package/src/commands/med-family/fast-forward.ts +80 -48
- package/src/commands/med-family/mesh-crud.ts +12 -2
- package/src/mesh/coordinator-prompt.ts +145 -0
- package/src/mesh/mesh-events-coordinator.ts +64 -1
- package/src/mesh/mesh-events-stale.ts +23 -0
- package/src/mesh/mesh-events-utils.ts +1 -1
- package/src/mesh/mesh-ledger.ts +5 -0
- package/src/mesh/mesh-reconcile-loop.ts +145 -10
- package/src/providers/chat-message-normalization.ts +1 -1
- package/src/providers/cli-provider-instance.ts +69 -2
|
@@ -33,6 +33,49 @@ import type {
|
|
|
33
33
|
} from '../repo-mesh-types.js';
|
|
34
34
|
import { DEFAULT_MESH_POLICY } from '../repo-mesh-types.js';
|
|
35
35
|
|
|
36
|
+
/**
|
|
37
|
+
* Cheap, locally-derived "what just happened" snapshot for the coordinator
|
|
38
|
+
* prompt. Built at launch from the local ledger + work-queue stats — no remote
|
|
39
|
+
* peer probe. Surfaces the gap a fresh coordinator otherwise misses: it can't
|
|
40
|
+
* see recent failures / queue depth until it manually calls mesh_task_history.
|
|
41
|
+
*
|
|
42
|
+
* All fields are optional so callers that have nothing to report (or fail to
|
|
43
|
+
* read the ledger) simply omit the section — the prompt output stays identical
|
|
44
|
+
* to the pre-activity form in that case.
|
|
45
|
+
*/
|
|
46
|
+
export interface CoordinatorRecentActivity {
|
|
47
|
+
/** task_failed entries from the recent window, newest last. */
|
|
48
|
+
recentFailures?: Array<{
|
|
49
|
+
timestamp?: string;
|
|
50
|
+
nodeId?: string;
|
|
51
|
+
/** Short task title/message, already truncated by the caller. */
|
|
52
|
+
summary?: string;
|
|
53
|
+
}>;
|
|
54
|
+
/** Count of task_failed entries inside the recent (30-min) window. */
|
|
55
|
+
recentFailureCount?: number;
|
|
56
|
+
/** Pending (unclaimed) tasks in the work queue. */
|
|
57
|
+
pendingTasks?: number;
|
|
58
|
+
/** Assigned-but-not-yet-terminal tasks in the work queue. */
|
|
59
|
+
assignedTasks?: number;
|
|
60
|
+
/** Stalled tasks recorded in the ledger. */
|
|
61
|
+
stalledTasks?: number;
|
|
62
|
+
/** ISO timestamp of the most recent ledger activity, if any. */
|
|
63
|
+
lastActivityAt?: string | null;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* One coordinator operating note — a runtime-accumulated lesson (provider
|
|
68
|
+
* quirk, pattern to avoid, recovery lesson) persisted in the ledger so it
|
|
69
|
+
* survives coordinator restarts and is provider-neutral (visible to codex /
|
|
70
|
+
* hermes / antigravity coordinators, not just Claude's memory).
|
|
71
|
+
*/
|
|
72
|
+
export interface CoordinatorOperatingNote {
|
|
73
|
+
text: string;
|
|
74
|
+
category?: 'provider_quirk' | 'pattern_to_avoid' | 'recovery_lesson';
|
|
75
|
+
createdAt?: string;
|
|
76
|
+
sourceCoordinator?: string;
|
|
77
|
+
}
|
|
78
|
+
|
|
36
79
|
// ─── Prompt Builder ─────────────────────────────
|
|
37
80
|
|
|
38
81
|
export interface CoordinatorPromptContext {
|
|
@@ -46,6 +89,18 @@ export interface CoordinatorPromptContext {
|
|
|
46
89
|
* stays identical to the pre-M3 form in that case.
|
|
47
90
|
*/
|
|
48
91
|
missionSection?: string;
|
|
92
|
+
/**
|
|
93
|
+
* Gap1: recent ledger/queue activity surfaced so a freshly-launched
|
|
94
|
+
* coordinator sees recent failures + queue depth without first calling
|
|
95
|
+
* mesh_task_history. Omitted → no "## Recent Activity" section.
|
|
96
|
+
*/
|
|
97
|
+
recentActivity?: CoordinatorRecentActivity;
|
|
98
|
+
/**
|
|
99
|
+
* Gap2-A: runtime-accumulated operating notes (provider-neutral lessons)
|
|
100
|
+
* read from the ledger at launch. Omitted/empty → no "## Operating Notes"
|
|
101
|
+
* section.
|
|
102
|
+
*/
|
|
103
|
+
operatingNotes?: CoordinatorOperatingNote[];
|
|
49
104
|
}
|
|
50
105
|
|
|
51
106
|
/**
|
|
@@ -132,6 +187,14 @@ Repository: \`${mesh.repoIdentity}\`${mesh.defaultBranch ? `\nDefault branch: \`
|
|
|
132
187
|
sections.push(ctx.missionSection.trim());
|
|
133
188
|
}
|
|
134
189
|
|
|
190
|
+
// ── Recent Activity (Gap1) — only present when there's something to show ──
|
|
191
|
+
const recentActivity = buildRecentActivitySection(ctx.recentActivity);
|
|
192
|
+
if (recentActivity) sections.push(recentActivity);
|
|
193
|
+
|
|
194
|
+
// ── Operating Notes (Gap2-A) — only present when notes exist ──
|
|
195
|
+
const operatingNotes = buildOperatingNotesSection(ctx.operatingNotes);
|
|
196
|
+
if (operatingNotes) sections.push(operatingNotes);
|
|
197
|
+
|
|
135
198
|
// ── Policy ──
|
|
136
199
|
sections.push(buildPolicySection({ ...DEFAULT_MESH_POLICY, ...(mesh.policy || {}) }));
|
|
137
200
|
|
|
@@ -187,6 +250,8 @@ function readUserPromptFile(cliType: string | undefined, suffix: string): string
|
|
|
187
250
|
* {{cliType}} — coordinator CLI type or empty
|
|
188
251
|
* {{nodes}} — full node section (status if known, otherwise config)
|
|
189
252
|
* {{mission}} — active mission summary section (empty when none)
|
|
253
|
+
* {{recentActivity}} — recent failures + queue depth section (empty when none)
|
|
254
|
+
* {{operatingNotes}} — accumulated operating notes section (empty when none)
|
|
190
255
|
* {{policy}} — full policy section
|
|
191
256
|
* {{tools}} — the canonical tools table
|
|
192
257
|
* {{workflow}} — the canonical orchestration workflow
|
|
@@ -211,6 +276,8 @@ function expandPromptPlaceholders(template: string, ctx: CoordinatorPromptContex
|
|
|
211
276
|
cliType: coordinatorCliType || '',
|
|
212
277
|
nodes: nodesSection,
|
|
213
278
|
mission: ctx.missionSection?.trim() || '',
|
|
279
|
+
recentActivity: buildRecentActivitySection(ctx.recentActivity) || '',
|
|
280
|
+
operatingNotes: buildOperatingNotesSection(ctx.operatingNotes) || '',
|
|
214
281
|
policy: buildPolicySection({ ...DEFAULT_MESH_POLICY, ...(mesh.policy || {}) }),
|
|
215
282
|
tools: TOOLS_SECTION,
|
|
216
283
|
workflow: WORKFLOW_SECTION,
|
|
@@ -303,6 +370,83 @@ function indentFollowing(text: string, pad: string): string {
|
|
|
303
370
|
return [lines[0], ...lines.slice(1).map(l => pad + l)].join('\n');
|
|
304
371
|
}
|
|
305
372
|
|
|
373
|
+
/**
|
|
374
|
+
* Gap1 — render the "## Recent Activity" section from the locally-derived
|
|
375
|
+
* activity snapshot. Returns '' (no section) when there's nothing worth
|
|
376
|
+
* surfacing: no recent failures, no queued work, no stalls. This keeps a quiet
|
|
377
|
+
* mesh's prompt identical to the pre-activity form.
|
|
378
|
+
*/
|
|
379
|
+
function buildRecentActivitySection(activity?: CoordinatorRecentActivity): string {
|
|
380
|
+
if (!activity) return '';
|
|
381
|
+
const failures = Array.isArray(activity.recentFailures) ? activity.recentFailures : [];
|
|
382
|
+
const pending = Number.isFinite(activity.pendingTasks) ? Number(activity.pendingTasks) : 0;
|
|
383
|
+
const assigned = Number.isFinite(activity.assignedTasks) ? Number(activity.assignedTasks) : 0;
|
|
384
|
+
const stalled = Number.isFinite(activity.stalledTasks) ? Number(activity.stalledTasks) : 0;
|
|
385
|
+
const recentFailureCount = Number.isFinite(activity.recentFailureCount)
|
|
386
|
+
? Number(activity.recentFailureCount)
|
|
387
|
+
: failures.length;
|
|
388
|
+
|
|
389
|
+
// Nothing actionable to show → omit the section entirely.
|
|
390
|
+
if (failures.length === 0 && pending === 0 && assigned === 0 && stalled === 0 && recentFailureCount === 0) {
|
|
391
|
+
return '';
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
const lines: string[] = ['## Recent Activity', ''];
|
|
395
|
+
lines.push('A snapshot of this mesh\'s recent ledger/queue state at launch. Use it to decide what needs attention first; call `mesh_task_history` / `mesh_view_queue` for full detail.');
|
|
396
|
+
lines.push('');
|
|
397
|
+
|
|
398
|
+
const counts: string[] = [];
|
|
399
|
+
if (pending > 0) counts.push(`**${pending}** pending`);
|
|
400
|
+
if (assigned > 0) counts.push(`**${assigned}** assigned`);
|
|
401
|
+
if (stalled > 0) counts.push(`**${stalled}** stalled`);
|
|
402
|
+
if (recentFailureCount > 0) counts.push(`**${recentFailureCount}** failed in the last 30 min`);
|
|
403
|
+
if (counts.length) lines.push(`- Queue/ledger: ${counts.join(', ')}.`);
|
|
404
|
+
if (activity.lastActivityAt) lines.push(`- Last ledger activity: ${activity.lastActivityAt}.`);
|
|
405
|
+
|
|
406
|
+
if (failures.length > 0) {
|
|
407
|
+
// Newest first, capped to the 5 most recent so the prompt stays lean.
|
|
408
|
+
const recent = failures.slice(-5).reverse();
|
|
409
|
+
lines.push('', 'Recent failures (newest first):');
|
|
410
|
+
for (const f of recent) {
|
|
411
|
+
const when = f.timestamp ? `${f.timestamp} ` : '';
|
|
412
|
+
const node = f.nodeId ? `node \`${f.nodeId}\`` : 'unknown node';
|
|
413
|
+
const summary = (f.summary || '').trim();
|
|
414
|
+
lines.push(`- ${when}${node}${summary ? ` — ${summary}` : ''}`);
|
|
415
|
+
}
|
|
416
|
+
lines.push('', '_Check `mesh_task_history` before retrying; repeated failures on the same node mean reassign or escalate, not retry._');
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
return lines.join('\n');
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
/**
|
|
423
|
+
* Gap2-A — render the "## Operating Notes" section from accumulated coordinator
|
|
424
|
+
* notes. Returns '' when there are none, so a mesh that has never recorded a
|
|
425
|
+
* note gets the unchanged prompt. Notes are runtime-accumulated lessons that
|
|
426
|
+
* persist across coordinator restarts and are provider-neutral.
|
|
427
|
+
*/
|
|
428
|
+
function buildOperatingNotesSection(notes?: CoordinatorOperatingNote[]): string {
|
|
429
|
+
const valid = Array.isArray(notes)
|
|
430
|
+
? notes.filter(n => n && typeof n.text === 'string' && n.text.trim())
|
|
431
|
+
: [];
|
|
432
|
+
if (valid.length === 0) return '';
|
|
433
|
+
|
|
434
|
+
const categoryLabel: Record<string, string> = {
|
|
435
|
+
provider_quirk: 'provider quirk',
|
|
436
|
+
pattern_to_avoid: 'pattern to avoid',
|
|
437
|
+
recovery_lesson: 'recovery lesson',
|
|
438
|
+
};
|
|
439
|
+
|
|
440
|
+
const lines: string[] = ['## Operating Notes', ''];
|
|
441
|
+
lines.push('Lessons earlier coordinators on this mesh recorded via `mesh_record_note`. Treat them as accumulated operating knowledge — apply them. When you learn a durable lesson (a provider quirk, a pattern to avoid, a recovery lesson), record it with `mesh_record_note` so future coordinators inherit it.');
|
|
442
|
+
lines.push('');
|
|
443
|
+
for (const n of valid) {
|
|
444
|
+
const cat = n.category && categoryLabel[n.category] ? `[${categoryLabel[n.category]}] ` : '';
|
|
445
|
+
lines.push(`- ${cat}${n.text.trim()}`);
|
|
446
|
+
}
|
|
447
|
+
return lines.join('\n');
|
|
448
|
+
}
|
|
449
|
+
|
|
306
450
|
function buildPolicySection(policy: RepoMeshPolicy): string {
|
|
307
451
|
const rules: string[] = [];
|
|
308
452
|
if (policy.requirePreTaskCheckpoint) rules.push('- Create a git checkpoint **before** starting each task');
|
|
@@ -340,6 +484,7 @@ const TOOLS_SECTION = `## Available Tools
|
|
|
340
484
|
| \`mesh_read_chat\` | Read recent chat messages from a delegated agent session |
|
|
341
485
|
| \`mesh_read_debug\` | Collect a daemon-side chat/parser debug bundle for a session |
|
|
342
486
|
| \`mesh_task_history\` | Read the task ledger — dispatches, completions, failures. Use to understand what has been done before deciding next steps |
|
|
487
|
+
| \`mesh_record_note\` | Record a durable, provider-neutral operating note (provider quirk / pattern to avoid / recovery lesson). Future coordinators see it under "## Operating Notes" at launch |
|
|
343
488
|
| \`mesh_git_status\` | Check git status on a specific node |
|
|
344
489
|
| \`mesh_read_node_logs\` | Fetch a remote node's daemon log tail directly over P2P (grep/since/byte-bounded, secrets redacted) — no session/PowerShell needed to debug a node's daemon |
|
|
345
490
|
| \`mesh_fast_forward_node\` | Safely dry-run or explicitly execute an obvious clean fast-forward without launching an agent session |
|
|
@@ -23,6 +23,7 @@ import type { RepoMeshSchedulingStrategy } from '../repo-mesh-types.js';
|
|
|
23
23
|
import { normalizeMeshNodeId, meshNodeIdMatches, daemonIdsEquivalent, expandDaemonIdForms, normalizeMeshWorkspaceForCompare, meshWorkspacesEquivalent, type MeshNodeIdentified } from '@adhdev/mesh-shared';
|
|
24
24
|
import {
|
|
25
25
|
findRecentTerminalLedgerEvidence,
|
|
26
|
+
findTerminalLedgerEvidenceForTask,
|
|
26
27
|
hasDispatchAfterTerminal,
|
|
27
28
|
hasUnterminalDirectDispatchLedgerEntry,
|
|
28
29
|
buildNoProgressCompletionReconciliation,
|
|
@@ -393,6 +394,42 @@ function isWeakTerminalLedgerPayload(payload: Record<string, unknown> | undefine
|
|
|
393
394
|
return diag?.finalAssistantPresent === false || diag?.blockReason === 'missing_final_assistant';
|
|
394
395
|
}
|
|
395
396
|
|
|
397
|
+
// (FALSEIDLE-BGCHILD-b) A later genuine completion of the SAME task that carries a
|
|
398
|
+
// substantively different — and fuller — final summary than the recorded terminal is the REAL
|
|
399
|
+
// final that an earlier (false-idle) completion pre-empted, not a duplicate. The background-child
|
|
400
|
+
// false idle is the nasty case the plain isWeakTerminalLedgerPayload supersession misses: the
|
|
401
|
+
// early completion's screen parser DID see a prior/intermediate standard assistant, so it is
|
|
402
|
+
// recorded as a STRONG terminal with a non-empty (but truncated) finalSummary. Without this the
|
|
403
|
+
// providerSessionId/finalSummary dedup below swallows the genuine final and the coordinator is
|
|
404
|
+
// stuck with the truncated mid-turn text forever (the one-shot-consumption symptom). Same-task,
|
|
405
|
+
// new event is genuine, prior terminal summary is a strict prefix of (or otherwise shorter than)
|
|
406
|
+
// the new one → treat as the corrected final and let it through. Conservative: requires the new
|
|
407
|
+
// summary to be genuine evidence AND meaningfully longer, so an identical re-arrival or a SHORTER
|
|
408
|
+
// later summary is still deduped.
|
|
409
|
+
function supersedesTruncatedTerminalSummary(args: {
|
|
410
|
+
terminalPayload: Record<string, unknown>;
|
|
411
|
+
metadataEvent: Record<string, unknown>;
|
|
412
|
+
terminalTaskId: string;
|
|
413
|
+
eventTaskId: string;
|
|
414
|
+
}): boolean {
|
|
415
|
+
// Only applies when both name the SAME task (a distinct task is handled by distinctTaskCompletion).
|
|
416
|
+
if (!args.terminalTaskId || !args.eventTaskId || args.terminalTaskId !== args.eventTaskId) return false;
|
|
417
|
+
if (!isGenuineCompletionEvidence(args.metadataEvent)) return false;
|
|
418
|
+
const terminalSummary = readNonEmptyString(args.terminalPayload.finalSummary);
|
|
419
|
+
const eventSummary = readNonEmptyString(args.metadataEvent.finalSummary);
|
|
420
|
+
if (!eventSummary) return false;
|
|
421
|
+
// Identical text → genuine duplicate, keep deduping.
|
|
422
|
+
if (terminalSummary === eventSummary) return false;
|
|
423
|
+
// The recorded terminal was a known-weak (false-idle) one → already handled by the weak
|
|
424
|
+
// supersession path; nothing extra to do here.
|
|
425
|
+
if (isWeakTerminalLedgerPayload(args.terminalPayload)) return false;
|
|
426
|
+
// No prior summary at all, or the new summary strictly extends / is meaningfully longer than
|
|
427
|
+
// the recorded one → the recorded terminal was the truncated pre-emption; supersede it.
|
|
428
|
+
if (!terminalSummary) return true;
|
|
429
|
+
if (eventSummary.startsWith(terminalSummary)) return true;
|
|
430
|
+
return eventSummary.length > terminalSummary.length + 32;
|
|
431
|
+
}
|
|
432
|
+
|
|
396
433
|
// The latest still-active direct-dispatch taskId for a session, resolved BEFORE the
|
|
397
434
|
// completion flips the dispatch row terminal. Direct dispatches (mesh_send_task) have no
|
|
398
435
|
// work-queue row, so this is the only taskId available to attribute the terminal ledger
|
|
@@ -654,6 +691,24 @@ export function tryAssignQueueTask(
|
|
|
654
691
|
return false;
|
|
655
692
|
}
|
|
656
693
|
|
|
694
|
+
const terminal = findTerminalLedgerEvidenceForTask({
|
|
695
|
+
meshId,
|
|
696
|
+
taskId: task.id,
|
|
697
|
+
});
|
|
698
|
+
if (terminal) {
|
|
699
|
+
const status = terminal.kind === 'task_completed' ? 'completed' : 'failed';
|
|
700
|
+
updateTaskStatus(meshId, task.id, status);
|
|
701
|
+
LOG.info('MeshQueue', `Skipped dispatch for terminal task ${task.id} on mesh ${meshId}; ${terminal.kind} ledger evidence already exists`);
|
|
702
|
+
traceMeshEventDrop('dispatch_terminal_ledger', {
|
|
703
|
+
taskId: task.id,
|
|
704
|
+
sessionId,
|
|
705
|
+
nodeId,
|
|
706
|
+
meshId,
|
|
707
|
+
event: 'agent_command',
|
|
708
|
+
}, terminal.kind);
|
|
709
|
+
return false;
|
|
710
|
+
}
|
|
711
|
+
|
|
657
712
|
LOG.info('MeshQueue', `Node ${nodeId} (${sessionId}) pulled task ${task.id}`);
|
|
658
713
|
|
|
659
714
|
if (node?.daemonId && components.dispatchMeshCommand) {
|
|
@@ -1944,7 +1999,15 @@ function evaluateMeshEventSuppression(
|
|
|
1944
1999
|
const terminalTaskId = readNonEmptyString(terminal.payload.taskId);
|
|
1945
2000
|
const eventTaskId = readNonEmptyString(args.metadataEvent.taskId);
|
|
1946
2001
|
const distinctTaskCompletion = !!eventTaskId && !!terminalTaskId && eventTaskId !== terminalTaskId;
|
|
1947
|
-
|
|
2002
|
+
// (FALSEIDLE-BGCHILD-b) Same-task genuine completion carrying a fuller summary than the
|
|
2003
|
+
// recorded (truncated, false-idle-pre-empted) terminal supersedes it — see helper.
|
|
2004
|
+
const supersedesTruncatedTerminal = supersedesTruncatedTerminalSummary({
|
|
2005
|
+
terminalPayload: terminal.payload,
|
|
2006
|
+
metadataEvent: args.metadataEvent,
|
|
2007
|
+
terminalTaskId,
|
|
2008
|
+
eventTaskId,
|
|
2009
|
+
});
|
|
2010
|
+
if (!newDispatchAfterTerminal && !supersedesWeakTerminal && !distinctTaskCompletion && !supersedesTruncatedTerminal) {
|
|
1948
2011
|
const terminalProviderSessionId = readNonEmptyString(terminal.payload.providerSessionId);
|
|
1949
2012
|
const terminalFinalSummary = readNonEmptyString(terminal.payload.finalSummary);
|
|
1950
2013
|
const eventProviderSessionId = readNonEmptyString(args.metadataEvent.providerSessionId);
|
|
@@ -85,6 +85,29 @@ function isWeakCompletionLedgerPayload(payload: Record<string, unknown> | undefi
|
|
|
85
85
|
return diag?.finalAssistantPresent === false || diag?.blockReason === 'missing_final_assistant';
|
|
86
86
|
}
|
|
87
87
|
|
|
88
|
+
export function findTerminalLedgerEvidenceForTask(args: {
|
|
89
|
+
meshId: string;
|
|
90
|
+
taskId?: string;
|
|
91
|
+
sessionId?: string;
|
|
92
|
+
nodeId?: string;
|
|
93
|
+
tail?: number;
|
|
94
|
+
}): { id: string; kind: Extract<MeshLedgerKind, 'task_completed' | 'task_failed' | 'task_stalled'>; payload: Record<string, unknown>; timestamp: string } | null {
|
|
95
|
+
const taskId = readNonEmptyString(args.taskId);
|
|
96
|
+
if (!taskId) return null;
|
|
97
|
+
const entries = readLedgerEntries(args.meshId, { tail: args.tail ?? 500 });
|
|
98
|
+
for (let i = entries.length - 1; i >= 0; i--) {
|
|
99
|
+
const entry = entries[i];
|
|
100
|
+
if (entry.kind !== 'task_completed' && entry.kind !== 'task_failed' && entry.kind !== 'task_stalled') continue;
|
|
101
|
+
const terminalTaskId = readNonEmptyString(entry.payload?.taskId);
|
|
102
|
+
if (terminalTaskId !== taskId) continue;
|
|
103
|
+
if (entry.kind === 'task_completed' && isWeakCompletionLedgerPayload(entry.payload)) continue;
|
|
104
|
+
if (args.sessionId && entry.sessionId && entry.sessionId !== args.sessionId) continue;
|
|
105
|
+
if (!args.sessionId && args.nodeId && entry.nodeId && !meshNodeIdMatches(entry as unknown as MeshNodeIdentified, args.nodeId)) continue;
|
|
106
|
+
return { id: entry.id, kind: entry.kind, payload: entry.payload || {}, timestamp: entry.timestamp };
|
|
107
|
+
}
|
|
108
|
+
return null;
|
|
109
|
+
}
|
|
110
|
+
|
|
88
111
|
function findDirectDispatchLedgerEntry(args: {
|
|
89
112
|
meshId: string;
|
|
90
113
|
taskId: string;
|
|
@@ -109,7 +109,7 @@ const MESH_SURFACED_PREVIEW_MAX_CHARS = 512;
|
|
|
109
109
|
// coordinator-facing payload that replaces a "go call mesh_read_chat" instruction —
|
|
110
110
|
// it should carry enough of the worker's result to act on without a second round-trip,
|
|
111
111
|
// while still bounding what is written into the coordinator PTY.
|
|
112
|
-
const MESH_COMPLETION_SURFACE_MAX_CHARS =
|
|
112
|
+
const MESH_COMPLETION_SURFACE_MAX_CHARS = 16000;
|
|
113
113
|
|
|
114
114
|
/**
|
|
115
115
|
* The worker's final assistant text carried on a completion event — read from
|
package/src/mesh/mesh-ledger.ts
CHANGED
|
@@ -44,6 +44,11 @@ export type MeshLedgerKind =
|
|
|
44
44
|
| 'direct_dispatch_pruned'
|
|
45
45
|
| 'event_held'
|
|
46
46
|
| 'task_reclaimed'
|
|
47
|
+
// Gap2-A: a coordinator-recorded operating note — a runtime-accumulated
|
|
48
|
+
// lesson (provider quirk, pattern to avoid, recovery lesson) persisted in
|
|
49
|
+
// the ledger so it survives coordinator restarts and is provider-neutral.
|
|
50
|
+
// payload: { text, category?, createdAt?, sourceCoordinator? }
|
|
51
|
+
| 'coordinator_operating_note'
|
|
47
52
|
;
|
|
48
53
|
|
|
49
54
|
export interface MeshLedgerEntry {
|
|
@@ -58,10 +58,10 @@ import {
|
|
|
58
58
|
import { readNonEmptyString, readMeshCompletionSummary } from './mesh-events-utils.js';
|
|
59
59
|
import { traceMeshEventStage, traceMeshEventDrop } from './mesh-event-trace.js';
|
|
60
60
|
import { expandDaemonIdForms, daemonIdsEquivalent } from '@adhdev/mesh-shared';
|
|
61
|
-
import { getActiveDirectDispatches, getQueue, reclaimStrandedAssignedTask } from './mesh-work-queue.js';
|
|
61
|
+
import { getActiveDirectDispatches, getQueue, reclaimStrandedAssignedTask, updateTaskStatus } from './mesh-work-queue.js';
|
|
62
62
|
import { readLedgerEntries } from './mesh-ledger.js';
|
|
63
63
|
import { pruneStaleDirectDispatches } from './mesh-active-work.js';
|
|
64
|
-
import { reconcileDirectDispatchCompletionFromTranscript } from './mesh-events-stale.js';
|
|
64
|
+
import { findTerminalLedgerEvidenceForTask, reconcileDirectDispatchCompletionFromTranscript } from './mesh-events-stale.js';
|
|
65
65
|
import { extractFinalAssistantSummaryEvidence } from '../providers/chat-message-normalization.js';
|
|
66
66
|
import type { ChatMessage } from '../types.js';
|
|
67
67
|
|
|
@@ -164,7 +164,12 @@ function daemonHostsMesh(mesh: LocalMeshEntry, daemonIds: string[]): boolean {
|
|
|
164
164
|
const hostDaemonId = readNonEmptyString(host.hostDaemonId);
|
|
165
165
|
// Host role but no pinned hostDaemonId → treat as host (single-daemon / legacy).
|
|
166
166
|
if (!hostDaemonId) return true;
|
|
167
|
-
return daemonIds
|
|
167
|
+
return daemonIdListIncludes(daemonIds, hostDaemonId);
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
function daemonIdListIncludes(ids: readonly string[], id: string | undefined): boolean {
|
|
171
|
+
if (!id) return false;
|
|
172
|
+
return ids.some(candidate => candidate === id || daemonIdsEquivalent(candidate, id));
|
|
168
173
|
}
|
|
169
174
|
|
|
170
175
|
// Resolve EVERY id-form this daemon answers to FOR A GIVEN MESH: the runtime drain
|
|
@@ -183,8 +188,8 @@ function resolveCoordinatorSelfIds(mesh: LocalMeshEntry, drainDaemonIds: string[
|
|
|
183
188
|
for (const node of mesh.nodes) {
|
|
184
189
|
const nodeDaemonId = readNonEmptyString(node.daemonId);
|
|
185
190
|
const nodeMachineId = readNonEmptyString(node.machineId);
|
|
186
|
-
const isSelf = (nodeDaemonId && drainDaemonIds
|
|
187
|
-
|| (nodeMachineId && drainDaemonIds
|
|
191
|
+
const isSelf = (nodeDaemonId && daemonIdListIncludes(drainDaemonIds, nodeDaemonId))
|
|
192
|
+
|| (nodeMachineId && daemonIdListIncludes(drainDaemonIds, nodeMachineId));
|
|
188
193
|
if (!isSelf) continue;
|
|
189
194
|
if (nodeDaemonId) ids.add(nodeDaemonId);
|
|
190
195
|
if (nodeMachineId) ids.add(nodeMachineId);
|
|
@@ -196,10 +201,17 @@ function resolveCoordinatorSelfIds(mesh: LocalMeshEntry, drainDaemonIds: string[
|
|
|
196
201
|
// this daemon does not make this daemon the host; daemonHostsMesh still honours a
|
|
197
202
|
// foreign hostDaemonId and rejects ownership.
|
|
198
203
|
const hostDaemonId = readNonEmptyString(mesh.meshHost?.hostDaemonId);
|
|
199
|
-
if (hostDaemonId && ids
|
|
204
|
+
if (hostDaemonId && daemonIdListIncludes([...ids], hostDaemonId)) ids.add(hostDaemonId);
|
|
200
205
|
return [...ids];
|
|
201
206
|
}
|
|
202
207
|
|
|
208
|
+
// Observability: last-seen modal-park state per coordinator session, so we LOG.info
|
|
209
|
+
// only on a TRANSITION (clear → parked, parked → cleared) instead of every 4s tick.
|
|
210
|
+
// Per-process; a restart re-logs the first observation, which is desirable — it
|
|
211
|
+
// re-confirms a coordinator that is still parked after the restart (the exact
|
|
212
|
+
// "restart does not clear it" symptom the operator needs visibility into).
|
|
213
|
+
const coordinatorModalParkState = new Map<string, boolean>();
|
|
214
|
+
|
|
203
215
|
// Find live CLI coordinator instances on THIS daemon, keyed by mesh.
|
|
204
216
|
function findLiveCoordinators(components: DaemonComponents): LiveCoordinator[] {
|
|
205
217
|
const out: LiveCoordinator[] = [];
|
|
@@ -217,6 +229,20 @@ function findLiveCoordinators(components: DaemonComponents): LiveCoordinator[] {
|
|
|
217
229
|
// and waiting_choice is absent from some of them (see cli-provider-instance).
|
|
218
230
|
const modalParked = status === 'waiting_choice' || status === 'waiting_approval';
|
|
219
231
|
const sessionId = readNonEmptyString(state.instanceId);
|
|
232
|
+
// Modal-park transition observability: a coordinator entering modal-park is what
|
|
233
|
+
// begins holding completion events under `modal_parked`; one leaving it is what
|
|
234
|
+
// drains them. Both transitions were previously SILENT (the operator had no log
|
|
235
|
+
// to diagnose a stuck/held completion), so emit a single line per edge.
|
|
236
|
+
const stateKey = `${meshId}::${sessionId || '?'}`;
|
|
237
|
+
const prevParked = coordinatorModalParkState.get(stateKey);
|
|
238
|
+
if (prevParked !== modalParked) {
|
|
239
|
+
coordinatorModalParkState.set(stateKey, modalParked);
|
|
240
|
+
if (modalParked) {
|
|
241
|
+
LOG.info('MeshReconcile', `Coordinator ${sessionId || '?'} (mesh ${meshId}) entered modal-park (status=${status}) — terminal events for it will be held until the modal is answered`);
|
|
242
|
+
} else if (prevParked === true) {
|
|
243
|
+
LOG.info('MeshReconcile', `Coordinator ${sessionId || '?'} (mesh ${meshId}) left modal-park (status=${status}) — held events will drain on this/next tick`);
|
|
244
|
+
}
|
|
245
|
+
}
|
|
220
246
|
out.push({ meshId, instance: inst, sessionId, idle: status === 'idle', modalParked });
|
|
221
247
|
}
|
|
222
248
|
return out;
|
|
@@ -358,6 +384,23 @@ function recoverStrandedAssignedDispatches(meshId: string, store: MeshRuntimeSto
|
|
|
358
384
|
const dispatchedAtMs = Date.parse(row.dispatchTimestamp ?? '');
|
|
359
385
|
if (!Number.isFinite(dispatchedAtMs)) continue; // no dispatch ts → can't age it
|
|
360
386
|
if (nowMs - dispatchedAtMs < ASSIGNED_STRANDED_DEADLINE_MS) continue; // still in confirm window
|
|
387
|
+
const terminal = findTerminalLedgerEvidenceForTask({
|
|
388
|
+
meshId,
|
|
389
|
+
taskId: row.id,
|
|
390
|
+
});
|
|
391
|
+
if (terminal) {
|
|
392
|
+
const status = terminal.kind === 'task_completed' ? 'completed' : 'failed';
|
|
393
|
+
updateTaskStatus(meshId, row.id, status);
|
|
394
|
+
LOG.warn('MeshReconcile', `Skipped stranded reclaim redispatch for terminal task ${row.id} on mesh ${meshId}; ${terminal.kind} ledger evidence already exists`);
|
|
395
|
+
traceMeshEventDrop('assigned_stranded_terminal_ledger', {
|
|
396
|
+
taskId: row.id,
|
|
397
|
+
sessionId: row.assignedSessionId,
|
|
398
|
+
nodeId: row.assignedNodeId,
|
|
399
|
+
meshId,
|
|
400
|
+
event: 'agent:generating_completed',
|
|
401
|
+
}, terminal.kind);
|
|
402
|
+
continue;
|
|
403
|
+
}
|
|
361
404
|
if (store.taskHasConfirmedDelivery(meshId, row.id)) continue; // dispatched → PHASE 4's job
|
|
362
405
|
const reclaimed = reclaimStrandedAssignedTask(meshId, row.id, {
|
|
363
406
|
reason: 'assigned_stranded_dispatch_unconfirmed',
|
|
@@ -577,7 +620,73 @@ export async function runMeshReconcileTick(components: DaemonComponents): Promis
|
|
|
577
620
|
// force-injected via generatingCoordinators — we never block the deadlock-break.)
|
|
578
621
|
if (targetCoordinators.length === 0) {
|
|
579
622
|
if (modalParkedCoordinators.length > 0) {
|
|
580
|
-
|
|
623
|
+
// ── orphan escape (MUST precede the blanket modal-park hold) ──────────
|
|
624
|
+
// A modal-parked coordinator with no idle/generating sibling otherwise
|
|
625
|
+
// wedges EVERY pending event under `modal_parked` until that modal resolves
|
|
626
|
+
// — including a STRICT-routed completion whose originating coordinator
|
|
627
|
+
// session is GONE (an orphan: the worktree/session that produced it was
|
|
628
|
+
// removed, or that coordinator session died). Such an event will never be
|
|
629
|
+
// deliverable to its target session no matter what the modal-parked sibling
|
|
630
|
+
// does, so holding it under modal_parked is a permanent-held leak (the very
|
|
631
|
+
// "data restart re-reproduces it" symptom — the gate is reconstructed live
|
|
632
|
+
// from the still-parked modal, so a restart does not clear it). Route those
|
|
633
|
+
// orphan events through the strict-route hold/expire path so the bounded
|
|
634
|
+
// STRICT_SESSION_MATCH_TTL eventually expires them (recoverable, ledgered)
|
|
635
|
+
// instead of leaving them held forever. A strict event whose target session
|
|
636
|
+
// IS live but merely modal-parked is left to the blanket hold below (it is
|
|
637
|
+
// genuinely transiently blocked, not orphaned).
|
|
638
|
+
const liveSessionIds = new Set(
|
|
639
|
+
meshCoordinators.map(c => readNonEmptyString(c.sessionId)).filter(Boolean),
|
|
640
|
+
);
|
|
641
|
+
let orphanEscaped = 0;
|
|
642
|
+
const hasPendingForOrphanPeek = !store
|
|
643
|
+
|| (() => { try { return store.pendingEventCount(meshId) > 0; } catch { return true; } })();
|
|
644
|
+
if (hasPendingForOrphanPeek) {
|
|
645
|
+
// Identify which pending event NAMES correspond to orphan-targeted events
|
|
646
|
+
// (a strict targetCoordinatorSessionId that matches no live coordinator).
|
|
647
|
+
let peeked: readonly PendingMeshCoordinatorEvent[] = [];
|
|
648
|
+
try {
|
|
649
|
+
peeked = getPendingMeshCoordinatorEvents(meshId, drainDaemonIds.length > 0 ? drainDaemonIds : undefined);
|
|
650
|
+
} catch { peeked = []; }
|
|
651
|
+
const isOrphan = (e: PendingMeshCoordinatorEvent): boolean => {
|
|
652
|
+
const want = readNonEmptyString(e.targetCoordinatorSessionId);
|
|
653
|
+
return !!want && !liveSessionIds.has(want);
|
|
654
|
+
};
|
|
655
|
+
const orphanEventNames = new Set(peeked.filter(isOrphan).map(e => e.event));
|
|
656
|
+
if (orphanEventNames.size > 0) {
|
|
657
|
+
// The drain filter is event-NAME scoped (not per-row), so draining by the
|
|
658
|
+
// orphan event names also pulls any non-orphan event sharing that name. Drain
|
|
659
|
+
// them all, then re-route: orphan-targeted events go through the strict-route
|
|
660
|
+
// hold/expire path (bounded TTL → eventually ledger-expired, recoverable);
|
|
661
|
+
// non-orphan events of the same name are re-queued unchanged (queuedAt
|
|
662
|
+
// preserved) so they remain genuinely held for their still-live, modal-parked
|
|
663
|
+
// target. This is the same per-event strict routing PHASE 2 does below — just
|
|
664
|
+
// reached here because the blanket modal-park short-circuit would otherwise
|
|
665
|
+
// wedge the orphans forever.
|
|
666
|
+
let drained: PendingMeshCoordinatorEvent[] = [];
|
|
667
|
+
try {
|
|
668
|
+
drained = drainPendingMeshCoordinatorEvents(
|
|
669
|
+
meshId,
|
|
670
|
+
drainDaemonIds.length > 0 ? drainDaemonIds : localDaemonId,
|
|
671
|
+
{ onlyEvents: orphanEventNames },
|
|
672
|
+
);
|
|
673
|
+
} catch (e: any) {
|
|
674
|
+
LOG.warn('MeshReconcile', `Orphan-escape drain failed for mesh ${meshId}: ${e?.message || e}`);
|
|
675
|
+
drained = [];
|
|
676
|
+
}
|
|
677
|
+
for (const pending of drained) {
|
|
678
|
+
if (isOrphan(pending)) {
|
|
679
|
+
holdOrExpireStrictUnmatchedEvent(pending, readNonEmptyString(pending.targetCoordinatorSessionId), meshId);
|
|
680
|
+
orphanEscaped++;
|
|
681
|
+
} else {
|
|
682
|
+
// Still-live (modal-parked) target — re-queue unchanged so it is held
|
|
683
|
+
// for the next modal-resolved tick, exactly like the blanket hold would.
|
|
684
|
+
try { queuePendingMeshCoordinatorEvent(pending); } catch { /* best-effort re-queue */ }
|
|
685
|
+
}
|
|
686
|
+
}
|
|
687
|
+
}
|
|
688
|
+
}
|
|
689
|
+
LOG.info('MeshReconcile', `Reconcile skip → modal-parked: holding pending event(s) for mesh ${meshId} (${modalParkedCoordinators.length} coordinator(s) awaiting a modal answer; events left queued${orphanEscaped > 0 ? `; ${orphanEscaped} orphan-targeted event(s) routed to strict-route TTL` : ''})`);
|
|
581
690
|
// C1: mirror held terminal events into the ledger so a held completion's
|
|
582
691
|
// worker summary is auditable/recoverable even if the modal is never
|
|
583
692
|
// resolved, the coordinator restarts, or the pending file is later trimmed.
|
|
@@ -896,7 +1005,7 @@ async function pullRemoteNodeQueues(
|
|
|
896
1005
|
// from ourselves over P2P is both wasteful and a self-dispatch hazard.
|
|
897
1006
|
if (!nodeDaemonId) continue;
|
|
898
1007
|
if (daemonIdsEquivalent(nodeDaemonId, localDaemonId)) continue;
|
|
899
|
-
if (candidateDaemonIds
|
|
1008
|
+
if (daemonIdListIncludes(candidateDaemonIds, nodeDaemonId)) continue;
|
|
900
1009
|
|
|
901
1010
|
for (const pendingEventArgs of pulls) {
|
|
902
1011
|
let events: unknown;
|
|
@@ -972,7 +1081,7 @@ async function reconcileUnterminatedDirectDispatches(
|
|
|
972
1081
|
// A node is local when it has no daemonId, names this daemon, or actually
|
|
973
1082
|
// has a live instance here. Anything else is reached over P2P.
|
|
974
1083
|
const isLocalNode = !nodeDaemonId
|
|
975
|
-
|| selfIds
|
|
1084
|
+
|| daemonIdListIncludes(selfIds, nodeDaemonId)
|
|
976
1085
|
|| daemonIdsEquivalent(nodeDaemonId, localDaemonId)
|
|
977
1086
|
|| !!components.instanceManager.getInstance(sessionId);
|
|
978
1087
|
|
|
@@ -1012,6 +1121,32 @@ async function reconcileUnterminatedDirectDispatches(
|
|
|
1012
1121
|
const evidence = extractFinalAssistantSummaryEvidence(messages);
|
|
1013
1122
|
if (!evidence.finalSummary) continue; // no assistant result yet — nothing to attribute
|
|
1014
1123
|
|
|
1124
|
+
// STALE-SUMMARY guard (modal-parked / reused-session misattribution): a direct
|
|
1125
|
+
// dispatch frequently reuses a session that already ran a PRIOR task. read_chat
|
|
1126
|
+
// returns the tail of the WHOLE session, so extractFinalAssistantSummaryEvidence
|
|
1127
|
+
// picks the latest user-facing assistant message — which, for a task that has
|
|
1128
|
+
// barely started (the session momentarily reads idle between turns), is the prior
|
|
1129
|
+
// task's final summary. The downstream reconcile proves the summary is after the
|
|
1130
|
+
// LEDGER task_dispatched entry; here we additionally have the AUTHORITATIVE per-task
|
|
1131
|
+
// dispatchedAt (the dispatch-store row, immune to ledger-ordering quirks), so when
|
|
1132
|
+
// the selected transcript message is provably BEFORE this task's own dispatch we
|
|
1133
|
+
// refuse it outright — it is a prior task's summary, not this task's output (the
|
|
1134
|
+
// 2843ms-duration stale-summary bug where task 2e3f501e copy-pasted 4eca2d9d's
|
|
1135
|
+
// summary). When the message carries no usable timestamp we do NOT block here: the
|
|
1136
|
+
// downstream reconcile already rejects a non-JSON summary it cannot prove is
|
|
1137
|
+
// post-dispatch (transcript_not_proven_after_dispatch), and a structured
|
|
1138
|
+
// final_summary_json is self-attributing — so a timeless provider is not
|
|
1139
|
+
// over-blocked while the provable-stale case is still caught.
|
|
1140
|
+
const dispatchedAtMs = Date.parse(readNonEmptyString(dispatch.dispatchedAt));
|
|
1141
|
+
const transcriptAtMs = Date.parse(evidence.transcriptMessageAt ?? '');
|
|
1142
|
+
if (Number.isFinite(dispatchedAtMs) && Number.isFinite(transcriptAtMs) && transcriptAtMs < dispatchedAtMs) {
|
|
1143
|
+
LOG.info('MeshReconcile', `Stale-summary guard: skipping transcript reconcile for task ${taskId} on node ${nodeId} (mesh ${mesh.id}) — final assistant message (${evidence.transcriptMessageAt}) predates this task's dispatch (${dispatch.dispatchedAt}); it is a prior task's summary`);
|
|
1144
|
+
traceMeshEventDrop('reconcile_stale_summary_before_dispatch', {
|
|
1145
|
+
taskId, sessionId, nodeId, meshId: mesh.id, event: 'agent:generating_completed',
|
|
1146
|
+
}, `transcriptAt=${evidence.transcriptMessageAt} < dispatchedAt=${dispatch.dispatchedAt}`);
|
|
1147
|
+
continue;
|
|
1148
|
+
}
|
|
1149
|
+
|
|
1015
1150
|
const providerSessionId = readNonEmptyString(payload.providerSessionId);
|
|
1016
1151
|
const coordinatorDaemonId = selfIds.find(id => !!id);
|
|
1017
1152
|
try {
|
|
@@ -1092,7 +1227,7 @@ async function collectLiveNodesWithSessions(
|
|
|
1092
1227
|
return Promise.all(mesh.nodes.map(async (node) => {
|
|
1093
1228
|
const nodeDaemonId = readNonEmptyString(node.daemonId);
|
|
1094
1229
|
const isLocalNode = !nodeDaemonId
|
|
1095
|
-
|| selfIds
|
|
1230
|
+
|| daemonIdListIncludes(selfIds, nodeDaemonId)
|
|
1096
1231
|
|| daemonIdsEquivalent(nodeDaemonId, localDaemonId);
|
|
1097
1232
|
let statusResult: unknown;
|
|
1098
1233
|
try {
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { ChatMessage } from '../types.js';
|
|
2
2
|
import { flattenContent } from './contracts.js';
|
|
3
3
|
|
|
4
|
-
export const DEFAULT_FINAL_SUMMARY_MAX_CHARS =
|
|
4
|
+
export const DEFAULT_FINAL_SUMMARY_MAX_CHARS = 16_000;
|
|
5
5
|
|
|
6
6
|
export function extractFinalSummaryFromMessages(
|
|
7
7
|
messages: ChatMessage[] | null | undefined,
|