@adhdev/daemon-core 0.9.82-rc.468 → 0.9.82-rc.469
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +7 -5
- package/dist/index.js +683 -495
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +674 -493
- package/dist/index.mjs.map +1 -1
- package/dist/mesh/mesh-completion-synthesis.d.ts +4 -0
- package/dist/mesh/mesh-delivery-policy.d.ts +0 -27
- package/dist/mesh/mesh-events-pending.d.ts +40 -0
- package/dist/mesh/mesh-events.d.ts +2 -2
- package/dist/mesh/mesh-ledger.d.ts +1 -1
- package/dist/mesh/mesh-reconcile-config.d.ts +6 -0
- package/dist/mesh/mesh-remote-event-pull.d.ts +15 -0
- package/dist/mesh/mesh-runtime-store.d.ts +0 -22
- package/dist/mesh/mesh-work-queue.d.ts +45 -0
- package/package.json +3 -3
- package/src/index.ts +11 -5
- package/src/mesh/coordinator-prompt.ts +15 -0
- package/src/mesh/mesh-completion-synthesis.ts +398 -0
- package/src/mesh/mesh-delivery-policy.ts +7 -38
- package/src/mesh/mesh-event-forwarding.ts +9 -10
- package/src/mesh/mesh-events-pending.ts +176 -0
- package/src/mesh/mesh-events.ts +6 -1
- package/src/mesh/mesh-ledger.ts +5 -0
- package/src/mesh/mesh-queue-assignment.ts +16 -2
- package/src/mesh/mesh-reconcile-config.ts +66 -0
- package/src/mesh/mesh-reconcile-loop.ts +21 -674
- package/src/mesh/mesh-remote-event-pull.ts +279 -0
- package/src/mesh/mesh-runtime-store.ts +46 -82
- package/src/mesh/mesh-work-queue.ts +90 -0
package/dist/index.mjs
CHANGED
|
@@ -404,10 +404,10 @@ function readInjected(value) {
|
|
|
404
404
|
}
|
|
405
405
|
function getDaemonBuildInfo() {
|
|
406
406
|
if (cached) return cached;
|
|
407
|
-
const commit = readInjected(true ? "
|
|
408
|
-
const commitShort = readInjected(true ? "
|
|
409
|
-
const version = readInjected(true ? "0.9.82-rc.
|
|
410
|
-
const builtAt = readInjected(true ? "2026-07-
|
|
407
|
+
const commit = readInjected(true ? "7e34ff6bc7c6528fca49478a57e0a83620f3a063" : void 0) ?? "unknown";
|
|
408
|
+
const commitShort = readInjected(true ? "7e34ff6b" : void 0) ?? (commit !== "unknown" ? commit.slice(0, 7) : "unknown");
|
|
409
|
+
const version = readInjected(true ? "0.9.82-rc.469" : void 0) ?? readInjected(typeof process !== "undefined" ? process.env?.ADHDEV_PKG_VERSION : void 0) ?? "unknown";
|
|
410
|
+
const builtAt = readInjected(true ? "2026-07-05T12:18:38.077Z" : void 0);
|
|
411
411
|
cached = builtAt ? { commit, commitShort, version, builtAt } : { commit, commitShort, version };
|
|
412
412
|
return cached;
|
|
413
413
|
}
|
|
@@ -2765,12 +2765,57 @@ function summarizeGitShape(status) {
|
|
|
2765
2765
|
submodules
|
|
2766
2766
|
};
|
|
2767
2767
|
}
|
|
2768
|
-
var DAEMON_ID_PREFIXES, MAGI_RAW_ANSWER_CAP;
|
|
2768
|
+
var DAEMON_ID_PREFIXES, MAGI_RAW_ANSWER_CAP, CANONICAL_MESH_TOOL_NAMES, CANONICAL_MESH_TOOL_COUNT;
|
|
2769
2769
|
var init_dist = __esm({
|
|
2770
2770
|
"../mesh-shared/dist/index.mjs"() {
|
|
2771
2771
|
"use strict";
|
|
2772
2772
|
DAEMON_ID_PREFIXES = ["daemon_", "standalone_"];
|
|
2773
2773
|
MAGI_RAW_ANSWER_CAP = 4e3;
|
|
2774
|
+
CANONICAL_MESH_TOOL_NAMES = [
|
|
2775
|
+
"mesh_status",
|
|
2776
|
+
"mesh_list_nodes",
|
|
2777
|
+
"mesh_enqueue_task",
|
|
2778
|
+
"mesh_view_queue",
|
|
2779
|
+
"mesh_queue_cancel",
|
|
2780
|
+
"mesh_queue_requeue",
|
|
2781
|
+
"mesh_send_task",
|
|
2782
|
+
"mesh_read_chat",
|
|
2783
|
+
"mesh_read_debug",
|
|
2784
|
+
"mesh_launch_session",
|
|
2785
|
+
"mesh_git_status",
|
|
2786
|
+
"mesh_read_node_logs",
|
|
2787
|
+
"mesh_fast_forward_node",
|
|
2788
|
+
"mesh_restart_daemon",
|
|
2789
|
+
"mesh_checkpoint",
|
|
2790
|
+
"mesh_approve",
|
|
2791
|
+
"mesh_clone_node",
|
|
2792
|
+
"mesh_remove_node",
|
|
2793
|
+
"mesh_refine_node",
|
|
2794
|
+
"mesh_refine_batch",
|
|
2795
|
+
"mesh_refine_config",
|
|
2796
|
+
"mesh_change_impact_config",
|
|
2797
|
+
"mesh_init",
|
|
2798
|
+
"mesh_reinit",
|
|
2799
|
+
"mesh_write_mesh_json_config",
|
|
2800
|
+
"mesh_refine_plan",
|
|
2801
|
+
"mesh_cleanup_sessions",
|
|
2802
|
+
"mesh_prune_stale_direct",
|
|
2803
|
+
"mesh_task_history",
|
|
2804
|
+
"mesh_record_note",
|
|
2805
|
+
"mesh_forget_note",
|
|
2806
|
+
"mesh_reconcile_ledger",
|
|
2807
|
+
"mesh_requeue_held_events",
|
|
2808
|
+
"mesh_mission_upsert",
|
|
2809
|
+
"mesh_mission_list",
|
|
2810
|
+
"mesh_review_inbox",
|
|
2811
|
+
"mesh_magi_review",
|
|
2812
|
+
"mesh_magi_collect",
|
|
2813
|
+
"mesh_magi_panel_set",
|
|
2814
|
+
"mesh_magi_panel_list",
|
|
2815
|
+
"mesh_magi_kind_panel_set",
|
|
2816
|
+
"mesh_magi_kind_panel_list"
|
|
2817
|
+
];
|
|
2818
|
+
CANONICAL_MESH_TOOL_COUNT = CANONICAL_MESH_TOOL_NAMES.length;
|
|
2774
2819
|
}
|
|
2775
2820
|
});
|
|
2776
2821
|
|
|
@@ -3792,24 +3837,39 @@ var init_coordinator_prompt = __esm({
|
|
|
3792
3837
|
| \`mesh_queue_cancel\` | Cancel a queue task without deleting audit history |
|
|
3793
3838
|
| \`mesh_queue_requeue\` | Return a task to pending for retry; clears stale session targets |
|
|
3794
3839
|
| \`mesh_send_task\` | Legacy push: enqueue a task targeted at a specific node |
|
|
3840
|
+
| \`mesh_mission_upsert\` | Create/update a persistent mission so a multi-task plan survives coordinator restarts; set status completed/abandoned when the outcome is decided |
|
|
3841
|
+
| \`mesh_mission_list\` | List every mission with goal, status, and live task progress \u2014 the authority for "what work remains" (never hidden by status) |
|
|
3795
3842
|
| \`mesh_launch_session\` | Start a new agent session on a node |
|
|
3796
3843
|
| \`mesh_read_chat\` | Read recent chat messages from a delegated agent session |
|
|
3797
3844
|
| \`mesh_read_debug\` | Collect a daemon-side chat/parser debug bundle for a session |
|
|
3798
3845
|
| \`mesh_task_history\` | Read the task ledger \u2014 dispatches, completions, failures. Use to understand what has been done before deciding next steps |
|
|
3846
|
+
| \`mesh_reconcile_ledger\` | Reconcile daemon-local ledgers over P2P \u2014 import missing entries from remote nodes into the coordinator local ledger |
|
|
3847
|
+
| \`mesh_requeue_held_events\` | Restore recoverable held coordinator events (T6 quarantine / pending-trim) back to the pending queue; lossless, no double-requeue |
|
|
3848
|
+
| \`mesh_review_inbox\` | List local worktree nodes needing human review \u2014 merge candidates and Refinery-blocked results with evidence/diff summaries |
|
|
3799
3849
|
| \`mesh_record_note\` | Record a durable, provider-neutral operating note (provider quirk / pattern to avoid / recovery lesson). Future coordinators see it under "## Operating Notes" at launch |
|
|
3800
3850
|
| \`mesh_forget_note\` | Retract a stale/wrong operating note by note_id or exact text so it stops riding into future coordinators' prompts (append-only tombstone; history preserved) |
|
|
3801
3851
|
| \`mesh_git_status\` | Check git status on a specific node |
|
|
3802
3852
|
| \`mesh_read_node_logs\` | Fetch a remote node's daemon log tail directly over P2P (grep/since/byte-bounded, secrets redacted) \u2014 no session/PowerShell needed to debug a node's daemon |
|
|
3803
3853
|
| \`mesh_fast_forward_node\` | Safely dry-run or explicitly execute an obvious clean fast-forward without launching an agent session |
|
|
3854
|
+
| \`mesh_restart_daemon\` | Update a node's daemon to the latest published version on its channel and restart it (the dashboard "preview update" path, as a mesh command) |
|
|
3804
3855
|
| \`mesh_checkpoint\` | Create a git checkpoint on a node |
|
|
3805
3856
|
| \`mesh_approve\` | Approve/reject a pending agent action |
|
|
3806
3857
|
| \`mesh_clone_node\` | Create a worktree node for isolated parallel branch work |
|
|
3807
3858
|
| \`mesh_refine_node\` | Validate and merge a completed worktree node back into its base branch |
|
|
3859
|
+
| \`mesh_refine_batch\` | Batch Refinery: converge multiple sibling worktree nodes onto the base branch in one conflict-aware sequential pipeline |
|
|
3860
|
+
| \`mesh_refine_plan\` | Dry-run Refinery plan for a worktree node \u2014 config source, validation commands, merge/cleanup intent \u2014 without executing validation or git merge |
|
|
3861
|
+
| \`mesh_refine_config\` | Refinery config helper (read-only) \u2014 unified entry for schema/validate/suggest via a required \`mode\` |
|
|
3862
|
+
| \`mesh_change_impact_config\` | Change Impact config helper \u2014 unified entry for schema/validate/suggest via a required \`mode\` |
|
|
3808
3863
|
| \`mesh_remove_node\` | Remove a node (cleans up worktree if applicable) |
|
|
3809
3864
|
| \`mesh_cleanup_sessions\` | Manually clean up delegated session records for a node |
|
|
3865
|
+
| \`mesh_prune_stale_direct\` | Prune orphaned staleDirect dispatch records (dry-run by default); live/pending work and audit history preserved |
|
|
3810
3866
|
| \`mesh_init\` | Guided onboarding for a fresh repo: dry-run scan \u2192 suggest \`.adhdev/*\` configs (refine/bootstrap/change-impact) + providerPriority + current-config echo; gated write on approval |
|
|
3811
3867
|
| \`mesh_reinit\` | Re-onboard an already-configured repo: re-suggest with overwrite semantics + current-vs-suggested diff; dry-run preview first, per-section approval before write |
|
|
3812
3868
|
| \`mesh_write_mesh_json_config\` | Gated write of \`.adhdev/mesh.json\` (repo coordinator-prompt config) from the mesh entry \u2014 dry-run/overwrite like mesh_init |
|
|
3869
|
+
| \`mesh_magi_review\` | Cross-verify a read-only investigation across a standing panel of independent mesh agents (different machines/providers) instead of a single worker |
|
|
3870
|
+
| \`mesh_magi_collect\` | Collect + synthesize a previously dispatched MAGI fan-out by its consensus group id (async companion to mesh_magi_review wait:false) |
|
|
3871
|
+
| \`mesh_magi_panel_set\` | Upsert a named MAGI panel (standing set of independent node\xD7provider members) into machine-local config |
|
|
3872
|
+
| \`mesh_magi_panel_list\` | List configured MAGI panels and resolve each member's availability against the current mesh (read-only) |
|
|
3813
3873
|
| \`mesh_magi_kind_panel_set\` | Bind a task_kind \u2192 MAGI kind-panel slots (machine-local, wholesale replacement \u2014 approve current-vs-new first) |
|
|
3814
3874
|
| \`mesh_magi_kind_panel_list\` | List configured task_kind \u2192 MAGI kind-panel slot bindings (machine-local, read-only) |`;
|
|
3815
3875
|
TOOL_EXPOSURE_PREFLIGHT_SECTION = `## Tool Exposure Preflight
|
|
@@ -5267,29 +5327,6 @@ function getActiveSessionDeliveries(meshId, sessionId) {
|
|
|
5267
5327
|
return [];
|
|
5268
5328
|
}
|
|
5269
5329
|
}
|
|
5270
|
-
function recordCompletionConflict(opts) {
|
|
5271
|
-
try {
|
|
5272
|
-
MeshRuntimeStore.getInstance().recordCompletionConflict({
|
|
5273
|
-
id: randomUUID5(),
|
|
5274
|
-
meshId: opts.meshId,
|
|
5275
|
-
fingerprint: opts.fingerprint,
|
|
5276
|
-
conflictingTaskId: opts.conflictingTaskId,
|
|
5277
|
-
conflictingSessionId: opts.conflictingSessionId,
|
|
5278
|
-
originalTaskId: opts.originalTaskId,
|
|
5279
|
-
originalSessionId: opts.originalSessionId,
|
|
5280
|
-
event: opts.event,
|
|
5281
|
-
createdAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
5282
|
-
});
|
|
5283
|
-
} catch {
|
|
5284
|
-
}
|
|
5285
|
-
}
|
|
5286
|
-
function getRecentCompletionConflicts(meshId, limitMs) {
|
|
5287
|
-
try {
|
|
5288
|
-
return MeshRuntimeStore.getInstance().getRecentCompletionConflicts(meshId, limitMs);
|
|
5289
|
-
} catch {
|
|
5290
|
-
return [];
|
|
5291
|
-
}
|
|
5292
|
-
}
|
|
5293
5330
|
function markSessionDeliveriesTerminal(meshId, sessionId, terminalStatus) {
|
|
5294
5331
|
try {
|
|
5295
5332
|
const active = MeshRuntimeStore.getInstance().getActiveSessionDeliveries(meshId, sessionId);
|
|
@@ -5363,6 +5400,8 @@ __export(mesh_work_queue_exports, {
|
|
|
5363
5400
|
ACTIVE_MESH_QUEUE_STATUSES: () => ACTIVE_MESH_QUEUE_STATUSES,
|
|
5364
5401
|
HISTORICAL_MESH_QUEUE_STATUSES: () => HISTORICAL_MESH_QUEUE_STATUSES,
|
|
5365
5402
|
MESH_TASK_MODES: () => MESH_TASK_MODES,
|
|
5403
|
+
MESH_TASK_PRIORITIES: () => MESH_TASK_PRIORITIES,
|
|
5404
|
+
NOT_BEFORE_RELATIVE_THRESHOLD_MS: () => NOT_BEFORE_RELATIVE_THRESHOLD_MS,
|
|
5366
5405
|
__clearDirectDispatchesForTests: () => __clearDirectDispatchesForTests,
|
|
5367
5406
|
__clearMeshQueueForTests: () => __clearMeshQueueForTests,
|
|
5368
5407
|
__replaceMeshQueueForTests: () => __replaceMeshQueueForTests,
|
|
@@ -5383,15 +5422,19 @@ __export(mesh_work_queue_exports, {
|
|
|
5383
5422
|
insertDirectDispatch: () => insertDirectDispatch,
|
|
5384
5423
|
isTaskReadonly: () => isTaskReadonly,
|
|
5385
5424
|
markStaleDirectDispatches: () => markStaleDirectDispatches,
|
|
5425
|
+
meshTaskNotBeforeReady: () => meshTaskNotBeforeReady,
|
|
5426
|
+
meshTaskPriorityRank: () => meshTaskPriorityRank,
|
|
5386
5427
|
nodeSatisfiesRequiredTags: () => nodeSatisfiesRequiredTags,
|
|
5387
5428
|
normalizeMeshCapabilityTags: () => normalizeMeshCapabilityTags,
|
|
5388
5429
|
normalizeMeshTaskMode: () => normalizeMeshTaskMode,
|
|
5430
|
+
normalizeMeshTaskPriority: () => normalizeMeshTaskPriority,
|
|
5389
5431
|
reclaimStrandedAssignedTask: () => reclaimStrandedAssignedTask,
|
|
5390
5432
|
recordDirectDispatchTask: () => recordDirectDispatchTask,
|
|
5391
5433
|
recordMeshToolCall: () => recordMeshToolCall,
|
|
5392
5434
|
recordTaskAutoLaunch: () => recordTaskAutoLaunch,
|
|
5393
5435
|
requeueTask: () => requeueTask,
|
|
5394
5436
|
resolveConvergeRequiredTags: () => resolveConvergeRequiredTags,
|
|
5437
|
+
resolveNotBefore: () => resolveNotBefore,
|
|
5395
5438
|
taskDependenciesSatisfied: () => taskDependenciesSatisfied,
|
|
5396
5439
|
updateDirectDispatchStatus: () => updateDirectDispatchStatus,
|
|
5397
5440
|
updateSessionTaskStatus: () => updateSessionTaskStatus,
|
|
@@ -5399,6 +5442,41 @@ __export(mesh_work_queue_exports, {
|
|
|
5399
5442
|
validateMeshTaskModeRequest: () => validateMeshTaskModeRequest
|
|
5400
5443
|
});
|
|
5401
5444
|
import { randomUUID as randomUUID6 } from "crypto";
|
|
5445
|
+
function meshTaskPriorityRank(priority) {
|
|
5446
|
+
switch (priority) {
|
|
5447
|
+
case "high":
|
|
5448
|
+
return 2;
|
|
5449
|
+
case "low":
|
|
5450
|
+
return 0;
|
|
5451
|
+
default:
|
|
5452
|
+
return 1;
|
|
5453
|
+
}
|
|
5454
|
+
}
|
|
5455
|
+
function normalizeMeshTaskPriority(value) {
|
|
5456
|
+
return value === "low" || value === "normal" || value === "high" ? value : void 0;
|
|
5457
|
+
}
|
|
5458
|
+
function resolveNotBefore(value, nowMs = Date.now()) {
|
|
5459
|
+
if (value === void 0 || value === null) return void 0;
|
|
5460
|
+
let absMs;
|
|
5461
|
+
if (typeof value === "number" && Number.isFinite(value)) {
|
|
5462
|
+
absMs = value < NOT_BEFORE_RELATIVE_THRESHOLD_MS ? nowMs + value : value;
|
|
5463
|
+
} else if (typeof value === "string" && value.trim()) {
|
|
5464
|
+
const parsed = Date.parse(value.trim());
|
|
5465
|
+
if (Number.isNaN(parsed)) return void 0;
|
|
5466
|
+
absMs = parsed;
|
|
5467
|
+
} else {
|
|
5468
|
+
return void 0;
|
|
5469
|
+
}
|
|
5470
|
+
if (absMs <= nowMs) return new Date(nowMs).toISOString();
|
|
5471
|
+
return new Date(absMs).toISOString();
|
|
5472
|
+
}
|
|
5473
|
+
function meshTaskNotBeforeReady(task, nowMs = Date.now()) {
|
|
5474
|
+
const nb = task?.notBefore;
|
|
5475
|
+
if (!nb) return true;
|
|
5476
|
+
const parsed = Date.parse(nb);
|
|
5477
|
+
if (Number.isNaN(parsed)) return true;
|
|
5478
|
+
return parsed <= nowMs;
|
|
5479
|
+
}
|
|
5402
5480
|
function isTaskReadonly(task) {
|
|
5403
5481
|
if (!task) return false;
|
|
5404
5482
|
return task.readonly === true || task.taskMode === "live_debug_readonly";
|
|
@@ -5726,6 +5804,9 @@ function enqueueTask(meshId, message, opts) {
|
|
|
5726
5804
|
}
|
|
5727
5805
|
const id = typeof opts?.id === "string" && opts.id.trim() ? opts.id.trim() : randomUUID6();
|
|
5728
5806
|
const dependsOn = normalizeDependsOn(opts?.dependsOn);
|
|
5807
|
+
const priority = normalizeMeshTaskPriority(opts?.priority);
|
|
5808
|
+
const notBefore = resolveNotBefore(opts?.notBefore);
|
|
5809
|
+
const maxRetries = typeof opts?.maxRetries === "number" && Number.isFinite(opts.maxRetries) && opts.maxRetries >= 0 ? Math.floor(opts.maxRetries) : void 0;
|
|
5729
5810
|
return withQueueLock(meshId, () => {
|
|
5730
5811
|
if (MeshRuntimeStore.getInstance().findQueueEntryById(meshId, id)) {
|
|
5731
5812
|
throw new Error(`duplicate_task_id: task '${id}' already exists in mesh '${meshId}'`);
|
|
@@ -5749,6 +5830,12 @@ function enqueueTask(meshId, message, opts) {
|
|
|
5749
5830
|
targetSessionId: opts?.targetSessionId,
|
|
5750
5831
|
requiredTags: resolvedRequiredTags,
|
|
5751
5832
|
...dependsOn.length > 0 ? { dependsOn } : {},
|
|
5833
|
+
// G6: only persist a non-default priority so legacy/normal rows stay minimal.
|
|
5834
|
+
...priority && priority !== "normal" ? { priority } : {},
|
|
5835
|
+
// G7: hold-until gate (stored ISO). Omitted when absent/immediate.
|
|
5836
|
+
...notBefore ? { notBefore } : {},
|
|
5837
|
+
// P3: explicit retry cap. Omitted → requeue path falls back to policy default.
|
|
5838
|
+
...maxRetries !== void 0 ? { maxRetries } : {},
|
|
5752
5839
|
...typeof opts?.missionId === "string" && opts.missionId.trim() ? { missionId: opts.missionId.trim() } : {},
|
|
5753
5840
|
...typeof opts?.consensusGroupId === "string" && opts.consensusGroupId.trim() ? { consensusGroupId: opts.consensusGroupId.trim() } : {},
|
|
5754
5841
|
...typeof opts?.model === "string" && opts.model.trim() ? { model: opts.model.trim() } : {},
|
|
@@ -6089,7 +6176,7 @@ function recordMeshToolCall(opts) {
|
|
|
6089
6176
|
return { rateLimitExceeded: false, callsInWindow: 0, advisory: null };
|
|
6090
6177
|
}
|
|
6091
6178
|
}
|
|
6092
|
-
var ACTIVE_MESH_QUEUE_STATUSES, HISTORICAL_MESH_QUEUE_STATUSES, MESH_TASK_MODES, LIVE_DEBUG_READONLY_FORBIDDEN, NEGATION_CUES, NEGATION_WINDOW_TOKENS, GIT_MUTATION_SUBCOMMANDS, GIT_STASH_READONLY_SUBCOMMANDS, DEPENDENCY_FAILURE_TERMINALS, MAX_STRANDED_RECLAIMS;
|
|
6179
|
+
var ACTIVE_MESH_QUEUE_STATUSES, HISTORICAL_MESH_QUEUE_STATUSES, MESH_TASK_MODES, MESH_TASK_PRIORITIES, NOT_BEFORE_RELATIVE_THRESHOLD_MS, LIVE_DEBUG_READONLY_FORBIDDEN, NEGATION_CUES, NEGATION_WINDOW_TOKENS, GIT_MUTATION_SUBCOMMANDS, GIT_STASH_READONLY_SUBCOMMANDS, DEPENDENCY_FAILURE_TERMINALS, MAX_STRANDED_RECLAIMS;
|
|
6093
6180
|
var init_mesh_work_queue = __esm({
|
|
6094
6181
|
"src/mesh/mesh-work-queue.ts"() {
|
|
6095
6182
|
"use strict";
|
|
@@ -6105,6 +6192,8 @@ var init_mesh_work_queue = __esm({
|
|
|
6105
6192
|
ACTIVE_MESH_QUEUE_STATUSES = ["pending", "assigned"];
|
|
6106
6193
|
HISTORICAL_MESH_QUEUE_STATUSES = ["completed", "failed", "cancelled"];
|
|
6107
6194
|
MESH_TASK_MODES = ["code_change", "validation", "live_debug_readonly", "launch_app", "convergence"];
|
|
6195
|
+
MESH_TASK_PRIORITIES = ["low", "normal", "high"];
|
|
6196
|
+
NOT_BEFORE_RELATIVE_THRESHOLD_MS = 365 * 24 * 60 * 60 * 1e3;
|
|
6108
6197
|
LIVE_DEBUG_READONLY_FORBIDDEN = [
|
|
6109
6198
|
{ label: "source_edit", pattern: /\b(edit|modify|patch|apply\s+patch|write\s+(?:to\s+)?(?:file|source)|overwrite|delete\s+file|remove\s+file|create\s+file|touch\s+file)\b/i },
|
|
6110
6199
|
{ label: "checkpoint", pattern: /\b(checkpoint|mesh_checkpoint)\b/i },
|
|
@@ -6377,20 +6466,9 @@ var init_mesh_runtime_store = __esm({
|
|
|
6377
6466
|
CREATE INDEX IF NOT EXISTS idx_mesh_session_delivery_task
|
|
6378
6467
|
ON mesh_session_delivery(mesh_id, task_id);
|
|
6379
6468
|
|
|
6380
|
-
|
|
6381
|
-
|
|
6382
|
-
|
|
6383
|
-
fingerprint TEXT NOT NULL,
|
|
6384
|
-
conflicting_task_id TEXT,
|
|
6385
|
-
conflicting_session_id TEXT,
|
|
6386
|
-
original_task_id TEXT,
|
|
6387
|
-
original_session_id TEXT,
|
|
6388
|
-
event TEXT NOT NULL,
|
|
6389
|
-
created_at TEXT NOT NULL
|
|
6390
|
-
);
|
|
6391
|
-
|
|
6392
|
-
CREATE INDEX IF NOT EXISTS idx_mesh_completion_conflicts_mesh
|
|
6393
|
-
ON mesh_completion_conflicts(mesh_id, created_at);
|
|
6469
|
+
-- MESH-COMPLEXITY-AUDIT Part 8-2: mesh_completion_conflicts removed
|
|
6470
|
+
-- (write-only fingerprint-collision diagnostic, no production reader,
|
|
6471
|
+
-- no no-loss role). Dropped in migrateMeshIsolationColumns step 6.
|
|
6394
6472
|
|
|
6395
6473
|
CREATE TABLE IF NOT EXISTS mesh_tool_call_log (
|
|
6396
6474
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
@@ -6572,6 +6650,7 @@ var init_mesh_runtime_store = __esm({
|
|
|
6572
6650
|
WHERE event_id IS NOT NULL
|
|
6573
6651
|
`);
|
|
6574
6652
|
this.db.exec(`DROP TABLE IF EXISTS mesh_direct_delivered_events`);
|
|
6653
|
+
this.db.exec(`DROP TABLE IF EXISTS mesh_completion_conflicts`);
|
|
6575
6654
|
} catch (err) {
|
|
6576
6655
|
if (!loggedMigrationFailure) {
|
|
6577
6656
|
loggedMigrationFailure = true;
|
|
@@ -6881,24 +6960,27 @@ var init_mesh_runtime_store = __esm({
|
|
|
6881
6960
|
}
|
|
6882
6961
|
const nodeIdForms = expandDaemonIdForms(nodeId);
|
|
6883
6962
|
const nodePinnedPlaceholders = nodeIdForms.map(() => "?").join(", ");
|
|
6884
|
-
const
|
|
6885
|
-
|
|
6963
|
+
const parseTier = (query, ...params) => {
|
|
6964
|
+
const tierRows = this.db.prepare(query).all(...params);
|
|
6965
|
+
return tierRows.map((row) => JSON.parse(row.payload)).sort((a, b) => meshTaskPriorityRank(b.priority) - meshTaskPriorityRank(a.priority));
|
|
6966
|
+
};
|
|
6967
|
+
const candidates = [
|
|
6968
|
+
...parseTier(`
|
|
6886
6969
|
SELECT payload FROM mesh_queue
|
|
6887
6970
|
WHERE mesh_id = ? AND status = 'pending' AND target_session_id = ?
|
|
6888
6971
|
ORDER BY created_at ASC
|
|
6889
|
-
|
|
6890
|
-
...
|
|
6972
|
+
`, meshId, sessionId),
|
|
6973
|
+
...parseTier(`
|
|
6891
6974
|
SELECT payload FROM mesh_queue
|
|
6892
6975
|
WHERE mesh_id = ? AND status = 'pending' AND target_node_id IN (${nodePinnedPlaceholders}) AND target_session_id IS NULL
|
|
6893
6976
|
ORDER BY created_at ASC
|
|
6894
|
-
|
|
6895
|
-
...
|
|
6977
|
+
`, meshId, ...nodeIdForms),
|
|
6978
|
+
...parseTier(`
|
|
6896
6979
|
SELECT payload FROM mesh_queue
|
|
6897
6980
|
WHERE mesh_id = ? AND status = 'pending' AND target_node_id IS NULL AND target_session_id IS NULL
|
|
6898
6981
|
ORDER BY created_at ASC
|
|
6899
|
-
|
|
6982
|
+
`, meshId)
|
|
6900
6983
|
];
|
|
6901
|
-
const candidates = rows.map((row) => JSON.parse(row.payload));
|
|
6902
6984
|
const depIds = [...new Set(candidates.flatMap((c) => Array.isArray(c.dependsOn) ? c.dependsOn : []))];
|
|
6903
6985
|
const depStatus = /* @__PURE__ */ new Map();
|
|
6904
6986
|
if (depIds.length > 0) {
|
|
@@ -6913,6 +6995,8 @@ var init_mesh_runtime_store = __esm({
|
|
|
6913
6995
|
if (isTaskReadonly(candidate)) return true;
|
|
6914
6996
|
return !nodeBusy;
|
|
6915
6997
|
};
|
|
6998
|
+
const claimNowMs = Date.now();
|
|
6999
|
+
const notBeforeReady = (candidate) => meshTaskNotBeforeReady(candidate, claimNowMs);
|
|
6916
7000
|
const nodeIsWorktree = opts?.nodeIsWorktree === true;
|
|
6917
7001
|
const convergenceAllows = (candidate) => candidate.taskMode !== "convergence" || !nodeIsWorktree;
|
|
6918
7002
|
const targetMatches = (candidate) => {
|
|
@@ -6922,7 +7006,7 @@ var init_mesh_runtime_store = __esm({
|
|
|
6922
7006
|
}
|
|
6923
7007
|
return true;
|
|
6924
7008
|
};
|
|
6925
|
-
const entry = candidates.find((candidate) => nodeSatisfiesRequiredTags(candidate.requiredTags, capabilityTags) && dependenciesSatisfied(candidate) && convergenceAllows(candidate) && targetMatches(candidate) && nodeConflictAllows(candidate));
|
|
7009
|
+
const entry = candidates.find((candidate) => nodeSatisfiesRequiredTags(candidate.requiredTags, capabilityTags) && dependenciesSatisfied(candidate) && notBeforeReady(candidate) && convergenceAllows(candidate) && targetMatches(candidate) && nodeConflictAllows(candidate));
|
|
6926
7010
|
if (!entry) return null;
|
|
6927
7011
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
6928
7012
|
entry.status = "assigned";
|
|
@@ -7282,43 +7366,11 @@ var init_mesh_runtime_store = __esm({
|
|
|
7282
7366
|
this.db.prepare("DELETE FROM mesh_session_delivery WHERE mesh_id = ?").run(meshId);
|
|
7283
7367
|
}
|
|
7284
7368
|
// ── Completion Conflict Diagnostics ──────────────────────────────────────
|
|
7285
|
-
recordCompletionConflict
|
|
7286
|
-
|
|
7287
|
-
|
|
7288
|
-
|
|
7289
|
-
|
|
7290
|
-
VALUES (@id, @meshId, @fingerprint, @conflictingTaskId, @conflictingSessionId,
|
|
7291
|
-
@originalTaskId, @originalSessionId, @event, @createdAt)
|
|
7292
|
-
`).run({
|
|
7293
|
-
id: entry.id,
|
|
7294
|
-
meshId: entry.meshId,
|
|
7295
|
-
fingerprint: entry.fingerprint,
|
|
7296
|
-
conflictingTaskId: entry.conflictingTaskId ?? null,
|
|
7297
|
-
conflictingSessionId: entry.conflictingSessionId ?? null,
|
|
7298
|
-
originalTaskId: entry.originalTaskId ?? null,
|
|
7299
|
-
originalSessionId: entry.originalSessionId ?? null,
|
|
7300
|
-
event: entry.event,
|
|
7301
|
-
createdAt: entry.createdAt
|
|
7302
|
-
});
|
|
7303
|
-
this.maybeCheckpointWal();
|
|
7304
|
-
}
|
|
7305
|
-
getRecentCompletionConflicts(meshId, limitMs = 60 * 60 * 1e3) {
|
|
7306
|
-
const cutoff = new Date(Date.now() - limitMs).toISOString();
|
|
7307
|
-
const rows = this.db.prepare(
|
|
7308
|
-
"SELECT * FROM mesh_completion_conflicts WHERE mesh_id = ? AND created_at >= ? ORDER BY created_at DESC LIMIT 50"
|
|
7309
|
-
).all(meshId, cutoff);
|
|
7310
|
-
return rows.map((r) => ({
|
|
7311
|
-
id: r.id,
|
|
7312
|
-
meshId: r.mesh_id,
|
|
7313
|
-
fingerprint: r.fingerprint,
|
|
7314
|
-
conflictingTaskId: r.conflicting_task_id,
|
|
7315
|
-
conflictingSessionId: r.conflicting_session_id,
|
|
7316
|
-
originalTaskId: r.original_task_id,
|
|
7317
|
-
originalSessionId: r.original_session_id,
|
|
7318
|
-
event: r.event,
|
|
7319
|
-
createdAt: r.created_at
|
|
7320
|
-
}));
|
|
7321
|
-
}
|
|
7369
|
+
// MESH-COMPLEXITY-AUDIT Part 8-2: recordCompletionConflict /
|
|
7370
|
+
// getRecentCompletionConflicts (and their mesh_completion_conflicts table)
|
|
7371
|
+
// were removed. They were a write-only diagnostic of fingerprint-dedup
|
|
7372
|
+
// collisions with no production reader and no part in the no-loss delivery
|
|
7373
|
+
// contract; the table is dropped in migrateMeshIsolationColumns (step 6).
|
|
7322
7374
|
/**
|
|
7323
7375
|
* Record a mesh tool call and check whether this mesh+tool combination is
|
|
7324
7376
|
* being called too rapidly (sliding window rate guard).
|
|
@@ -11559,7 +11611,11 @@ function ledgerRecordQuarantinedEvent(event, reason) {
|
|
|
11559
11611
|
targetCoordinatorDaemonId: event.targetCoordinatorDaemonId ?? null,
|
|
11560
11612
|
...readNonEmptyString2(event.eventId) ? { eventId: event.eventId } : {},
|
|
11561
11613
|
queuedAt: event.queuedAt,
|
|
11562
|
-
...finalSummary ? { finalSummary } : {}
|
|
11614
|
+
...finalSummary ? { finalSummary } : {},
|
|
11615
|
+
// Full original event so mesh_requeue_held_events can restore it
|
|
11616
|
+
// losslessly (event_held→pending). The summary/label fields above stay
|
|
11617
|
+
// for human-readable audit; `heldEvent` is the machine recovery copy.
|
|
11618
|
+
heldEvent: event
|
|
11563
11619
|
}
|
|
11564
11620
|
});
|
|
11565
11621
|
} catch (e) {
|
|
@@ -11870,8 +11926,11 @@ function trimPendingEventsIfNeeded(path45) {
|
|
|
11870
11926
|
nodeLabel: event.nodeLabel,
|
|
11871
11927
|
...event.workspace ? { workspace: event.workspace } : {},
|
|
11872
11928
|
targetCoordinatorDaemonId: event.targetCoordinatorDaemonId ?? null,
|
|
11929
|
+
...readNonEmptyString2(event.eventId) ? { eventId: event.eventId } : {},
|
|
11873
11930
|
queuedAt: event.queuedAt,
|
|
11874
|
-
...finalSummary ? { finalSummary } : {}
|
|
11931
|
+
...finalSummary ? { finalSummary } : {},
|
|
11932
|
+
// Full original event for lossless mesh_requeue_held_events restore.
|
|
11933
|
+
heldEvent: event
|
|
11875
11934
|
}
|
|
11876
11935
|
});
|
|
11877
11936
|
LOG.warn("MeshEvents", `Pending-events trim dropping undelivered ${event.event} for mesh ${event.meshId} \u2014 recorded to ledger (recoverable)`);
|
|
@@ -12198,6 +12257,83 @@ function getPendingMeshCoordinatorEvents(meshId, coordinatorDaemonId, opts) {
|
|
|
12198
12257
|
});
|
|
12199
12258
|
return reconcilePendingMeshCoordinatorEvents(meshId, routed);
|
|
12200
12259
|
}
|
|
12260
|
+
function readHeldTaskId(restored, payload) {
|
|
12261
|
+
const fromMeta = restored?.metadataEvent && typeof restored.metadataEvent === "object" ? readNonEmptyString2(restored.metadataEvent.taskId) : "";
|
|
12262
|
+
return fromMeta || readNonEmptyString2(payload.taskId) || "";
|
|
12263
|
+
}
|
|
12264
|
+
function requeueHeldMeshCoordinatorEvents(meshId, filter) {
|
|
12265
|
+
const result = {
|
|
12266
|
+
meshId,
|
|
12267
|
+
matched: 0,
|
|
12268
|
+
alreadyRequeued: 0,
|
|
12269
|
+
unrecoverable: 0,
|
|
12270
|
+
requeued: 0,
|
|
12271
|
+
dedupSuppressed: 0,
|
|
12272
|
+
entries: []
|
|
12273
|
+
};
|
|
12274
|
+
const all = readLedgerEntries(meshId);
|
|
12275
|
+
const requeuedIds = /* @__PURE__ */ new Set();
|
|
12276
|
+
for (const entry of all) {
|
|
12277
|
+
if (entry.kind !== "event_held_requeued") continue;
|
|
12278
|
+
const id = readNonEmptyString2(entry.payload?.heldEntryId);
|
|
12279
|
+
if (id) requeuedIds.add(id);
|
|
12280
|
+
}
|
|
12281
|
+
const sinceMs = filter?.since ? new Date(filter.since).getTime() : NaN;
|
|
12282
|
+
const wantEvent = readNonEmptyString2(filter?.event);
|
|
12283
|
+
const wantNode = readNonEmptyString2(filter?.nodeId);
|
|
12284
|
+
const wantTask = readNonEmptyString2(filter?.taskId);
|
|
12285
|
+
const wantReason = readNonEmptyString2(filter?.reason);
|
|
12286
|
+
for (const entry of all) {
|
|
12287
|
+
if (entry.kind !== "event_held") continue;
|
|
12288
|
+
const payload = entry.payload && typeof entry.payload === "object" ? entry.payload : {};
|
|
12289
|
+
if (payload.recoverable !== true) continue;
|
|
12290
|
+
const restored = payload.heldEvent && typeof payload.heldEvent === "object" ? { ...payload.heldEvent } : void 0;
|
|
12291
|
+
const eventName = restored?.event || readNonEmptyString2(payload.event);
|
|
12292
|
+
const nodeId = restored?.nodeId || entry.nodeId || readNonEmptyString2(payload.nodeId) || void 0;
|
|
12293
|
+
const taskId = readHeldTaskId(restored, payload);
|
|
12294
|
+
const reason = readNonEmptyString2(payload.reason) || void 0;
|
|
12295
|
+
if (wantEvent && eventName !== wantEvent) continue;
|
|
12296
|
+
if (wantNode && nodeId !== wantNode) continue;
|
|
12297
|
+
if (wantTask && taskId !== wantTask) continue;
|
|
12298
|
+
if (wantReason && reason !== wantReason) continue;
|
|
12299
|
+
if (filter?.since && !Number.isNaN(sinceMs) && new Date(entry.timestamp).getTime() < sinceMs) continue;
|
|
12300
|
+
result.matched++;
|
|
12301
|
+
if (requeuedIds.has(entry.id)) {
|
|
12302
|
+
result.alreadyRequeued++;
|
|
12303
|
+
result.entries.push({ heldEntryId: entry.id, event: eventName, ...nodeId ? { nodeId } : {}, ...taskId ? { taskId } : {}, ...reason ? { reason } : {}, outcome: "already_requeued" });
|
|
12304
|
+
continue;
|
|
12305
|
+
}
|
|
12306
|
+
if (!restored || !readNonEmptyString2(restored.event) || !readNonEmptyString2(restored.meshId)) {
|
|
12307
|
+
result.unrecoverable++;
|
|
12308
|
+
result.entries.push({ heldEntryId: entry.id, event: eventName, ...nodeId ? { nodeId } : {}, ...taskId ? { taskId } : {}, ...reason ? { reason } : {}, outcome: "unrecoverable" });
|
|
12309
|
+
continue;
|
|
12310
|
+
}
|
|
12311
|
+
const beforeDup = hasPendingCoordinatorEventDuplicate(restored);
|
|
12312
|
+
let ok = false;
|
|
12313
|
+
try {
|
|
12314
|
+
ok = queuePendingMeshCoordinatorEvent(restored);
|
|
12315
|
+
} catch (e) {
|
|
12316
|
+
LOG.warn("MeshEvents", `Requeue of held ${eventName} for mesh ${meshId} failed: ${e?.message || e}`);
|
|
12317
|
+
}
|
|
12318
|
+
appendLedgerEntry(meshId, {
|
|
12319
|
+
kind: "event_held_requeued",
|
|
12320
|
+
...nodeId ? { nodeId } : {},
|
|
12321
|
+
payload: {
|
|
12322
|
+
heldEntryId: entry.id,
|
|
12323
|
+
event: eventName,
|
|
12324
|
+
requeued: ok,
|
|
12325
|
+
...taskId ? { taskId } : {},
|
|
12326
|
+
...reason ? { reason } : {},
|
|
12327
|
+
...beforeDup ? { dedupSuppressed: true } : {}
|
|
12328
|
+
}
|
|
12329
|
+
});
|
|
12330
|
+
requeuedIds.add(entry.id);
|
|
12331
|
+
result.requeued++;
|
|
12332
|
+
if (beforeDup) result.dedupSuppressed++;
|
|
12333
|
+
result.entries.push({ heldEntryId: entry.id, event: eventName, ...nodeId ? { nodeId } : {}, ...taskId ? { taskId } : {}, ...reason ? { reason } : {}, outcome: "requeued" });
|
|
12334
|
+
}
|
|
12335
|
+
return result;
|
|
12336
|
+
}
|
|
12201
12337
|
function clearPendingMeshCoordinatorEvents(meshId, coordinatorDaemonId) {
|
|
12202
12338
|
if (!meshId) return;
|
|
12203
12339
|
try {
|
|
@@ -14233,7 +14369,7 @@ function driveExpiredAwaitClaim(components, meshId, task, ctx) {
|
|
|
14233
14369
|
async function maybeAutoLaunchOneQueueSession(components, meshId, mesh) {
|
|
14234
14370
|
const queue = getQueue(meshId);
|
|
14235
14371
|
const statusById = new Map(queue.map((task) => [task.id, task.status]));
|
|
14236
|
-
const pending = queue.filter((task) => task.status === "pending");
|
|
14372
|
+
const pending = queue.filter((task) => task.status === "pending").sort((a, b) => meshTaskPriorityRank(b.priority) - meshTaskPriorityRank(a.priority));
|
|
14237
14373
|
{
|
|
14238
14374
|
const pendingIds = new Set(pending.map((t) => t.id));
|
|
14239
14375
|
const prefix = `${meshId}::`;
|
|
@@ -14249,6 +14385,10 @@ async function maybeAutoLaunchOneQueueSession(components, meshId, mesh) {
|
|
|
14249
14385
|
markAutoLaunch(meshId, task.id, { status: "skipped", reason: "dependencies_unsatisfied" });
|
|
14250
14386
|
continue;
|
|
14251
14387
|
}
|
|
14388
|
+
if (!meshTaskNotBeforeReady(task)) {
|
|
14389
|
+
markAutoLaunch(meshId, task.id, { status: "skipped", reason: "not_before_delayed" });
|
|
14390
|
+
continue;
|
|
14391
|
+
}
|
|
14252
14392
|
const isReadonly = isTaskReadonly(task);
|
|
14253
14393
|
if (isReadonly) {
|
|
14254
14394
|
if (activeReadonlyAssignedCount(meshId) >= maxReadonlyParallelTasks) {
|
|
@@ -17384,15 +17524,6 @@ function isDuplicateMeshCompletionEvent(args) {
|
|
|
17384
17524
|
const fingerprint = buildMeshCompletionFingerprint(args);
|
|
17385
17525
|
if (!fingerprint) return false;
|
|
17386
17526
|
if (hasFingerprintSeen(args.meshId, fingerprint)) {
|
|
17387
|
-
if (args.taskId) {
|
|
17388
|
-
recordCompletionConflict({
|
|
17389
|
-
meshId: args.meshId,
|
|
17390
|
-
fingerprint,
|
|
17391
|
-
conflictingTaskId: args.taskId,
|
|
17392
|
-
conflictingSessionId: args.sessionId,
|
|
17393
|
-
event: args.event
|
|
17394
|
-
});
|
|
17395
|
-
}
|
|
17396
17527
|
return true;
|
|
17397
17528
|
}
|
|
17398
17529
|
recordFingerprintSeen(args.meshId, fingerprint);
|
|
@@ -18470,38 +18601,6 @@ var init_mesh_reconcile_identity = __esm({
|
|
|
18470
18601
|
}
|
|
18471
18602
|
});
|
|
18472
18603
|
|
|
18473
|
-
// src/mesh/mesh-reconcile-v2-backstop.ts
|
|
18474
|
-
function getMeshV2BackstopCounters() {
|
|
18475
|
-
return { ...meshV2BackstopCounters };
|
|
18476
|
-
}
|
|
18477
|
-
function meshProtocolV2EnforceOn() {
|
|
18478
|
-
const raw = process.env.MESH_PROTOCOL_V2_ENFORCE;
|
|
18479
|
-
if (typeof raw !== "string" || !raw.trim()) return true;
|
|
18480
|
-
const v = raw.trim().toLowerCase();
|
|
18481
|
-
return !(v === "0" || v === "false" || v === "off" || v === "no");
|
|
18482
|
-
}
|
|
18483
|
-
function recordBackstopFire(kind, detail) {
|
|
18484
|
-
meshV2BackstopCounters[kind]++;
|
|
18485
|
-
if (meshProtocolV2EnforceOn()) {
|
|
18486
|
-
LOG.warn("MeshReconcileV2", `v2 ENFORCE last-resort backstop fired (${kind}): ${detail}. Under a healthy v2 completion contract this should be 0 \u2014 a worker's real terminal emit was lost/late.`);
|
|
18487
|
-
}
|
|
18488
|
-
}
|
|
18489
|
-
var meshV2BackstopCounters;
|
|
18490
|
-
var init_mesh_reconcile_v2_backstop = __esm({
|
|
18491
|
-
"src/mesh/mesh-reconcile-v2-backstop.ts"() {
|
|
18492
|
-
"use strict";
|
|
18493
|
-
init_logger();
|
|
18494
|
-
meshV2BackstopCounters = {
|
|
18495
|
-
/** PHASE-4 transcript synthesis actually reconciled a missing completion. */
|
|
18496
|
-
phase4SynthesisFired: 0,
|
|
18497
|
-
/** Acked-hold transcript fast-track promoted a synth ahead of the death deadline. */
|
|
18498
|
-
ackedHoldFastTrackFired: 0,
|
|
18499
|
-
/** Acked-hold death-deadline backstop released a held synth (the 8-min net). */
|
|
18500
|
-
ackedHoldDeathDeadlineFired: 0
|
|
18501
|
-
};
|
|
18502
|
-
}
|
|
18503
|
-
});
|
|
18504
|
-
|
|
18505
18604
|
// src/mesh/mesh-reconcile-acked-hold.ts
|
|
18506
18605
|
function resolveTunedReconcileMs(envName, def, min, max) {
|
|
18507
18606
|
const raw = readNonEmptyString2(process.env[envName]);
|
|
@@ -18628,7 +18727,7 @@ var init_mesh_reconcile_acked_hold = __esm({
|
|
|
18628
18727
|
}
|
|
18629
18728
|
});
|
|
18630
18729
|
|
|
18631
|
-
// src/mesh/mesh-reconcile-
|
|
18730
|
+
// src/mesh/mesh-reconcile-config.ts
|
|
18632
18731
|
function resolveAutoPruneMinAgeMs() {
|
|
18633
18732
|
const raw = readNonEmptyString2(process.env.MESH_AUTO_PRUNE_MIN_AGE_MS);
|
|
18634
18733
|
if (raw) {
|
|
@@ -18648,6 +18747,432 @@ function resolveReconcileIntervalMs() {
|
|
|
18648
18747
|
}
|
|
18649
18748
|
return DEFAULT_RECONCILE_INTERVAL_MS;
|
|
18650
18749
|
}
|
|
18750
|
+
var DEFAULT_RECONCILE_INTERVAL_MS, DEFAULT_AUTO_PRUNE_MIN_AGE_MS, DEFAULT_PENDING_HELD_DRAIN_ESCALATE_MS;
|
|
18751
|
+
var init_mesh_reconcile_config = __esm({
|
|
18752
|
+
"src/mesh/mesh-reconcile-config.ts"() {
|
|
18753
|
+
"use strict";
|
|
18754
|
+
init_mesh_events_utils();
|
|
18755
|
+
init_mesh_reconcile_acked_hold();
|
|
18756
|
+
DEFAULT_RECONCILE_INTERVAL_MS = 4e3;
|
|
18757
|
+
DEFAULT_AUTO_PRUNE_MIN_AGE_MS = 24 * 60 * 6e4;
|
|
18758
|
+
DEFAULT_PENDING_HELD_DRAIN_ESCALATE_MS = 12e3;
|
|
18759
|
+
}
|
|
18760
|
+
});
|
|
18761
|
+
|
|
18762
|
+
// src/mesh/mesh-remote-event-pull.ts
|
|
18763
|
+
async function pullRemoteNodeQueues(components, mesh, localDaemonId, candidateDaemonIds) {
|
|
18764
|
+
const dispatchMeshCommand = components.dispatchMeshCommand;
|
|
18765
|
+
if (!dispatchMeshCommand) return;
|
|
18766
|
+
const meshId = mesh.id;
|
|
18767
|
+
const pulls = candidateDaemonIds.length > 0 ? candidateDaemonIds.map((id) => ({ meshId, coordinatorDaemonId: id })) : [{ meshId }];
|
|
18768
|
+
await Promise.allSettled(mesh.nodes.map(async (node) => {
|
|
18769
|
+
const nodeDaemonId = readNonEmptyString2(node.daemonId);
|
|
18770
|
+
if (!nodeDaemonId) return;
|
|
18771
|
+
if (daemonIdsEquivalent(nodeDaemonId, localDaemonId)) return;
|
|
18772
|
+
if (daemonIdListIncludes(candidateDaemonIds, nodeDaemonId)) return;
|
|
18773
|
+
const peerSnapshot = components.getMeshPeerConnectionStatus?.(nodeDaemonId);
|
|
18774
|
+
if (peerSnapshot && String(peerSnapshot.state) !== "connected") return;
|
|
18775
|
+
for (const pendingEventArgs of pulls) {
|
|
18776
|
+
let events;
|
|
18777
|
+
try {
|
|
18778
|
+
events = await dispatchMeshCommand(nodeDaemonId, "get_pending_mesh_events", pendingEventArgs);
|
|
18779
|
+
} catch {
|
|
18780
|
+
break;
|
|
18781
|
+
}
|
|
18782
|
+
const list = extractPendingEvents(events).filter((e) => readNonEmptyString2(e?.meshId) === meshId);
|
|
18783
|
+
for (const event of list) {
|
|
18784
|
+
const payload = buildForwardPayloadFromPending(event);
|
|
18785
|
+
if (!payload.event || !payload.meshId) continue;
|
|
18786
|
+
try {
|
|
18787
|
+
handleMeshForwardEvent(components, payload);
|
|
18788
|
+
} catch {
|
|
18789
|
+
}
|
|
18790
|
+
}
|
|
18791
|
+
}
|
|
18792
|
+
}));
|
|
18793
|
+
}
|
|
18794
|
+
function unwrapReadChatPayload(raw) {
|
|
18795
|
+
let cursor = raw;
|
|
18796
|
+
for (let depth = 0; depth < 4 && cursor && typeof cursor === "object"; depth++) {
|
|
18797
|
+
const record = cursor;
|
|
18798
|
+
if (Array.isArray(record.messages)) return record;
|
|
18799
|
+
if (record.payload && typeof record.payload === "object") {
|
|
18800
|
+
cursor = record.payload;
|
|
18801
|
+
continue;
|
|
18802
|
+
}
|
|
18803
|
+
if (record.result && typeof record.result === "object") {
|
|
18804
|
+
cursor = record.result;
|
|
18805
|
+
continue;
|
|
18806
|
+
}
|
|
18807
|
+
if (record.data && typeof record.data === "object") {
|
|
18808
|
+
cursor = record.data;
|
|
18809
|
+
continue;
|
|
18810
|
+
}
|
|
18811
|
+
break;
|
|
18812
|
+
}
|
|
18813
|
+
return cursor && typeof cursor === "object" ? cursor : null;
|
|
18814
|
+
}
|
|
18815
|
+
function readChatPayloadStatus(payload) {
|
|
18816
|
+
return readNonEmptyString2(payload?.status).toLowerCase();
|
|
18817
|
+
}
|
|
18818
|
+
function realTerminalEmitPendingForTask(meshId, taskId) {
|
|
18819
|
+
let pending;
|
|
18820
|
+
try {
|
|
18821
|
+
pending = getPendingMeshCoordinatorEvents(meshId);
|
|
18822
|
+
} catch {
|
|
18823
|
+
return false;
|
|
18824
|
+
}
|
|
18825
|
+
return pending.some((e) => readNonEmptyString2(e.metadataEvent?.taskId) === taskId && (e.event === "agent:generating_completed" || e.event === "agent:stopped"));
|
|
18826
|
+
}
|
|
18827
|
+
async function reprobeWorkerStatus(components, args) {
|
|
18828
|
+
try {
|
|
18829
|
+
if (args.isLocalNode) {
|
|
18830
|
+
const r = await components.commandHandler.handle("read_chat", args.readArgs);
|
|
18831
|
+
if (r && r.success === false) return null;
|
|
18832
|
+
return readChatPayloadStatus(unwrapReadChatPayload(r));
|
|
18833
|
+
}
|
|
18834
|
+
if (components.dispatchMeshCommand) {
|
|
18835
|
+
const r = await components.dispatchMeshCommand(args.nodeDaemonId, "read_chat", args.readArgs);
|
|
18836
|
+
const p = unwrapReadChatPayload(r);
|
|
18837
|
+
if (p && p.success === false) return null;
|
|
18838
|
+
return readChatPayloadStatus(p);
|
|
18839
|
+
}
|
|
18840
|
+
} catch {
|
|
18841
|
+
return null;
|
|
18842
|
+
}
|
|
18843
|
+
return null;
|
|
18844
|
+
}
|
|
18845
|
+
async function collectLiveNodesWithSessions(components, mesh, selfIds, localDaemonId) {
|
|
18846
|
+
const dispatchMeshCommand = components.dispatchMeshCommand;
|
|
18847
|
+
return Promise.all(mesh.nodes.map(async (node) => {
|
|
18848
|
+
const nodeDaemonId = readNonEmptyString2(node.daemonId);
|
|
18849
|
+
const isLocalNode = !nodeDaemonId || daemonIdListIncludes(selfIds, nodeDaemonId) || daemonIdsEquivalent(nodeDaemonId, localDaemonId);
|
|
18850
|
+
if (!isLocalNode) {
|
|
18851
|
+
const peerSnapshot = components.getMeshPeerConnectionStatus?.(nodeDaemonId);
|
|
18852
|
+
if (peerSnapshot && String(peerSnapshot.state) !== "connected") return node;
|
|
18853
|
+
}
|
|
18854
|
+
let statusResult;
|
|
18855
|
+
try {
|
|
18856
|
+
if (isLocalNode) {
|
|
18857
|
+
statusResult = await components.commandHandler.handle("get_status_metadata", {});
|
|
18858
|
+
} else if (dispatchMeshCommand) {
|
|
18859
|
+
statusResult = await dispatchMeshCommand(nodeDaemonId, "get_status_metadata", {});
|
|
18860
|
+
} else {
|
|
18861
|
+
return node;
|
|
18862
|
+
}
|
|
18863
|
+
} catch {
|
|
18864
|
+
return node;
|
|
18865
|
+
}
|
|
18866
|
+
const sessions = extractStatusMetadataSessions(statusResult);
|
|
18867
|
+
return sessions.length > 0 ? { ...node, sessions } : node;
|
|
18868
|
+
}));
|
|
18869
|
+
}
|
|
18870
|
+
function extractStatusMetadataSessions(raw) {
|
|
18871
|
+
let cursor = raw;
|
|
18872
|
+
for (let depth = 0; depth < 4 && cursor && typeof cursor === "object"; depth++) {
|
|
18873
|
+
const record = cursor;
|
|
18874
|
+
const status = record.status && typeof record.status === "object" ? record.status : void 0;
|
|
18875
|
+
if (status && Array.isArray(status.sessions)) return status.sessions;
|
|
18876
|
+
if (Array.isArray(record.sessions)) return record.sessions;
|
|
18877
|
+
if (record.payload && typeof record.payload === "object") {
|
|
18878
|
+
cursor = record.payload;
|
|
18879
|
+
continue;
|
|
18880
|
+
}
|
|
18881
|
+
if (record.result && typeof record.result === "object") {
|
|
18882
|
+
cursor = record.result;
|
|
18883
|
+
continue;
|
|
18884
|
+
}
|
|
18885
|
+
if (record.data && typeof record.data === "object") {
|
|
18886
|
+
cursor = record.data;
|
|
18887
|
+
continue;
|
|
18888
|
+
}
|
|
18889
|
+
break;
|
|
18890
|
+
}
|
|
18891
|
+
return [];
|
|
18892
|
+
}
|
|
18893
|
+
function extractPendingEvents(raw) {
|
|
18894
|
+
if (Array.isArray(raw)) return raw;
|
|
18895
|
+
if (raw && typeof raw === "object") {
|
|
18896
|
+
const events = raw.events;
|
|
18897
|
+
if (Array.isArray(events)) return events;
|
|
18898
|
+
}
|
|
18899
|
+
return [];
|
|
18900
|
+
}
|
|
18901
|
+
function buildForwardPayloadFromPending(event) {
|
|
18902
|
+
const metadata = event?.metadataEvent && typeof event.metadataEvent === "object" ? event.metadataEvent : {};
|
|
18903
|
+
return {
|
|
18904
|
+
event: readNonEmptyString2(event?.event),
|
|
18905
|
+
meshId: readNonEmptyString2(event?.meshId),
|
|
18906
|
+
nodeId: readNonEmptyString2(event?.nodeId) || readNonEmptyString2(metadata.meshNodeId),
|
|
18907
|
+
workspace: readNonEmptyString2(event?.workspace) || readNonEmptyString2(metadata.workspace),
|
|
18908
|
+
// Preserve the originating coordinator session id across the relay. It is normally
|
|
18909
|
+
// carried inside metadataEvent.meshCoordinatorSessionId (spread below), but pass the
|
|
18910
|
+
// top-level field through explicitly too so the handleMeshForwardEvent whitelist
|
|
18911
|
+
// recovers it regardless of which carrier the producing daemon used.
|
|
18912
|
+
...readNonEmptyString2(event?.targetCoordinatorSessionId) ? { targetCoordinatorSessionId: readNonEmptyString2(event.targetCoordinatorSessionId) } : {},
|
|
18913
|
+
...metadata,
|
|
18914
|
+
// NOTIF-MISS (FIX 3): surface the dispatch task id at the TOP LEVEL so the relay's
|
|
18915
|
+
// received-stage trace (and buildRelayMetadataEvent) recovers it regardless of which
|
|
18916
|
+
// carrier the producing daemon used. The metadata spread above may carry the id only as
|
|
18917
|
+
// `meshActiveTaskId` (a worker provider event), leaving top-level `taskId` unset and the
|
|
18918
|
+
// received stage rendering `task=-`. Resolve both carriers into an explicit `taskId` so
|
|
18919
|
+
// dedup stays task-scoped end-to-end. Only set when a non-empty id exists (no clobber to
|
|
18920
|
+
// undefined when neither is present).
|
|
18921
|
+
...(() => {
|
|
18922
|
+
const tid = readNonEmptyString2(metadata.taskId) || readNonEmptyString2(metadata.meshActiveTaskId);
|
|
18923
|
+
return tid ? { taskId: tid } : {};
|
|
18924
|
+
})(),
|
|
18925
|
+
// T4 (B3b): carry the v2 envelope (protocolVersion/eventId/scope/dispatchedBy/
|
|
18926
|
+
// intendedFor) across the P2P relay boundary at the TOP LEVEL. These live on the
|
|
18927
|
+
// pending event itself, not inside metadataEvent, so without this the remote pull
|
|
18928
|
+
// re-queue would re-stamp a fresh eventId — breaking cross-machine idempotency and
|
|
18929
|
+
// downgrading the relayed completion to v1 broadcast routing. Spread LAST so the
|
|
18930
|
+
// authoritative envelope always wins over any stale key the metadata spread carried.
|
|
18931
|
+
...serializeV2EnvelopeToWire(event)
|
|
18932
|
+
};
|
|
18933
|
+
}
|
|
18934
|
+
var init_mesh_remote_event_pull = __esm({
|
|
18935
|
+
"src/mesh/mesh-remote-event-pull.ts"() {
|
|
18936
|
+
"use strict";
|
|
18937
|
+
init_mesh_events_pending();
|
|
18938
|
+
init_mesh_events_coordinator();
|
|
18939
|
+
init_mesh_events_utils();
|
|
18940
|
+
init_dist();
|
|
18941
|
+
init_mesh_reconcile_identity();
|
|
18942
|
+
}
|
|
18943
|
+
});
|
|
18944
|
+
|
|
18945
|
+
// src/mesh/mesh-reconcile-v2-backstop.ts
|
|
18946
|
+
function getMeshV2BackstopCounters() {
|
|
18947
|
+
return { ...meshV2BackstopCounters };
|
|
18948
|
+
}
|
|
18949
|
+
function meshProtocolV2EnforceOn() {
|
|
18950
|
+
const raw = process.env.MESH_PROTOCOL_V2_ENFORCE;
|
|
18951
|
+
if (typeof raw !== "string" || !raw.trim()) return true;
|
|
18952
|
+
const v = raw.trim().toLowerCase();
|
|
18953
|
+
return !(v === "0" || v === "false" || v === "off" || v === "no");
|
|
18954
|
+
}
|
|
18955
|
+
function recordBackstopFire(kind, detail) {
|
|
18956
|
+
meshV2BackstopCounters[kind]++;
|
|
18957
|
+
if (meshProtocolV2EnforceOn()) {
|
|
18958
|
+
LOG.warn("MeshReconcileV2", `v2 ENFORCE last-resort backstop fired (${kind}): ${detail}. Under a healthy v2 completion contract this should be 0 \u2014 a worker's real terminal emit was lost/late.`);
|
|
18959
|
+
}
|
|
18960
|
+
}
|
|
18961
|
+
var meshV2BackstopCounters;
|
|
18962
|
+
var init_mesh_reconcile_v2_backstop = __esm({
|
|
18963
|
+
"src/mesh/mesh-reconcile-v2-backstop.ts"() {
|
|
18964
|
+
"use strict";
|
|
18965
|
+
init_logger();
|
|
18966
|
+
meshV2BackstopCounters = {
|
|
18967
|
+
/** PHASE-4 transcript synthesis actually reconciled a missing completion. */
|
|
18968
|
+
phase4SynthesisFired: 0,
|
|
18969
|
+
/** Acked-hold transcript fast-track promoted a synth ahead of the death deadline. */
|
|
18970
|
+
ackedHoldFastTrackFired: 0,
|
|
18971
|
+
/** Acked-hold death-deadline backstop released a held synth (the 8-min net). */
|
|
18972
|
+
ackedHoldDeathDeadlineFired: 0
|
|
18973
|
+
};
|
|
18974
|
+
}
|
|
18975
|
+
});
|
|
18976
|
+
|
|
18977
|
+
// src/mesh/mesh-completion-synthesis.ts
|
|
18978
|
+
async function reconcileUnterminatedDirectDispatches(components, mesh, selfIds, localDaemonId) {
|
|
18979
|
+
const dispatches = getActiveDirectDispatches(mesh.id);
|
|
18980
|
+
rehydrateAckedHoldsForMesh(mesh.id);
|
|
18981
|
+
const activeTaskKeys = new Set(
|
|
18982
|
+
dispatches.map((d) => readNonEmptyString2(d.taskId)).filter(Boolean).map((taskId) => inFlightSynthKey(mesh.id, taskId))
|
|
18983
|
+
);
|
|
18984
|
+
const heldKeys = collectHeldSynthKeysForMesh(mesh.id);
|
|
18985
|
+
for (const key2 of heldKeys) {
|
|
18986
|
+
if (!activeTaskKeys.has(key2)) deleteHoldState(key2, mesh.id);
|
|
18987
|
+
}
|
|
18988
|
+
if (dispatches.length === 0) return;
|
|
18989
|
+
const dispatchMeshCommand = components.dispatchMeshCommand;
|
|
18990
|
+
const nodeById = new Map(mesh.nodes.map((n) => [n.id, n]));
|
|
18991
|
+
for (const dispatch of dispatches) {
|
|
18992
|
+
const sessionId = readNonEmptyString2(dispatch.sessionId);
|
|
18993
|
+
const nodeId = readNonEmptyString2(dispatch.nodeId);
|
|
18994
|
+
const taskId = readNonEmptyString2(dispatch.taskId);
|
|
18995
|
+
if (!sessionId || !nodeId || !taskId) continue;
|
|
18996
|
+
const node = nodeById.get(nodeId);
|
|
18997
|
+
const nodeDaemonId = readNonEmptyString2(node?.daemonId);
|
|
18998
|
+
const isLocalNode = !nodeDaemonId || daemonIdListIncludes(selfIds, nodeDaemonId) || daemonIdsEquivalent(nodeDaemonId, localDaemonId) || !!components.instanceManager.getInstance(sessionId);
|
|
18999
|
+
const providerType = readNonEmptyString2(dispatch.providerType);
|
|
19000
|
+
const readArgs = {
|
|
19001
|
+
sessionId,
|
|
19002
|
+
targetSessionId: sessionId,
|
|
19003
|
+
tailLimit: 10,
|
|
19004
|
+
...node?.workspace ? { workspace: node.workspace } : {},
|
|
19005
|
+
...providerType ? { agentType: providerType, providerType } : {}
|
|
19006
|
+
};
|
|
19007
|
+
const synthKey = inFlightSynthKey(mesh.id, taskId);
|
|
19008
|
+
const isAcked = dispatch.status === "acked";
|
|
19009
|
+
let backstopKind;
|
|
19010
|
+
let payload = null;
|
|
19011
|
+
let readFailed = false;
|
|
19012
|
+
try {
|
|
19013
|
+
if (isLocalNode) {
|
|
19014
|
+
const result = await components.commandHandler.handle("read_chat", readArgs);
|
|
19015
|
+
if (result && result.success === false) {
|
|
19016
|
+
readFailed = true;
|
|
19017
|
+
} else {
|
|
19018
|
+
payload = unwrapReadChatPayload(result);
|
|
19019
|
+
}
|
|
19020
|
+
} else if (dispatchMeshCommand) {
|
|
19021
|
+
const result = await dispatchMeshCommand(nodeDaemonId, "read_chat", readArgs);
|
|
19022
|
+
payload = unwrapReadChatPayload(result);
|
|
19023
|
+
if (payload && payload.success === false) {
|
|
19024
|
+
payload = null;
|
|
19025
|
+
readFailed = true;
|
|
19026
|
+
}
|
|
19027
|
+
} else {
|
|
19028
|
+
continue;
|
|
19029
|
+
}
|
|
19030
|
+
} catch {
|
|
19031
|
+
readFailed = true;
|
|
19032
|
+
}
|
|
19033
|
+
if (!payload && !readFailed) continue;
|
|
19034
|
+
if (readFailed || !payload) {
|
|
19035
|
+
if (isAcked) {
|
|
19036
|
+
const prior = getHoldState(synthKey, mesh.id);
|
|
19037
|
+
const failures = (prior?.consecutiveReadFailures ?? 0) + 1;
|
|
19038
|
+
const liveConfirmedSinceAck = prior?.liveConfirmedSinceAck ?? false;
|
|
19039
|
+
setHoldState(synthKey, mesh.id, { liveConfirmedSinceAck, consecutiveReadFailures: failures });
|
|
19040
|
+
if (liveConfirmedSinceAck && failures >= ACKED_DEATH_CONSECUTIVE_READ_FAILURES) {
|
|
19041
|
+
LOG.warn("MeshReconcile", `Acked-hold death signal: task ${taskId} on node ${nodeId} (mesh ${mesh.id}) read_chat failed ${failures}x consecutively after a live-confirmed ack \u2014 worker session presumed gone mid-turn; releasing the indefinite synth hold to the stranded-reclaim / orphan-prune nets`);
|
|
19042
|
+
}
|
|
19043
|
+
}
|
|
19044
|
+
continue;
|
|
19045
|
+
}
|
|
19046
|
+
const priorHoldState = getHoldState(synthKey, mesh.id);
|
|
19047
|
+
setHoldState(synthKey, mesh.id, {
|
|
19048
|
+
liveConfirmedSinceAck: true,
|
|
19049
|
+
consecutiveReadFailures: 0,
|
|
19050
|
+
...priorHoldState?.transcriptIdleSinceMs !== void 0 ? { transcriptIdleSinceMs: priorHoldState.transcriptIdleSinceMs } : {}
|
|
19051
|
+
});
|
|
19052
|
+
const nowMs = Date.now();
|
|
19053
|
+
if (readChatPayloadStatus(payload) !== "idle") {
|
|
19054
|
+
setHoldState(synthKey, mesh.id, { liveConfirmedSinceAck: true, consecutiveReadFailures: 0 });
|
|
19055
|
+
continue;
|
|
19056
|
+
}
|
|
19057
|
+
const messages = Array.isArray(payload.messages) ? payload.messages : [];
|
|
19058
|
+
const evidence = extractFinalAssistantSummaryEvidence(messages);
|
|
19059
|
+
if (isAcked) {
|
|
19060
|
+
const ackedAtMs = Date.parse(readNonEmptyString2(dispatch.updatedAt));
|
|
19061
|
+
const sinceAckMs = Number.isFinite(ackedAtMs) ? nowMs - ackedAtMs : Number.POSITIVE_INFINITY;
|
|
19062
|
+
const deathDeadlineMs = resolveAckedDeathDeadlineMs();
|
|
19063
|
+
const holdState = getHoldState(synthKey, mesh.id);
|
|
19064
|
+
let fastTrackReady = false;
|
|
19065
|
+
if (evidence.finalSummary) {
|
|
19066
|
+
const idleSinceMs = holdState?.transcriptIdleSinceMs ?? nowMs;
|
|
19067
|
+
if (holdState && holdState.transcriptIdleSinceMs === void 0) {
|
|
19068
|
+
setHoldState(synthKey, mesh.id, { ...holdState, transcriptIdleSinceMs: idleSinceMs });
|
|
19069
|
+
}
|
|
19070
|
+
const fastTrackGraceMs = resolveAckedTranscriptFastTrackGraceMs();
|
|
19071
|
+
const idleHeldMs = nowMs - idleSinceMs;
|
|
19072
|
+
if (idleHeldMs >= fastTrackGraceMs) {
|
|
19073
|
+
fastTrackReady = true;
|
|
19074
|
+
backstopKind = "ackedHoldFastTrackFired";
|
|
19075
|
+
LOG.info("MeshReconcile", `Acked-hold transcript fast-track: task ${taskId} on node ${nodeId} (mesh ${mesh.id}) read idle WITH a final assistant message for ${Math.round(idleHeldMs / 1e3)}s continuous (grace ${Math.round(fastTrackGraceMs / 1e3)}s) \u2014 promoting the synth ahead of the ${Math.round(deathDeadlineMs / 1e3)}s death backstop; the worker's real emit was lost/late and a later one no-ops idempotently.`);
|
|
19076
|
+
}
|
|
19077
|
+
} else if (holdState?.transcriptIdleSinceMs !== void 0) {
|
|
19078
|
+
setHoldState(synthKey, mesh.id, { ...holdState, transcriptIdleSinceMs: void 0 });
|
|
19079
|
+
}
|
|
19080
|
+
if (!fastTrackReady && sinceAckMs < deathDeadlineMs) {
|
|
19081
|
+
LOG.info("MeshReconcile", `Acked-hold: task ${taskId} on node ${nodeId} (mesh ${mesh.id}) read idle ${Number.isFinite(sinceAckMs) ? Math.round(sinceAckMs / 1e3) + "s" : "\u221E"} since the generating_started ack \u2014 HOLDING synth (worker presumed alive; a later real emit is idempotent). Transcript fast-track promotes at ${Math.round(resolveAckedTranscriptFastTrackGraceMs() / 1e3)}s continuous idle-with-final-assistant; death backstop at ${Math.round(deathDeadlineMs / 1e3)}s or on consecutive read failures.`);
|
|
19082
|
+
continue;
|
|
19083
|
+
}
|
|
19084
|
+
if (!fastTrackReady) {
|
|
19085
|
+
backstopKind = "ackedHoldDeathDeadlineFired";
|
|
19086
|
+
LOG.warn("MeshReconcile", `Acked-hold death deadline reached: task ${taskId} on node ${nodeId} (mesh ${mesh.id}) still idle ${Math.round(sinceAckMs / 1e3)}s after the ack (deadline ${Math.round(deathDeadlineMs / 1e3)}s) \u2014 synthesizing the missing completion as a notification-loss net (a real emit, if it ever lands, no-ops idempotently).`);
|
|
19087
|
+
}
|
|
19088
|
+
}
|
|
19089
|
+
if (realTerminalEmitPendingForTask(mesh.id, taskId)) {
|
|
19090
|
+
deleteHoldState(synthKey, mesh.id);
|
|
19091
|
+
LOG.info("MeshReconcile", `Worker-emit priority: task ${taskId} on node ${nodeId} (mesh ${mesh.id}) has a real terminal completion already queued \u2014 yielding synth to the worker's own emit`);
|
|
19092
|
+
continue;
|
|
19093
|
+
}
|
|
19094
|
+
if (!evidence.finalSummary) continue;
|
|
19095
|
+
const dispatchedAtMs = Date.parse(readNonEmptyString2(dispatch.dispatchedAt));
|
|
19096
|
+
const transcriptAtMs = Date.parse(evidence.transcriptMessageAt ?? "");
|
|
19097
|
+
if (Number.isFinite(dispatchedAtMs) && Number.isFinite(transcriptAtMs) && transcriptAtMs < dispatchedAtMs) {
|
|
19098
|
+
LOG.info("MeshReconcile", `Stale-summary guard: skipping transcript reconcile for task ${taskId} on node ${nodeId} (mesh ${mesh.id}) \u2014 final assistant message (${evidence.transcriptMessageAt}) predates this task's dispatch (${dispatch.dispatchedAt}); it is a prior task's summary`);
|
|
19099
|
+
traceMeshEventDrop("reconcile_stale_summary_before_dispatch", {
|
|
19100
|
+
taskId,
|
|
19101
|
+
sessionId,
|
|
19102
|
+
nodeId,
|
|
19103
|
+
meshId: mesh.id,
|
|
19104
|
+
event: "agent:generating_completed"
|
|
19105
|
+
}, `transcriptAt=${evidence.transcriptMessageAt} < dispatchedAt=${dispatch.dispatchedAt}`);
|
|
19106
|
+
continue;
|
|
19107
|
+
}
|
|
19108
|
+
const reprobeStatus = await reprobeWorkerStatus(components, { isLocalNode, nodeDaemonId, readArgs });
|
|
19109
|
+
if (reprobeStatus && reprobeStatus !== "idle") {
|
|
19110
|
+
deleteHoldState(synthKey, mesh.id);
|
|
19111
|
+
LOG.info("MeshReconcile", `Live re-probe defer: task ${taskId} on node ${nodeId} (mesh ${mesh.id}) read '${reprobeStatus}' at synth-commit time \u2014 worker resumed generating; deferring synth to a later tick`);
|
|
19112
|
+
continue;
|
|
19113
|
+
}
|
|
19114
|
+
const providerSessionId = readNonEmptyString2(payload.providerSessionId);
|
|
19115
|
+
const coordinatorDaemonId = selfIds.find((id) => !!id);
|
|
19116
|
+
try {
|
|
19117
|
+
const result = reconcileDirectDispatchCompletionFromTranscript({
|
|
19118
|
+
meshId: mesh.id,
|
|
19119
|
+
nodeId,
|
|
19120
|
+
sessionId,
|
|
19121
|
+
providerType: providerType || void 0,
|
|
19122
|
+
providerSessionId: providerSessionId || void 0,
|
|
19123
|
+
taskId,
|
|
19124
|
+
finalSummary: evidence.finalSummary,
|
|
19125
|
+
...evidence.transcriptMessageAt ? { transcriptMessageAt: evidence.transcriptMessageAt } : {},
|
|
19126
|
+
...coordinatorDaemonId ? { targetCoordinatorDaemonId: coordinatorDaemonId } : {},
|
|
19127
|
+
source: "daemon_reconcile_transcript_completion"
|
|
19128
|
+
});
|
|
19129
|
+
if (result.reconciled) {
|
|
19130
|
+
recordBackstopFire(backstopKind ?? "phase4SynthesisFired", `task ${taskId} on node ${nodeId} (mesh ${mesh.id}), kind=${result.kind}`);
|
|
19131
|
+
LOG.info("MeshReconcile", `Synthesized missing completion (${result.kind}) for task ${taskId} on node ${nodeId} (mesh ${mesh.id})`);
|
|
19132
|
+
}
|
|
19133
|
+
} catch (e) {
|
|
19134
|
+
LOG.warn("MeshReconcile", `Transcript completion reconcile threw for task ${taskId}: ${e?.message || e}`);
|
|
19135
|
+
}
|
|
19136
|
+
}
|
|
19137
|
+
}
|
|
19138
|
+
async function autoPruneStaleDirectDispatches(components, mesh, selfIds, localDaemonId, minAgeMs) {
|
|
19139
|
+
const directDispatches = getActiveDirectDispatches(mesh.id);
|
|
19140
|
+
if (directDispatches.length === 0) return;
|
|
19141
|
+
const liveNodes = await collectLiveNodesWithSessions(components, mesh, selfIds, localDaemonId);
|
|
19142
|
+
const result = pruneStaleDirectDispatches({
|
|
19143
|
+
meshId: mesh.id,
|
|
19144
|
+
queue: getQueue(mesh.id),
|
|
19145
|
+
ledgerEntries: readLedgerEntries(mesh.id, { tail: 500 }),
|
|
19146
|
+
directDispatches,
|
|
19147
|
+
nodes: liveNodes,
|
|
19148
|
+
execute: true,
|
|
19149
|
+
minAgeMs,
|
|
19150
|
+
source: "daemon_reconcile_auto_prune"
|
|
19151
|
+
});
|
|
19152
|
+
if (result.prunedCount > 0) {
|
|
19153
|
+
LOG.info("MeshReconcile", `Auto-pruned ${result.prunedCount} orphaned direct dispatch record(s) for mesh ${mesh.id}`);
|
|
19154
|
+
}
|
|
19155
|
+
}
|
|
19156
|
+
var init_mesh_completion_synthesis = __esm({
|
|
19157
|
+
"src/mesh/mesh-completion-synthesis.ts"() {
|
|
19158
|
+
"use strict";
|
|
19159
|
+
init_logger();
|
|
19160
|
+
init_mesh_events_utils();
|
|
19161
|
+
init_mesh_event_trace();
|
|
19162
|
+
init_dist();
|
|
19163
|
+
init_mesh_reconcile_identity();
|
|
19164
|
+
init_mesh_work_queue();
|
|
19165
|
+
init_mesh_ledger();
|
|
19166
|
+
init_mesh_active_work();
|
|
19167
|
+
init_mesh_events_stale();
|
|
19168
|
+
init_chat_message_normalization();
|
|
19169
|
+
init_mesh_reconcile_v2_backstop();
|
|
19170
|
+
init_mesh_reconcile_acked_hold();
|
|
19171
|
+
init_mesh_remote_event_pull();
|
|
19172
|
+
}
|
|
19173
|
+
});
|
|
19174
|
+
|
|
19175
|
+
// src/mesh/mesh-reconcile-loop.ts
|
|
18651
19176
|
function findLiveCoordinators(components) {
|
|
18652
19177
|
const out = [];
|
|
18653
19178
|
for (const inst of components.instanceManager.getByCategory("cli")) {
|
|
@@ -19326,355 +19851,6 @@ async function retryUnresolvedDelegateForwards(components) {
|
|
|
19326
19851
|
LOG.info("MeshReconcile", `Retried+delivered unresolved-delegate ${readNonEmptyString2(entry.payload.event)} to coordinator ${entry.coordinatorDaemonId}`);
|
|
19327
19852
|
}
|
|
19328
19853
|
}
|
|
19329
|
-
async function pullRemoteNodeQueues(components, mesh, localDaemonId, candidateDaemonIds) {
|
|
19330
|
-
const dispatchMeshCommand = components.dispatchMeshCommand;
|
|
19331
|
-
if (!dispatchMeshCommand) return;
|
|
19332
|
-
const meshId = mesh.id;
|
|
19333
|
-
const pulls = candidateDaemonIds.length > 0 ? candidateDaemonIds.map((id) => ({ meshId, coordinatorDaemonId: id })) : [{ meshId }];
|
|
19334
|
-
await Promise.allSettled(mesh.nodes.map(async (node) => {
|
|
19335
|
-
const nodeDaemonId = readNonEmptyString2(node.daemonId);
|
|
19336
|
-
if (!nodeDaemonId) return;
|
|
19337
|
-
if (daemonIdsEquivalent(nodeDaemonId, localDaemonId)) return;
|
|
19338
|
-
if (daemonIdListIncludes(candidateDaemonIds, nodeDaemonId)) return;
|
|
19339
|
-
const peerSnapshot = components.getMeshPeerConnectionStatus?.(nodeDaemonId);
|
|
19340
|
-
if (peerSnapshot && String(peerSnapshot.state) !== "connected") return;
|
|
19341
|
-
for (const pendingEventArgs of pulls) {
|
|
19342
|
-
let events;
|
|
19343
|
-
try {
|
|
19344
|
-
events = await dispatchMeshCommand(nodeDaemonId, "get_pending_mesh_events", pendingEventArgs);
|
|
19345
|
-
} catch {
|
|
19346
|
-
break;
|
|
19347
|
-
}
|
|
19348
|
-
const list = extractPendingEvents(events).filter((e) => readNonEmptyString2(e?.meshId) === meshId);
|
|
19349
|
-
for (const event of list) {
|
|
19350
|
-
const payload = buildForwardPayloadFromPending(event);
|
|
19351
|
-
if (!payload.event || !payload.meshId) continue;
|
|
19352
|
-
try {
|
|
19353
|
-
handleMeshForwardEvent(components, payload);
|
|
19354
|
-
} catch {
|
|
19355
|
-
}
|
|
19356
|
-
}
|
|
19357
|
-
}
|
|
19358
|
-
}));
|
|
19359
|
-
}
|
|
19360
|
-
function unwrapReadChatPayload(raw) {
|
|
19361
|
-
let cursor = raw;
|
|
19362
|
-
for (let depth = 0; depth < 4 && cursor && typeof cursor === "object"; depth++) {
|
|
19363
|
-
const record = cursor;
|
|
19364
|
-
if (Array.isArray(record.messages)) return record;
|
|
19365
|
-
if (record.payload && typeof record.payload === "object") {
|
|
19366
|
-
cursor = record.payload;
|
|
19367
|
-
continue;
|
|
19368
|
-
}
|
|
19369
|
-
if (record.result && typeof record.result === "object") {
|
|
19370
|
-
cursor = record.result;
|
|
19371
|
-
continue;
|
|
19372
|
-
}
|
|
19373
|
-
if (record.data && typeof record.data === "object") {
|
|
19374
|
-
cursor = record.data;
|
|
19375
|
-
continue;
|
|
19376
|
-
}
|
|
19377
|
-
break;
|
|
19378
|
-
}
|
|
19379
|
-
return cursor && typeof cursor === "object" ? cursor : null;
|
|
19380
|
-
}
|
|
19381
|
-
function readChatPayloadStatus(payload) {
|
|
19382
|
-
return readNonEmptyString2(payload?.status).toLowerCase();
|
|
19383
|
-
}
|
|
19384
|
-
function realTerminalEmitPendingForTask(meshId, taskId) {
|
|
19385
|
-
let pending;
|
|
19386
|
-
try {
|
|
19387
|
-
pending = getPendingMeshCoordinatorEvents(meshId);
|
|
19388
|
-
} catch {
|
|
19389
|
-
return false;
|
|
19390
|
-
}
|
|
19391
|
-
return pending.some((e) => readNonEmptyString2(e.metadataEvent?.taskId) === taskId && (e.event === "agent:generating_completed" || e.event === "agent:stopped"));
|
|
19392
|
-
}
|
|
19393
|
-
async function reprobeWorkerStatus(components, args) {
|
|
19394
|
-
try {
|
|
19395
|
-
if (args.isLocalNode) {
|
|
19396
|
-
const r = await components.commandHandler.handle("read_chat", args.readArgs);
|
|
19397
|
-
if (r && r.success === false) return null;
|
|
19398
|
-
return readChatPayloadStatus(unwrapReadChatPayload(r));
|
|
19399
|
-
}
|
|
19400
|
-
if (components.dispatchMeshCommand) {
|
|
19401
|
-
const r = await components.dispatchMeshCommand(args.nodeDaemonId, "read_chat", args.readArgs);
|
|
19402
|
-
const p = unwrapReadChatPayload(r);
|
|
19403
|
-
if (p && p.success === false) return null;
|
|
19404
|
-
return readChatPayloadStatus(p);
|
|
19405
|
-
}
|
|
19406
|
-
} catch {
|
|
19407
|
-
return null;
|
|
19408
|
-
}
|
|
19409
|
-
return null;
|
|
19410
|
-
}
|
|
19411
|
-
async function reconcileUnterminatedDirectDispatches(components, mesh, selfIds, localDaemonId) {
|
|
19412
|
-
const dispatches = getActiveDirectDispatches(mesh.id);
|
|
19413
|
-
rehydrateAckedHoldsForMesh(mesh.id);
|
|
19414
|
-
const activeTaskKeys = new Set(
|
|
19415
|
-
dispatches.map((d) => readNonEmptyString2(d.taskId)).filter(Boolean).map((taskId) => inFlightSynthKey(mesh.id, taskId))
|
|
19416
|
-
);
|
|
19417
|
-
const heldKeys = collectHeldSynthKeysForMesh(mesh.id);
|
|
19418
|
-
for (const key2 of heldKeys) {
|
|
19419
|
-
if (!activeTaskKeys.has(key2)) deleteHoldState(key2, mesh.id);
|
|
19420
|
-
}
|
|
19421
|
-
if (dispatches.length === 0) return;
|
|
19422
|
-
const dispatchMeshCommand = components.dispatchMeshCommand;
|
|
19423
|
-
const nodeById = new Map(mesh.nodes.map((n) => [n.id, n]));
|
|
19424
|
-
for (const dispatch of dispatches) {
|
|
19425
|
-
const sessionId = readNonEmptyString2(dispatch.sessionId);
|
|
19426
|
-
const nodeId = readNonEmptyString2(dispatch.nodeId);
|
|
19427
|
-
const taskId = readNonEmptyString2(dispatch.taskId);
|
|
19428
|
-
if (!sessionId || !nodeId || !taskId) continue;
|
|
19429
|
-
const node = nodeById.get(nodeId);
|
|
19430
|
-
const nodeDaemonId = readNonEmptyString2(node?.daemonId);
|
|
19431
|
-
const isLocalNode = !nodeDaemonId || daemonIdListIncludes(selfIds, nodeDaemonId) || daemonIdsEquivalent(nodeDaemonId, localDaemonId) || !!components.instanceManager.getInstance(sessionId);
|
|
19432
|
-
const providerType = readNonEmptyString2(dispatch.providerType);
|
|
19433
|
-
const readArgs = {
|
|
19434
|
-
sessionId,
|
|
19435
|
-
targetSessionId: sessionId,
|
|
19436
|
-
tailLimit: 10,
|
|
19437
|
-
...node?.workspace ? { workspace: node.workspace } : {},
|
|
19438
|
-
...providerType ? { agentType: providerType, providerType } : {}
|
|
19439
|
-
};
|
|
19440
|
-
const synthKey = inFlightSynthKey(mesh.id, taskId);
|
|
19441
|
-
const isAcked = dispatch.status === "acked";
|
|
19442
|
-
let backstopKind;
|
|
19443
|
-
let payload = null;
|
|
19444
|
-
let readFailed = false;
|
|
19445
|
-
try {
|
|
19446
|
-
if (isLocalNode) {
|
|
19447
|
-
const result = await components.commandHandler.handle("read_chat", readArgs);
|
|
19448
|
-
if (result && result.success === false) {
|
|
19449
|
-
readFailed = true;
|
|
19450
|
-
} else {
|
|
19451
|
-
payload = unwrapReadChatPayload(result);
|
|
19452
|
-
}
|
|
19453
|
-
} else if (dispatchMeshCommand) {
|
|
19454
|
-
const result = await dispatchMeshCommand(nodeDaemonId, "read_chat", readArgs);
|
|
19455
|
-
payload = unwrapReadChatPayload(result);
|
|
19456
|
-
if (payload && payload.success === false) {
|
|
19457
|
-
payload = null;
|
|
19458
|
-
readFailed = true;
|
|
19459
|
-
}
|
|
19460
|
-
} else {
|
|
19461
|
-
continue;
|
|
19462
|
-
}
|
|
19463
|
-
} catch {
|
|
19464
|
-
readFailed = true;
|
|
19465
|
-
}
|
|
19466
|
-
if (!payload && !readFailed) continue;
|
|
19467
|
-
if (readFailed || !payload) {
|
|
19468
|
-
if (isAcked) {
|
|
19469
|
-
const prior = getHoldState(synthKey, mesh.id);
|
|
19470
|
-
const failures = (prior?.consecutiveReadFailures ?? 0) + 1;
|
|
19471
|
-
const liveConfirmedSinceAck = prior?.liveConfirmedSinceAck ?? false;
|
|
19472
|
-
setHoldState(synthKey, mesh.id, { liveConfirmedSinceAck, consecutiveReadFailures: failures });
|
|
19473
|
-
if (liveConfirmedSinceAck && failures >= ACKED_DEATH_CONSECUTIVE_READ_FAILURES) {
|
|
19474
|
-
LOG.warn("MeshReconcile", `Acked-hold death signal: task ${taskId} on node ${nodeId} (mesh ${mesh.id}) read_chat failed ${failures}x consecutively after a live-confirmed ack \u2014 worker session presumed gone mid-turn; releasing the indefinite synth hold to the stranded-reclaim / orphan-prune nets`);
|
|
19475
|
-
}
|
|
19476
|
-
}
|
|
19477
|
-
continue;
|
|
19478
|
-
}
|
|
19479
|
-
const priorHoldState = getHoldState(synthKey, mesh.id);
|
|
19480
|
-
setHoldState(synthKey, mesh.id, {
|
|
19481
|
-
liveConfirmedSinceAck: true,
|
|
19482
|
-
consecutiveReadFailures: 0,
|
|
19483
|
-
...priorHoldState?.transcriptIdleSinceMs !== void 0 ? { transcriptIdleSinceMs: priorHoldState.transcriptIdleSinceMs } : {}
|
|
19484
|
-
});
|
|
19485
|
-
const nowMs = Date.now();
|
|
19486
|
-
if (readChatPayloadStatus(payload) !== "idle") {
|
|
19487
|
-
setHoldState(synthKey, mesh.id, { liveConfirmedSinceAck: true, consecutiveReadFailures: 0 });
|
|
19488
|
-
continue;
|
|
19489
|
-
}
|
|
19490
|
-
const messages = Array.isArray(payload.messages) ? payload.messages : [];
|
|
19491
|
-
const evidence = extractFinalAssistantSummaryEvidence(messages);
|
|
19492
|
-
if (isAcked) {
|
|
19493
|
-
const ackedAtMs = Date.parse(readNonEmptyString2(dispatch.updatedAt));
|
|
19494
|
-
const sinceAckMs = Number.isFinite(ackedAtMs) ? nowMs - ackedAtMs : Number.POSITIVE_INFINITY;
|
|
19495
|
-
const deathDeadlineMs = resolveAckedDeathDeadlineMs();
|
|
19496
|
-
const holdState = getHoldState(synthKey, mesh.id);
|
|
19497
|
-
let fastTrackReady = false;
|
|
19498
|
-
if (evidence.finalSummary) {
|
|
19499
|
-
const idleSinceMs = holdState?.transcriptIdleSinceMs ?? nowMs;
|
|
19500
|
-
if (holdState && holdState.transcriptIdleSinceMs === void 0) {
|
|
19501
|
-
setHoldState(synthKey, mesh.id, { ...holdState, transcriptIdleSinceMs: idleSinceMs });
|
|
19502
|
-
}
|
|
19503
|
-
const fastTrackGraceMs = resolveAckedTranscriptFastTrackGraceMs();
|
|
19504
|
-
const idleHeldMs = nowMs - idleSinceMs;
|
|
19505
|
-
if (idleHeldMs >= fastTrackGraceMs) {
|
|
19506
|
-
fastTrackReady = true;
|
|
19507
|
-
backstopKind = "ackedHoldFastTrackFired";
|
|
19508
|
-
LOG.info("MeshReconcile", `Acked-hold transcript fast-track: task ${taskId} on node ${nodeId} (mesh ${mesh.id}) read idle WITH a final assistant message for ${Math.round(idleHeldMs / 1e3)}s continuous (grace ${Math.round(fastTrackGraceMs / 1e3)}s) \u2014 promoting the synth ahead of the ${Math.round(deathDeadlineMs / 1e3)}s death backstop; the worker's real emit was lost/late and a later one no-ops idempotently.`);
|
|
19509
|
-
}
|
|
19510
|
-
} else if (holdState?.transcriptIdleSinceMs !== void 0) {
|
|
19511
|
-
setHoldState(synthKey, mesh.id, { ...holdState, transcriptIdleSinceMs: void 0 });
|
|
19512
|
-
}
|
|
19513
|
-
if (!fastTrackReady && sinceAckMs < deathDeadlineMs) {
|
|
19514
|
-
LOG.info("MeshReconcile", `Acked-hold: task ${taskId} on node ${nodeId} (mesh ${mesh.id}) read idle ${Number.isFinite(sinceAckMs) ? Math.round(sinceAckMs / 1e3) + "s" : "\u221E"} since the generating_started ack \u2014 HOLDING synth (worker presumed alive; a later real emit is idempotent). Transcript fast-track promotes at ${Math.round(resolveAckedTranscriptFastTrackGraceMs() / 1e3)}s continuous idle-with-final-assistant; death backstop at ${Math.round(deathDeadlineMs / 1e3)}s or on consecutive read failures.`);
|
|
19515
|
-
continue;
|
|
19516
|
-
}
|
|
19517
|
-
if (!fastTrackReady) {
|
|
19518
|
-
backstopKind = "ackedHoldDeathDeadlineFired";
|
|
19519
|
-
LOG.warn("MeshReconcile", `Acked-hold death deadline reached: task ${taskId} on node ${nodeId} (mesh ${mesh.id}) still idle ${Math.round(sinceAckMs / 1e3)}s after the ack (deadline ${Math.round(deathDeadlineMs / 1e3)}s) \u2014 synthesizing the missing completion as a notification-loss net (a real emit, if it ever lands, no-ops idempotently).`);
|
|
19520
|
-
}
|
|
19521
|
-
}
|
|
19522
|
-
if (realTerminalEmitPendingForTask(mesh.id, taskId)) {
|
|
19523
|
-
deleteHoldState(synthKey, mesh.id);
|
|
19524
|
-
LOG.info("MeshReconcile", `Worker-emit priority: task ${taskId} on node ${nodeId} (mesh ${mesh.id}) has a real terminal completion already queued \u2014 yielding synth to the worker's own emit`);
|
|
19525
|
-
continue;
|
|
19526
|
-
}
|
|
19527
|
-
if (!evidence.finalSummary) continue;
|
|
19528
|
-
const dispatchedAtMs = Date.parse(readNonEmptyString2(dispatch.dispatchedAt));
|
|
19529
|
-
const transcriptAtMs = Date.parse(evidence.transcriptMessageAt ?? "");
|
|
19530
|
-
if (Number.isFinite(dispatchedAtMs) && Number.isFinite(transcriptAtMs) && transcriptAtMs < dispatchedAtMs) {
|
|
19531
|
-
LOG.info("MeshReconcile", `Stale-summary guard: skipping transcript reconcile for task ${taskId} on node ${nodeId} (mesh ${mesh.id}) \u2014 final assistant message (${evidence.transcriptMessageAt}) predates this task's dispatch (${dispatch.dispatchedAt}); it is a prior task's summary`);
|
|
19532
|
-
traceMeshEventDrop("reconcile_stale_summary_before_dispatch", {
|
|
19533
|
-
taskId,
|
|
19534
|
-
sessionId,
|
|
19535
|
-
nodeId,
|
|
19536
|
-
meshId: mesh.id,
|
|
19537
|
-
event: "agent:generating_completed"
|
|
19538
|
-
}, `transcriptAt=${evidence.transcriptMessageAt} < dispatchedAt=${dispatch.dispatchedAt}`);
|
|
19539
|
-
continue;
|
|
19540
|
-
}
|
|
19541
|
-
const reprobeStatus = await reprobeWorkerStatus(components, { isLocalNode, nodeDaemonId, readArgs });
|
|
19542
|
-
if (reprobeStatus && reprobeStatus !== "idle") {
|
|
19543
|
-
deleteHoldState(synthKey, mesh.id);
|
|
19544
|
-
LOG.info("MeshReconcile", `Live re-probe defer: task ${taskId} on node ${nodeId} (mesh ${mesh.id}) read '${reprobeStatus}' at synth-commit time \u2014 worker resumed generating; deferring synth to a later tick`);
|
|
19545
|
-
continue;
|
|
19546
|
-
}
|
|
19547
|
-
const providerSessionId = readNonEmptyString2(payload.providerSessionId);
|
|
19548
|
-
const coordinatorDaemonId = selfIds.find((id) => !!id);
|
|
19549
|
-
try {
|
|
19550
|
-
const result = reconcileDirectDispatchCompletionFromTranscript({
|
|
19551
|
-
meshId: mesh.id,
|
|
19552
|
-
nodeId,
|
|
19553
|
-
sessionId,
|
|
19554
|
-
providerType: providerType || void 0,
|
|
19555
|
-
providerSessionId: providerSessionId || void 0,
|
|
19556
|
-
taskId,
|
|
19557
|
-
finalSummary: evidence.finalSummary,
|
|
19558
|
-
...evidence.transcriptMessageAt ? { transcriptMessageAt: evidence.transcriptMessageAt } : {},
|
|
19559
|
-
...coordinatorDaemonId ? { targetCoordinatorDaemonId: coordinatorDaemonId } : {},
|
|
19560
|
-
source: "daemon_reconcile_transcript_completion"
|
|
19561
|
-
});
|
|
19562
|
-
if (result.reconciled) {
|
|
19563
|
-
recordBackstopFire(backstopKind ?? "phase4SynthesisFired", `task ${taskId} on node ${nodeId} (mesh ${mesh.id}), kind=${result.kind}`);
|
|
19564
|
-
LOG.info("MeshReconcile", `Synthesized missing completion (${result.kind}) for task ${taskId} on node ${nodeId} (mesh ${mesh.id})`);
|
|
19565
|
-
}
|
|
19566
|
-
} catch (e) {
|
|
19567
|
-
LOG.warn("MeshReconcile", `Transcript completion reconcile threw for task ${taskId}: ${e?.message || e}`);
|
|
19568
|
-
}
|
|
19569
|
-
}
|
|
19570
|
-
}
|
|
19571
|
-
async function autoPruneStaleDirectDispatches(components, mesh, selfIds, localDaemonId, minAgeMs) {
|
|
19572
|
-
const directDispatches = getActiveDirectDispatches(mesh.id);
|
|
19573
|
-
if (directDispatches.length === 0) return;
|
|
19574
|
-
const liveNodes = await collectLiveNodesWithSessions(components, mesh, selfIds, localDaemonId);
|
|
19575
|
-
const result = pruneStaleDirectDispatches({
|
|
19576
|
-
meshId: mesh.id,
|
|
19577
|
-
queue: getQueue(mesh.id),
|
|
19578
|
-
ledgerEntries: readLedgerEntries(mesh.id, { tail: 500 }),
|
|
19579
|
-
directDispatches,
|
|
19580
|
-
nodes: liveNodes,
|
|
19581
|
-
execute: true,
|
|
19582
|
-
minAgeMs,
|
|
19583
|
-
source: "daemon_reconcile_auto_prune"
|
|
19584
|
-
});
|
|
19585
|
-
if (result.prunedCount > 0) {
|
|
19586
|
-
LOG.info("MeshReconcile", `Auto-pruned ${result.prunedCount} orphaned direct dispatch record(s) for mesh ${mesh.id}`);
|
|
19587
|
-
}
|
|
19588
|
-
}
|
|
19589
|
-
async function collectLiveNodesWithSessions(components, mesh, selfIds, localDaemonId) {
|
|
19590
|
-
const dispatchMeshCommand = components.dispatchMeshCommand;
|
|
19591
|
-
return Promise.all(mesh.nodes.map(async (node) => {
|
|
19592
|
-
const nodeDaemonId = readNonEmptyString2(node.daemonId);
|
|
19593
|
-
const isLocalNode = !nodeDaemonId || daemonIdListIncludes(selfIds, nodeDaemonId) || daemonIdsEquivalent(nodeDaemonId, localDaemonId);
|
|
19594
|
-
if (!isLocalNode) {
|
|
19595
|
-
const peerSnapshot = components.getMeshPeerConnectionStatus?.(nodeDaemonId);
|
|
19596
|
-
if (peerSnapshot && String(peerSnapshot.state) !== "connected") return node;
|
|
19597
|
-
}
|
|
19598
|
-
let statusResult;
|
|
19599
|
-
try {
|
|
19600
|
-
if (isLocalNode) {
|
|
19601
|
-
statusResult = await components.commandHandler.handle("get_status_metadata", {});
|
|
19602
|
-
} else if (dispatchMeshCommand) {
|
|
19603
|
-
statusResult = await dispatchMeshCommand(nodeDaemonId, "get_status_metadata", {});
|
|
19604
|
-
} else {
|
|
19605
|
-
return node;
|
|
19606
|
-
}
|
|
19607
|
-
} catch {
|
|
19608
|
-
return node;
|
|
19609
|
-
}
|
|
19610
|
-
const sessions = extractStatusMetadataSessions(statusResult);
|
|
19611
|
-
return sessions.length > 0 ? { ...node, sessions } : node;
|
|
19612
|
-
}));
|
|
19613
|
-
}
|
|
19614
|
-
function extractStatusMetadataSessions(raw) {
|
|
19615
|
-
let cursor = raw;
|
|
19616
|
-
for (let depth = 0; depth < 4 && cursor && typeof cursor === "object"; depth++) {
|
|
19617
|
-
const record = cursor;
|
|
19618
|
-
const status = record.status && typeof record.status === "object" ? record.status : void 0;
|
|
19619
|
-
if (status && Array.isArray(status.sessions)) return status.sessions;
|
|
19620
|
-
if (Array.isArray(record.sessions)) return record.sessions;
|
|
19621
|
-
if (record.payload && typeof record.payload === "object") {
|
|
19622
|
-
cursor = record.payload;
|
|
19623
|
-
continue;
|
|
19624
|
-
}
|
|
19625
|
-
if (record.result && typeof record.result === "object") {
|
|
19626
|
-
cursor = record.result;
|
|
19627
|
-
continue;
|
|
19628
|
-
}
|
|
19629
|
-
if (record.data && typeof record.data === "object") {
|
|
19630
|
-
cursor = record.data;
|
|
19631
|
-
continue;
|
|
19632
|
-
}
|
|
19633
|
-
break;
|
|
19634
|
-
}
|
|
19635
|
-
return [];
|
|
19636
|
-
}
|
|
19637
|
-
function extractPendingEvents(raw) {
|
|
19638
|
-
if (Array.isArray(raw)) return raw;
|
|
19639
|
-
if (raw && typeof raw === "object") {
|
|
19640
|
-
const events = raw.events;
|
|
19641
|
-
if (Array.isArray(events)) return events;
|
|
19642
|
-
}
|
|
19643
|
-
return [];
|
|
19644
|
-
}
|
|
19645
|
-
function buildForwardPayloadFromPending(event) {
|
|
19646
|
-
const metadata = event?.metadataEvent && typeof event.metadataEvent === "object" ? event.metadataEvent : {};
|
|
19647
|
-
return {
|
|
19648
|
-
event: readNonEmptyString2(event?.event),
|
|
19649
|
-
meshId: readNonEmptyString2(event?.meshId),
|
|
19650
|
-
nodeId: readNonEmptyString2(event?.nodeId) || readNonEmptyString2(metadata.meshNodeId),
|
|
19651
|
-
workspace: readNonEmptyString2(event?.workspace) || readNonEmptyString2(metadata.workspace),
|
|
19652
|
-
// Preserve the originating coordinator session id across the relay. It is normally
|
|
19653
|
-
// carried inside metadataEvent.meshCoordinatorSessionId (spread below), but pass the
|
|
19654
|
-
// top-level field through explicitly too so the handleMeshForwardEvent whitelist
|
|
19655
|
-
// recovers it regardless of which carrier the producing daemon used.
|
|
19656
|
-
...readNonEmptyString2(event?.targetCoordinatorSessionId) ? { targetCoordinatorSessionId: readNonEmptyString2(event.targetCoordinatorSessionId) } : {},
|
|
19657
|
-
...metadata,
|
|
19658
|
-
// NOTIF-MISS (FIX 3): surface the dispatch task id at the TOP LEVEL so the relay's
|
|
19659
|
-
// received-stage trace (and buildRelayMetadataEvent) recovers it regardless of which
|
|
19660
|
-
// carrier the producing daemon used. The metadata spread above may carry the id only as
|
|
19661
|
-
// `meshActiveTaskId` (a worker provider event), leaving top-level `taskId` unset and the
|
|
19662
|
-
// received stage rendering `task=-`. Resolve both carriers into an explicit `taskId` so
|
|
19663
|
-
// dedup stays task-scoped end-to-end. Only set when a non-empty id exists (no clobber to
|
|
19664
|
-
// undefined when neither is present).
|
|
19665
|
-
...(() => {
|
|
19666
|
-
const tid = readNonEmptyString2(metadata.taskId) || readNonEmptyString2(metadata.meshActiveTaskId);
|
|
19667
|
-
return tid ? { taskId: tid } : {};
|
|
19668
|
-
})(),
|
|
19669
|
-
// T4 (B3b): carry the v2 envelope (protocolVersion/eventId/scope/dispatchedBy/
|
|
19670
|
-
// intendedFor) across the P2P relay boundary at the TOP LEVEL. These live on the
|
|
19671
|
-
// pending event itself, not inside metadataEvent, so without this the remote pull
|
|
19672
|
-
// re-queue would re-stamp a fresh eventId — breaking cross-machine idempotency and
|
|
19673
|
-
// downgrading the relayed completion to v1 broadcast routing. Spread LAST so the
|
|
19674
|
-
// authoritative envelope always wins over any stale key the metadata spread carried.
|
|
19675
|
-
...serializeV2EnvelopeToWire(event)
|
|
19676
|
-
};
|
|
19677
|
-
}
|
|
19678
19854
|
function setupMeshReconcileLoop(components) {
|
|
19679
19855
|
const intervalMs = resolveReconcileIntervalMs();
|
|
19680
19856
|
let running = false;
|
|
@@ -19694,7 +19870,7 @@ function setupMeshReconcileLoop(components) {
|
|
|
19694
19870
|
}
|
|
19695
19871
|
};
|
|
19696
19872
|
}
|
|
19697
|
-
var
|
|
19873
|
+
var coordinatorModalParkState, heldEventLedgerRecorded, ASSIGNED_STRANDED_DEADLINE_MS, DELIVERED_NO_TURN_DEADLINE_MS, RECLAIM_UNKNOWN_GRACE_TICKS, deliveredNoTurnUnknownStreak, STRICT_SESSION_MATCH_TTL_MS, unresolvedForwardRejectionCounts, MAX_FORWARD_REJECTIONS;
|
|
19698
19874
|
var init_mesh_reconcile_loop = __esm({
|
|
19699
19875
|
"src/mesh/mesh-reconcile-loop.ts"() {
|
|
19700
19876
|
"use strict";
|
|
@@ -19713,17 +19889,13 @@ var init_mesh_reconcile_loop = __esm({
|
|
|
19713
19889
|
init_mesh_work_queue();
|
|
19714
19890
|
init_mesh_queue_assignment();
|
|
19715
19891
|
init_mesh_ledger();
|
|
19716
|
-
init_mesh_active_work();
|
|
19717
19892
|
init_mesh_events_stale();
|
|
19718
|
-
init_chat_message_normalization();
|
|
19719
19893
|
init_mesh_reconcile_identity();
|
|
19894
|
+
init_mesh_reconcile_config();
|
|
19895
|
+
init_mesh_remote_event_pull();
|
|
19896
|
+
init_mesh_completion_synthesis();
|
|
19720
19897
|
init_mesh_reconcile_v2_backstop();
|
|
19721
19898
|
init_mesh_reconcile_acked_hold();
|
|
19722
|
-
init_mesh_reconcile_v2_backstop();
|
|
19723
|
-
init_mesh_reconcile_acked_hold();
|
|
19724
|
-
DEFAULT_RECONCILE_INTERVAL_MS = 4e3;
|
|
19725
|
-
DEFAULT_AUTO_PRUNE_MIN_AGE_MS = 24 * 60 * 6e4;
|
|
19726
|
-
DEFAULT_PENDING_HELD_DRAIN_ESCALATE_MS = 12e3;
|
|
19727
19899
|
coordinatorModalParkState = /* @__PURE__ */ new Map();
|
|
19728
19900
|
heldEventLedgerRecorded = /* @__PURE__ */ new Set();
|
|
19729
19901
|
ASSIGNED_STRANDED_DEADLINE_MS = 5 * 6e4;
|
|
@@ -19753,6 +19925,7 @@ __export(mesh_events_exports, {
|
|
|
19753
19925
|
queuePendingMeshCoordinatorEvent: () => queuePendingMeshCoordinatorEvent,
|
|
19754
19926
|
readV2EnvelopeFromWire: () => readV2EnvelopeFromWire,
|
|
19755
19927
|
reconcileDirectDispatchCompletionFromTranscript: () => reconcileDirectDispatchCompletionFromTranscript,
|
|
19928
|
+
requeueHeldMeshCoordinatorEvents: () => requeueHeldMeshCoordinatorEvents,
|
|
19756
19929
|
resolveCoordinatorDrainDeliverability: () => resolveCoordinatorDrainDeliverability,
|
|
19757
19930
|
runMeshReconcileTick: () => runMeshReconcileTick,
|
|
19758
19931
|
serializeV2EnvelopeToWire: () => serializeV2EnvelopeToWire,
|
|
@@ -26639,6 +26812,7 @@ init_mesh_config();
|
|
|
26639
26812
|
init_dist();
|
|
26640
26813
|
init_dist();
|
|
26641
26814
|
init_dist();
|
|
26815
|
+
init_dist();
|
|
26642
26816
|
init_coordinator_prompt();
|
|
26643
26817
|
init_mesh_missions();
|
|
26644
26818
|
init_mesh_task_stats();
|
|
@@ -68363,6 +68537,8 @@ export {
|
|
|
68363
68537
|
AcpProviderInstance,
|
|
68364
68538
|
AgentStreamPoller,
|
|
68365
68539
|
BUILTIN_CHAT_MESSAGE_KINDS,
|
|
68540
|
+
CANONICAL_MESH_TOOL_COUNT,
|
|
68541
|
+
CANONICAL_MESH_TOOL_NAMES,
|
|
68366
68542
|
CHANGE_IMPACT_CONFIG_LOCATIONS,
|
|
68367
68543
|
CHANGE_IMPACT_CONFIG_SCHEMA,
|
|
68368
68544
|
CHAT_MESSAGE_ACTIVITY_SOURCES,
|
|
@@ -68420,11 +68596,13 @@ export {
|
|
|
68420
68596
|
MESH_REFINE_CONFIG_LOCATIONS,
|
|
68421
68597
|
MESH_REFINE_CONFIG_SCHEMA,
|
|
68422
68598
|
MESH_SCHEDULING_STRATEGIES,
|
|
68599
|
+
MESH_TASK_PRIORITIES,
|
|
68423
68600
|
MESH_WORKTREE_BOOTSTRAP_CONFIG_LOCATIONS,
|
|
68424
68601
|
MESH_WORKTREE_BOOTSTRAP_CONFIG_SCHEMA,
|
|
68425
68602
|
MIN_GIT_WORKSPACE_POLL_INTERVAL_MS,
|
|
68426
68603
|
MIN_MACHINE_RUNTIME_SUBSCRIPTION_INTERVAL_MS,
|
|
68427
68604
|
MIN_SESSION_HOST_DIAGNOSTICS_SUBSCRIPTION_INTERVAL_MS,
|
|
68605
|
+
NOT_BEFORE_RELATIVE_THRESHOLD_MS,
|
|
68428
68606
|
NodePtyTransportFactory,
|
|
68429
68607
|
OPERATING_NOTE_DEDUPE_WINDOW,
|
|
68430
68608
|
OPERATING_NOTE_KEEP_LATEST,
|
|
@@ -68572,7 +68750,6 @@ export {
|
|
|
68572
68750
|
getQueue,
|
|
68573
68751
|
getRecentActivity,
|
|
68574
68752
|
getRecentCommands,
|
|
68575
|
-
getRecentCompletionConflicts,
|
|
68576
68753
|
getRecentDebugTrace,
|
|
68577
68754
|
getRecentLogs,
|
|
68578
68755
|
getSavedProviderSessions,
|
|
@@ -68636,6 +68813,8 @@ export {
|
|
|
68636
68813
|
maybeRunDaemonUpgradeHelperFromEnv,
|
|
68637
68814
|
mergeAndNormalizePolicy,
|
|
68638
68815
|
meshNodeIdMatches,
|
|
68816
|
+
meshTaskNotBeforeReady,
|
|
68817
|
+
meshTaskPriorityRank,
|
|
68639
68818
|
namedKeyToAnsi,
|
|
68640
68819
|
namedKeysToAnsi,
|
|
68641
68820
|
nodeSatisfiesRequiredTags,
|
|
@@ -68658,6 +68837,7 @@ export {
|
|
|
68658
68837
|
normalizeMeshNodeId,
|
|
68659
68838
|
normalizeMeshSchedulingStrategy,
|
|
68660
68839
|
normalizeMeshTaskMode,
|
|
68840
|
+
normalizeMeshTaskPriority,
|
|
68661
68841
|
normalizeMeshWorkerResult,
|
|
68662
68842
|
normalizeMessageParts,
|
|
68663
68843
|
normalizeRepoIdentity,
|
|
@@ -68686,7 +68866,6 @@ export {
|
|
|
68686
68866
|
readOperatingNotes,
|
|
68687
68867
|
readV2EnvelopeFromWire,
|
|
68688
68868
|
reconcileDirectDispatchCompletionFromTranscript,
|
|
68689
|
-
recordCompletionConflict,
|
|
68690
68869
|
recordDebugTrace,
|
|
68691
68870
|
recordDirectDispatchTask,
|
|
68692
68871
|
recordMeshToolCall,
|
|
@@ -68696,6 +68875,7 @@ export {
|
|
|
68696
68875
|
removeMagiPanel,
|
|
68697
68876
|
removeNode,
|
|
68698
68877
|
removeWorktree,
|
|
68878
|
+
requeueHeldMeshCoordinatorEvents,
|
|
68699
68879
|
requeueTask,
|
|
68700
68880
|
requireMeshHostQueueOwner,
|
|
68701
68881
|
resetConfig,
|
|
@@ -68716,6 +68896,7 @@ export {
|
|
|
68716
68896
|
resolveMeshRefineValidationPlan,
|
|
68717
68897
|
resolveMeshSurfacedSessionPreview,
|
|
68718
68898
|
resolveNodeSchedulingPriority,
|
|
68899
|
+
resolveNotBefore,
|
|
68719
68900
|
resolveProviderMaxParallel,
|
|
68720
68901
|
resolveSessionHostAppName,
|
|
68721
68902
|
resolveSessionHostAppNameResolution,
|