@adhdev/daemon-core 0.9.82-rc.291 → 0.9.82-rc.293
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/router.d.ts +50 -0
- package/dist/index.js +637 -327
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +639 -329
- package/dist/index.mjs.map +1 -1
- package/dist/mesh/mesh-missions.d.ts +25 -1
- package/dist/mesh/mesh-runtime-store.d.ts +15 -0
- package/dist/mesh/mesh-unresolved-forward-outbox.d.ts +30 -0
- package/package.json +2 -2
- package/src/commands/router.ts +274 -58
- package/src/git/git-status.ts +46 -11
- package/src/mesh/coordinator-prompt.ts +2 -2
- package/src/mesh/mesh-active-work.ts +2 -1
- package/src/mesh/mesh-events-coordinator.ts +58 -10
- package/src/mesh/mesh-missions.ts +41 -3
- package/src/mesh/mesh-reconcile-loop.ts +55 -0
- package/src/mesh/mesh-runtime-store.ts +30 -0
- package/src/mesh/mesh-unresolved-forward-outbox.ts +185 -0
- package/src/providers/cli-provider-instance.ts +21 -16
- package/src/providers/extension-provider-instance.ts +5 -1
- package/src/providers/ide-provider-instance.ts +6 -1
package/src/git/git-status.ts
CHANGED
|
@@ -143,11 +143,39 @@ const WEB_ONLY_PACKAGES = new Set([
|
|
|
143
143
|
'terminal-render-web',
|
|
144
144
|
]);
|
|
145
145
|
|
|
146
|
+
/**
|
|
147
|
+
* Root-level (non-package) files that demonstrably cannot change what the daemon
|
|
148
|
+
* runtime executes: convergence/verify markers, documentation, and license/notice
|
|
149
|
+
* text. A root commit that moves the oss gitlink while the oss commit only touched
|
|
150
|
+
* one of these is NOT a reason to rebuild/restart the daemon — flagging it produces
|
|
151
|
+
* the staleDaemonBuild false-positive this guard exists to suppress.
|
|
152
|
+
*
|
|
153
|
+
* Deliberately conservative: anything NOT matched here (root config like
|
|
154
|
+
* package.json / tsconfig / build scripts, `.txt` fixtures, lockfiles, unknown
|
|
155
|
+
* dotfiles) stays daemon-affecting, because a false-negative — staying silent when
|
|
156
|
+
* the daemon really IS stale — is worse than an over-warn. Markers are the common
|
|
157
|
+
* real-world case (e.g. `.verify-patch-equiv-rc292`), so they are matched broadly;
|
|
158
|
+
* docs are matched by the `docs/` prefix or a markdown/text-doc extension at a
|
|
159
|
+
* filename we recognize as documentation (README/CHANGELOG/LICENSE/NOTICE).
|
|
160
|
+
*/
|
|
161
|
+
function isNonRuntimeRootFile(file: string): boolean {
|
|
162
|
+
const base = file.slice(file.lastIndexOf('/') + 1);
|
|
163
|
+
// Verify/convergence markers: dotfiles whose name signals a transient marker.
|
|
164
|
+
if (/^\.(?:verify|marker|converge|ff-verify|patch-equiv|live-verify)\b/i.test(base)) return true;
|
|
165
|
+
// Documentation living under a docs/ tree (any depth, root or nested).
|
|
166
|
+
if (/(?:^|\/)docs\//i.test(file)) return true;
|
|
167
|
+
// Recognized top-level documentation / license files.
|
|
168
|
+
if (/^(?:README|CHANGELOG|LICENSE|NOTICE|AUTHORS|CONTRIBUTING|CODEOWNERS)(?:\.[A-Za-z0-9]+)?$/i.test(base)) {
|
|
169
|
+
return true;
|
|
170
|
+
}
|
|
171
|
+
return false;
|
|
172
|
+
}
|
|
173
|
+
|
|
146
174
|
/**
|
|
147
175
|
* Determine whether the changes between buildCommit..HEAD touch any daemon-runtime
|
|
148
176
|
* package. Returns isDaemonAffecting:true conservatively when the changed-file set
|
|
149
177
|
* can't be obtained or any changed path is outside the known web-only package set
|
|
150
|
-
*
|
|
178
|
+
* AND is not a recognized non-runtime root file (marker/doc/license).
|
|
151
179
|
*/
|
|
152
180
|
async function classifyDaemonBuildChange(
|
|
153
181
|
repoPath: string,
|
|
@@ -166,24 +194,28 @@ async function classifyDaemonBuildChange(
|
|
|
166
194
|
return { isDaemonAffecting: true, affectedPackages: [] };
|
|
167
195
|
}
|
|
168
196
|
const pkgs = new Set<string>();
|
|
169
|
-
|
|
197
|
+
// A non-package path that is NOT a recognized benign root file (marker/doc).
|
|
198
|
+
// Only these force daemon-affecting; benign markers/docs are ignored so a
|
|
199
|
+
// gitlink-moving root commit over a marker-only oss commit no longer over-warns.
|
|
200
|
+
let sawRuntimeAmbiguousNonPackage = false;
|
|
170
201
|
for (const file of files) {
|
|
171
202
|
const match = file.match(/(?:^|\/)packages\/([^/]+)\//);
|
|
172
203
|
if (!match) {
|
|
173
|
-
|
|
204
|
+
if (!isNonRuntimeRootFile(file)) sawRuntimeAmbiguousNonPackage = true;
|
|
174
205
|
continue;
|
|
175
206
|
}
|
|
176
207
|
pkgs.add(match[1]);
|
|
177
208
|
}
|
|
178
209
|
const affectedPackages = [...pkgs].sort();
|
|
179
|
-
// Daemon-affecting if: any non-package
|
|
180
|
-
// changed, or any explicit daemon-runtime package changed.
|
|
181
|
-
//
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
210
|
+
// Daemon-affecting if: any runtime-ambiguous non-package file changed, any
|
|
211
|
+
// unknown package changed, or any explicit daemon-runtime package changed.
|
|
212
|
+
// The daemon is unaffected only when every changed file is either a known
|
|
213
|
+
// web-only package or a recognized benign root file (and at least one such
|
|
214
|
+
// file changed) — i.e. nothing runtime-ambiguous remains.
|
|
215
|
+
const allBenign =
|
|
216
|
+
!sawRuntimeAmbiguousNonPackage &&
|
|
185
217
|
affectedPackages.every((p) => WEB_ONLY_PACKAGES.has(p) && !DAEMON_RUNTIME_PACKAGES.has(p));
|
|
186
|
-
return { isDaemonAffecting: !
|
|
218
|
+
return { isDaemonAffecting: !allBenign, affectedPackages };
|
|
187
219
|
} catch {
|
|
188
220
|
// diff probe failed → can't prove web-only; stay conservative.
|
|
189
221
|
return { isDaemonAffecting: true, affectedPackages: [] };
|
|
@@ -229,11 +261,14 @@ async function detectDaemonBuildBehind(
|
|
|
229
261
|
options,
|
|
230
262
|
);
|
|
231
263
|
const scopeLabel = scope === 'root' ? 'workspace' : scope;
|
|
264
|
+
const benignDetail = affectedPackages.length > 0
|
|
265
|
+
? `only web packages changed (${affectedPackages.join(', ')})`
|
|
266
|
+
: 'only non-runtime files changed (markers/docs)';
|
|
232
267
|
const warning = isDaemonAffecting
|
|
233
268
|
? `Live daemon was built from ${build.commitShort} which is behind ${scopeLabel} HEAD ${head.slice(0, 7)}. ` +
|
|
234
269
|
`Merged code is NOT live until the daemon is rebuilt/redeployed and restarted — a local dist rebuild alone does not update a cloud daemon.`
|
|
235
270
|
: `Live daemon was built from ${build.commitShort} which is behind ${scopeLabel} HEAD ${head.slice(0, 7)}, ` +
|
|
236
|
-
`but
|
|
271
|
+
`but ${benignDetail}. ` +
|
|
237
272
|
`Daemon restart NOT required — redeploy the web app to reflect the change.`;
|
|
238
273
|
return {
|
|
239
274
|
buildCommit: build.commit,
|
|
@@ -350,7 +350,7 @@ const WORKFLOW_SECTION = `## Orchestration Workflow
|
|
|
350
350
|
c. **Targeted Tasks**: Use \`mesh_send_task\` only when you need to bypass the queue and force a specific node to execute a task immediately.
|
|
351
351
|
d. For the first dispatch of a new task, provide a **complete, self-contained** instruction that includes all context the agent needs (file paths, line numbers, what to change, why). Do not send partial instructions expecting future follow-up.
|
|
352
352
|
e. For a continuation of the same issue in an existing session, send a concise **delta instruction**: current verified state, the exact failed/blocked step, the newly approved action, and final reporting requirements. Do not resend the full original task or open a new chat solely to continue the same work; that wastes coordinator and worker context.
|
|
353
|
-
4. **Monitor** — Prefer event-driven completion/status notifications. Do **not** poll \`mesh_read_chat\` repeatedly. Do **not** repeatedly call \`mesh_status\` or \`mesh_view_queue\` just to wait for assigned/generating work. After dispatching a direct or queued task, send one progress update with the task/session handle, then stop. Wait for \`pendingCoordinatorEvents\` or another completion/approval/status signal, an explicit user status request, or a real timeout/stall signal before reading status/chat/queue again. Use at most one compact \`mesh_read_chat\` check after a terminal signal. Handle approvals via \`mesh_approve\`.
|
|
353
|
+
4. **Monitor** — Prefer event-driven completion/status notifications. Do **not** poll \`mesh_read_chat\` repeatedly. Do **not** repeatedly call \`mesh_status\` or \`mesh_view_queue\` just to wait for assigned/generating work. After dispatching a direct or queued task, send one progress update with the task/session handle, then stop. Wait for \`pendingCoordinatorEvents\` or another completion/approval/status signal, an explicit user status request, or a real timeout/stall signal before reading status/chat/queue again. Use at most one compact \`mesh_read_chat\` check after a terminal signal. Handle approvals via \`mesh_approve\`. **Proactively parallelize new work.** When the user reports a new bug or asks for new work, start it immediately if it is independent of in-flight tasks and there is headroom under \`maxParallelTasks\` — do not wait for a current task to finish or for the user to prompt you to parallelize. Read-only diagnosis (\`live_debug_readonly\`) has no isolation or merge cost, so dispatch it in parallel right away. The no-polling / concurrency-limit rules constrain *re-checking or duplicating already-dispatched work*; they are **not** a reason to defer starting a new, independent task.
|
|
354
354
|
5. **Verify** — When a task reports completion or git work is visible, call \`mesh_git_status\` to verify changes were made.
|
|
355
355
|
6. **Checkpoint** — Call \`mesh_checkpoint\` to save the work.
|
|
356
356
|
7. **Converge branches** — Before marking any task complete, classify every touched node/branch into exactly one final state: \`merged_to_main\`, \`pushed_feature_branch_needs_merge\`, \`blocked_review\`, \`cleanup_candidate\`, or \`not_mergeable\`. Use \`mesh_status\` branchConvergenceSummary. For obvious clean branch catch-up (ahead 0, behind > 0, upstream fresh, no dirty/stash/submodule issues), use \`mesh_fast_forward_node\` dry-run first and execute only when explicitly safe/approved; this avoids consuming an agent session. Use \`mesh_refine_node\` for clean worktree branches when safe. Before/refine merging root commits that contain submodule gitlink changes, require each submodule commit to be reachable from the configured submodule remote main branch, not merely present on a feature ref or local checkout. If \`mesh_refine_node\` returns \`submodule_reachability_failed\` or publish-required evidence, keep the public convergence bucket as \`blocked_review\`; unless \`allowAutoPublishSubmoduleMainCommits\` is explicitly enabled and Refinery reports successful non-force publish plus post-publish verification, ask the user for explicit approval to push/publish the unreachable submodule commit(s) to submodule main, then rerun \`mesh_refine_node\`. Do not merge the root branch until the submodule commit(s) are reachable from submodule origin/main. A task that remains on a non-main branch is not fully complete unless the final report names the follow-up state and next step.
|
|
@@ -382,7 +382,7 @@ function buildRulesSection(coordinatorCliType?: string): string {
|
|
|
382
382
|
- **Reuse idle sessions.** For follow-up, retry, commit/push, or cleanup on the same issue, send only the delta to the existing idle session. Start fresh only for independent work, provider mismatch, transcript contamination, or required worktree isolation.
|
|
383
383
|
- **Respect explicit provider requests.** Map: Hermes → \`hermes-cli\`, Claude/Claude Code → \`claude-cli\`, Codex → \`codex-cli\`, Gemini → \`gemini-cli\`, Antigravity → \`antigravity-cli\`. Never substitute the coordinator's own runtime.
|
|
384
384
|
- **Verify via git, not source.** Use \`mesh_git_status\` to confirm side effects. Treat agent summaries as self-reports, not verification.
|
|
385
|
-
- **Limit parallelism.** Start with 1–2 tasks; scale only on success. Never duplicate a session because \`mesh_read_chat\` shows no final message while tool/terminal activity is ongoing.
|
|
385
|
+
- **Limit parallelism.** Start with 1–2 tasks; scale only on success. Never duplicate a session because \`mesh_read_chat\` shows no final message while tool/terminal activity is ongoing. This caps *concurrent* load — it does not mean serialize independent work: when a new, independent request arrives and there is headroom under \`maxParallelTasks\`, dispatch it right away rather than waiting for an in-flight task or a user nudge (read-only diagnosis especially, since it has no merge cost).
|
|
386
386
|
- **Check history first.** Call \`mesh_task_history\` at session start to avoid duplicate work and inform recovery. On failure, read task history before retrying.
|
|
387
387
|
- **Converge branches.** After worktree tasks: refine/fast-forward, or classify as \`pushed_feature_branch_needs_merge\` / \`blocked_review\` / \`cleanup_candidate\` / \`not_mergeable\`. Clean up with \`mesh_remove_node\`.
|
|
388
388
|
- **Refinery is config-driven.** \`mesh_refine_node\` must run validation from \`.adhdev/refine.{json,yaml,yml}\` or \`repo-mesh.refine.*\`. Heuristics are scaffolding only.
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { MeshLedgerEntry } from './mesh-ledger.js';
|
|
2
2
|
import type { MeshWorkQueueEntry, DirectDispatchRecord } from './mesh-work-queue.js';
|
|
3
|
+
import { meshNodeIdMatches } from '@adhdev/mesh-shared';
|
|
3
4
|
|
|
4
5
|
export type MeshActiveWorkSource = 'queue' | 'direct';
|
|
5
6
|
export type MeshActiveWorkStatus = 'pending' | 'assigned' | 'generating' | 'idle' | 'failed' | 'awaiting_approval';
|
|
@@ -102,7 +103,7 @@ function elapsedSince(value: string | undefined, now: number): number {
|
|
|
102
103
|
function sessionStatusFromNodes(nodes: any[] | undefined, nodeId?: string, sessionId?: string): { status?: MeshActiveWorkStatus; staleReason?: string } {
|
|
103
104
|
if (!Array.isArray(nodes)) return {};
|
|
104
105
|
if (!nodeId) return { staleReason: 'direct task has no node id' };
|
|
105
|
-
const node = nodes.find(item =>
|
|
106
|
+
const node = nodes.find(item => meshNodeIdMatches(item, nodeId));
|
|
106
107
|
if (!node) return { staleReason: 'direct task node is no longer in the live mesh' };
|
|
107
108
|
if (!sessionId) return {};
|
|
108
109
|
const candidates: any[] = [];
|
|
@@ -13,7 +13,9 @@ import { MeshRuntimeStore } from './mesh-runtime-store.js';
|
|
|
13
13
|
import { queuePendingMeshCoordinatorEvent, drainPendingMeshCoordinatorEvents } from './mesh-events-pending.js';
|
|
14
14
|
import type { PendingMeshCoordinatorEvent } from './mesh-events-pending.js';
|
|
15
15
|
import { resolveWorkerDelegateRouting, recordUnroutableDelegateEvent, isUnroutableDelegateRejection } from './mesh-routing.js';
|
|
16
|
+
import { enqueueUnresolvedDelegateForward, peekUnresolvedDelegateForwards, ackUnresolvedDelegateForward } from './mesh-unresolved-forward-outbox.js';
|
|
16
17
|
import { resolveDelegatedWorkerAutoApprove } from '../repo-mesh-types.js';
|
|
18
|
+
import { normalizeMeshNodeId, meshNodeIdMatches } from '@adhdev/mesh-shared';
|
|
17
19
|
import {
|
|
18
20
|
findRecentTerminalLedgerEvidence,
|
|
19
21
|
hasDispatchAfterTerminal,
|
|
@@ -610,9 +612,11 @@ async function resolveUsableProvider(
|
|
|
610
612
|
// target-routed task permanently pending with a misleading
|
|
611
613
|
// `no_node_satisfies_required_tags` skip.
|
|
612
614
|
function readMeshNodeId(node: any): string {
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
615
|
+
// Delegate to the shared 3-way (id / nodeId / node_id) normalizer so this
|
|
616
|
+
// and every other mesh node-id read agree on identity. Coalesce to '' to
|
|
617
|
+
// preserve the existing string return contract for callers that do
|
|
618
|
+
// `=== task.targetNodeId` / `if (!nodeId)`.
|
|
619
|
+
return normalizeMeshNodeId(node) ?? '';
|
|
616
620
|
}
|
|
617
621
|
|
|
618
622
|
async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, meshId: string, mesh: any): Promise<boolean> {
|
|
@@ -895,7 +899,7 @@ async function maybeAutoFastForwardIdleNode(components: DaemonComponents, args:
|
|
|
895
899
|
providerType?: string;
|
|
896
900
|
}): Promise<void> {
|
|
897
901
|
const mesh = getMeshWithCache(components, args.meshId);
|
|
898
|
-
const node = mesh?.nodes?.find((candidate: any) => candidate
|
|
902
|
+
const node = mesh?.nodes?.find((candidate: any) => meshNodeIdMatches(candidate, args.nodeId));
|
|
899
903
|
const workspace = readNonEmptyString(node?.workspace);
|
|
900
904
|
if (!workspace) return;
|
|
901
905
|
if (!existsSync(workspace)) return;
|
|
@@ -1536,8 +1540,16 @@ export function handleMeshForwardEvent(components: DaemonComponents, payload: Re
|
|
|
1536
1540
|
// - It fires only when the normal queue path did NOT run (isDelegate=false), so the
|
|
1537
1541
|
// event is never both queued locally and forwarded.
|
|
1538
1542
|
//
|
|
1539
|
-
// Returns true when the event was
|
|
1540
|
-
// skips the delivery_unroutable diagnostic); false when no
|
|
1543
|
+
// Returns true when the event was durably accepted for delivery to the coordinator
|
|
1544
|
+
// daemon (so the caller skips the delivery_unroutable diagnostic); false when no
|
|
1545
|
+
// fallback was possible (no coordinator anchor / no dispatch transport).
|
|
1546
|
+
//
|
|
1547
|
+
// Durability: the directed push to the coordinator is the ONLY delivery route for an
|
|
1548
|
+
// unresolved-mesh worker (it is in no mesh.node the coordinator can pull). So instead
|
|
1549
|
+
// of a fire-and-forget push that drops on one transient P2P failure, the event is
|
|
1550
|
+
// persisted to the worker-side outbox FIRST and only acked after a successful push.
|
|
1551
|
+
// A best-effort immediate push keeps latency low on the happy path; a failed or
|
|
1552
|
+
// un-acked push leaves the durable row for setupMeshReconcileLoop's PHASE 0 to retry.
|
|
1541
1553
|
function forwardUnresolvedDelegateEvent(
|
|
1542
1554
|
components: DaemonComponents,
|
|
1543
1555
|
routing: ReturnType<typeof resolveWorkerDelegateRouting>,
|
|
@@ -1561,16 +1573,52 @@ function forwardUnresolvedDelegateEvent(
|
|
|
1561
1573
|
workspace: readNonEmptyString(routing.workspace) || readNonEmptyString(event.workspace) || undefined,
|
|
1562
1574
|
};
|
|
1563
1575
|
|
|
1576
|
+
// 1) Persist durably FIRST. Idempotent on fingerprint, so a re-fired completion
|
|
1577
|
+
// does not duplicate the outbox row. If persistence fails we still attempt the
|
|
1578
|
+
// push below (degrades to the old at-most-once behaviour rather than dropping
|
|
1579
|
+
// the chance entirely).
|
|
1580
|
+
const persisted = enqueueUnresolvedDelegateForward(coordinatorDaemonId, eventName, payload);
|
|
1581
|
+
|
|
1582
|
+
// 2) Best-effort immediate push for low latency. On success, ack the outbox row so
|
|
1583
|
+
// the retry loop won't re-send it. On failure, leave it queued — PHASE 0 retries.
|
|
1564
1584
|
Promise.resolve(components.dispatchMeshCommand(coordinatorDaemonId, 'mesh_forward_event', payload))
|
|
1585
|
+
.then((result: any) => {
|
|
1586
|
+
if (result && result.success === false) {
|
|
1587
|
+
LOG.warn('MeshEvents', `Immediate forward of ${eventName} to coordinator ${coordinatorDaemonId} rejected (${readNonEmptyString(result.error) || 'no reason'}) — left queued for retry`);
|
|
1588
|
+
return;
|
|
1589
|
+
}
|
|
1590
|
+
// Acked. Mark the durable copy delivered so the retry loop skips it.
|
|
1591
|
+
if (persisted) ackUnresolvedDelegateForwardByFingerprint(coordinatorDaemonId, eventName, payload);
|
|
1592
|
+
})
|
|
1565
1593
|
.catch((e: any) => {
|
|
1566
|
-
//
|
|
1567
|
-
//
|
|
1568
|
-
LOG.warn('MeshEvents', `
|
|
1594
|
+
// Coordinator momentarily unreachable; the durable row stays queued and the
|
|
1595
|
+
// reconcile loop retries it. Trace so the relay attempt is visible.
|
|
1596
|
+
LOG.warn('MeshEvents', `Immediate forward of ${eventName} to coordinator ${coordinatorDaemonId} failed: ${e?.message || e} — left queued for retry`);
|
|
1569
1597
|
});
|
|
1570
|
-
LOG.info('MeshEvents', `
|
|
1598
|
+
LOG.info('MeshEvents', `Durably forwarded ${eventName} for unresolved-mesh worker at ${routing.workspace || '(no workspace)'} to coordinator daemon ${coordinatorDaemonId}`);
|
|
1571
1599
|
return true;
|
|
1572
1600
|
}
|
|
1573
1601
|
|
|
1602
|
+
// Ack a just-pushed outbox entry by re-deriving its row from the same coordinator +
|
|
1603
|
+
// event + payload. We don't thread the row id back from enqueue (the immediate push is
|
|
1604
|
+
// fire-then-ack), so locate it among the undrained entries by matching coordinator and
|
|
1605
|
+
// the flat payload's forward identity. A miss is harmless — the retry loop's own
|
|
1606
|
+
// receiver-side dedup suppresses a duplicate delivery.
|
|
1607
|
+
function ackUnresolvedDelegateForwardByFingerprint(
|
|
1608
|
+
coordinatorDaemonId: string,
|
|
1609
|
+
eventName: string,
|
|
1610
|
+
payload: Record<string, unknown>,
|
|
1611
|
+
): void {
|
|
1612
|
+
const match = peekUnresolvedDelegateForwards().find(entry =>
|
|
1613
|
+
entry.coordinatorDaemonId === coordinatorDaemonId
|
|
1614
|
+
&& readNonEmptyString(entry.payload.event) === eventName
|
|
1615
|
+
&& readNonEmptyString(entry.payload.targetSessionId || entry.payload.sessionId || entry.payload.instanceId)
|
|
1616
|
+
=== readNonEmptyString(payload.targetSessionId || payload.sessionId || payload.instanceId)
|
|
1617
|
+
&& readNonEmptyString(entry.payload.workspace) === readNonEmptyString(payload.workspace),
|
|
1618
|
+
);
|
|
1619
|
+
if (match) ackUnresolvedDelegateForward(match.id);
|
|
1620
|
+
}
|
|
1621
|
+
|
|
1574
1622
|
export function setupMeshEventForwarding(components: DaemonComponents) {
|
|
1575
1623
|
components.instanceManager.onEvent((event) => {
|
|
1576
1624
|
// --- Coordinator idle auto-flush (fast path) ---
|
|
@@ -47,6 +47,25 @@ export interface MeshMissionSummary extends MeshMissionRecord {
|
|
|
47
47
|
tasks: MeshMissionTaskAggregate;
|
|
48
48
|
}
|
|
49
49
|
|
|
50
|
+
/**
|
|
51
|
+
* Slim mission summary for the mesh_status compact (default) surface. Drops the
|
|
52
|
+
* full `goal` text — which can be hundreds of chars per mission and is repeated
|
|
53
|
+
* for every mission on every status call — keeping only a short preview plus a
|
|
54
|
+
* `goalTruncated` flag when the original was longer. The stored goal is never
|
|
55
|
+
* mutated; this is an output-only projection. Coordinators that need the full
|
|
56
|
+
* goal call mesh_status with verbose=true, or read the mission directly via
|
|
57
|
+
* mesh_mission_upsert / getMeshMission.
|
|
58
|
+
*/
|
|
59
|
+
export interface MeshMissionSlimSummary extends Omit<MeshMissionSummary, 'goal'> {
|
|
60
|
+
/** Short preview of the goal (≤ GOAL_PREVIEW_MAX chars), '' when goal empty. */
|
|
61
|
+
goalPreview: string;
|
|
62
|
+
/** True when the stored goal was longer than the preview (full text elided). */
|
|
63
|
+
goalTruncated: boolean;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/** Max chars of goal text retained in the slim (compact) mission summary. */
|
|
67
|
+
export const GOAL_PREVIEW_MAX = 120;
|
|
68
|
+
|
|
50
69
|
function normalizeMissionStatus(value: unknown): MeshMissionStatus {
|
|
51
70
|
return MESH_MISSION_STATUSES.includes(value as MeshMissionStatus)
|
|
52
71
|
? value as MeshMissionStatus
|
|
@@ -125,17 +144,35 @@ export function getActiveMeshMissionSummaries(meshId: string): MeshMissionSummar
|
|
|
125
144
|
return getMeshMissions(meshId, ['active']).map(mission => summarizeMeshMission(meshId, mission));
|
|
126
145
|
}
|
|
127
146
|
|
|
147
|
+
/** Project a full mission summary down to the slim (goal-elided) shape. */
|
|
148
|
+
function slimMissionSummary(summary: MeshMissionSummary): MeshMissionSlimSummary {
|
|
149
|
+
const goal = typeof summary.goal === 'string' ? summary.goal : '';
|
|
150
|
+
const goalTruncated = goal.length > GOAL_PREVIEW_MAX;
|
|
151
|
+
const { goal: _omitGoal, ...rest } = summary;
|
|
152
|
+
return {
|
|
153
|
+
...rest,
|
|
154
|
+
goalPreview: goalTruncated ? goal.slice(0, GOAL_PREVIEW_MAX) : goal,
|
|
155
|
+
goalTruncated,
|
|
156
|
+
};
|
|
157
|
+
}
|
|
158
|
+
|
|
128
159
|
/**
|
|
129
160
|
* Mission summaries for the mesh_status dashboard surface: every active/paused
|
|
130
161
|
* mission plus a capped, newest-first slice of completed/abandoned history so
|
|
131
162
|
* the dashboard can render a collapsible "history" section without unbounded
|
|
132
163
|
* payload growth. Returned newest-first within each group (active/paused first,
|
|
133
164
|
* then history), so the frontend can split on `status` directly.
|
|
165
|
+
*
|
|
166
|
+
* Compact mode (the default) elides each mission's full `goal` text — which is
|
|
167
|
+
* repeated verbatim on every status poll and dominates the payload when a mesh
|
|
168
|
+
* has many missions — returning only a short `goalPreview` + `goalTruncated`
|
|
169
|
+
* flag. Pass `verbose: true` to get the full `goal` text per mission. The stored
|
|
170
|
+
* goal is untouched in both modes; this is an output-only projection.
|
|
134
171
|
*/
|
|
135
172
|
export function getMeshStatusMissionSummaries(
|
|
136
173
|
meshId: string,
|
|
137
|
-
options?: { historyLimit?: number },
|
|
138
|
-
): MeshMissionSummary[] {
|
|
174
|
+
options?: { historyLimit?: number; verbose?: boolean },
|
|
175
|
+
): MeshMissionSummary[] | MeshMissionSlimSummary[] {
|
|
139
176
|
const historyLimit = Math.max(0, options?.historyLimit ?? 10);
|
|
140
177
|
const all = getMeshMissions(meshId);
|
|
141
178
|
const live = all.filter(m => m.status === 'active' || m.status === 'paused');
|
|
@@ -143,7 +180,8 @@ export function getMeshStatusMissionSummaries(
|
|
|
143
180
|
.filter(m => m.status === 'completed' || m.status === 'abandoned')
|
|
144
181
|
.sort((a, b) => (b.updatedAt || '').localeCompare(a.updatedAt || ''))
|
|
145
182
|
.slice(0, historyLimit);
|
|
146
|
-
|
|
183
|
+
const full = [...live, ...history].map(mission => summarizeMeshMission(meshId, mission));
|
|
184
|
+
return options?.verbose ? full : full.map(slimMissionSummary);
|
|
147
185
|
}
|
|
148
186
|
|
|
149
187
|
/**
|
|
@@ -49,6 +49,11 @@ import { drainPendingMeshCoordinatorEvents } from './mesh-events-pending.js';
|
|
|
49
49
|
import type { PendingMeshCoordinatorEvent } from './mesh-events-pending.js';
|
|
50
50
|
import { MeshRuntimeStore } from './mesh-runtime-store.js';
|
|
51
51
|
import { handleMeshForwardEvent, shouldForceInjectMeshEvent, MESH_FORCE_INJECT_EVENTS } from './mesh-events-coordinator.js';
|
|
52
|
+
import {
|
|
53
|
+
peekUnresolvedDelegateForwards,
|
|
54
|
+
ackUnresolvedDelegateForward,
|
|
55
|
+
expireStaleUnresolvedDelegateForwards,
|
|
56
|
+
} from './mesh-unresolved-forward-outbox.js';
|
|
52
57
|
import { readNonEmptyString } from './mesh-events-utils.js';
|
|
53
58
|
|
|
54
59
|
// Default reconcile cadence. approval/completion notifications to a live CLI
|
|
@@ -213,6 +218,20 @@ export async function runMeshReconcileTick(components: DaemonComponents): Promis
|
|
|
213
218
|
try { return MeshRuntimeStore.getInstance(); } catch { return undefined; }
|
|
214
219
|
})();
|
|
215
220
|
|
|
221
|
+
// ── PHASE 0: retry the worker-side unresolved-delegate forward outbox ──────
|
|
222
|
+
// Cloud-only (needs dispatchMeshCommand). A worker that is NOT a member of the
|
|
223
|
+
// coordinator's mesh cannot be reached by the coordinator's PHASE 1 pull (it is
|
|
224
|
+
// in no mesh.node), so its completion must be PUSHED to the coordinator. This
|
|
225
|
+
// drains the durable outbox enqueued by forwardUnresolvedDelegateEvent and retries
|
|
226
|
+
// any push that has not yet been acked. See mesh-unresolved-forward-outbox.ts.
|
|
227
|
+
if (dispatchMeshCommand) {
|
|
228
|
+
try {
|
|
229
|
+
await retryUnresolvedDelegateForwards(components);
|
|
230
|
+
} catch (e: any) {
|
|
231
|
+
LOG.warn('MeshReconcile', `Unresolved-delegate forward retry failed: ${e?.message || e}`);
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
|
|
216
235
|
// ── PHASE 1: pull remote node queues for every mesh this daemon hosts ──────
|
|
217
236
|
// Cloud-only (dispatchMeshCommand present). Runs whether or not a live CLI
|
|
218
237
|
// coordinator exists — this is what lets an MCP/LLM coordinator ever see a
|
|
@@ -295,6 +314,42 @@ export async function runMeshReconcileTick(components: DaemonComponents): Promis
|
|
|
295
314
|
}
|
|
296
315
|
}
|
|
297
316
|
|
|
317
|
+
// Cloud-only: retry the worker-side unresolved-delegate forward outbox. For each
|
|
318
|
+
// durably-queued entry, push it to its coordinator daemon over P2P (mesh_forward_event)
|
|
319
|
+
// and ack (mark drained) ONLY on a successful, non-rejected response. A failed or
|
|
320
|
+
// rejected push leaves the entry queued for the next tick — at-least-once delivery.
|
|
321
|
+
// Stale entries (coordinator unreachable past the max age) are expired first so the
|
|
322
|
+
// outbox can't grow without bound. The coordinator dedups duplicate deliveries on its
|
|
323
|
+
// own fingerprint, so a retry that races the original immediate push is harmless.
|
|
324
|
+
async function retryUnresolvedDelegateForwards(components: DaemonComponents): Promise<void> {
|
|
325
|
+
const dispatchMeshCommand = components.dispatchMeshCommand;
|
|
326
|
+
if (!dispatchMeshCommand) return;
|
|
327
|
+
|
|
328
|
+
// Drop entries that have exhausted their retry budget (fail-loud inside).
|
|
329
|
+
expireStaleUnresolvedDelegateForwards();
|
|
330
|
+
|
|
331
|
+
const entries = peekUnresolvedDelegateForwards();
|
|
332
|
+
if (entries.length === 0) return;
|
|
333
|
+
|
|
334
|
+
for (const entry of entries) {
|
|
335
|
+
let result: any;
|
|
336
|
+
try {
|
|
337
|
+
result = await dispatchMeshCommand(entry.coordinatorDaemonId, 'mesh_forward_event', entry.payload);
|
|
338
|
+
} catch (e: any) {
|
|
339
|
+
// Coordinator unreachable — keep the entry queued and try again next tick.
|
|
340
|
+
LOG.warn('MeshReconcile', `Retry forward to coordinator ${entry.coordinatorDaemonId} failed: ${e?.message || e} — left queued`);
|
|
341
|
+
continue;
|
|
342
|
+
}
|
|
343
|
+
if (result && result.success === false) {
|
|
344
|
+
LOG.warn('MeshReconcile', `Retry forward to coordinator ${entry.coordinatorDaemonId} rejected (${readNonEmptyString(result.error) || 'no reason'}) — left queued`);
|
|
345
|
+
continue;
|
|
346
|
+
}
|
|
347
|
+
// Acked — mark the durable copy delivered.
|
|
348
|
+
ackUnresolvedDelegateForward(entry.id);
|
|
349
|
+
LOG.info('MeshReconcile', `Retried+delivered unresolved-delegate ${readNonEmptyString(entry.payload.event)} to coordinator ${entry.coordinatorDaemonId}`);
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
|
|
298
353
|
// Cloud-only: poll each remote worker node daemon for pending coordinator events
|
|
299
354
|
// and re-inject them locally via handleMeshForwardEvent (which re-queues +
|
|
300
355
|
// surfaces to the live coordinator on the next tick / immediately if idle).
|
|
@@ -1396,4 +1396,34 @@ export class MeshRuntimeStore {
|
|
|
1396
1396
|
).get(meshId) as { cnt: number } | undefined;
|
|
1397
1397
|
return row?.cnt ?? 0;
|
|
1398
1398
|
}
|
|
1399
|
+
|
|
1400
|
+
/**
|
|
1401
|
+
* Mark specific pending-event rows drained by id (ack). Used by the
|
|
1402
|
+
* unresolved-delegate durable-forward outbox: an event is peeked (not drained)
|
|
1403
|
+
* while its push to the coordinator is unconfirmed, then marked drained ONLY
|
|
1404
|
+
* after the push is acked. A failed push leaves the row undrained so the next
|
|
1405
|
+
* reconcile tick retries it. Returns the number of rows newly marked drained.
|
|
1406
|
+
*/
|
|
1407
|
+
markPendingEventsDrainedById(ids: ReadonlyArray<string>): number {
|
|
1408
|
+
const idList = ids.filter((id): id is string => typeof id === 'string' && id.length > 0);
|
|
1409
|
+
if (idList.length === 0) return 0;
|
|
1410
|
+
const now = Date.now();
|
|
1411
|
+
return this.db.prepare(
|
|
1412
|
+
`UPDATE mesh_pending_events SET drained = 1, drained_at = ? WHERE drained = 0 AND id IN (${idList.map(() => '?').join(',')})`
|
|
1413
|
+
).run(now, ...idList).changes;
|
|
1414
|
+
}
|
|
1415
|
+
|
|
1416
|
+
/**
|
|
1417
|
+
* Hard-delete pending-event rows by id (including the dedup fingerprint history).
|
|
1418
|
+
* Used to expire an unresolved-delegate outbox entry that has exhausted its retry
|
|
1419
|
+
* budget — fully removing it frees the fingerprint so a genuinely new completion
|
|
1420
|
+
* for the same task could be re-queued later. Returns the number of rows deleted.
|
|
1421
|
+
*/
|
|
1422
|
+
deletePendingEventsById(ids: ReadonlyArray<string>): number {
|
|
1423
|
+
const idList = ids.filter((id): id is string => typeof id === 'string' && id.length > 0);
|
|
1424
|
+
if (idList.length === 0) return 0;
|
|
1425
|
+
return this.db.prepare(
|
|
1426
|
+
`DELETE FROM mesh_pending_events WHERE id IN (${idList.map(() => '?').join(',')})`
|
|
1427
|
+
).run(...idList).changes;
|
|
1428
|
+
}
|
|
1399
1429
|
}
|
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
// ---------------------------------------------------------------------------
|
|
2
|
+
// mesh-unresolved-forward-outbox — durable retry for unresolved-delegate forwards
|
|
3
|
+
// ---------------------------------------------------------------------------
|
|
4
|
+
// A REMOTE worker daemon that is P2P-remote-controlled by a coordinator is NOT a
|
|
5
|
+
// member of the coordinator's mesh — it has no local mesh record. So when its
|
|
6
|
+
// completion event reaches setupMeshEventForwarding, resolveWorkerDelegateRouting
|
|
7
|
+
// returns mesh_unresolved: the worker carries the coordinator daemon anchor but no
|
|
8
|
+
// resolvable mesh id, so the normal queue-then-coordinator-pull path can't be used
|
|
9
|
+
// (the coordinator's reconcile PHASE 1 only pulls mesh.nodes, and this worker is in
|
|
10
|
+
// none of them — see docs/refactoring/2026-06-16-mesh-completion-polling-single-model.md).
|
|
11
|
+
//
|
|
12
|
+
// The ONLY delivery route for this case is a directed push to the coordinator
|
|
13
|
+
// daemon (mesh_forward_event over P2P). Previously that push was fire-and-forget:
|
|
14
|
+
// a single dispatchMeshCommand whose rejection was only logged, so one transient
|
|
15
|
+
// P2P failure dropped the completion forever (delivery_unroutable).
|
|
16
|
+
//
|
|
17
|
+
// This outbox makes that push DURABLE without inventing a new persistence layer:
|
|
18
|
+
// it reuses the existing mesh_pending_events SQLite table (the same store every
|
|
19
|
+
// other mesh coordinator event is persisted to), under a synthetic, reserved
|
|
20
|
+
// "mesh id" namespace so it never collides with a real mesh's queue. The worker's
|
|
21
|
+
// reconcile tick (PHASE 0) peeks the outbox, pushes each entry to its coordinator,
|
|
22
|
+
// and marks it drained (acked) ONLY on a successful push — a failed push leaves the
|
|
23
|
+
// row undrained for the next tick. Entries that exceed a max age are expired so the
|
|
24
|
+
// outbox can't grow unbounded when a coordinator stays permanently unreachable.
|
|
25
|
+
//
|
|
26
|
+
// Idempotency:
|
|
27
|
+
// - Enqueue is idempotent on the event fingerprint (mesh_pending_events has a
|
|
28
|
+
// UNIQUE (mesh_id, fingerprint) index + INSERT OR IGNORE), so the same
|
|
29
|
+
// completion fired twice on the worker queues once.
|
|
30
|
+
// - The receiver dedups independently: handleMeshForwardEvent → injectMeshSystemMessage
|
|
31
|
+
// → queuePendingMeshCoordinatorEvent recomputes the fingerprint and suppresses a
|
|
32
|
+
// duplicate, so an at-least-once retry that double-delivers is harmless.
|
|
33
|
+
// ---------------------------------------------------------------------------
|
|
34
|
+
|
|
35
|
+
import { randomUUID } from 'crypto';
|
|
36
|
+
import { LOG } from '../logging/logger.js';
|
|
37
|
+
import { MeshRuntimeStore } from './mesh-runtime-store.js';
|
|
38
|
+
import { buildPendingEventFingerprint, type PendingMeshCoordinatorEvent } from './mesh-events-pending.js';
|
|
39
|
+
import { readNonEmptyString } from './mesh-events-utils.js';
|
|
40
|
+
|
|
41
|
+
// Reserved synthetic mesh id for the worker-side unresolved-forward outbox. The `::`
|
|
42
|
+
// and leading `__` make it impossible to collide with a real mesh id (which are
|
|
43
|
+
// `mesh_*` / `mreg_*` slugs). Scoping rows by coordinator_daemon_id inside this one
|
|
44
|
+
// namespace lets a single worker forward to multiple coordinators.
|
|
45
|
+
export const UNRESOLVED_FORWARD_OUTBOX_MESH_ID = '__unresolved_forward_outbox__';
|
|
46
|
+
|
|
47
|
+
// Entries older than this are expired (dropped) rather than retried forever — a
|
|
48
|
+
// coordinator that has been unreachable this long is treated as gone. Generous
|
|
49
|
+
// enough to ride out a long coordinator outage / restart, bounded enough that a
|
|
50
|
+
// permanently-dead coordinator can't accumulate outbox rows indefinitely.
|
|
51
|
+
const UNRESOLVED_FORWARD_MAX_AGE_MS = 30 * 60 * 1000; // 30 minutes
|
|
52
|
+
|
|
53
|
+
export interface UnresolvedForwardEntry {
|
|
54
|
+
/** Row id in mesh_pending_events; pass back to ack/expire after a push attempt. */
|
|
55
|
+
id: string;
|
|
56
|
+
/** The coordinator daemon to push this event to (mesh_forward_event target). */
|
|
57
|
+
coordinatorDaemonId: string;
|
|
58
|
+
/** The flat payload to forward (already shaped for handleMeshForwardEvent). */
|
|
59
|
+
payload: Record<string, unknown>;
|
|
60
|
+
/** When the entry was first enqueued (epoch ms) — used for age-based expiry. */
|
|
61
|
+
queuedAt: number;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function getStore(): MeshRuntimeStore | undefined {
|
|
65
|
+
try { return MeshRuntimeStore.getInstance(); } catch { return undefined; }
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Durably enqueue an unresolved-delegate forward for a coordinator daemon. The
|
|
70
|
+
* `forwardPayload` is the flat shape handleMeshForwardEvent reads on the coordinator.
|
|
71
|
+
* Idempotent on the event fingerprint. Returns true when a new row was written (or a
|
|
72
|
+
* duplicate was harmlessly ignored), false on a hard persistence failure.
|
|
73
|
+
*/
|
|
74
|
+
export function enqueueUnresolvedDelegateForward(
|
|
75
|
+
coordinatorDaemonId: string,
|
|
76
|
+
eventName: string,
|
|
77
|
+
forwardPayload: Record<string, unknown>,
|
|
78
|
+
): boolean {
|
|
79
|
+
const target = readNonEmptyString(coordinatorDaemonId);
|
|
80
|
+
const event = readNonEmptyString(eventName);
|
|
81
|
+
if (!target || !event) return false;
|
|
82
|
+
const store = getStore();
|
|
83
|
+
if (!store) return false;
|
|
84
|
+
|
|
85
|
+
const queuedAt = Date.now();
|
|
86
|
+
// Reuse the standard pending-event fingerprint so enqueue is idempotent AND the
|
|
87
|
+
// receiver's own dedup keys on a comparable identity. We synthesise the minimal
|
|
88
|
+
// PendingMeshCoordinatorEvent shape buildPendingEventFingerprint needs.
|
|
89
|
+
const fingerprintSource: PendingMeshCoordinatorEvent = {
|
|
90
|
+
event,
|
|
91
|
+
meshId: UNRESOLVED_FORWARD_OUTBOX_MESH_ID,
|
|
92
|
+
nodeLabel: readNonEmptyString(forwardPayload.nodeId) || readNonEmptyString(forwardPayload.workspace) || 'unresolved-delegate',
|
|
93
|
+
nodeId: readNonEmptyString(forwardPayload.nodeId) || undefined,
|
|
94
|
+
workspace: readNonEmptyString(forwardPayload.workspace) || undefined,
|
|
95
|
+
metadataEvent: forwardPayload,
|
|
96
|
+
queuedAt,
|
|
97
|
+
targetCoordinatorDaemonId: target,
|
|
98
|
+
};
|
|
99
|
+
// Scope the fingerprint to the coordinator so two coordinators awaiting the same
|
|
100
|
+
// worker session don't collapse into one outbox row.
|
|
101
|
+
const fingerprint = `${target}::${buildPendingEventFingerprint(fingerprintSource)}`;
|
|
102
|
+
|
|
103
|
+
try {
|
|
104
|
+
const inserted = store.insertPendingEvent({
|
|
105
|
+
id: randomUUID(),
|
|
106
|
+
meshId: UNRESOLVED_FORWARD_OUTBOX_MESH_ID,
|
|
107
|
+
coordinatorDaemonId: target,
|
|
108
|
+
event,
|
|
109
|
+
// Store the flat forward payload + the queue timestamp so the retry tick can
|
|
110
|
+
// rebuild the push args and apply age-based expiry without a schema change.
|
|
111
|
+
payload: { forwardPayload, coordinatorDaemonId: target, queuedAt },
|
|
112
|
+
fingerprint,
|
|
113
|
+
queuedAt,
|
|
114
|
+
});
|
|
115
|
+
if (inserted) {
|
|
116
|
+
LOG.info('MeshEvents', `Durably queued unresolved-delegate ${event} for coordinator ${target} (outbox)`);
|
|
117
|
+
}
|
|
118
|
+
return true;
|
|
119
|
+
} catch (e: any) {
|
|
120
|
+
LOG.warn('MeshEvents', `Failed to persist unresolved-delegate forward to outbox: ${e?.message || e}`);
|
|
121
|
+
return false;
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/** Peek (non-destructive) every undrained outbox entry across all coordinators. */
|
|
126
|
+
export function peekUnresolvedDelegateForwards(): UnresolvedForwardEntry[] {
|
|
127
|
+
const store = getStore();
|
|
128
|
+
if (!store) return [];
|
|
129
|
+
let rows: Array<{ id: string; event: string; payload: unknown }>;
|
|
130
|
+
try {
|
|
131
|
+
rows = store.peekPendingEvents(UNRESOLVED_FORWARD_OUTBOX_MESH_ID);
|
|
132
|
+
} catch {
|
|
133
|
+
return [];
|
|
134
|
+
}
|
|
135
|
+
const out: UnresolvedForwardEntry[] = [];
|
|
136
|
+
for (const row of rows) {
|
|
137
|
+
const stored = row.payload && typeof row.payload === 'object' ? row.payload as Record<string, unknown> : {};
|
|
138
|
+
const coordinatorDaemonId = readNonEmptyString(stored.coordinatorDaemonId);
|
|
139
|
+
const forwardPayload = stored.forwardPayload && typeof stored.forwardPayload === 'object'
|
|
140
|
+
? stored.forwardPayload as Record<string, unknown>
|
|
141
|
+
: undefined;
|
|
142
|
+
if (!coordinatorDaemonId || !forwardPayload) continue;
|
|
143
|
+
const queuedAt = typeof stored.queuedAt === 'number' ? stored.queuedAt : 0;
|
|
144
|
+
out.push({ id: row.id, coordinatorDaemonId, payload: forwardPayload, queuedAt });
|
|
145
|
+
}
|
|
146
|
+
return out;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/** Mark an outbox entry delivered (acked) after a successful push. */
|
|
150
|
+
export function ackUnresolvedDelegateForward(id: string): void {
|
|
151
|
+
const store = getStore();
|
|
152
|
+
if (!store) return;
|
|
153
|
+
try { store.markPendingEventsDrainedById([id]); } catch { /* best-effort */ }
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
/**
|
|
157
|
+
* Drop outbox entries that have exceeded the max retry age. Returns the count
|
|
158
|
+
* expired so the caller can log a fail-loud trace (a dropped completion is a real
|
|
159
|
+
* loss; it must be visible, not silent).
|
|
160
|
+
*/
|
|
161
|
+
export function expireStaleUnresolvedDelegateForwards(nowMs: number = Date.now()): number {
|
|
162
|
+
const entries = peekUnresolvedDelegateForwards();
|
|
163
|
+
const staleIds = entries
|
|
164
|
+
.filter(e => e.queuedAt > 0 && nowMs - e.queuedAt >= UNRESOLVED_FORWARD_MAX_AGE_MS)
|
|
165
|
+
.map(e => e.id);
|
|
166
|
+
if (staleIds.length === 0) return 0;
|
|
167
|
+
const store = getStore();
|
|
168
|
+
if (!store) return 0;
|
|
169
|
+
try {
|
|
170
|
+
const removed = store.deletePendingEventsById(staleIds);
|
|
171
|
+
if (removed > 0) {
|
|
172
|
+
LOG.warn('MeshEvents', `Expired ${removed} unresolved-delegate forward(s) after ${Math.round(UNRESOLVED_FORWARD_MAX_AGE_MS / 60000)}m of failed retries — coordinator unreachable, completion dropped`);
|
|
173
|
+
}
|
|
174
|
+
return removed;
|
|
175
|
+
} catch {
|
|
176
|
+
return 0;
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
/** Test helper: purge the entire outbox. */
|
|
181
|
+
export function __clearUnresolvedDelegateForwardOutboxForTests(): void {
|
|
182
|
+
const store = getStore();
|
|
183
|
+
if (!store) return;
|
|
184
|
+
try { store.clearPendingEventsForMesh(UNRESOLVED_FORWARD_OUTBOX_MESH_ID); } catch { /* nothing to clear */ }
|
|
185
|
+
}
|