@adhdev/daemon-core 0.9.82-rc.461 → 0.9.82-rc.463
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/router-aggregate-status.d.ts +24 -0
- package/dist/commands/router-mesh-session-owner.d.ts +59 -0
- package/dist/commands/router.d.ts +7 -47
- package/dist/index.js +388 -222
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +388 -222
- package/dist/index.mjs.map +1 -1
- package/dist/mesh/mesh-events-pending.d.ts +34 -2
- package/dist/mesh/mesh-events.d.ts +2 -2
- package/dist/mesh/mesh-reconcile-loop.d.ts +12 -0
- package/dist/providers/approval-utils.d.ts +1 -0
- package/dist/providers/cli-provider-instance.d.ts +9 -0
- package/dist/providers/spec/types.d.ts +22 -0
- package/dist/repo-mesh-types.d.ts +31 -0
- package/package.json +3 -3
- package/src/commands/high-family/mesh-events.ts +14 -1
- package/src/commands/high-family/mesh-status.ts +19 -2
- package/src/commands/router-aggregate-status.ts +209 -0
- package/src/commands/router-mesh-session-owner.ts +114 -0
- package/src/commands/router.ts +27 -255
- package/src/mesh/mesh-events-pending.ts +99 -5
- package/src/mesh/mesh-events.ts +3 -0
- package/src/mesh/mesh-reconcile-loop.ts +66 -0
- package/src/providers/approval-utils.ts +1 -1
- package/src/providers/cli-provider-instance.ts +41 -12
- package/src/providers/native-history/hermes-cli-transcript.ts +54 -2
- package/src/providers/spec/native-history-executor.ts +73 -6
- package/src/providers/spec/types.ts +22 -0
- package/src/repo-mesh-types.ts +32 -0
package/src/commands/router.ts
CHANGED
|
@@ -22,7 +22,7 @@ import { DaemonCliManager } from './cli-manager.js';
|
|
|
22
22
|
import type { ProviderLoader } from '../providers/provider-loader.js';
|
|
23
23
|
import type { ProviderInstanceManager } from '../providers/provider-instance-manager.js';
|
|
24
24
|
import { killIdeProcess, isIdeRunning } from '../launch.js';
|
|
25
|
-
import { normalizeMeshNodeId, meshNodeIdMatches
|
|
25
|
+
import { normalizeMeshNodeId, meshNodeIdMatches } from '@adhdev/mesh-shared';
|
|
26
26
|
import { SessionRegistry } from '../sessions/registry.js';
|
|
27
27
|
import { LOG } from '../logging/logger.js';
|
|
28
28
|
import { logCommand } from '../logging/command-log.js';
|
|
@@ -42,27 +42,16 @@ import { execFileSync } from 'node:child_process';
|
|
|
42
42
|
|
|
43
43
|
// ─── Extracted-module imports (symbols the dispatch class consumes) ───
|
|
44
44
|
import {
|
|
45
|
-
applyInlineMeshBranchConvergence,
|
|
46
|
-
buildInlineMeshTransitGitStatus,
|
|
47
|
-
buildLivePeerGitConnection,
|
|
48
|
-
collectMeshNodeHostedSessionIds,
|
|
49
|
-
deriveMeshNodeHealthFromGit,
|
|
50
45
|
foldMeshNodeIdentityToCanonical,
|
|
51
46
|
inlineMeshCarriesTransientNodeTruth,
|
|
52
|
-
isDeadLocalWorktreeNode,
|
|
53
47
|
MESH_DIRECT_PROBE_REUSE_MS,
|
|
54
48
|
MeshGitProbeCache,
|
|
55
49
|
normalizeInlineMeshNodeIdentity,
|
|
56
|
-
readBooleanValue,
|
|
57
|
-
readCachedInlineMeshActiveSessions,
|
|
58
50
|
readInlineMeshNodeId,
|
|
59
|
-
readMeshNodeDaemonId,
|
|
60
51
|
readObjectRecord,
|
|
61
52
|
readStringValue,
|
|
62
53
|
reconcileInlineMeshCache,
|
|
63
54
|
sanitizeInlineMesh,
|
|
64
|
-
shouldRefreshStalePendingAggregate,
|
|
65
|
-
summarizeInlineMeshBranchConvergence,
|
|
66
55
|
} from '../mesh/mesh-node-identity.js';
|
|
67
56
|
import {
|
|
68
57
|
alignRefinerySubmodulesAfterMerge,
|
|
@@ -105,6 +94,14 @@ import {
|
|
|
105
94
|
recordIntentionalMeshSessionStop,
|
|
106
95
|
sessionMatchesMeshNode,
|
|
107
96
|
} from './router-worktree-cleanup.js';
|
|
97
|
+
// ─── Aggregate mesh-status cache (bodies extracted from this file) ───
|
|
98
|
+
import {
|
|
99
|
+
getCachedAggregateMeshStatus,
|
|
100
|
+
hydrateCachedAggregateMeshStatusFromInline,
|
|
101
|
+
rememberAggregateMeshStatus,
|
|
102
|
+
} from './router-aggregate-status.js';
|
|
103
|
+
// ─── Remote mesh-session owner resolution (bodies extracted from this file) ───
|
|
104
|
+
import { resolveRemoteMeshSessionOwnerDaemonId } from './router-mesh-session-owner.js';
|
|
108
105
|
|
|
109
106
|
// ─── Barrel re-exports: node-identity / git-freshness, refine gates, coordinator config ───
|
|
110
107
|
// These modules were split out of router.ts. Re-export their public surface so the
|
|
@@ -299,8 +296,9 @@ export class DaemonCommandRouter {
|
|
|
299
296
|
* on disk) clears the tombstone and merges normally, preserving clone
|
|
300
297
|
* worktree visibility and legitimate node re-creation. */
|
|
301
298
|
private removedInlineMeshNodeIds = new Map<string, Set<string>>();
|
|
302
|
-
/** Coordinator-owned whole-mesh aggregate status snapshots. Browser callers read this by default.
|
|
303
|
-
|
|
299
|
+
/** Coordinator-owned whole-mesh aggregate status snapshots. Browser callers read this by default.
|
|
300
|
+
* Public (not private) so the extracted ./router-aggregate-status.ts orchestration can reach it via `self`. */
|
|
301
|
+
aggregateMeshStatusCache = new Map<string, { builtAt: number; snapshot: any; queueRevision: string }>();
|
|
304
302
|
/** Shared per-peer git_status probe dedup + recently-probed reuse gate.
|
|
305
303
|
* Spans separate mesh_status/get_mesh calls so the dashboard auto-retry
|
|
306
304
|
* loop cannot storm a slow peer with back-to-back refreshUpstream probes. */
|
|
@@ -323,108 +321,14 @@ export class DaemonCommandRouter {
|
|
|
323
321
|
this.deps = deps;
|
|
324
322
|
}
|
|
325
323
|
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
324
|
+
// ─── Aggregate mesh-status cache ────────────────────────────────────
|
|
325
|
+
// Implementation lives in ./router-aggregate-status.ts (behavior-preserving
|
|
326
|
+
// code move). Kept here as thin delegators: getCachedAggregateMeshStatus /
|
|
327
|
+
// rememberAggregateMeshStatus are bound into HighFamilyContext, so callers
|
|
328
|
+
// reach these via `self.` for correct instance dispatch.
|
|
330
329
|
|
|
331
330
|
private hydrateCachedAggregateMeshStatusFromInline(snapshot: any, mesh: any, options?: { requireDirectPeerTruth?: boolean }): any {
|
|
332
|
-
|
|
333
|
-
const inlineNodesById = new Map<string, any>();
|
|
334
|
-
for (const node of mesh.nodes) {
|
|
335
|
-
const nodeId = readInlineMeshNodeId(node);
|
|
336
|
-
if (nodeId) inlineNodesById.set(nodeId, node);
|
|
337
|
-
}
|
|
338
|
-
if (!inlineNodesById.size) return snapshot;
|
|
339
|
-
|
|
340
|
-
let changed = false;
|
|
341
|
-
const unavailableNodeIds = new Set<string>();
|
|
342
|
-
const sourceOfTruth = readObjectRecord(snapshot.sourceOfTruth);
|
|
343
|
-
const directPeerTruth = readObjectRecord(sourceOfTruth.directPeerTruth);
|
|
344
|
-
// Dead local worktree nodes (isLocalWorktree, workspace deleted from disk)
|
|
345
|
-
// carry no live truth and must never gate the aggregate as unavailable.
|
|
346
|
-
// A cached snapshot built before the worktree was removed can still list
|
|
347
|
-
// such a node in unavailableNodeIds, which would wedge the graph in a
|
|
348
|
-
// permanent direct_peer_truth_unavailable; drop them here so the held
|
|
349
|
-
// standing-state truth for the surviving nodes satisfies the aggregate.
|
|
350
|
-
const deadNodeIds = new Set<string>();
|
|
351
|
-
for (const node of mesh.nodes) {
|
|
352
|
-
if (!isDeadLocalWorktreeNode(node)) continue;
|
|
353
|
-
const deadId = readInlineMeshNodeId(node);
|
|
354
|
-
if (deadId) deadNodeIds.add(deadId);
|
|
355
|
-
}
|
|
356
|
-
let droppedDeadUnavailable = false;
|
|
357
|
-
for (const entry of Array.isArray(directPeerTruth.unavailableNodeIds) ? directPeerTruth.unavailableNodeIds : []) {
|
|
358
|
-
const nodeId = readStringValue(entry);
|
|
359
|
-
if (!nodeId) continue;
|
|
360
|
-
if (deadNodeIds.has(nodeId)) {
|
|
361
|
-
droppedDeadUnavailable = true;
|
|
362
|
-
continue;
|
|
363
|
-
}
|
|
364
|
-
unavailableNodeIds.add(nodeId);
|
|
365
|
-
}
|
|
366
|
-
// Force a rewrite when a dead worktree was filtered out of a previously
|
|
367
|
-
// built unavailable set, even if no live git was re-hydrated this pass —
|
|
368
|
-
// otherwise the early-return below would hand back the stale snapshot that
|
|
369
|
-
// still says direct_peer_truth_unavailable.
|
|
370
|
-
if (droppedDeadUnavailable) changed = true;
|
|
371
|
-
|
|
372
|
-
const nodes = snapshot.nodes.map((statusNode: any) => {
|
|
373
|
-
const nodeId = normalizeMeshNodeId(statusNode);
|
|
374
|
-
const inlineNode = nodeId ? inlineNodesById.get(nodeId) : undefined;
|
|
375
|
-
if (!inlineNode) return statusNode;
|
|
376
|
-
const liveGit = buildInlineMeshTransitGitStatus(inlineNode);
|
|
377
|
-
if (!liveGit) return statusNode;
|
|
378
|
-
const nextStatus = { ...statusNode };
|
|
379
|
-
nextStatus.git = liveGit;
|
|
380
|
-
nextStatus.health = deriveMeshNodeHealthFromGit(liveGit);
|
|
381
|
-
applyInlineMeshBranchConvergence(mesh, inlineNode, nextStatus);
|
|
382
|
-
nextStatus.launchReady = readBooleanValue(nextStatus.launchReady) ?? true;
|
|
383
|
-
const connection = readObjectRecord(nextStatus.connection);
|
|
384
|
-
const connectionState = readStringValue(connection.state);
|
|
385
|
-
const connectionReported = readBooleanValue(connection.reported) ?? false;
|
|
386
|
-
if (!connectionReported || connectionState === 'unknown') {
|
|
387
|
-
nextStatus.connection = buildLivePeerGitConnection(connection);
|
|
388
|
-
}
|
|
389
|
-
delete nextStatus.gitProbePending;
|
|
390
|
-
const error = readStringValue(nextStatus.error);
|
|
391
|
-
if (error && /pending_git|git probe|live peer git snapshot|no peer git snapshot/i.test(error)) delete nextStatus.error;
|
|
392
|
-
if (!readStringValue(nextStatus.machineStatus)) nextStatus.machineStatus = 'online';
|
|
393
|
-
if (nodeId) unavailableNodeIds.delete(nodeId);
|
|
394
|
-
changed = true;
|
|
395
|
-
return nextStatus;
|
|
396
|
-
});
|
|
397
|
-
|
|
398
|
-
const aggregateDirectTruthSatisfied = sourceOfTruth.coordinatorOwnsLiveTruth === true
|
|
399
|
-
|| directPeerTruth.satisfied === true;
|
|
400
|
-
if (!changed && !(options?.requireDirectPeerTruth && unavailableNodeIds.size > 0 && !aggregateDirectTruthSatisfied)) return snapshot;
|
|
401
|
-
const nextSourceOfTruth = {
|
|
402
|
-
...sourceOfTruth,
|
|
403
|
-
...(Object.keys(directPeerTruth).length ? {
|
|
404
|
-
directPeerTruth: {
|
|
405
|
-
...directPeerTruth,
|
|
406
|
-
satisfied: options?.requireDirectPeerTruth === true
|
|
407
|
-
? aggregateDirectTruthSatisfied || unavailableNodeIds.size === 0
|
|
408
|
-
: directPeerTruth.satisfied,
|
|
409
|
-
unavailableNodeIds: [...unavailableNodeIds],
|
|
410
|
-
},
|
|
411
|
-
...(options?.requireDirectPeerTruth === true ? {
|
|
412
|
-
coordinatorOwnsLiveTruth: aggregateDirectTruthSatisfied || unavailableNodeIds.size === 0,
|
|
413
|
-
currentStatus: aggregateDirectTruthSatisfied || unavailableNodeIds.size === 0 ? 'live_git_and_session_probes' : 'direct_peer_truth_unavailable',
|
|
414
|
-
} : {}),
|
|
415
|
-
} : {}),
|
|
416
|
-
};
|
|
417
|
-
return {
|
|
418
|
-
...snapshot,
|
|
419
|
-
...(options?.requireDirectPeerTruth === true && unavailableNodeIds.size > 0 && !aggregateDirectTruthSatisfied ? {
|
|
420
|
-
success: false,
|
|
421
|
-
code: 'mesh_direct_peer_truth_unavailable',
|
|
422
|
-
error: 'Selected coordinator could not confirm direct mesh truth for every remote node yet.',
|
|
423
|
-
} : {}),
|
|
424
|
-
sourceOfTruth: nextSourceOfTruth,
|
|
425
|
-
branchConvergenceSummary: summarizeInlineMeshBranchConvergence(nodes),
|
|
426
|
-
nodes,
|
|
427
|
-
};
|
|
331
|
+
return hydrateCachedAggregateMeshStatusFromInline(this, snapshot, mesh, options);
|
|
428
332
|
}
|
|
429
333
|
|
|
430
334
|
private getCachedAggregateMeshStatus(
|
|
@@ -432,63 +336,11 @@ export class DaemonCommandRouter {
|
|
|
432
336
|
mesh?: any,
|
|
433
337
|
options?: { requireDirectPeerTruth?: boolean; allowStalePending?: boolean },
|
|
434
338
|
): any | null {
|
|
435
|
-
|
|
436
|
-
if (!cached?.snapshot || cached.snapshot.success !== true || !Array.isArray(cached.snapshot.nodes)) return null;
|
|
437
|
-
// Genuine invalidation still forces truth: a queue mutation bumps the
|
|
438
|
-
// revision, so a stale-revision snapshot is never served (even under the
|
|
439
|
-
// SWR allowStalePending path below).
|
|
440
|
-
if (cached.queueRevision !== getMeshQueueRevision(meshId)) return null;
|
|
441
|
-
let snapshot = this.cloneJsonValue(cached.snapshot);
|
|
442
|
-
snapshot = this.hydrateCachedAggregateMeshStatusFromInline(snapshot, mesh, options);
|
|
443
|
-
// SWR: allowStalePending lets the interactive detail-open serve a snapshot
|
|
444
|
-
// that still has pending peer-git nodes (would otherwise miss here) so the
|
|
445
|
-
// graph paints instantly; the caller fires a background freshen. The
|
|
446
|
-
// queueRevision guard above is NOT relaxed — only the pending-git freshness
|
|
447
|
-
// gate is, so a genuine queue/identity mutation still forces a live rebuild.
|
|
448
|
-
if (!options?.allowStalePending && shouldRefreshStalePendingAggregate(snapshot, options)) return null;
|
|
449
|
-
const ageMs = Math.max(0, Date.now() - cached.builtAt);
|
|
450
|
-
const sourceOfTruth = snapshot.sourceOfTruth && typeof snapshot.sourceOfTruth === 'object'
|
|
451
|
-
? snapshot.sourceOfTruth
|
|
452
|
-
: {};
|
|
453
|
-
snapshot.sourceOfTruth = {
|
|
454
|
-
...sourceOfTruth,
|
|
455
|
-
aggregateSnapshot: {
|
|
456
|
-
...(sourceOfTruth.aggregateSnapshot && typeof sourceOfTruth.aggregateSnapshot === 'object'
|
|
457
|
-
? sourceOfTruth.aggregateSnapshot
|
|
458
|
-
: {}),
|
|
459
|
-
owner: 'coordinator_daemon_memory',
|
|
460
|
-
cached: true,
|
|
461
|
-
source: 'memory',
|
|
462
|
-
refreshReason: 'memory_cache_hit',
|
|
463
|
-
ageMs,
|
|
464
|
-
cachedAt: new Date(cached.builtAt).toISOString(),
|
|
465
|
-
returnedAt: new Date().toISOString(),
|
|
466
|
-
},
|
|
467
|
-
};
|
|
468
|
-
return snapshot;
|
|
339
|
+
return getCachedAggregateMeshStatus(this, meshId, mesh, options);
|
|
469
340
|
}
|
|
470
341
|
|
|
471
342
|
private rememberAggregateMeshStatus(meshId: string, snapshot: any, refreshReason: string): any {
|
|
472
|
-
|
|
473
|
-
const builtAt = Date.now();
|
|
474
|
-
const next = this.cloneJsonValue(snapshot);
|
|
475
|
-
const sourceOfTruth = next.sourceOfTruth && typeof next.sourceOfTruth === 'object'
|
|
476
|
-
? next.sourceOfTruth
|
|
477
|
-
: {};
|
|
478
|
-
next.sourceOfTruth = {
|
|
479
|
-
...sourceOfTruth,
|
|
480
|
-
aggregateSnapshot: {
|
|
481
|
-
owner: 'coordinator_daemon_memory',
|
|
482
|
-
cached: false,
|
|
483
|
-
source: 'live_refresh',
|
|
484
|
-
refreshReason,
|
|
485
|
-
ageMs: 0,
|
|
486
|
-
cachedAt: new Date(builtAt).toISOString(),
|
|
487
|
-
returnedAt: new Date(builtAt).toISOString(),
|
|
488
|
-
},
|
|
489
|
-
};
|
|
490
|
-
this.aggregateMeshStatusCache.set(meshId, { builtAt, snapshot: this.cloneJsonValue(next), queueRevision: getMeshQueueRevision(meshId) });
|
|
491
|
-
return next;
|
|
343
|
+
return rememberAggregateMeshStatus(this, meshId, snapshot, refreshReason);
|
|
492
344
|
}
|
|
493
345
|
|
|
494
346
|
public getCachedInlineMeshNodes(): any[] {
|
|
@@ -501,94 +353,14 @@ export class DaemonCommandRouter {
|
|
|
501
353
|
return nodes;
|
|
502
354
|
}
|
|
503
355
|
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
* instanceManager/sessionRegistry — only their cached mesh-node metadata. A
|
|
510
|
-
* dashboard-issued session-scoped command (invoke_provider_script / resolve_action /
|
|
511
|
-
* set_mode / …) lands on the coordinator with a targetSessionId the coordinator can't
|
|
512
|
-
* find locally, and without forwarding it dies as "Live session not found". send_chat
|
|
513
|
-
* happens to survive (its target resolves to the worker by another route), but the
|
|
514
|
-
* controlbar commands do not — so the controlbar buttons appear to do nothing.
|
|
515
|
-
*
|
|
516
|
-
* Mirror the existing node-level remote-forward pattern (fast_forward_mesh_node etc.):
|
|
517
|
-
* scan the candidate mesh nodes for the one hosting the targetSessionId, and return its
|
|
518
|
-
* daemonId when that daemonId is a remote daemon (i.e. not this coordinator's own
|
|
519
|
-
* statusInstanceId). Returns undefined for a locally-hosted session (no forward — execute
|
|
520
|
-
* locally as before) or when ownership can't be resolved.
|
|
521
|
-
*
|
|
522
|
-
* The candidate set spans BOTH the cached inline-mesh nodes and the live aggregate
|
|
523
|
-
* mesh-status snapshots. The inline cache reliably carries only each node's single primary
|
|
524
|
-
* session (cachedStatus.activeSession), so a worker hosting more than one session exposes its
|
|
525
|
-
* non-primary sessions only on the aggregate snapshot nodes (status.activeSessions /
|
|
526
|
-
* activeSessionDetails, built from live session records). collectMeshNodeHostedSessionIds does
|
|
527
|
-
* the wider plural-shape scan so a controlbar/modal command targeting a non-primary remote
|
|
528
|
-
* session still resolves its owner — the singular readCachedInlineMeshActiveSessions semantics
|
|
529
|
-
* other consumers depend on stay untouched.
|
|
530
|
-
*
|
|
531
|
-
* CANCEL-STOP-RELAY: the session-id cache scan above only matches when the coordinator's
|
|
532
|
-
* cached status snapshot already lists the worker's session id in a recognized active-sessions
|
|
533
|
-
* shape. A worktree-clone worker session whose id form/timing differs from the cached snapshot
|
|
534
|
-
* (or is simply not yet reflected) misses the scan, so a stop that carries the authoritative
|
|
535
|
-
* owning nodeId (mesh_queue_cancel knows assignedNodeId) used to silently fail to forward.
|
|
536
|
-
* `ownerNodeIdHint` adds a deterministic fallback: when the session-id scan misses, resolve the
|
|
537
|
-
* owner daemonId by matching the node by id (meshNodeIdMatches — same form-tolerant compare the
|
|
538
|
-
* rest of the router uses, no new raw compare). The same self-loopback guard applies to both
|
|
539
|
-
* paths, so a coordinator-hosted node is still never force-forwarded to a remote form of itself.
|
|
540
|
-
*/
|
|
541
|
-
public resolveRemoteMeshSessionOwnerDaemonId(sessionId: string, ownerNodeIdHint?: string): string | undefined {
|
|
542
|
-
const trimmed = typeof sessionId === 'string' ? sessionId.trim() : '';
|
|
543
|
-
const nodeHint = typeof ownerNodeIdHint === 'string' ? ownerNodeIdHint.trim() : '';
|
|
544
|
-
if (!trimmed && !nodeHint) return undefined;
|
|
545
|
-
const selfDaemonId = this.deps.statusInstanceId;
|
|
546
|
-
const candidates = this.collectMeshSessionOwnerCandidateNodes();
|
|
547
|
-
if (trimmed) {
|
|
548
|
-
for (const node of candidates) {
|
|
549
|
-
if (!collectMeshNodeHostedSessionIds(node).has(trimmed)) continue;
|
|
550
|
-
const nodeDaemonId = readMeshNodeDaemonId(readObjectRecord(node));
|
|
551
|
-
// A matching node with no readable daemonId can't be attributed — keep scanning
|
|
552
|
-
// the remaining candidates (e.g. the same session on an aggregate node that does
|
|
553
|
-
// carry the daemonId) rather than bailing on the whole resolution.
|
|
554
|
-
if (!nodeDaemonId) continue;
|
|
555
|
-
// Only forward to a genuinely remote daemon. When the owning node is this
|
|
556
|
-
// coordinator itself (locally hosted worker), fall through to local handling.
|
|
557
|
-
// id-form robust: the node daemonId and selfDaemonId may be stored in different
|
|
558
|
-
// forms of the same machine — a strict `===` would miss the self-match and forward
|
|
559
|
-
// a local session to a remote form of THIS daemon (loopback).
|
|
560
|
-
if (selfDaemonId && daemonIdsEquivalent(nodeDaemonId, selfDaemonId)) return undefined;
|
|
561
|
-
return nodeDaemonId;
|
|
562
|
-
}
|
|
563
|
-
}
|
|
564
|
-
// Deterministic fallback: the session-id scan missed (cache lag / id-form mismatch on a
|
|
565
|
-
// worktree-clone worker), but the caller knows the authoritative owning nodeId. Resolve the
|
|
566
|
-
// owner daemonId straight off that node — never the fuzzy session cache.
|
|
567
|
-
if (nodeHint) {
|
|
568
|
-
for (const node of candidates) {
|
|
569
|
-
if (!meshNodeIdMatches(node, nodeHint)) continue;
|
|
570
|
-
const nodeDaemonId = readMeshNodeDaemonId(readObjectRecord(node));
|
|
571
|
-
if (!nodeDaemonId) continue;
|
|
572
|
-
if (selfDaemonId && daemonIdsEquivalent(nodeDaemonId, selfDaemonId)) return undefined;
|
|
573
|
-
return nodeDaemonId;
|
|
574
|
-
}
|
|
575
|
-
}
|
|
576
|
-
return undefined;
|
|
577
|
-
}
|
|
356
|
+
// ─── Remote mesh-session owner resolution ───────────────────────────
|
|
357
|
+
// Implementation lives in ./router-mesh-session-owner.ts (behavior-preserving
|
|
358
|
+
// code move). resolveRemoteMeshSessionOwnerDaemonId stays public (the [Z]
|
|
359
|
+
// session-scoped forward in executeDaemonCommand and a unit test call it), so
|
|
360
|
+
// it's kept here as a thin delegator.
|
|
578
361
|
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
* carry each node's primary session) plus the nodes from every cached aggregate mesh-status
|
|
582
|
-
* snapshot (which carry each node's full live session list). getCachedInlineMeshNodes()
|
|
583
|
-
* returns a fresh array, so appending the aggregate nodes never mutates cached state.
|
|
584
|
-
*/
|
|
585
|
-
private collectMeshSessionOwnerCandidateNodes(): any[] {
|
|
586
|
-
const nodes: any[] = this.getCachedInlineMeshNodes();
|
|
587
|
-
for (const cached of this.aggregateMeshStatusCache.values()) {
|
|
588
|
-
const snapshotNodes = cached?.snapshot?.nodes;
|
|
589
|
-
if (Array.isArray(snapshotNodes)) nodes.push(...snapshotNodes);
|
|
590
|
-
}
|
|
591
|
-
return nodes;
|
|
362
|
+
public resolveRemoteMeshSessionOwnerDaemonId(sessionId: string, ownerNodeIdHint?: string): string | undefined {
|
|
363
|
+
return resolveRemoteMeshSessionOwnerDaemonId(this, sessionId, ownerNodeIdHint);
|
|
592
364
|
}
|
|
593
365
|
|
|
594
366
|
public getCachedInlineMesh(meshId: string, inlineMesh?: unknown): any | undefined {
|
|
@@ -134,8 +134,68 @@ function normalizeCoordinatorDaemonIds(
|
|
|
134
134
|
// SAME machine as the drainer (a coordinatorRunId change from a restart
|
|
135
135
|
// orphaned it): it is delivered to the current coordinator on that daemon.
|
|
136
136
|
|
|
137
|
-
/**
|
|
138
|
-
*
|
|
137
|
+
/**
|
|
138
|
+
* T6 (B3c) enforce switch. When ON, the drain path stops passing an unversioned
|
|
139
|
+
* (v1) or a validation-failing v2 event through to the coordinator — it QUARANTINES
|
|
140
|
+
* it instead (excluded from the delivered batch + WARN + counter), and unicast
|
|
141
|
+
* routing is the only delivery path (there is no v1 broadcast fallback). Off (the
|
|
142
|
+
* default) preserves the accept-and-warn rollout behaviour exactly.
|
|
143
|
+
*
|
|
144
|
+
* Per the rollout plan (§1 decision 4): enforce is `MESH_PROTOCOL_V2_ENFORCE` (env)
|
|
145
|
+
* — its activation is a deliberate operational step taken ONLY after daemonBuilds
|
|
146
|
+
* confirms every node emits v2 (§배포 게이트 1 / risk §4). So the code default is
|
|
147
|
+
* OFF; flipping the env back to accept mode is a pure-env rollback (no code change,
|
|
148
|
+
* no data migration — the schema is additive). Read at call time so a test /
|
|
149
|
+
* operator can toggle it without a restart.
|
|
150
|
+
*
|
|
151
|
+
* Quarantine (not drop) keeps the loss-free invariant. The DESTRUCTIVE drain has
|
|
152
|
+
* already consumed the event from its store by the time routing runs, so "held
|
|
153
|
+
* back" here means: excluded from the delivered batch AND mirrored into the mesh
|
|
154
|
+
* ledger as a recoverable `event_held` entry (the same recovery channel the
|
|
155
|
+
* pending-trim path uses). It is observable via the counters + the ledger, so an
|
|
156
|
+
* operator can requeue it after fixing the producer. The non-destructive PEEK path
|
|
157
|
+
* (countMetrics=false) merely omits the event from the returned list — it never
|
|
158
|
+
* consumed it and must not ledger-record on every status poll.
|
|
159
|
+
*/
|
|
160
|
+
export function isMeshProtocolV2EnforceEnabled(): boolean {
|
|
161
|
+
const raw = readNonEmptyString(process.env.MESH_PROTOCOL_V2_ENFORCE);
|
|
162
|
+
if (!raw) return false;
|
|
163
|
+
const v = raw.trim().toLowerCase();
|
|
164
|
+
return v === '1' || v === 'true' || v === 'on' || v === 'yes';
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
/**
|
|
168
|
+
* Record a v2-enforce-quarantined event into the mesh ledger as recoverable, so a
|
|
169
|
+
* destructively-drained event held back by enforce is auditable and requeue-able
|
|
170
|
+
* (loss-free invariant). Mirrors the pending-trim `event_held` shape. Best-effort:
|
|
171
|
+
* a ledger write failure must not break the drain. Called ONLY on the destructive
|
|
172
|
+
* drain path (the peek path never consumed the event, so nothing to recover).
|
|
173
|
+
*/
|
|
174
|
+
function ledgerRecordQuarantinedEvent(event: PendingMeshCoordinatorEvent, reason: string): void {
|
|
175
|
+
try {
|
|
176
|
+
const finalSummary = readMeshCompletionSummary(event.metadataEvent || {});
|
|
177
|
+
appendLedgerEntry(event.meshId, {
|
|
178
|
+
kind: 'event_held',
|
|
179
|
+
...(event.nodeId ? { nodeId: event.nodeId } : {}),
|
|
180
|
+
payload: {
|
|
181
|
+
event: event.event,
|
|
182
|
+
reason,
|
|
183
|
+
recoverable: true,
|
|
184
|
+
nodeLabel: event.nodeLabel,
|
|
185
|
+
...(event.workspace ? { workspace: event.workspace } : {}),
|
|
186
|
+
targetCoordinatorDaemonId: event.targetCoordinatorDaemonId ?? null,
|
|
187
|
+
...(readNonEmptyString(event.eventId) ? { eventId: event.eventId } : {}),
|
|
188
|
+
queuedAt: event.queuedAt,
|
|
189
|
+
...(finalSummary ? { finalSummary } : {}),
|
|
190
|
+
},
|
|
191
|
+
});
|
|
192
|
+
} catch (e: any) {
|
|
193
|
+
LOG.warn('MeshEventsV2', `Failed to ledger-record v2-quarantined ${event.event} for mesh ${event.meshId}: ${e?.message || e}`);
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
/** Observability counters for the v2 drain path. Read by tests and surfaced in
|
|
198
|
+
* mesh_status (B4/T6). Process-lifetime totals — never reset in production. */
|
|
139
199
|
const meshV2DrainCounters = {
|
|
140
200
|
/** v2 events that passed validation and unicast/broadcast routing → delivered. */
|
|
141
201
|
v2Delivered: 0,
|
|
@@ -152,6 +212,14 @@ const meshV2DrainCounters = {
|
|
|
152
212
|
v2ReattributedToDrainer: 0,
|
|
153
213
|
/** v1 (unversioned) events passed through as broadcast (rollout baseline). */
|
|
154
214
|
v1BroadcastAccepted: 0,
|
|
215
|
+
/** T6 enforce: v2 events that FAILED validation and were QUARANTINED (held back
|
|
216
|
+
* from delivery, not dropped). Non-zero here means a producer is still emitting a
|
|
217
|
+
* malformed envelope after enforce was turned on. */
|
|
218
|
+
v2ValidationFailedQuarantined: 0,
|
|
219
|
+
/** T6 enforce: v1 (unversioned) events QUARANTINED because no v2 envelope could be
|
|
220
|
+
* derived at emit time. Non-zero here means a producer path still emits v1 after
|
|
221
|
+
* enforce — it should reach 0 once every node is on a v2-stamping build. */
|
|
222
|
+
v1UnversionedQuarantined: 0,
|
|
155
223
|
};
|
|
156
224
|
|
|
157
225
|
/** Test/observability accessor for the v2 drain counters (snapshot copy). */
|
|
@@ -271,23 +339,49 @@ function routeV2EventsForDrainer(
|
|
|
271
339
|
},
|
|
272
340
|
): PendingMeshCoordinatorEvent[] {
|
|
273
341
|
if (!drainer) return events;
|
|
342
|
+
// Read the enforce flag ONCE per drain so the whole batch is classified under a
|
|
343
|
+
// single, consistent policy (a mid-batch env flip cannot split one drain).
|
|
344
|
+
const enforce = isMeshProtocolV2EnforceEnabled();
|
|
274
345
|
const bump = (k: keyof typeof meshV2DrainCounters) => { if (ctx.countMetrics) meshV2DrainCounters[k]++; };
|
|
275
346
|
const kept: PendingMeshCoordinatorEvent[] = [];
|
|
276
347
|
for (const event of events) {
|
|
277
348
|
if (!isV2Event(event)) {
|
|
278
|
-
// v1 / unversioned event
|
|
349
|
+
// v1 / unversioned event. ACCEPT MODE: broadcast during rollout (existing
|
|
350
|
+
// policy). ENFORCE MODE: quarantine — an unversioned event has no scope, so
|
|
351
|
+
// there is no safe unicast target; hold it back (not delivered) and mirror
|
|
352
|
+
// it to the ledger as recoverable, with a one-shot WARN + counter.
|
|
353
|
+
if (enforce) {
|
|
354
|
+
bump('v1UnversionedQuarantined');
|
|
355
|
+
if (ctx.countMetrics) ledgerRecordQuarantinedEvent(event, 'v2_enforce_unversioned_quarantined');
|
|
356
|
+
warnV2Once(
|
|
357
|
+
`${event.meshId}::${event.eventId ?? event.event}::v1-quarantined`,
|
|
358
|
+
`v2 ENFORCE: unversioned ${event.event} on mesh ${event.meshId} QUARANTINED (no v2 envelope — held back, not delivered; ledger-recorded recoverable). A producer path still emits v1.`,
|
|
359
|
+
);
|
|
360
|
+
continue;
|
|
361
|
+
}
|
|
279
362
|
bump('v1BroadcastAccepted');
|
|
280
363
|
kept.push(event);
|
|
281
364
|
continue;
|
|
282
365
|
}
|
|
283
366
|
|
|
284
367
|
// Validate the v2 envelope. ACCEPT MODE: a validation failure does NOT drop
|
|
285
|
-
// the event — it passes through with a one-shot WARN + counter.
|
|
286
|
-
//
|
|
368
|
+
// the event — it passes through with a one-shot WARN + counter. ENFORCE MODE:
|
|
369
|
+
// a validation failure is QUARANTINED (held back, not delivered) — the malformed
|
|
370
|
+
// envelope carries no trustworthy scope/target, so delivering it risks a
|
|
371
|
+
// cross-surface. It is ledger-recorded recoverable on the destructive path.
|
|
287
372
|
let validated: PendingMeshCoordinatorEventV2;
|
|
288
373
|
try {
|
|
289
374
|
validated = assertPendingMeshCoordinatorEventV2(event);
|
|
290
375
|
} catch (e: any) {
|
|
376
|
+
if (enforce) {
|
|
377
|
+
bump('v2ValidationFailedQuarantined');
|
|
378
|
+
if (ctx.countMetrics) ledgerRecordQuarantinedEvent(event, 'v2_enforce_validation_failed_quarantined');
|
|
379
|
+
warnV2Once(
|
|
380
|
+
`${event.meshId}::${event.eventId ?? event.event}::invalid-quarantined`,
|
|
381
|
+
`v2 ENFORCE: envelope validation failed for ${event.event} on mesh ${event.meshId} — QUARANTINED (held back, not delivered; ledger-recorded recoverable): ${e?.message || e}`,
|
|
382
|
+
);
|
|
383
|
+
continue;
|
|
384
|
+
}
|
|
291
385
|
bump('v2ValidationFailedAccepted');
|
|
292
386
|
warnV2Once(
|
|
293
387
|
`${event.meshId}::${event.eventId ?? event.event}::invalid`,
|
package/src/mesh/mesh-events.ts
CHANGED
|
@@ -13,6 +13,8 @@ export {
|
|
|
13
13
|
clearPendingMeshCoordinatorEvents,
|
|
14
14
|
serializeV2EnvelopeToWire,
|
|
15
15
|
readV2EnvelopeFromWire,
|
|
16
|
+
getMeshV2DrainCounters,
|
|
17
|
+
isMeshProtocolV2EnforceEnabled,
|
|
16
18
|
} from './mesh-events-pending.js';
|
|
17
19
|
|
|
18
20
|
export {
|
|
@@ -24,6 +26,7 @@ export {
|
|
|
24
26
|
runMeshReconcileTick,
|
|
25
27
|
resolveCoordinatorDrainDeliverability,
|
|
26
28
|
shouldHoldPendingDrainForBusyLocalCoordinator,
|
|
29
|
+
getMeshV2BackstopCounters,
|
|
27
30
|
} from './mesh-reconcile-loop.js';
|
|
28
31
|
|
|
29
32
|
export type { MeshQueueTriggerResult } from './mesh-events-coordinator.js';
|
|
@@ -258,6 +258,59 @@ const inFlightAckedHoldState = new Map<string, AckedHoldState>();
|
|
|
258
258
|
// A restart resets this set, so the first touch of each mesh reloads from disk.
|
|
259
259
|
const rehydratedHoldMeshes = new Set<string>();
|
|
260
260
|
|
|
261
|
+
// ─── T6 (B3c): PHASE-4 synthesis + acked-hold fast-track demoted to last-resort ──
|
|
262
|
+
//
|
|
263
|
+
// Under mesh-protocol-v2 enforce, the completion contract is explicit: a worker's
|
|
264
|
+
// terminal emit is a v2 unicast event drained straight to the coordinator. The
|
|
265
|
+
// PHASE-4 transcript-synthesis backstop and the acked-hold fast-track exist to
|
|
266
|
+
// paper over a LOST emit — they should NEVER fire once v2 delivery is healthy. So
|
|
267
|
+
// their firing is now a demoted last-resort signal: every fire bumps a counter, and
|
|
268
|
+
// under enforce a fire additionally emits a WARN naming it a v2-contract violation
|
|
269
|
+
// (a real emit was expected but never arrived). Target = 0 fires in steady state.
|
|
270
|
+
//
|
|
271
|
+
// The code is NOT removed — it stays as the correctness net for a genuinely lost
|
|
272
|
+
// emit (rollout plan §B3c: "코드 삭제는 하지 않고 관측 후 다음 사이클에 판단"). Process-
|
|
273
|
+
// lifetime totals; read by tests + surfaced in mesh_status.
|
|
274
|
+
const meshV2BackstopCounters = {
|
|
275
|
+
/** PHASE-4 transcript synthesis actually reconciled a missing completion. */
|
|
276
|
+
phase4SynthesisFired: 0,
|
|
277
|
+
/** Acked-hold transcript fast-track promoted a synth ahead of the death deadline. */
|
|
278
|
+
ackedHoldFastTrackFired: 0,
|
|
279
|
+
/** Acked-hold death-deadline backstop released a held synth (the 8-min net). */
|
|
280
|
+
ackedHoldDeathDeadlineFired: 0,
|
|
281
|
+
};
|
|
282
|
+
|
|
283
|
+
/** Test/observability accessor for the v2 last-resort backstop counters (snapshot). */
|
|
284
|
+
export function getMeshV2BackstopCounters(): Readonly<typeof meshV2BackstopCounters> {
|
|
285
|
+
return { ...meshV2BackstopCounters };
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
/** Test helper: zero the backstop counters so a case starts from a clean slate. */
|
|
289
|
+
export function __resetMeshV2BackstopCountersForTests(): void {
|
|
290
|
+
for (const k of Object.keys(meshV2BackstopCounters) as Array<keyof typeof meshV2BackstopCounters>) {
|
|
291
|
+
meshV2BackstopCounters[k] = 0;
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
/** Enforce switch mirror (see isMeshProtocolV2EnforceEnabled in mesh-events-pending);
|
|
296
|
+
* re-read here (not imported) to keep the reconcile loop free of a cross-file coupling
|
|
297
|
+
* and to read env at fire time. Same truthy vocabulary. */
|
|
298
|
+
function meshProtocolV2EnforceOn(): boolean {
|
|
299
|
+
const raw = process.env.MESH_PROTOCOL_V2_ENFORCE;
|
|
300
|
+
if (typeof raw !== 'string') return false;
|
|
301
|
+
const v = raw.trim().toLowerCase();
|
|
302
|
+
return v === '1' || v === 'true' || v === 'on' || v === 'yes';
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
/** Bump a backstop counter and, under enforce, WARN that a last-resort net fired —
|
|
306
|
+
* which under a healthy v2 contract should not happen (the real emit was lost). */
|
|
307
|
+
function recordBackstopFire(kind: keyof typeof meshV2BackstopCounters, detail: string): void {
|
|
308
|
+
meshV2BackstopCounters[kind]++;
|
|
309
|
+
if (meshProtocolV2EnforceOn()) {
|
|
310
|
+
LOG.warn('MeshReconcileV2', `v2 ENFORCE last-resort backstop fired (${kind}): ${detail}. Under a healthy v2 completion contract this should be 0 — a worker's real terminal emit was lost/late.`);
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
|
|
261
314
|
function inFlightSynthKey(meshId: string, taskId: string): string {
|
|
262
315
|
return `${meshId}::${taskId}`;
|
|
263
316
|
}
|
|
@@ -2011,6 +2064,12 @@ async function reconcileUnterminatedDirectDispatches(
|
|
|
2011
2064
|
|
|
2012
2065
|
const synthKey = inFlightSynthKey(mesh.id, taskId);
|
|
2013
2066
|
const isAcked = dispatch.status === 'acked';
|
|
2067
|
+
// T6: which last-resort backstop (if any) drove this synth. Set when the
|
|
2068
|
+
// acked-hold fast-track / death-deadline promotes the synth; the counter is
|
|
2069
|
+
// bumped only if the synth actually COMMITS (result.reconciled), so a
|
|
2070
|
+
// deferred/re-probed-away synth is not miscounted. A never-acked dispatch
|
|
2071
|
+
// that reaches the commit is a plain PHASE-4 transcript synthesis.
|
|
2072
|
+
let backstopKind: keyof ReturnType<typeof getMeshV2BackstopCounters> | undefined;
|
|
2014
2073
|
|
|
2015
2074
|
// R4f: read the worker session. A FAILED read (transport error / success:false / no payload)
|
|
2016
2075
|
// is no longer silently swallowed for an acked task — it is the liveness side of the
|
|
@@ -2130,6 +2189,7 @@ async function reconcileUnterminatedDirectDispatches(
|
|
|
2130
2189
|
const idleHeldMs = nowMs - idleSinceMs;
|
|
2131
2190
|
if (idleHeldMs >= fastTrackGraceMs) {
|
|
2132
2191
|
fastTrackReady = true;
|
|
2192
|
+
backstopKind = 'ackedHoldFastTrackFired';
|
|
2133
2193
|
LOG.info('MeshReconcile', `Acked-hold transcript fast-track: task ${taskId} on node ${nodeId} (mesh ${mesh.id}) read idle WITH a final assistant message for ${Math.round(idleHeldMs / 1000)}s continuous (grace ${Math.round(fastTrackGraceMs / 1000)}s) — promoting the synth ahead of the ${Math.round(deathDeadlineMs / 1000)}s death backstop; the worker's real emit was lost/late and a later one no-ops idempotently.`);
|
|
2134
2194
|
}
|
|
2135
2195
|
} else if (holdState?.transcriptIdleSinceMs !== undefined) {
|
|
@@ -2144,6 +2204,7 @@ async function reconcileUnterminatedDirectDispatches(
|
|
|
2144
2204
|
continue;
|
|
2145
2205
|
}
|
|
2146
2206
|
if (!fastTrackReady) {
|
|
2207
|
+
backstopKind = 'ackedHoldDeathDeadlineFired';
|
|
2147
2208
|
LOG.warn('MeshReconcile', `Acked-hold death deadline reached: task ${taskId} on node ${nodeId} (mesh ${mesh.id}) still idle ${Math.round(sinceAckMs / 1000)}s after the ack (deadline ${Math.round(deathDeadlineMs / 1000)}s) — synthesizing the missing completion as a notification-loss net (a real emit, if it ever lands, no-ops idempotently).`);
|
|
2148
2209
|
}
|
|
2149
2210
|
}
|
|
@@ -2221,6 +2282,11 @@ async function reconcileUnterminatedDirectDispatches(
|
|
|
2221
2282
|
source: 'daemon_reconcile_transcript_completion',
|
|
2222
2283
|
});
|
|
2223
2284
|
if (result.reconciled) {
|
|
2285
|
+
// T6: this synth actually committed → count the last-resort backstop fire.
|
|
2286
|
+
// An acked hold routes to the fast-track / death-deadline kind captured
|
|
2287
|
+
// above; a never-acked dispatch is a plain PHASE-4 transcript synthesis.
|
|
2288
|
+
// Under enforce, recordBackstopFire additionally WARNs (target = 0 fires).
|
|
2289
|
+
recordBackstopFire(backstopKind ?? 'phase4SynthesisFired', `task ${taskId} on node ${nodeId} (mesh ${mesh.id}), kind=${result.kind}`);
|
|
2224
2290
|
LOG.info('MeshReconcile', `Synthesized missing completion (${result.kind}) for task ${taskId} on node ${nodeId} (mesh ${mesh.id})`);
|
|
2225
2291
|
}
|
|
2226
2292
|
} catch (e: any) {
|
|
@@ -16,7 +16,7 @@ const DEFAULT_APPROVAL_POSITIVE_HINTS = [
|
|
|
16
16
|
'always allow',
|
|
17
17
|
];
|
|
18
18
|
|
|
19
|
-
function normalizeApprovalLabel(value: string): string {
|
|
19
|
+
export function normalizeApprovalLabel(value: string): string {
|
|
20
20
|
return String(value || '')
|
|
21
21
|
.toLowerCase()
|
|
22
22
|
.replace(/^[\s\[(<{]*\d+(?:\s*[.)\]}>:-]|\s)+/, '')
|