@adhdev/daemon-core 0.9.82-rc.461 → 0.9.82-rc.462
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +111 -11
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +111 -11
- package/dist/index.mjs.map +1 -1
- package/dist/mesh/mesh-events-pending.d.ts +34 -2
- package/dist/mesh/mesh-events.d.ts +2 -2
- package/dist/mesh/mesh-reconcile-loop.d.ts +12 -0
- package/dist/providers/approval-utils.d.ts +1 -0
- package/dist/repo-mesh-types.d.ts +31 -0
- package/package.json +3 -3
- package/src/commands/high-family/mesh-events.ts +14 -1
- package/src/commands/high-family/mesh-status.ts +19 -2
- package/src/mesh/mesh-events-pending.ts +99 -5
- package/src/mesh/mesh-events.ts +3 -0
- package/src/mesh/mesh-reconcile-loop.ts +66 -0
- package/src/providers/approval-utils.ts +1 -1
- package/src/providers/cli-provider-instance.ts +24 -12
- package/src/repo-mesh-types.ts +32 -0
|
@@ -52,8 +52,32 @@ export interface PendingEventEmitHint {
|
|
|
52
52
|
/** coordinatorRunId to fold into the derived identity when the event lacks one. */
|
|
53
53
|
coordinatorRunId?: string;
|
|
54
54
|
}
|
|
55
|
-
/**
|
|
56
|
-
*
|
|
55
|
+
/**
|
|
56
|
+
* T6 (B3c) enforce switch. When ON, the drain path stops passing an unversioned
|
|
57
|
+
* (v1) or a validation-failing v2 event through to the coordinator — it QUARANTINES
|
|
58
|
+
* it instead (excluded from the delivered batch + WARN + counter), and unicast
|
|
59
|
+
* routing is the only delivery path (there is no v1 broadcast fallback). Off (the
|
|
60
|
+
* default) preserves the accept-and-warn rollout behaviour exactly.
|
|
61
|
+
*
|
|
62
|
+
* Per the rollout plan (§1 decision 4): enforce is `MESH_PROTOCOL_V2_ENFORCE` (env)
|
|
63
|
+
* — its activation is a deliberate operational step taken ONLY after daemonBuilds
|
|
64
|
+
* confirms every node emits v2 (§배포 게이트 1 / risk §4). So the code default is
|
|
65
|
+
* OFF; flipping the env back to accept mode is a pure-env rollback (no code change,
|
|
66
|
+
* no data migration — the schema is additive). Read at call time so a test /
|
|
67
|
+
* operator can toggle it without a restart.
|
|
68
|
+
*
|
|
69
|
+
* Quarantine (not drop) keeps the loss-free invariant. The DESTRUCTIVE drain has
|
|
70
|
+
* already consumed the event from its store by the time routing runs, so "held
|
|
71
|
+
* back" here means: excluded from the delivered batch AND mirrored into the mesh
|
|
72
|
+
* ledger as a recoverable `event_held` entry (the same recovery channel the
|
|
73
|
+
* pending-trim path uses). It is observable via the counters + the ledger, so an
|
|
74
|
+
* operator can requeue it after fixing the producer. The non-destructive PEEK path
|
|
75
|
+
* (countMetrics=false) merely omits the event from the returned list — it never
|
|
76
|
+
* consumed it and must not ledger-record on every status poll.
|
|
77
|
+
*/
|
|
78
|
+
export declare function isMeshProtocolV2EnforceEnabled(): boolean;
|
|
79
|
+
/** Observability counters for the v2 drain path. Read by tests and surfaced in
|
|
80
|
+
* mesh_status (B4/T6). Process-lifetime totals — never reset in production. */
|
|
57
81
|
declare const meshV2DrainCounters: {
|
|
58
82
|
/** v2 events that passed validation and unicast/broadcast routing → delivered. */
|
|
59
83
|
v2Delivered: number;
|
|
@@ -70,6 +94,14 @@ declare const meshV2DrainCounters: {
|
|
|
70
94
|
v2ReattributedToDrainer: number;
|
|
71
95
|
/** v1 (unversioned) events passed through as broadcast (rollout baseline). */
|
|
72
96
|
v1BroadcastAccepted: number;
|
|
97
|
+
/** T6 enforce: v2 events that FAILED validation and were QUARANTINED (held back
|
|
98
|
+
* from delivery, not dropped). Non-zero here means a producer is still emitting a
|
|
99
|
+
* malformed envelope after enforce was turned on. */
|
|
100
|
+
v2ValidationFailedQuarantined: number;
|
|
101
|
+
/** T6 enforce: v1 (unversioned) events QUARANTINED because no v2 envelope could be
|
|
102
|
+
* derived at emit time. Non-zero here means a producer path still emits v1 after
|
|
103
|
+
* enforce — it should reach 0 once every node is on a v2-stamping build. */
|
|
104
|
+
v1UnversionedQuarantined: number;
|
|
73
105
|
};
|
|
74
106
|
/** Test/observability accessor for the v2 drain counters (snapshot copy). */
|
|
75
107
|
export declare function getMeshV2DrainCounters(): Readonly<typeof meshV2DrainCounters>;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
export type { PendingMeshCoordinatorEvent } from './mesh-events-pending.js';
|
|
2
|
-
export { queuePendingMeshCoordinatorEvent, drainPendingMeshCoordinatorEvents, getPendingMeshCoordinatorEvents, clearPendingMeshCoordinatorEvents, serializeV2EnvelopeToWire, readV2EnvelopeFromWire, } from './mesh-events-pending.js';
|
|
2
|
+
export { queuePendingMeshCoordinatorEvent, drainPendingMeshCoordinatorEvents, getPendingMeshCoordinatorEvents, clearPendingMeshCoordinatorEvents, serializeV2EnvelopeToWire, readV2EnvelopeFromWire, getMeshV2DrainCounters, isMeshProtocolV2EnforceEnabled, } from './mesh-events-pending.js';
|
|
3
3
|
export { reconcileDirectDispatchCompletionFromTranscript, } from './mesh-events-stale.js';
|
|
4
|
-
export { setupMeshReconcileLoop, runMeshReconcileTick, resolveCoordinatorDrainDeliverability, shouldHoldPendingDrainForBusyLocalCoordinator, } from './mesh-reconcile-loop.js';
|
|
4
|
+
export { setupMeshReconcileLoop, runMeshReconcileTick, resolveCoordinatorDrainDeliverability, shouldHoldPendingDrainForBusyLocalCoordinator, getMeshV2BackstopCounters, } from './mesh-reconcile-loop.js';
|
|
5
5
|
export type { MeshQueueTriggerResult } from './mesh-events-coordinator.js';
|
|
6
6
|
export { tryAssignQueueTask, isSessionActivelyGenerating, triggerMeshQueue, handleMeshForwardEvent, setupMeshEventForwarding, isMeshCoordinatorEvent, __resetIdleAutoFastForwardForTests, __resetMeshWorkspaceCacheForTests, } from './mesh-events-coordinator.js';
|
|
@@ -1,4 +1,16 @@
|
|
|
1
1
|
import type { DaemonComponents } from '../boot/daemon-lifecycle.js';
|
|
2
|
+
declare const meshV2BackstopCounters: {
|
|
3
|
+
/** PHASE-4 transcript synthesis actually reconciled a missing completion. */
|
|
4
|
+
phase4SynthesisFired: number;
|
|
5
|
+
/** Acked-hold transcript fast-track promoted a synth ahead of the death deadline. */
|
|
6
|
+
ackedHoldFastTrackFired: number;
|
|
7
|
+
/** Acked-hold death-deadline backstop released a held synth (the 8-min net). */
|
|
8
|
+
ackedHoldDeathDeadlineFired: number;
|
|
9
|
+
};
|
|
10
|
+
/** Test/observability accessor for the v2 last-resort backstop counters (snapshot). */
|
|
11
|
+
export declare function getMeshV2BackstopCounters(): Readonly<typeof meshV2BackstopCounters>;
|
|
12
|
+
/** Test helper: zero the backstop counters so a case starts from a clean slate. */
|
|
13
|
+
export declare function __resetMeshV2BackstopCountersForTests(): void;
|
|
2
14
|
export declare function __resetReconcileInFlightSynthDebounceForTests(): void;
|
|
3
15
|
/**
|
|
4
16
|
* DRAIN-WITHOUT-INJECT guard. Classify, for a mesh on THIS daemon, whether a
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { ProviderModule } from './contracts.js';
|
|
2
|
+
export declare function normalizeApprovalLabel(value: string): string;
|
|
2
3
|
/**
|
|
3
4
|
* True when any of the given button labels reads as a decline/negative option
|
|
4
5
|
* (No / Deny / Cancel / Skip / …). Used as the second half of an approval-modal
|
|
@@ -743,6 +743,37 @@ export interface RepoMeshStatus {
|
|
|
743
743
|
* Omitted when nothing was drained. Mirrors the MCP tool's meshProtocolMetrics.
|
|
744
744
|
*/
|
|
745
745
|
meshProtocolMetrics?: MeshProtocolMetrics;
|
|
746
|
+
/**
|
|
747
|
+
* T6 (B3c): live process-lifetime mesh-protocol-v2 enforce counters from THIS
|
|
748
|
+
* daemon — the enforce flag state, drain-routing tallies (deliver / route-away /
|
|
749
|
+
* dedup / quarantine), and the last-resort backstop fire counts (PHASE-4 synth,
|
|
750
|
+
* acked-hold fast-track / death-deadline). Diagnostic-only and never cached (a
|
|
751
|
+
* live snapshot). Under enforce, non-zero quarantine or backstop counts are the
|
|
752
|
+
* rollout-health signal (target 0). Omitted when unavailable.
|
|
753
|
+
*/
|
|
754
|
+
meshProtocolV2Counters?: MeshProtocolV2Counters;
|
|
755
|
+
}
|
|
756
|
+
/** T6 (B3c) live v2 enforce/observability counters (see RepoMeshStatus.meshProtocolV2Counters). */
|
|
757
|
+
export interface MeshProtocolV2Counters {
|
|
758
|
+
/** True when MESH_PROTOCOL_V2_ENFORCE is active on this daemon. */
|
|
759
|
+
enforce: boolean;
|
|
760
|
+
/** Drain-path routing tallies (accept + enforce). Process-lifetime totals. */
|
|
761
|
+
drain: {
|
|
762
|
+
v2Delivered: number;
|
|
763
|
+
v2RoutedAway: number;
|
|
764
|
+
v2DedupSkipped: number;
|
|
765
|
+
v2ValidationFailedAccepted: number;
|
|
766
|
+
v2ReattributedToDrainer: number;
|
|
767
|
+
v1BroadcastAccepted: number;
|
|
768
|
+
v2ValidationFailedQuarantined: number;
|
|
769
|
+
v1UnversionedQuarantined: number;
|
|
770
|
+
};
|
|
771
|
+
/** Last-resort backstop fire counts. Target 0 under a healthy v2 contract. */
|
|
772
|
+
backstop: {
|
|
773
|
+
phase4SynthesisFired: number;
|
|
774
|
+
ackedHoldFastTrackFired: number;
|
|
775
|
+
ackedHoldDeathDeadlineFired: number;
|
|
776
|
+
};
|
|
746
777
|
}
|
|
747
778
|
/** One provider's version skew across mesh nodes (see RepoMeshStatus.providerVersionSkew). */
|
|
748
779
|
export interface MeshProviderVersionSkew {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@adhdev/daemon-core",
|
|
3
|
-
"version": "0.9.82-rc.
|
|
3
|
+
"version": "0.9.82-rc.462",
|
|
4
4
|
"description": "ADHDev daemon core — CDP, IDE detection, providers, command execution",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -47,8 +47,8 @@
|
|
|
47
47
|
"author": "vilmire",
|
|
48
48
|
"license": "AGPL-3.0-or-later",
|
|
49
49
|
"dependencies": {
|
|
50
|
-
"@adhdev/mesh-shared": "0.9.82-rc.
|
|
51
|
-
"@adhdev/session-host-core": "0.9.82-rc.
|
|
50
|
+
"@adhdev/mesh-shared": "0.9.82-rc.462",
|
|
51
|
+
"@adhdev/session-host-core": "0.9.82-rc.462",
|
|
52
52
|
"@agentclientprotocol/sdk": "^0.16.1",
|
|
53
53
|
"ajv": "^8.20.0",
|
|
54
54
|
"ajv-formats": "^3.0.1",
|
|
@@ -12,6 +12,9 @@ import {
|
|
|
12
12
|
drainPendingMeshCoordinatorEvents,
|
|
13
13
|
shouldHoldPendingDrainForBusyLocalCoordinator,
|
|
14
14
|
resolveCoordinatorDrainDeliverability,
|
|
15
|
+
getMeshV2DrainCounters,
|
|
16
|
+
getMeshV2BackstopCounters,
|
|
17
|
+
isMeshProtocolV2EnforceEnabled,
|
|
15
18
|
} from '../../mesh/mesh-events.js';
|
|
16
19
|
import { normalizeInteractivePromptResponse } from '../../providers/types/interactive-prompt.js';
|
|
17
20
|
import type { HighFamilyContext, HighFamilyHandler } from './types.js';
|
|
@@ -66,11 +69,21 @@ export const meshEventsHandlers: Record<string, HighFamilyHandler> = {
|
|
|
66
69
|
return { success: true, events: [], heldForBusyLocalCoordinator: true, hasLiveCliCoordinator };
|
|
67
70
|
}
|
|
68
71
|
const events = drainPendingMeshCoordinatorEvents(meshId || undefined, coordinatorDaemonId);
|
|
72
|
+
// T6 (B3c): ride the live v2 enforce/backstop counters on the drain response so a
|
|
73
|
+
// pure stdio MCP coordinator (which reads its inbox via this IPC call, not the
|
|
74
|
+
// daemon-core mesh_status command) sees the same enforce state + quarantine /
|
|
75
|
+
// last-resort-backstop tallies. Process-lifetime snapshot; the counters were just
|
|
76
|
+
// updated by the drain above. Additive — omitting it keeps version-skewed pullers safe.
|
|
77
|
+
const meshProtocolV2Counters = {
|
|
78
|
+
enforce: isMeshProtocolV2EnforceEnabled(),
|
|
79
|
+
drain: { ...getMeshV2DrainCounters() },
|
|
80
|
+
backstop: { ...getMeshV2BackstopCounters() },
|
|
81
|
+
};
|
|
69
82
|
// SELF-COORDINATOR INBOX LEVEL-DRAIN: when the busy local coordinator drained its OWN
|
|
70
83
|
// inbox (selfCoordinatorInboxRead), tell the puller these events were surfaced through
|
|
71
84
|
// the caller's tool result — it must NOT re-forward them into the (busy) PTY (that is the
|
|
72
85
|
// lossy path). Absent the flag, delivery is unchanged (reconcile-owned PTY / remote pull).
|
|
73
|
-
return { success: true, events, hasLiveCliCoordinator, ...(selfCoordinatorInboxRead ? { surfacedForSelfCoordinator: true } : {}) };
|
|
86
|
+
return { success: true, events, hasLiveCliCoordinator, meshProtocolV2Counters, ...(selfCoordinatorInboxRead ? { surfacedForSelfCoordinator: true } : {}) };
|
|
74
87
|
},
|
|
75
88
|
|
|
76
89
|
interactive_prompt_response: async (ctx: HighFamilyContext, args: any) => {
|
|
@@ -19,7 +19,13 @@ import {
|
|
|
19
19
|
normalizeMeshNodeId,
|
|
20
20
|
daemonIdsEquivalent,
|
|
21
21
|
} from '@adhdev/mesh-shared';
|
|
22
|
-
import {
|
|
22
|
+
import {
|
|
23
|
+
getPendingMeshCoordinatorEvents,
|
|
24
|
+
getMeshV2DrainCounters,
|
|
25
|
+
getMeshV2BackstopCounters,
|
|
26
|
+
isMeshProtocolV2EnforceEnabled,
|
|
27
|
+
} from '../../mesh/mesh-events.js';
|
|
28
|
+
import type { MeshProtocolV2Counters } from '../../repo-mesh-types.js';
|
|
23
29
|
import { getRecentUnroutableDeliveries } from '../../mesh/mesh-routing.js';
|
|
24
30
|
import { normalizeMeshDaemonRole, resolveMeshHostStatus } from '../../mesh/mesh-host-ownership.js';
|
|
25
31
|
import { buildPreviewFreshness } from '../../mesh/preview-freshness.js';
|
|
@@ -592,6 +598,15 @@ export const meshStatusHandlers: Record<string, HighFamilyHandler> = {
|
|
|
592
598
|
// that a worker completion was lost (envelope present, mesh unresolved) instead
|
|
593
599
|
// of it vanishing silently. Diagnostic-only — never cached (see omit below).
|
|
594
600
|
const unroutableDeliveries = getRecentUnroutableDeliveries();
|
|
601
|
+
// T6 (B3c): live enforce/observability counters from this daemon. A
|
|
602
|
+
// process-lifetime snapshot (never cached — like unroutableDeliveries)
|
|
603
|
+
// so an operator/coordinator can read enforce state, quarantine tallies,
|
|
604
|
+
// and last-resort backstop fires straight from the aggregate status.
|
|
605
|
+
const meshProtocolV2Counters: MeshProtocolV2Counters = {
|
|
606
|
+
enforce: isMeshProtocolV2EnforceEnabled(),
|
|
607
|
+
drain: { ...getMeshV2DrainCounters() },
|
|
608
|
+
backstop: { ...getMeshV2BackstopCounters() },
|
|
609
|
+
};
|
|
595
610
|
const previewFreshness = (() => {
|
|
596
611
|
const localRepoRoot = nodeStatuses
|
|
597
612
|
.map((node: any) => readStringValue(node?.git?.repoRoot, node?.repoRoot, node?.workspace))
|
|
@@ -707,6 +722,7 @@ export const meshStatusHandlers: Record<string, HighFamilyHandler> = {
|
|
|
707
722
|
...(historicalSessions ? { historicalSessions } : {}),
|
|
708
723
|
...(pendingCoordinatorEvents.length > 0 ? { pendingCoordinatorEvents } : {}),
|
|
709
724
|
...(unroutableDeliveries.length > 0 ? { unroutableDeliveries } : {}),
|
|
725
|
+
meshProtocolV2Counters,
|
|
710
726
|
activeRefineJobs: Array.from(ctx.runningRefineJobs.values())
|
|
711
727
|
.filter(job => job.meshId === meshId)
|
|
712
728
|
.map(job => ({
|
|
@@ -718,7 +734,7 @@ export const meshStatusHandlers: Record<string, HighFamilyHandler> = {
|
|
|
718
734
|
targetCoordinatorDaemonId: job.targetCoordinatorDaemonId,
|
|
719
735
|
})),
|
|
720
736
|
};
|
|
721
|
-
const { pendingCoordinatorEvents: _pendingCoordinatorEvents, unroutableDeliveries: _unroutableDeliveries, ...cacheableStatusResult } = statusResult as any;
|
|
737
|
+
const { pendingCoordinatorEvents: _pendingCoordinatorEvents, unroutableDeliveries: _unroutableDeliveries, meshProtocolV2Counters: _meshProtocolV2Counters, ...cacheableStatusResult } = statusResult as any;
|
|
722
738
|
// Verbose carries full mission goals; never store it in the shared
|
|
723
739
|
// (compact) aggregate cache or a later compact poll would return the
|
|
724
740
|
// heavy goals from cache. Return it without caching.
|
|
@@ -729,6 +745,7 @@ export const meshStatusHandlers: Record<string, HighFamilyHandler> = {
|
|
|
729
745
|
...rememberedStatus,
|
|
730
746
|
...(pendingCoordinatorEvents.length > 0 ? { pendingCoordinatorEvents } : {}),
|
|
731
747
|
...(unroutableDeliveries.length > 0 ? { unroutableDeliveries } : {}),
|
|
748
|
+
meshProtocolV2Counters,
|
|
732
749
|
};
|
|
733
750
|
logRepoMeshStatusDebug('return_live', {
|
|
734
751
|
meshId,
|
|
@@ -134,8 +134,68 @@ function normalizeCoordinatorDaemonIds(
|
|
|
134
134
|
// SAME machine as the drainer (a coordinatorRunId change from a restart
|
|
135
135
|
// orphaned it): it is delivered to the current coordinator on that daemon.
|
|
136
136
|
|
|
137
|
-
/**
|
|
138
|
-
*
|
|
137
|
+
/**
|
|
138
|
+
* T6 (B3c) enforce switch. When ON, the drain path stops passing an unversioned
|
|
139
|
+
* (v1) or a validation-failing v2 event through to the coordinator — it QUARANTINES
|
|
140
|
+
* it instead (excluded from the delivered batch + WARN + counter), and unicast
|
|
141
|
+
* routing is the only delivery path (there is no v1 broadcast fallback). Off (the
|
|
142
|
+
* default) preserves the accept-and-warn rollout behaviour exactly.
|
|
143
|
+
*
|
|
144
|
+
* Per the rollout plan (§1 decision 4): enforce is `MESH_PROTOCOL_V2_ENFORCE` (env)
|
|
145
|
+
* — its activation is a deliberate operational step taken ONLY after daemonBuilds
|
|
146
|
+
* confirms every node emits v2 (§배포 게이트 1 / risk §4). So the code default is
|
|
147
|
+
* OFF; flipping the env back to accept mode is a pure-env rollback (no code change,
|
|
148
|
+
* no data migration — the schema is additive). Read at call time so a test /
|
|
149
|
+
* operator can toggle it without a restart.
|
|
150
|
+
*
|
|
151
|
+
* Quarantine (not drop) keeps the loss-free invariant. The DESTRUCTIVE drain has
|
|
152
|
+
* already consumed the event from its store by the time routing runs, so "held
|
|
153
|
+
* back" here means: excluded from the delivered batch AND mirrored into the mesh
|
|
154
|
+
* ledger as a recoverable `event_held` entry (the same recovery channel the
|
|
155
|
+
* pending-trim path uses). It is observable via the counters + the ledger, so an
|
|
156
|
+
* operator can requeue it after fixing the producer. The non-destructive PEEK path
|
|
157
|
+
* (countMetrics=false) merely omits the event from the returned list — it never
|
|
158
|
+
* consumed it and must not ledger-record on every status poll.
|
|
159
|
+
*/
|
|
160
|
+
export function isMeshProtocolV2EnforceEnabled(): boolean {
|
|
161
|
+
const raw = readNonEmptyString(process.env.MESH_PROTOCOL_V2_ENFORCE);
|
|
162
|
+
if (!raw) return false;
|
|
163
|
+
const v = raw.trim().toLowerCase();
|
|
164
|
+
return v === '1' || v === 'true' || v === 'on' || v === 'yes';
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
/**
|
|
168
|
+
* Record a v2-enforce-quarantined event into the mesh ledger as recoverable, so a
|
|
169
|
+
* destructively-drained event held back by enforce is auditable and requeue-able
|
|
170
|
+
* (loss-free invariant). Mirrors the pending-trim `event_held` shape. Best-effort:
|
|
171
|
+
* a ledger write failure must not break the drain. Called ONLY on the destructive
|
|
172
|
+
* drain path (the peek path never consumed the event, so nothing to recover).
|
|
173
|
+
*/
|
|
174
|
+
function ledgerRecordQuarantinedEvent(event: PendingMeshCoordinatorEvent, reason: string): void {
|
|
175
|
+
try {
|
|
176
|
+
const finalSummary = readMeshCompletionSummary(event.metadataEvent || {});
|
|
177
|
+
appendLedgerEntry(event.meshId, {
|
|
178
|
+
kind: 'event_held',
|
|
179
|
+
...(event.nodeId ? { nodeId: event.nodeId } : {}),
|
|
180
|
+
payload: {
|
|
181
|
+
event: event.event,
|
|
182
|
+
reason,
|
|
183
|
+
recoverable: true,
|
|
184
|
+
nodeLabel: event.nodeLabel,
|
|
185
|
+
...(event.workspace ? { workspace: event.workspace } : {}),
|
|
186
|
+
targetCoordinatorDaemonId: event.targetCoordinatorDaemonId ?? null,
|
|
187
|
+
...(readNonEmptyString(event.eventId) ? { eventId: event.eventId } : {}),
|
|
188
|
+
queuedAt: event.queuedAt,
|
|
189
|
+
...(finalSummary ? { finalSummary } : {}),
|
|
190
|
+
},
|
|
191
|
+
});
|
|
192
|
+
} catch (e: any) {
|
|
193
|
+
LOG.warn('MeshEventsV2', `Failed to ledger-record v2-quarantined ${event.event} for mesh ${event.meshId}: ${e?.message || e}`);
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
/** Observability counters for the v2 drain path. Read by tests and surfaced in
|
|
198
|
+
* mesh_status (B4/T6). Process-lifetime totals — never reset in production. */
|
|
139
199
|
const meshV2DrainCounters = {
|
|
140
200
|
/** v2 events that passed validation and unicast/broadcast routing → delivered. */
|
|
141
201
|
v2Delivered: 0,
|
|
@@ -152,6 +212,14 @@ const meshV2DrainCounters = {
|
|
|
152
212
|
v2ReattributedToDrainer: 0,
|
|
153
213
|
/** v1 (unversioned) events passed through as broadcast (rollout baseline). */
|
|
154
214
|
v1BroadcastAccepted: 0,
|
|
215
|
+
/** T6 enforce: v2 events that FAILED validation and were QUARANTINED (held back
|
|
216
|
+
* from delivery, not dropped). Non-zero here means a producer is still emitting a
|
|
217
|
+
* malformed envelope after enforce was turned on. */
|
|
218
|
+
v2ValidationFailedQuarantined: 0,
|
|
219
|
+
/** T6 enforce: v1 (unversioned) events QUARANTINED because no v2 envelope could be
|
|
220
|
+
* derived at emit time. Non-zero here means a producer path still emits v1 after
|
|
221
|
+
* enforce — it should reach 0 once every node is on a v2-stamping build. */
|
|
222
|
+
v1UnversionedQuarantined: 0,
|
|
155
223
|
};
|
|
156
224
|
|
|
157
225
|
/** Test/observability accessor for the v2 drain counters (snapshot copy). */
|
|
@@ -271,23 +339,49 @@ function routeV2EventsForDrainer(
|
|
|
271
339
|
},
|
|
272
340
|
): PendingMeshCoordinatorEvent[] {
|
|
273
341
|
if (!drainer) return events;
|
|
342
|
+
// Read the enforce flag ONCE per drain so the whole batch is classified under a
|
|
343
|
+
// single, consistent policy (a mid-batch env flip cannot split one drain).
|
|
344
|
+
const enforce = isMeshProtocolV2EnforceEnabled();
|
|
274
345
|
const bump = (k: keyof typeof meshV2DrainCounters) => { if (ctx.countMetrics) meshV2DrainCounters[k]++; };
|
|
275
346
|
const kept: PendingMeshCoordinatorEvent[] = [];
|
|
276
347
|
for (const event of events) {
|
|
277
348
|
if (!isV2Event(event)) {
|
|
278
|
-
// v1 / unversioned event
|
|
349
|
+
// v1 / unversioned event. ACCEPT MODE: broadcast during rollout (existing
|
|
350
|
+
// policy). ENFORCE MODE: quarantine — an unversioned event has no scope, so
|
|
351
|
+
// there is no safe unicast target; hold it back (not delivered) and mirror
|
|
352
|
+
// it to the ledger as recoverable, with a one-shot WARN + counter.
|
|
353
|
+
if (enforce) {
|
|
354
|
+
bump('v1UnversionedQuarantined');
|
|
355
|
+
if (ctx.countMetrics) ledgerRecordQuarantinedEvent(event, 'v2_enforce_unversioned_quarantined');
|
|
356
|
+
warnV2Once(
|
|
357
|
+
`${event.meshId}::${event.eventId ?? event.event}::v1-quarantined`,
|
|
358
|
+
`v2 ENFORCE: unversioned ${event.event} on mesh ${event.meshId} QUARANTINED (no v2 envelope — held back, not delivered; ledger-recorded recoverable). A producer path still emits v1.`,
|
|
359
|
+
);
|
|
360
|
+
continue;
|
|
361
|
+
}
|
|
279
362
|
bump('v1BroadcastAccepted');
|
|
280
363
|
kept.push(event);
|
|
281
364
|
continue;
|
|
282
365
|
}
|
|
283
366
|
|
|
284
367
|
// Validate the v2 envelope. ACCEPT MODE: a validation failure does NOT drop
|
|
285
|
-
// the event — it passes through with a one-shot WARN + counter.
|
|
286
|
-
//
|
|
368
|
+
// the event — it passes through with a one-shot WARN + counter. ENFORCE MODE:
|
|
369
|
+
// a validation failure is QUARANTINED (held back, not delivered) — the malformed
|
|
370
|
+
// envelope carries no trustworthy scope/target, so delivering it risks a
|
|
371
|
+
// cross-surface. It is ledger-recorded recoverable on the destructive path.
|
|
287
372
|
let validated: PendingMeshCoordinatorEventV2;
|
|
288
373
|
try {
|
|
289
374
|
validated = assertPendingMeshCoordinatorEventV2(event);
|
|
290
375
|
} catch (e: any) {
|
|
376
|
+
if (enforce) {
|
|
377
|
+
bump('v2ValidationFailedQuarantined');
|
|
378
|
+
if (ctx.countMetrics) ledgerRecordQuarantinedEvent(event, 'v2_enforce_validation_failed_quarantined');
|
|
379
|
+
warnV2Once(
|
|
380
|
+
`${event.meshId}::${event.eventId ?? event.event}::invalid-quarantined`,
|
|
381
|
+
`v2 ENFORCE: envelope validation failed for ${event.event} on mesh ${event.meshId} — QUARANTINED (held back, not delivered; ledger-recorded recoverable): ${e?.message || e}`,
|
|
382
|
+
);
|
|
383
|
+
continue;
|
|
384
|
+
}
|
|
291
385
|
bump('v2ValidationFailedAccepted');
|
|
292
386
|
warnV2Once(
|
|
293
387
|
`${event.meshId}::${event.eventId ?? event.event}::invalid`,
|
package/src/mesh/mesh-events.ts
CHANGED
|
@@ -13,6 +13,8 @@ export {
|
|
|
13
13
|
clearPendingMeshCoordinatorEvents,
|
|
14
14
|
serializeV2EnvelopeToWire,
|
|
15
15
|
readV2EnvelopeFromWire,
|
|
16
|
+
getMeshV2DrainCounters,
|
|
17
|
+
isMeshProtocolV2EnforceEnabled,
|
|
16
18
|
} from './mesh-events-pending.js';
|
|
17
19
|
|
|
18
20
|
export {
|
|
@@ -24,6 +26,7 @@ export {
|
|
|
24
26
|
runMeshReconcileTick,
|
|
25
27
|
resolveCoordinatorDrainDeliverability,
|
|
26
28
|
shouldHoldPendingDrainForBusyLocalCoordinator,
|
|
29
|
+
getMeshV2BackstopCounters,
|
|
27
30
|
} from './mesh-reconcile-loop.js';
|
|
28
31
|
|
|
29
32
|
export type { MeshQueueTriggerResult } from './mesh-events-coordinator.js';
|
|
@@ -258,6 +258,59 @@ const inFlightAckedHoldState = new Map<string, AckedHoldState>();
|
|
|
258
258
|
// A restart resets this set, so the first touch of each mesh reloads from disk.
|
|
259
259
|
const rehydratedHoldMeshes = new Set<string>();
|
|
260
260
|
|
|
261
|
+
// ─── T6 (B3c): PHASE-4 synthesis + acked-hold fast-track demoted to last-resort ──
|
|
262
|
+
//
|
|
263
|
+
// Under mesh-protocol-v2 enforce, the completion contract is explicit: a worker's
|
|
264
|
+
// terminal emit is a v2 unicast event drained straight to the coordinator. The
|
|
265
|
+
// PHASE-4 transcript-synthesis backstop and the acked-hold fast-track exist to
|
|
266
|
+
// paper over a LOST emit — they should NEVER fire once v2 delivery is healthy. So
|
|
267
|
+
// their firing is now a demoted last-resort signal: every fire bumps a counter, and
|
|
268
|
+
// under enforce a fire additionally emits a WARN naming it a v2-contract violation
|
|
269
|
+
// (a real emit was expected but never arrived). Target = 0 fires in steady state.
|
|
270
|
+
//
|
|
271
|
+
// The code is NOT removed — it stays as the correctness net for a genuinely lost
|
|
272
|
+
// emit (rollout plan §B3c: "코드 삭제는 하지 않고 관측 후 다음 사이클에 판단"). Process-
|
|
273
|
+
// lifetime totals; read by tests + surfaced in mesh_status.
|
|
274
|
+
const meshV2BackstopCounters = {
|
|
275
|
+
/** PHASE-4 transcript synthesis actually reconciled a missing completion. */
|
|
276
|
+
phase4SynthesisFired: 0,
|
|
277
|
+
/** Acked-hold transcript fast-track promoted a synth ahead of the death deadline. */
|
|
278
|
+
ackedHoldFastTrackFired: 0,
|
|
279
|
+
/** Acked-hold death-deadline backstop released a held synth (the 8-min net). */
|
|
280
|
+
ackedHoldDeathDeadlineFired: 0,
|
|
281
|
+
};
|
|
282
|
+
|
|
283
|
+
/** Test/observability accessor for the v2 last-resort backstop counters (snapshot). */
|
|
284
|
+
export function getMeshV2BackstopCounters(): Readonly<typeof meshV2BackstopCounters> {
|
|
285
|
+
return { ...meshV2BackstopCounters };
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
/** Test helper: zero the backstop counters so a case starts from a clean slate. */
|
|
289
|
+
export function __resetMeshV2BackstopCountersForTests(): void {
|
|
290
|
+
for (const k of Object.keys(meshV2BackstopCounters) as Array<keyof typeof meshV2BackstopCounters>) {
|
|
291
|
+
meshV2BackstopCounters[k] = 0;
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
/** Enforce switch mirror (see isMeshProtocolV2EnforceEnabled in mesh-events-pending);
|
|
296
|
+
* re-read here (not imported) to keep the reconcile loop free of a cross-file coupling
|
|
297
|
+
* and to read env at fire time. Same truthy vocabulary. */
|
|
298
|
+
function meshProtocolV2EnforceOn(): boolean {
|
|
299
|
+
const raw = process.env.MESH_PROTOCOL_V2_ENFORCE;
|
|
300
|
+
if (typeof raw !== 'string') return false;
|
|
301
|
+
const v = raw.trim().toLowerCase();
|
|
302
|
+
return v === '1' || v === 'true' || v === 'on' || v === 'yes';
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
/** Bump a backstop counter and, under enforce, WARN that a last-resort net fired —
|
|
306
|
+
* which under a healthy v2 contract should not happen (the real emit was lost). */
|
|
307
|
+
function recordBackstopFire(kind: keyof typeof meshV2BackstopCounters, detail: string): void {
|
|
308
|
+
meshV2BackstopCounters[kind]++;
|
|
309
|
+
if (meshProtocolV2EnforceOn()) {
|
|
310
|
+
LOG.warn('MeshReconcileV2', `v2 ENFORCE last-resort backstop fired (${kind}): ${detail}. Under a healthy v2 completion contract this should be 0 — a worker's real terminal emit was lost/late.`);
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
|
|
261
314
|
function inFlightSynthKey(meshId: string, taskId: string): string {
|
|
262
315
|
return `${meshId}::${taskId}`;
|
|
263
316
|
}
|
|
@@ -2011,6 +2064,12 @@ async function reconcileUnterminatedDirectDispatches(
|
|
|
2011
2064
|
|
|
2012
2065
|
const synthKey = inFlightSynthKey(mesh.id, taskId);
|
|
2013
2066
|
const isAcked = dispatch.status === 'acked';
|
|
2067
|
+
// T6: which last-resort backstop (if any) drove this synth. Set when the
|
|
2068
|
+
// acked-hold fast-track / death-deadline promotes the synth; the counter is
|
|
2069
|
+
// bumped only if the synth actually COMMITS (result.reconciled), so a
|
|
2070
|
+
// deferred/re-probed-away synth is not miscounted. A never-acked dispatch
|
|
2071
|
+
// that reaches the commit is a plain PHASE-4 transcript synthesis.
|
|
2072
|
+
let backstopKind: keyof ReturnType<typeof getMeshV2BackstopCounters> | undefined;
|
|
2014
2073
|
|
|
2015
2074
|
// R4f: read the worker session. A FAILED read (transport error / success:false / no payload)
|
|
2016
2075
|
// is no longer silently swallowed for an acked task — it is the liveness side of the
|
|
@@ -2130,6 +2189,7 @@ async function reconcileUnterminatedDirectDispatches(
|
|
|
2130
2189
|
const idleHeldMs = nowMs - idleSinceMs;
|
|
2131
2190
|
if (idleHeldMs >= fastTrackGraceMs) {
|
|
2132
2191
|
fastTrackReady = true;
|
|
2192
|
+
backstopKind = 'ackedHoldFastTrackFired';
|
|
2133
2193
|
LOG.info('MeshReconcile', `Acked-hold transcript fast-track: task ${taskId} on node ${nodeId} (mesh ${mesh.id}) read idle WITH a final assistant message for ${Math.round(idleHeldMs / 1000)}s continuous (grace ${Math.round(fastTrackGraceMs / 1000)}s) — promoting the synth ahead of the ${Math.round(deathDeadlineMs / 1000)}s death backstop; the worker's real emit was lost/late and a later one no-ops idempotently.`);
|
|
2134
2194
|
}
|
|
2135
2195
|
} else if (holdState?.transcriptIdleSinceMs !== undefined) {
|
|
@@ -2144,6 +2204,7 @@ async function reconcileUnterminatedDirectDispatches(
|
|
|
2144
2204
|
continue;
|
|
2145
2205
|
}
|
|
2146
2206
|
if (!fastTrackReady) {
|
|
2207
|
+
backstopKind = 'ackedHoldDeathDeadlineFired';
|
|
2147
2208
|
LOG.warn('MeshReconcile', `Acked-hold death deadline reached: task ${taskId} on node ${nodeId} (mesh ${mesh.id}) still idle ${Math.round(sinceAckMs / 1000)}s after the ack (deadline ${Math.round(deathDeadlineMs / 1000)}s) — synthesizing the missing completion as a notification-loss net (a real emit, if it ever lands, no-ops idempotently).`);
|
|
2148
2209
|
}
|
|
2149
2210
|
}
|
|
@@ -2221,6 +2282,11 @@ async function reconcileUnterminatedDirectDispatches(
|
|
|
2221
2282
|
source: 'daemon_reconcile_transcript_completion',
|
|
2222
2283
|
});
|
|
2223
2284
|
if (result.reconciled) {
|
|
2285
|
+
// T6: this synth actually committed → count the last-resort backstop fire.
|
|
2286
|
+
// An acked hold routes to the fast-track / death-deadline kind captured
|
|
2287
|
+
// above; a never-acked dispatch is a plain PHASE-4 transcript synthesis.
|
|
2288
|
+
// Under enforce, recordBackstopFire additionally WARNs (target = 0 fires).
|
|
2289
|
+
recordBackstopFire(backstopKind ?? 'phase4SynthesisFired', `task ${taskId} on node ${nodeId} (mesh ${mesh.id}), kind=${result.kind}`);
|
|
2224
2290
|
LOG.info('MeshReconcile', `Synthesized missing completion (${result.kind}) for task ${taskId} on node ${nodeId} (mesh ${mesh.id})`);
|
|
2225
2291
|
}
|
|
2226
2292
|
} catch (e: any) {
|
|
@@ -16,7 +16,7 @@ const DEFAULT_APPROVAL_POSITIVE_HINTS = [
|
|
|
16
16
|
'always allow',
|
|
17
17
|
];
|
|
18
18
|
|
|
19
|
-
function normalizeApprovalLabel(value: string): string {
|
|
19
|
+
export function normalizeApprovalLabel(value: string): string {
|
|
20
20
|
return String(value || '')
|
|
21
21
|
.toLowerCase()
|
|
22
22
|
.replace(/^[\s\[(<{]*\d+(?:\s*[.)\]}>:-]|\s)+/, '')
|
|
@@ -27,7 +27,7 @@ import { shouldCollectTraceCategory } from '../logging/debug-config.js';
|
|
|
27
27
|
import { traceMeshEventStage, traceMeshEventDrop } from '../mesh/mesh-event-trace.js';
|
|
28
28
|
import type { ChatMessage } from '../types.js';
|
|
29
29
|
import { buildPersistedProviderEffectMessage, normalizeProviderEffects } from './control-effects.js';
|
|
30
|
-
import { formatAutoApprovalMessage, pickApprovalButton, hasNegativeApprovalOption, hasReliableApprovalAffirmative, looksLikeActiveApprovalPromptText } from './approval-utils.js';
|
|
30
|
+
import { formatAutoApprovalMessage, pickApprovalButton, hasNegativeApprovalOption, hasReliableApprovalAffirmative, looksLikeActiveApprovalPromptText, normalizeApprovalLabel } from './approval-utils.js';
|
|
31
31
|
import { getCliScriptCommand, parseCliScriptResult } from './cli-script-results.js';
|
|
32
32
|
import { mergeProviderPatchState, resolveProviderStateSurface } from './provider-patch-state.js';
|
|
33
33
|
import { normalizeProviderSessionId } from './provider-session-id.js';
|
|
@@ -2317,19 +2317,31 @@ export class CliProviderInstance implements ProviderInstance {
|
|
|
2317
2317
|
// kind gate). Surface the modal so the user decides; never pick blindly.
|
|
2318
2318
|
return autoApproveActive;
|
|
2319
2319
|
}
|
|
2320
|
-
// Modal *identity* signature — the question
|
|
2321
|
-
//
|
|
2322
|
-
// approvalEntrySeq on every fresh
|
|
2323
|
-
// modal→generating→modal flap (the question
|
|
2324
|
-
// captured frame while the button block stays)
|
|
2325
|
-
// again. Folding that seq into the settle signature
|
|
2326
|
-
// settle clock restart on every flap, so the modal never
|
|
2327
|
-
// long enough to fire — the gate was never satisfied.
|
|
2328
|
-
// the seq so
|
|
2320
|
+
// Modal *identity* signature — the question plus the STABLE affirmative
|
|
2321
|
+
// anchor only, NO volatile counters and NO raw button set. This is what
|
|
2322
|
+
// the settle gate tracks: the FSM bumps approvalEntrySeq on every fresh
|
|
2323
|
+
// waiting_approval entry, and a modal→generating→modal flap (the question
|
|
2324
|
+
// line scrolled out of the captured frame while the button block stays)
|
|
2325
|
+
// re-enters and bumps it again. Folding that seq into the settle signature
|
|
2326
|
+
// made the 600ms settle clock restart on every flap, so the modal never
|
|
2327
|
+
// stayed stable long enough to fire — the gate was never satisfied.
|
|
2328
|
+
// Identity excludes the seq so seq flap of the SAME modal keeps one clock.
|
|
2329
|
+
//
|
|
2330
|
+
// The raw button set is also excluded: on a TALL Write/Edit diff claude's
|
|
2331
|
+
// TUI repaints the button block 3↔5↔none between frames (buttons scroll in
|
|
2332
|
+
// and out of the captured region), which flipped both buttons.join('|') and
|
|
2333
|
+
// the positional buttonIndex every frame → signature flap → the settle
|
|
2334
|
+
// clock reset 4–9s and only mask-stalled episodes leaked to the coordinator
|
|
2335
|
+
// (AUTOAPPROVE-SETTLE-FLAP). The affirmative the auto-approve will actually
|
|
2336
|
+
// press is the invariant across those repaints, so we anchor on its
|
|
2337
|
+
// NORMALIZED label (numbers/bullets/punctuation stripped, so "1. Yes" and
|
|
2338
|
+
// "3. Yes" collapse to "yes"). Message + affirmative label uniquely
|
|
2339
|
+
// identifies the consent question without tracking the volatile button
|
|
2340
|
+
// positions.
|
|
2341
|
+
const affirmativeAnchor = normalizeApprovalLabel(buttonLabel);
|
|
2329
2342
|
const modalSignature = [
|
|
2330
2343
|
typeof modal?.message === 'string' ? modal.message.trim() : '',
|
|
2331
|
-
|
|
2332
|
-
buttonIndex,
|
|
2344
|
+
affirmativeAnchor,
|
|
2333
2345
|
].join('::');
|
|
2334
2346
|
// Busy-window re-entry guard still needs the seq: two DISTINCT
|
|
2335
2347
|
// back-to-back approvals can carry identical message/buttons (common
|
package/src/repo-mesh-types.ts
CHANGED
|
@@ -1020,6 +1020,38 @@ export interface RepoMeshStatus {
|
|
|
1020
1020
|
* Omitted when nothing was drained. Mirrors the MCP tool's meshProtocolMetrics.
|
|
1021
1021
|
*/
|
|
1022
1022
|
meshProtocolMetrics?: MeshProtocolMetrics;
|
|
1023
|
+
/**
|
|
1024
|
+
* T6 (B3c): live process-lifetime mesh-protocol-v2 enforce counters from THIS
|
|
1025
|
+
* daemon — the enforce flag state, drain-routing tallies (deliver / route-away /
|
|
1026
|
+
* dedup / quarantine), and the last-resort backstop fire counts (PHASE-4 synth,
|
|
1027
|
+
* acked-hold fast-track / death-deadline). Diagnostic-only and never cached (a
|
|
1028
|
+
* live snapshot). Under enforce, non-zero quarantine or backstop counts are the
|
|
1029
|
+
* rollout-health signal (target 0). Omitted when unavailable.
|
|
1030
|
+
*/
|
|
1031
|
+
meshProtocolV2Counters?: MeshProtocolV2Counters;
|
|
1032
|
+
}
|
|
1033
|
+
|
|
1034
|
+
/** T6 (B3c) live v2 enforce/observability counters (see RepoMeshStatus.meshProtocolV2Counters). */
|
|
1035
|
+
export interface MeshProtocolV2Counters {
|
|
1036
|
+
/** True when MESH_PROTOCOL_V2_ENFORCE is active on this daemon. */
|
|
1037
|
+
enforce: boolean;
|
|
1038
|
+
/** Drain-path routing tallies (accept + enforce). Process-lifetime totals. */
|
|
1039
|
+
drain: {
|
|
1040
|
+
v2Delivered: number;
|
|
1041
|
+
v2RoutedAway: number;
|
|
1042
|
+
v2DedupSkipped: number;
|
|
1043
|
+
v2ValidationFailedAccepted: number;
|
|
1044
|
+
v2ReattributedToDrainer: number;
|
|
1045
|
+
v1BroadcastAccepted: number;
|
|
1046
|
+
v2ValidationFailedQuarantined: number;
|
|
1047
|
+
v1UnversionedQuarantined: number;
|
|
1048
|
+
};
|
|
1049
|
+
/** Last-resort backstop fire counts. Target 0 under a healthy v2 contract. */
|
|
1050
|
+
backstop: {
|
|
1051
|
+
phase4SynthesisFired: number;
|
|
1052
|
+
ackedHoldFastTrackFired: number;
|
|
1053
|
+
ackedHoldDeathDeadlineFired: number;
|
|
1054
|
+
};
|
|
1023
1055
|
}
|
|
1024
1056
|
|
|
1025
1057
|
/** One provider's version skew across mesh nodes (see RepoMeshStatus.providerVersionSkew). */
|