@adhdev/daemon-core 0.9.82-rc.407 → 0.9.82-rc.409
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/git/git-status.d.ts +34 -0
- package/dist/index.d.ts +1 -1
- package/dist/index.js +221 -95
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +220 -95
- package/dist/index.mjs.map +1 -1
- package/dist/mesh/mesh-queue-assignment.d.ts +1 -1
- package/dist/mesh/mesh-work-queue.d.ts +35 -1
- package/dist/providers/chat-message-normalization.d.ts +18 -0
- package/package.json +2 -2
- package/src/cli-adapters/terminal-backends/ghostty-vt-backend.ts +11 -1
- package/src/commands/chat-commands-write.ts +11 -0
- package/src/git/git-commands.ts +4 -1
- package/src/git/git-status.ts +219 -31
- package/src/index.ts +1 -1
- package/src/mesh/mesh-fast-forward.ts +5 -1
- package/src/mesh/mesh-queue-assignment.ts +8 -8
- package/src/mesh/mesh-reconcile-loop.ts +151 -94
- package/src/mesh/mesh-refine-gates.ts +6 -0
- package/src/mesh/mesh-runtime-store.ts +6 -5
- package/src/mesh/mesh-scheduling-runtime.ts +4 -7
- package/src/mesh/mesh-work-queue.ts +49 -8
- package/src/providers/chat-message-normalization.ts +53 -5
- package/src/providers/cli-provider-instance.ts +49 -8
|
@@ -6,7 +6,7 @@ import { getMesh } from '../config/mesh-config.js';
|
|
|
6
6
|
import { detectCLI } from '../detection/cli-detector.js';
|
|
7
7
|
import { LOG } from '../logging/logger.js';
|
|
8
8
|
import { appendLedgerEntry } from './mesh-ledger.js';
|
|
9
|
-
import { buildMeshNodeCapabilityTags, nodeSatisfiesRequiredTags, claimNextTask, updateTaskStatus, getQueue, recordTaskAutoLaunch, getActiveDirectDispatches } from './mesh-work-queue.js';
|
|
9
|
+
import { buildMeshNodeCapabilityTags, nodeSatisfiesRequiredTags, claimNextTask, updateTaskStatus, getQueue, recordTaskAutoLaunch, getActiveDirectDispatches, isTaskReadonly } from './mesh-work-queue.js';
|
|
10
10
|
import type { MeshWorkQueueEntry } from './mesh-work-queue.js';
|
|
11
11
|
import { fastForwardMeshNode } from './mesh-fast-forward.js';
|
|
12
12
|
import { createSessionDelivery, updateSessionDeliveryStatus } from './mesh-delivery-policy.js';
|
|
@@ -805,13 +805,13 @@ function activeAssignedCount(meshId: string): number {
|
|
|
805
805
|
* (everything except read-only diagnoses, which run unbounded by the write cap). */
|
|
806
806
|
export function activeWriteAssignedCount(meshId: string): number {
|
|
807
807
|
return getQueue(meshId, { status: ['assigned'] as any })
|
|
808
|
-
.filter(task => task
|
|
808
|
+
.filter(task => !isTaskReadonly(task)).length;
|
|
809
809
|
}
|
|
810
810
|
|
|
811
|
-
/** Active read-only
|
|
811
|
+
/** Active read-only assignments, for the read-only safety cap. */
|
|
812
812
|
export function activeReadonlyAssignedCount(meshId: string): number {
|
|
813
813
|
return getQueue(meshId, { status: ['assigned'] as any })
|
|
814
|
-
.filter(
|
|
814
|
+
.filter(isTaskReadonly).length;
|
|
815
815
|
}
|
|
816
816
|
|
|
817
817
|
function nodeHasActiveAssignment(meshId: string, nodeId: string): boolean {
|
|
@@ -1145,7 +1145,7 @@ async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, mesh
|
|
|
1145
1145
|
// higher safety cap (readonlyMultiplier × the write cap, default 2×).
|
|
1146
1146
|
const maxReadonlyParallelTasks = resolveMaxReadonlyParallelTasks(maxParallelTasks, schedulingOverride?.readonlyMultiplier);
|
|
1147
1147
|
for (const task of pending) {
|
|
1148
|
-
const isReadonly = task
|
|
1148
|
+
const isReadonly = isTaskReadonly(task);
|
|
1149
1149
|
if (isReadonly) {
|
|
1150
1150
|
if (activeReadonlyAssignedCount(meshId) >= maxReadonlyParallelTasks) {
|
|
1151
1151
|
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'max_readonly_parallel_tasks_reached' });
|
|
@@ -1293,9 +1293,9 @@ async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, mesh
|
|
|
1293
1293
|
continue;
|
|
1294
1294
|
}
|
|
1295
1295
|
// Write tasks keep the one-active-per-node invariant (worktree isolation);
|
|
1296
|
-
// read-only
|
|
1297
|
-
//
|
|
1298
|
-
if (task
|
|
1296
|
+
// read-only diagnoses may auto-launch onto a node that already has an active
|
|
1297
|
+
// assignment. Classified by the shared isTaskReadonly predicate.
|
|
1298
|
+
if (!isTaskReadonly(task) && nodeHasActiveAssignment(meshId, nodeId)) {
|
|
1299
1299
|
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'node_has_active_assignment', nodeId });
|
|
1300
1300
|
continue;
|
|
1301
1301
|
}
|
|
@@ -97,40 +97,56 @@ function resolveReconcileIntervalMs(): number {
|
|
|
97
97
|
return DEFAULT_RECONCILE_INTERVAL_MS;
|
|
98
98
|
}
|
|
99
99
|
|
|
100
|
-
//
|
|
101
|
-
//
|
|
102
|
-
//
|
|
103
|
-
//
|
|
104
|
-
//
|
|
105
|
-
//
|
|
106
|
-
// terminal that then masks the REAL completion when it lands seconds later
|
|
100
|
+
// R4f (GENERATING-BOUNDARY, acked-hold redesign). PHASE 4 only synthesizes a missing completion
|
|
101
|
+
// when the worker session reads `idle`. But a worker that is GENUINELY generating (it emitted
|
|
102
|
+
// agent:generating_started — the dispatch row is 'acked' — and has not yet completed) can
|
|
103
|
+
// momentarily read `idle` mid-turn (a CLI PTY inter-tool-call settle, or the final assistant text
|
|
104
|
+
// already rendered while the turn's generating_completed lifecycle close still lags). A premature
|
|
105
|
+
// synth writes a terminal that then masks the worker's REAL completion when it lands seconds later
|
|
107
106
|
// (drop:duplicate_completion_terminal_ledger; the observed 71s task a250fb44 lost its [System]
|
|
108
|
-
// notification this way
|
|
109
|
-
// before synthesizing for an `acked` (started-generating) dispatch — a transient mid-turn idle
|
|
110
|
-
// flicker clears on the next ~4s tick, whereas a genuinely-settled (completed-but-lost) or dead
|
|
111
|
-
// session reads idle every tick. A dispatch that was never acked (worker never started) is NOT
|
|
112
|
-
// debounced here: there is no in-flight generation to protect, and the downstream grace gate +
|
|
113
|
-
// stale-summary guard remain the backstops for that lost-dispatch case.
|
|
107
|
+
// notification this way; the R4e 53s task synth fired 16s BEFORE the worker's real emit).
|
|
114
108
|
//
|
|
115
|
-
//
|
|
116
|
-
//
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
//
|
|
120
|
-
//
|
|
121
|
-
//
|
|
122
|
-
//
|
|
123
|
-
//
|
|
124
|
-
//
|
|
125
|
-
//
|
|
126
|
-
//
|
|
127
|
-
//
|
|
128
|
-
//
|
|
129
|
-
//
|
|
130
|
-
//
|
|
131
|
-
//
|
|
132
|
-
//
|
|
133
|
-
//
|
|
109
|
+
// R4 → R4e used FINITE timers (consecutive ticks / MIN_IDLE_SETTLE / ACKED_TURN_SETTLE) to delay the
|
|
110
|
+
// synth. That class of fix is fundamentally a RACE: the worker's real emit latency is variable and
|
|
111
|
+
// unbounded (win32 idle reads can flip before the emit arrives), so ANY finite timer eventually
|
|
112
|
+
// loses to a slow-enough turn — and the synth pre-empts the real completion. R4e live-FAILED for
|
|
113
|
+
// exactly this reason.
|
|
114
|
+
//
|
|
115
|
+
// R4f redesign (direction B). An `acked` task means the worker ECHOED generating_started (the
|
|
116
|
+
// taskId flip) — it is alive and mid-turn, so it WILL eventually emit a real terminal. We therefore
|
|
117
|
+
// HOLD the synth INDEFINITELY for an acked task. This is safe against the emit actually arriving:
|
|
118
|
+
// when the worker's real generating_completed lands, it writes a terminal ledger, and
|
|
119
|
+
// reconcileDirectDispatchCompletionFromTranscript's hasTerminalLedgerAfterDispatch check makes any
|
|
120
|
+
// later synth an idempotent no-op (alreadyTerminal). So the hold never costs a missed notification —
|
|
121
|
+
// the real emit always wins, no matter how late.
|
|
122
|
+
//
|
|
123
|
+
// The indefinite hold is released ONLY by a genuine-DEATH / emit-loss BACKSTOP — never a finite
|
|
124
|
+
// timer that races normal lag:
|
|
125
|
+
// (a) liveness failure — read_chat reports the session is gone, OR N consecutive read failures
|
|
126
|
+
// accumulate (a transport/session-gone signal, counted as death rather than swallowed via
|
|
127
|
+
// `continue`). A worker that died mid-turn will never emit, so the synth must eventually fire.
|
|
128
|
+
// (b) an absolute LONG death-deadline — time since the generating_started ack exceeds
|
|
129
|
+
// ACKED_DEATH_DEADLINE_MS, a backstop set FAR above any observed emit latency (default 8 min)
|
|
130
|
+
// so it does not race a normal slow turn; it only catches a worker that is genuinely wedged or
|
|
131
|
+
// whose emit was permanently lost. This is a notification-loss net, not a completion timer.
|
|
132
|
+
//
|
|
133
|
+
// A dispatch that was never acked (worker never started) is NOT held here: there is no in-flight
|
|
134
|
+
// generation to protect, so it keeps the existing first-idle-tick synth behavior (its lost-dispatch
|
|
135
|
+
// case is covered by the downstream grace + stale-summary guards). The map is pruned each PHASE-4
|
|
136
|
+
// pass to the set of currently active dispatches, so a completed/pruned task's state is dropped (no
|
|
137
|
+
// unbounded growth). Keyed by `${meshId}::${taskId}`.
|
|
138
|
+
|
|
139
|
+
// R4f backstop (a): how many CONSECUTIVE read_chat failures (transport error / success:false /
|
|
140
|
+
// no payload) for an acked task are treated as a death signal that releases the indefinite hold.
|
|
141
|
+
// A single failed read is a transient probe blip; a session that genuinely died reads-fail every
|
|
142
|
+
// tick, so a small streak distinguishes the two without racing a live-but-slow worker.
|
|
143
|
+
const ACKED_DEATH_CONSECUTIVE_READ_FAILURES = 3;
|
|
144
|
+
|
|
145
|
+
// R4f backstop (b): the absolute death-deadline. An acked task is held indefinitely until this much
|
|
146
|
+
// time has elapsed since its generating_started ack (dispatch.updatedAt); past it, a persistently
|
|
147
|
+
// idle session is synthesized as a notification-loss net. This is set FAR above any observed emit
|
|
148
|
+
// latency (R4e's worst case was ~16s) so it does NOT race a normal slow turn — it only catches a
|
|
149
|
+
// genuinely wedged worker or a permanently-lost emit. Read at call time so tests can tune it.
|
|
134
150
|
function resolveTunedReconcileMs(envName: string, def: number, min: number, max: number): number {
|
|
135
151
|
const raw = readNonEmptyString(process.env[envName]);
|
|
136
152
|
if (raw) {
|
|
@@ -139,24 +155,28 @@ function resolveTunedReconcileMs(envName: string, def: number, min: number, max:
|
|
|
139
155
|
}
|
|
140
156
|
return def;
|
|
141
157
|
}
|
|
142
|
-
function
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
158
|
+
function resolveAckedDeathDeadlineMs(): number {
|
|
159
|
+
// Default 8 min — FAR above the variable emit latency the finite R4..R4e timers raced (R4e's
|
|
160
|
+
// worst case was ~16s); by the time this fires a live worker would long since have emitted its
|
|
161
|
+
// real terminal. The env-override floor is 0 so tests can force the deadline (production never
|
|
162
|
+
// sets it that low); the ceiling is 60min so a mis-set env cannot disable the loss-net forever.
|
|
163
|
+
return resolveTunedReconcileMs('MESH_INFLIGHT_ACKED_DEATH_DEADLINE_MS', 8 * 60_000, 0, 60 * 60_000);
|
|
147
164
|
}
|
|
148
165
|
|
|
149
|
-
// Per-task in-flight
|
|
150
|
-
//
|
|
151
|
-
|
|
166
|
+
// Per-task in-flight hold state for an acked dispatch:
|
|
167
|
+
// - liveConfirmedSinceAck: we have seen at least one conclusive read (idle OR generating) since
|
|
168
|
+
// the ack — proves the session is reachable, so a later read FAILURE is a genuine liveness loss
|
|
169
|
+
// rather than a node that was never reachable.
|
|
170
|
+
// - consecutiveReadFailures: streak of inconclusive read_chat results (death backstop (a)).
|
|
171
|
+
const inFlightAckedHoldState = new Map<string, { liveConfirmedSinceAck: boolean; consecutiveReadFailures: number }>();
|
|
152
172
|
|
|
153
173
|
function inFlightSynthKey(meshId: string, taskId: string): string {
|
|
154
174
|
return `${meshId}::${taskId}`;
|
|
155
175
|
}
|
|
156
176
|
|
|
157
|
-
// Test hook: clear the in-flight
|
|
177
|
+
// Test hook: clear the in-flight acked-hold state between cases.
|
|
158
178
|
export function __resetReconcileInFlightSynthDebounceForTests(): void {
|
|
159
|
-
|
|
179
|
+
inFlightAckedHoldState.clear();
|
|
160
180
|
}
|
|
161
181
|
|
|
162
182
|
interface LiveCoordinator {
|
|
@@ -1328,17 +1348,17 @@ async function reconcileUnterminatedDirectDispatches(
|
|
|
1328
1348
|
const dispatches = getActiveDirectDispatches(mesh.id);
|
|
1329
1349
|
if (dispatches.length === 0) return; // cheap exit — nothing dispatched, nothing to reconcile
|
|
1330
1350
|
|
|
1331
|
-
// Prune the in-flight
|
|
1332
|
-
// completed/pruned task's
|
|
1351
|
+
// Prune the in-flight acked-hold map to the tasks still active in THIS mesh, so a
|
|
1352
|
+
// completed/pruned task's state is dropped (the map never grows without bound).
|
|
1333
1353
|
const activeTaskKeys = new Set(
|
|
1334
1354
|
dispatches
|
|
1335
1355
|
.map(d => readNonEmptyString(d.taskId))
|
|
1336
1356
|
.filter(Boolean)
|
|
1337
1357
|
.map(taskId => inFlightSynthKey(mesh.id, taskId)),
|
|
1338
1358
|
);
|
|
1339
|
-
for (const key of
|
|
1359
|
+
for (const key of inFlightAckedHoldState.keys()) {
|
|
1340
1360
|
if (key.startsWith(`${mesh.id}::`) && !activeTaskKeys.has(key)) {
|
|
1341
|
-
|
|
1361
|
+
inFlightAckedHoldState.delete(key);
|
|
1342
1362
|
}
|
|
1343
1363
|
}
|
|
1344
1364
|
|
|
@@ -1369,74 +1389,109 @@ async function reconcileUnterminatedDirectDispatches(
|
|
|
1369
1389
|
...(providerType ? { agentType: providerType, providerType } : {}),
|
|
1370
1390
|
};
|
|
1371
1391
|
|
|
1392
|
+
const synthKey = inFlightSynthKey(mesh.id, taskId);
|
|
1393
|
+
const isAcked = dispatch.status === 'acked';
|
|
1394
|
+
|
|
1395
|
+
// R4f: read the worker session. A FAILED read (transport error / success:false / no payload)
|
|
1396
|
+
// is no longer silently swallowed for an acked task — it is the liveness side of the
|
|
1397
|
+
// death backstop (a). We classify the read result and route an acked failure into the
|
|
1398
|
+
// failure counter; a never-acked (or non-acked) failure keeps the old best-effort `continue`.
|
|
1372
1399
|
let payload: Record<string, unknown> | null = null;
|
|
1400
|
+
let readFailed = false;
|
|
1373
1401
|
try {
|
|
1374
1402
|
if (isLocalNode) {
|
|
1375
1403
|
const result = await components.commandHandler.handle('read_chat', readArgs);
|
|
1376
|
-
if (result && (result as { success?: boolean }).success === false)
|
|
1377
|
-
|
|
1404
|
+
if (result && (result as { success?: boolean }).success === false) {
|
|
1405
|
+
readFailed = true;
|
|
1406
|
+
} else {
|
|
1407
|
+
payload = unwrapReadChatPayload(result);
|
|
1408
|
+
}
|
|
1378
1409
|
} else if (dispatchMeshCommand) {
|
|
1379
1410
|
const result = await dispatchMeshCommand(nodeDaemonId, 'read_chat', readArgs);
|
|
1380
1411
|
payload = unwrapReadChatPayload(result);
|
|
1381
|
-
if (payload && (payload as { success?: boolean }).success === false)
|
|
1412
|
+
if (payload && (payload as { success?: boolean }).success === false) { payload = null; readFailed = true; }
|
|
1382
1413
|
} else {
|
|
1383
|
-
continue; // remote node but no P2P transport — can't read; retry next tick
|
|
1414
|
+
continue; // remote node but no P2P transport — can't read; retry next tick (not a death signal)
|
|
1384
1415
|
}
|
|
1385
1416
|
} catch {
|
|
1386
|
-
|
|
1417
|
+
readFailed = true; // session may be gone or node offline
|
|
1418
|
+
}
|
|
1419
|
+
if (!payload && !readFailed) continue; // null payload that wasn't a hard failure — retry next tick
|
|
1420
|
+
|
|
1421
|
+
if (readFailed || !payload) {
|
|
1422
|
+
// R4f backstop (a) — liveness failure. For a never-acked dispatch there is no in-flight
|
|
1423
|
+
// turn to protect, so a read failure is a transient probe blip → retry next tick (old
|
|
1424
|
+
// behavior). For an ACKED dispatch that we had previously confirmed live, a streak of
|
|
1425
|
+
// consecutive read failures means the worker session genuinely went away mid-turn and
|
|
1426
|
+
// will never emit its real completion — count it. The actual terminal cleanup of a
|
|
1427
|
+
// gone session is owned by PHASE 2.5 (stranded reclaim) / PHASE 5 (orphan prune); here
|
|
1428
|
+
// we only record the death observation and STOP holding so those nets can take over,
|
|
1429
|
+
// rather than pinning the row on an indefinite hold for a session that is already gone.
|
|
1430
|
+
if (isAcked) {
|
|
1431
|
+
const prior = inFlightAckedHoldState.get(synthKey);
|
|
1432
|
+
const failures = (prior?.consecutiveReadFailures ?? 0) + 1;
|
|
1433
|
+
const liveConfirmedSinceAck = prior?.liveConfirmedSinceAck ?? false;
|
|
1434
|
+
inFlightAckedHoldState.set(synthKey, { liveConfirmedSinceAck, consecutiveReadFailures: failures });
|
|
1435
|
+
if (liveConfirmedSinceAck && failures >= ACKED_DEATH_CONSECUTIVE_READ_FAILURES) {
|
|
1436
|
+
LOG.warn('MeshReconcile', `Acked-hold death signal: task ${taskId} on node ${nodeId} (mesh ${mesh.id}) read_chat failed ${failures}x consecutively after a live-confirmed ack — worker session presumed gone mid-turn; releasing the indefinite synth hold to the stranded-reclaim / orphan-prune nets`);
|
|
1437
|
+
}
|
|
1438
|
+
}
|
|
1439
|
+
continue; // no readable transcript this tick → cannot synth here; retry / let backstops act
|
|
1387
1440
|
}
|
|
1388
|
-
|
|
1441
|
+
|
|
1442
|
+
// Read succeeded (a conclusive idle/generating status) → the session is reachable: reset the
|
|
1443
|
+
// failure streak and mark it live-confirmed-since-ack, so a LATER read failure is recognized
|
|
1444
|
+
// as a genuine liveness loss (backstop a) rather than a node that was never reachable.
|
|
1445
|
+
inFlightAckedHoldState.set(synthKey, { liveConfirmedSinceAck: true, consecutiveReadFailures: 0 });
|
|
1389
1446
|
|
|
1390
1447
|
// Only act on a session that has actually settled to idle. A generating /
|
|
1391
1448
|
// waiting_approval session is mid-turn — synthesizing a completion now would
|
|
1392
1449
|
// be wrong. (idle is the only status the MCP poll path reconciles too.)
|
|
1393
|
-
const synthKey = inFlightSynthKey(mesh.id, taskId);
|
|
1394
1450
|
const nowMs = Date.now();
|
|
1395
1451
|
if (readChatPayloadStatus(payload) !== 'idle') {
|
|
1396
|
-
// Not idle → the worker is mid-turn
|
|
1397
|
-
//
|
|
1398
|
-
inFlightIdleObservationCounts.delete(synthKey);
|
|
1452
|
+
// Not idle → the worker is genuinely mid-turn (a clear live signal). Keep the
|
|
1453
|
+
// live-confirmed flag set (above) but otherwise just wait for the real emit.
|
|
1399
1454
|
continue;
|
|
1400
1455
|
}
|
|
1401
1456
|
|
|
1402
|
-
//
|
|
1403
|
-
//
|
|
1404
|
-
//
|
|
1405
|
-
// mid-turn window
|
|
1406
|
-
//
|
|
1407
|
-
//
|
|
1408
|
-
//
|
|
1409
|
-
//
|
|
1410
|
-
//
|
|
1411
|
-
//
|
|
1412
|
-
//
|
|
1413
|
-
//
|
|
1414
|
-
//
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
|
|
1418
|
-
|
|
1419
|
-
|
|
1420
|
-
const idleSettleMs = nowMs - firstIdleAtMs;
|
|
1421
|
-
const minIdleSettleMs = resolveMinIdleSettleMs();
|
|
1422
|
-
const ackedTurnSettleMs = resolveAckedTurnSettleMs();
|
|
1457
|
+
// R4f GENERATING-BOUNDARY (acked-hold): a dispatch whose worker was OBSERVED to start
|
|
1458
|
+
// generating (the agent:generating_started ack flipped the row to 'acked') is ALIVE and
|
|
1459
|
+
// mid-turn — it WILL eventually emit a real terminal. An `idle` read here is therefore
|
|
1460
|
+
// presumed a TRANSIENT mid-turn window (a PTY inter-tool-call settle, or final text already
|
|
1461
|
+
// rendered while the lifecycle close lags), NOT a settled completion. We HOLD the synth
|
|
1462
|
+
// INDEFINITELY rather than racing the worker's (variable, unbounded) emit latency with a
|
|
1463
|
+
// finite timer — the failure mode of R4..R4e. This is safe: when the worker's real emit
|
|
1464
|
+
// lands it writes a terminal ledger, and reconcileDirectDispatchCompletionFromTranscript's
|
|
1465
|
+
// hasTerminalLedgerAfterDispatch makes any later synth an idempotent no-op, so the real emit
|
|
1466
|
+
// always wins no matter how late. The hold is released ONLY by the death backstops:
|
|
1467
|
+
// (a) consecutive read failures after a live-confirmed ack (handled above), or
|
|
1468
|
+
// (b) the absolute ACKED_DEATH_DEADLINE_MS since the ack — a notification-loss net set FAR
|
|
1469
|
+
// above any observed emit latency, so it catches a genuinely-wedged worker / lost emit
|
|
1470
|
+
// without racing a normal slow turn.
|
|
1471
|
+
// A never-acked dispatch (worker never started) is exempt — no in-flight generation to
|
|
1472
|
+
// pre-empt; it keeps the first-idle-tick synth, with the downstream grace + stale-summary
|
|
1473
|
+
// guards as its backstops.
|
|
1474
|
+
if (isAcked) {
|
|
1423
1475
|
const ackedAtMs = Date.parse(readNonEmptyString(dispatch.updatedAt));
|
|
1424
1476
|
const sinceAckMs = Number.isFinite(ackedAtMs) ? nowMs - ackedAtMs : Number.POSITIVE_INFINITY;
|
|
1425
|
-
const
|
|
1426
|
-
|
|
1427
|
-
|
|
1428
|
-
if (!tickGuardMet || !settleGuardMet || !ackGuardMet) {
|
|
1429
|
-
LOG.info('MeshReconcile', `In-flight synth hold: task ${taskId} on node ${nodeId} (mesh ${mesh.id}) idle ${idleStreak}/${REQUIRED_CONSECUTIVE_IDLE_TICKS_FOR_INFLIGHT_SYNTH} tick(s), settle ${Math.round(idleSettleMs / 1000)}s/${Math.round(minIdleSettleMs / 1000)}s, since-ack ${Number.isFinite(sinceAckMs) ? Math.round(sinceAckMs / 1000) + 's' : '∞'}/${Math.round(ackedTurnSettleMs / 1000)}s — deferring completion synth until the worker's turn genuinely settles (guards against a mid-turn idle window pre-empting the real completion)`);
|
|
1477
|
+
const deathDeadlineMs = resolveAckedDeathDeadlineMs();
|
|
1478
|
+
if (sinceAckMs < deathDeadlineMs) {
|
|
1479
|
+
LOG.info('MeshReconcile', `Acked-hold: task ${taskId} on node ${nodeId} (mesh ${mesh.id}) read idle ${Number.isFinite(sinceAckMs) ? Math.round(sinceAckMs / 1000) + 's' : '∞'} since the generating_started ack — HOLDING synth indefinitely (worker is alive and will emit; a later real emit is idempotent). Death backstop fires at ${Math.round(deathDeadlineMs / 1000)}s or on consecutive read failures.`);
|
|
1430
1480
|
continue;
|
|
1431
1481
|
}
|
|
1482
|
+
LOG.warn('MeshReconcile', `Acked-hold death deadline reached: task ${taskId} on node ${nodeId} (mesh ${mesh.id}) still idle ${Math.round(sinceAckMs / 1000)}s after the ack (deadline ${Math.round(deathDeadlineMs / 1000)}s) — synthesizing the missing completion as a notification-loss net (a real emit, if it ever lands, no-ops idempotently).`);
|
|
1432
1483
|
}
|
|
1433
1484
|
|
|
1434
|
-
// R4e fix
|
|
1435
|
-
// already arrived in the pending-events queue (queued
|
|
1436
|
-
// not yet written a terminal ledger, YIELD
|
|
1437
|
-
// it with a synth that would win the taskId-anchored
|
|
1485
|
+
// R4f (auxiliary, was R4e fix 3) — worker-emit priority. Secondary check: if the worker's
|
|
1486
|
+
// REAL terminal emit for this task has already arrived in the pending-events queue (queued
|
|
1487
|
+
// for delivery to the coordinator) but not yet written a terminal ledger, YIELD — let the
|
|
1488
|
+
// genuine emit surface rather than racing it with a synth that would win the taskId-anchored
|
|
1489
|
+
// fingerprint dedup and mask it. Under the R4f acked-hold this is now an auxiliary belt-and-
|
|
1490
|
+
// suspenders check (the indefinite hold already defers an acked synth); it still guards the
|
|
1491
|
+
// never-acked path and the post-death-deadline acked synth from racing an emit caught in
|
|
1492
|
+
// flight at synth-commit time.
|
|
1438
1493
|
if (realTerminalEmitPendingForTask(mesh.id, taskId)) {
|
|
1439
|
-
|
|
1494
|
+
inFlightAckedHoldState.delete(synthKey);
|
|
1440
1495
|
LOG.info('MeshReconcile', `Worker-emit priority: task ${taskId} on node ${nodeId} (mesh ${mesh.id}) has a real terminal completion already queued — yielding synth to the worker's own emit`);
|
|
1441
1496
|
continue;
|
|
1442
1497
|
}
|
|
@@ -1471,15 +1526,17 @@ async function reconcileUnterminatedDirectDispatches(
|
|
|
1471
1526
|
continue;
|
|
1472
1527
|
}
|
|
1473
1528
|
|
|
1474
|
-
// R4e fix
|
|
1475
|
-
//
|
|
1476
|
-
//
|
|
1477
|
-
//
|
|
1478
|
-
//
|
|
1479
|
-
//
|
|
1529
|
+
// R4f (auxiliary, was R4e fix 2) — live re-probe immediately before committing the synth. A
|
|
1530
|
+
// fresh read right now catches a worker that resumed generating since this tick's first read
|
|
1531
|
+
// so it is never falsely completed off a stale snapshot. Best-effort: an inconclusive
|
|
1532
|
+
// re-probe (transport error/null) falls through to the synth — we already hold a valid idle
|
|
1533
|
+
// read from the top of THIS tick, so a re-probe failure must not re-introduce a
|
|
1534
|
+
// notification-miss. Under the R4f acked-hold this matters mainly for the never-acked path
|
|
1535
|
+
// and the post-death-deadline acked synth (the indefinite hold already deferred a live acked
|
|
1536
|
+
// turn); it stays as a final live-state guard at synth-commit time.
|
|
1480
1537
|
const reprobeStatus = await reprobeWorkerStatus(components, { isLocalNode, nodeDaemonId, readArgs });
|
|
1481
1538
|
if (reprobeStatus && reprobeStatus !== 'idle') {
|
|
1482
|
-
|
|
1539
|
+
inFlightAckedHoldState.delete(synthKey);
|
|
1483
1540
|
LOG.info('MeshReconcile', `Live re-probe defer: task ${taskId} on node ${nodeId} (mesh ${mesh.id}) read '${reprobeStatus}' at synth-commit time — worker resumed generating; deferring synth to a later tick`);
|
|
1484
1541
|
continue;
|
|
1485
1542
|
}
|
|
@@ -1160,6 +1160,9 @@ export async function alignRefinerySubmodulesAfterMerge(
|
|
|
1160
1160
|
includeSubmodules: true,
|
|
1161
1161
|
submoduleIgnorePaths: options.submoduleIgnorePaths,
|
|
1162
1162
|
timeoutMs: 15_000,
|
|
1163
|
+
// Decision path — the out-of-sync submodule set drives a mutating `submodule
|
|
1164
|
+
// update`. Must not act on a TTL-cached status; bypass the C1 cache.
|
|
1165
|
+
forceFresh: true,
|
|
1163
1166
|
});
|
|
1164
1167
|
const outOfSyncPaths = (preStatus.submodules || [])
|
|
1165
1168
|
.filter(submodule => submodule.dirty || submodule.outOfSync || !!submodule.error)
|
|
@@ -1193,6 +1196,9 @@ export async function alignRefinerySubmodulesAfterMerge(
|
|
|
1193
1196
|
includeSubmodules: true,
|
|
1194
1197
|
submoduleIgnorePaths: options.submoduleIgnorePaths,
|
|
1195
1198
|
timeoutMs: 15_000,
|
|
1199
|
+
// Re-read AFTER `submodule update` mutated the tree — MUST be fresh, never the
|
|
1200
|
+
// cached preStatus from moments ago (which would falsely report still-dirty).
|
|
1201
|
+
forceFresh: true,
|
|
1196
1202
|
});
|
|
1197
1203
|
const remaining = (postStatus.submodules || [])
|
|
1198
1204
|
.filter(submodule => updatePaths.includes(submodule.path) && (submodule.dirty || submodule.outOfSync || !!submodule.error));
|
|
@@ -3,7 +3,7 @@ import { dirname, join } from 'path';
|
|
|
3
3
|
import { LOG } from '../logging/logger.js';
|
|
4
4
|
import { loadBetterSqlite3 } from '../system/load-better-sqlite3.js';
|
|
5
5
|
import { getLedgerDir } from './mesh-ledger.js';
|
|
6
|
-
import { nodeSatisfiesRequiredTags } from './mesh-work-queue.js';
|
|
6
|
+
import { nodeSatisfiesRequiredTags, isTaskReadonly } from './mesh-work-queue.js';
|
|
7
7
|
import { meshNodeIdMatches, daemonIdsEquivalent, expandDaemonIdForms } from '@adhdev/mesh-shared';
|
|
8
8
|
import type { MeshTaskStatus, MeshWorkQueueEntry } from './mesh-work-queue.js';
|
|
9
9
|
import type BetterSqlite3 from 'better-sqlite3';
|
|
@@ -752,11 +752,12 @@ export class MeshRuntimeStore {
|
|
|
752
752
|
return deps.every(depId => depStatus.get(depId) === 'completed');
|
|
753
753
|
};
|
|
754
754
|
|
|
755
|
-
// Per-candidate node-conflict gate: write tasks
|
|
756
|
-
//
|
|
757
|
-
//
|
|
755
|
+
// Per-candidate node-conflict gate: write tasks require an idle node; read-only
|
|
756
|
+
// tasks bypass the node-busy check so N read-only diagnoses can run on one node
|
|
757
|
+
// at once. Read-only classification is decided solely by isTaskReadonly (the
|
|
758
|
+
// single predicate shared with the cap counters / auto-launch / guardrail).
|
|
758
759
|
const nodeConflictAllows = (candidate: MeshWorkQueueEntry): boolean => {
|
|
759
|
-
if (candidate
|
|
760
|
+
if (isTaskReadonly(candidate)) return true;
|
|
760
761
|
return !nodeBusy;
|
|
761
762
|
};
|
|
762
763
|
|
|
@@ -22,6 +22,7 @@ import {
|
|
|
22
22
|
} from '../repo-mesh-types.js';
|
|
23
23
|
import { normalizeMeshNodeId } from '@adhdev/mesh-shared';
|
|
24
24
|
import type { MeshWorkQueueEntry } from './mesh-work-queue.js';
|
|
25
|
+
import { isTaskReadonly } from './mesh-work-queue.js';
|
|
25
26
|
|
|
26
27
|
/** Per-(node, provider) cap and its current consumption. */
|
|
27
28
|
export interface MeshNodeProviderSchedulingRuntime {
|
|
@@ -80,10 +81,6 @@ interface MeshLike {
|
|
|
80
81
|
nodes?: Array<{ id?: string; nodeId?: string; node_id?: string; policy?: RepoMeshNodePolicy | null; isLocalWorktree?: boolean }> | null;
|
|
81
82
|
}
|
|
82
83
|
|
|
83
|
-
function isReadonly(task: MeshWorkQueueEntry): boolean {
|
|
84
|
-
return task.taskMode === 'live_debug_readonly';
|
|
85
|
-
}
|
|
86
|
-
|
|
87
84
|
function isAssigned(task: MeshWorkQueueEntry): boolean {
|
|
88
85
|
return task.status === 'assigned';
|
|
89
86
|
}
|
|
@@ -112,8 +109,8 @@ export function buildMeshSchedulingRuntime(
|
|
|
112
109
|
const maxReadonlyParallelTasks = resolveMaxReadonlyParallelTasks(maxParallelTasks);
|
|
113
110
|
|
|
114
111
|
const assignedTasks = (Array.isArray(queue) ? queue : []).filter(isAssigned);
|
|
115
|
-
const activeWriteAssigned = assignedTasks.filter(t => !
|
|
116
|
-
const activeReadonlyAssigned = assignedTasks.filter(
|
|
112
|
+
const activeWriteAssigned = assignedTasks.filter(t => !isTaskReadonly(t)).length;
|
|
113
|
+
const activeReadonlyAssigned = assignedTasks.filter(isTaskReadonly).length;
|
|
117
114
|
const globalWriteCapReached = activeWriteAssigned >= maxParallelTasks;
|
|
118
115
|
const globalReadonlyCapReached = activeReadonlyAssigned >= maxReadonlyParallelTasks;
|
|
119
116
|
|
|
@@ -126,7 +123,7 @@ export function buildMeshSchedulingRuntime(
|
|
|
126
123
|
const nodeId = typeof task.assignedNodeId === 'string' ? task.assignedNodeId.trim() : '';
|
|
127
124
|
if (!nodeId) continue;
|
|
128
125
|
assignedByNode.set(nodeId, (assignedByNode.get(nodeId) ?? 0) + 1);
|
|
129
|
-
if (!
|
|
126
|
+
if (!isTaskReadonly(task)) {
|
|
130
127
|
writeAssignedByNode.set(nodeId, (writeAssignedByNode.get(nodeId) ?? 0) + 1);
|
|
131
128
|
}
|
|
132
129
|
const provider = typeof task.assignedProviderType === 'string' ? task.assignedProviderType : '';
|
|
@@ -18,6 +18,29 @@ export const ACTIVE_MESH_QUEUE_STATUSES: MeshActiveTaskStatus[] = ['pending', 'a
|
|
|
18
18
|
export const HISTORICAL_MESH_QUEUE_STATUSES: MeshHistoricalTaskStatus[] = ['completed', 'failed', 'cancelled'];
|
|
19
19
|
export const MESH_TASK_MODES: MeshTaskMode[] = ['code_change', 'validation', 'live_debug_readonly', 'launch_app', 'convergence'];
|
|
20
20
|
|
|
21
|
+
/**
|
|
22
|
+
* QUEUE-NODE-SERIALIZATION: single source of truth for "is this task read-only?".
|
|
23
|
+
*
|
|
24
|
+
* Read-only classification used to be inlined as `task.taskMode === 'live_debug_readonly'`
|
|
25
|
+
* at every enforcement site (node-conflict claim gate, auto-launch isolation, the
|
|
26
|
+
* write/readonly cap counters, the write guardrail). That spread-out comparison is the
|
|
27
|
+
* exact recurring-defect class — one site drifting from the others silently makes the same
|
|
28
|
+
* task read-only at some gates and write at others, i.e. partial serialization. All sites
|
|
29
|
+
* MUST call this predicate so the classification is decided in exactly one place.
|
|
30
|
+
*
|
|
31
|
+
* Two orthogonal inputs feed the same boolean axis (kept backward-compatible):
|
|
32
|
+
* • `readonly === true` — the explicit boolean axis (new API surface).
|
|
33
|
+
* • `taskMode === 'live_debug_readonly'` — the original enum value, preserved as an
|
|
34
|
+
* OR-fallback so existing live_debug_readonly tasks keep behaving identically.
|
|
35
|
+
*
|
|
36
|
+
* Accepts any task-like shape (full {@link MeshWorkQueueEntry} or a bare
|
|
37
|
+
* `{ readonly?, taskMode? }`) so the daemon-core and mcp-server boundaries can share it.
|
|
38
|
+
*/
|
|
39
|
+
export function isTaskReadonly(task: { readonly?: boolean; taskMode?: MeshTaskMode | string } | null | undefined): boolean {
|
|
40
|
+
if (!task) return false;
|
|
41
|
+
return task.readonly === true || task.taskMode === 'live_debug_readonly';
|
|
42
|
+
}
|
|
43
|
+
|
|
21
44
|
export interface MeshTaskModeValidationResult {
|
|
22
45
|
valid: boolean;
|
|
23
46
|
taskMode?: MeshTaskMode;
|
|
@@ -410,13 +433,15 @@ export function normalizeMeshTaskMode(value: unknown): MeshTaskMode | undefined
|
|
|
410
433
|
return (MESH_TASK_MODES as string[]).includes(normalized) ? normalized : undefined;
|
|
411
434
|
}
|
|
412
435
|
|
|
413
|
-
export function validateMeshTaskModeRequest(mode: unknown, message: string): MeshTaskModeValidationResult {
|
|
436
|
+
export function validateMeshTaskModeRequest(mode: unknown, message: string, readonly?: boolean): MeshTaskModeValidationResult {
|
|
414
437
|
const taskMode = normalizeMeshTaskMode(mode);
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
438
|
+
// QUEUE-NODE-SERIALIZATION: the write guardrail (reject deploy/push/edit commands on a
|
|
439
|
+
// read-only task) is driven by the unified read-only axis, not by the enum alone — so a
|
|
440
|
+
// task flagged read-only via the explicit `readonly:true` boolean is guarded identically
|
|
441
|
+
// to a legacy live_debug_readonly task. isTaskReadonly is the single classifier.
|
|
442
|
+
const isReadonly = isTaskReadonly({ readonly, taskMode });
|
|
443
|
+
if (!isReadonly) {
|
|
444
|
+
return taskMode ? { valid: true, taskMode, violations: [] } : { valid: true, violations: [] };
|
|
420
445
|
}
|
|
421
446
|
const text = message || '';
|
|
422
447
|
// Only flag keywords that look like real commands (code/command context) and
|
|
@@ -447,6 +472,14 @@ export interface MeshWorkQueueEntry {
|
|
|
447
472
|
message: string;
|
|
448
473
|
status: MeshTaskStatus;
|
|
449
474
|
taskMode?: MeshTaskMode;
|
|
475
|
+
/**
|
|
476
|
+
* QUEUE-NODE-SERIALIZATION: explicit read-only axis, orthogonal to taskMode. When
|
|
477
|
+
* true the task is treated as read-only by every scheduling gate (no node-busy
|
|
478
|
+
* isolation, counted under the read-only cap, write commands rejected) regardless of
|
|
479
|
+
* its taskMode. Decided exclusively through {@link isTaskReadonly}; `taskMode ===
|
|
480
|
+
* 'live_debug_readonly'` remains an OR-fallback so legacy rows behave unchanged.
|
|
481
|
+
*/
|
|
482
|
+
readonly?: boolean;
|
|
450
483
|
/** If specified, only this node can claim the task (used by legacy mesh_send_task) */
|
|
451
484
|
targetNodeId?: string;
|
|
452
485
|
/** If specified, only this runtime session can claim the task */
|
|
@@ -722,6 +755,8 @@ export function enqueueTask(
|
|
|
722
755
|
targetNodeId?: string;
|
|
723
756
|
targetSessionId?: string;
|
|
724
757
|
taskMode?: MeshTaskMode | string;
|
|
758
|
+
/** QUEUE-NODE-SERIALIZATION: explicit read-only axis (orthogonal to taskMode). */
|
|
759
|
+
readonly?: boolean;
|
|
725
760
|
requiredTags?: string[];
|
|
726
761
|
/** M1: tasks that must complete before this one is claimable. */
|
|
727
762
|
dependsOn?: string[];
|
|
@@ -734,7 +769,8 @@ export function enqueueTask(
|
|
|
734
769
|
} & MeshQueueMutationOptions,
|
|
735
770
|
): MeshWorkQueueEntry {
|
|
736
771
|
requireMeshHostQueueOwner(opts);
|
|
737
|
-
const
|
|
772
|
+
const readonly = opts?.readonly === true;
|
|
773
|
+
const modeValidation = validateMeshTaskModeRequest(opts?.taskMode, message, readonly);
|
|
738
774
|
if (!modeValidation.valid) {
|
|
739
775
|
throw new Error(`live_debug_readonly_guardrail_violation: forbidden operations (${modeValidation.violations.join(', ')})`);
|
|
740
776
|
}
|
|
@@ -763,6 +799,7 @@ export function enqueueTask(
|
|
|
763
799
|
message,
|
|
764
800
|
status: 'pending',
|
|
765
801
|
taskMode: modeValidation.taskMode,
|
|
802
|
+
...(readonly ? { readonly: true } : {}),
|
|
766
803
|
targetNodeId: opts?.targetNodeId,
|
|
767
804
|
targetSessionId: opts?.targetSessionId,
|
|
768
805
|
requiredTags: resolvedRequiredTags,
|
|
@@ -806,6 +843,8 @@ export function recordDirectDispatchTask(
|
|
|
806
843
|
assignedNodeId?: string;
|
|
807
844
|
assignedSessionId?: string;
|
|
808
845
|
taskMode?: MeshTaskMode | string;
|
|
846
|
+
/** QUEUE-NODE-SERIALIZATION: explicit read-only axis (orthogonal to taskMode). */
|
|
847
|
+
readonly?: boolean;
|
|
809
848
|
dispatchedAt?: string;
|
|
810
849
|
},
|
|
811
850
|
): MeshWorkQueueEntry | null {
|
|
@@ -813,7 +852,8 @@ export function recordDirectDispatchTask(
|
|
|
813
852
|
if (!missionId) return null;
|
|
814
853
|
const taskId = typeof opts.id === 'string' ? opts.id.trim() : '';
|
|
815
854
|
if (!taskId) return null;
|
|
816
|
-
const
|
|
855
|
+
const readonly = opts.readonly === true;
|
|
856
|
+
const modeValidation = validateMeshTaskModeRequest(opts.taskMode, message, readonly);
|
|
817
857
|
if (!modeValidation.valid) {
|
|
818
858
|
throw new Error(`live_debug_readonly_guardrail_violation: forbidden operations (${modeValidation.violations.join(', ')})`);
|
|
819
859
|
}
|
|
@@ -829,6 +869,7 @@ export function recordDirectDispatchTask(
|
|
|
829
869
|
message,
|
|
830
870
|
status: 'assigned',
|
|
831
871
|
...(modeValidation.taskMode ? { taskMode: modeValidation.taskMode } : {}),
|
|
872
|
+
...(readonly ? { readonly: true } : {}),
|
|
832
873
|
missionId,
|
|
833
874
|
...(opts.assignedNodeId ? { targetNodeId: opts.assignedNodeId, assignedNodeId: opts.assignedNodeId } : {}),
|
|
834
875
|
...(opts.assignedSessionId ? { targetSessionId: opts.assignedSessionId, assignedSessionId: opts.assignedSessionId } : {}),
|