@adhdev/daemon-core 0.9.82-rc.532 → 0.9.82-rc.534
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli-adapters/provider-cli-adapter.d.ts +26 -0
- package/dist/index.js +156 -25
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +156 -25
- package/dist/index.mjs.map +1 -1
- package/dist/mesh/mesh-delivery-policy.d.ts +7 -0
- package/dist/mesh/mesh-runtime-store.d.ts +25 -0
- package/package.json +3 -3
- package/src/cli-adapters/provider-cli-adapter.ts +40 -3
- package/src/mesh/mesh-delivery-policy.ts +24 -4
- package/src/mesh/mesh-event-forwarding.ts +9 -11
- package/src/mesh/mesh-reconcile-loop.ts +42 -2
- package/src/mesh/mesh-runtime-store.ts +103 -1
|
@@ -96,6 +96,13 @@ export declare function getActiveSessionDeliveries(meshId: string, sessionId?: s
|
|
|
96
96
|
createdAt: string;
|
|
97
97
|
updatedAt: string;
|
|
98
98
|
}[];
|
|
99
|
+
/**
|
|
100
|
+
* DELIVERED-NOT-CONSUMED-REDRIVE consume path. Advance a task's delivery record(s) to a
|
|
101
|
+
* CONSUMED status ('acked'/'completed') by (mesh, session[, task]), INCLUDING rows already in
|
|
102
|
+
* 'delivered' — unlike getActiveSessionDeliveries which excludes 'delivered'. The store's
|
|
103
|
+
* monotonic guard only ever advances the row. Returns the number of rows advanced.
|
|
104
|
+
*/
|
|
105
|
+
export declare function consumeSessionDelivery(meshId: string, sessionId: string, status: 'acked' | 'completed', taskId?: string): number;
|
|
99
106
|
export declare function __clearSessionDeliveriesForTests(meshId: string): void;
|
|
100
107
|
/**
|
|
101
108
|
* Mark all active (queued/delivering/delivered/acked) deliveries for a session as completed or failed.
|
|
@@ -207,10 +207,35 @@ export declare class MeshRuntimeStore {
|
|
|
207
207
|
createdAt: string;
|
|
208
208
|
updatedAt: string;
|
|
209
209
|
}): void;
|
|
210
|
+
private static readonly DELIVERY_PROGRESS_RANK;
|
|
210
211
|
updateSessionDeliveryStatus(id: string, status: string, opts?: {
|
|
211
212
|
lastError?: string;
|
|
212
213
|
incrementAttempt?: boolean;
|
|
213
214
|
}): void;
|
|
215
|
+
/**
|
|
216
|
+
* DELIVERED-NOT-CONSUMED-REDRIVE consume path. Advance a task's delivery record(s) to a
|
|
217
|
+
* CONSUMED status ('acked' or 'completed'), matching on mesh + session (+ taskId when the
|
|
218
|
+
* event names one) and INCLUDING rows already in 'delivered'/'acked'/'delivering'.
|
|
219
|
+
*
|
|
220
|
+
* The ack/terminal callers previously routed through getActiveSessionDeliveries(), whose SQL
|
|
221
|
+
* EXCLUDES 'delivered' — so in the normal event order (transport confirm flips 'delivered'
|
|
222
|
+
* BEFORE the worker's generating_started fires) the ack matched zero rows and the delivery
|
|
223
|
+
* was stranded 'delivered', never 'acked'. This finds the row by (mesh, session[, task])
|
|
224
|
+
* directly and relies on updateSessionDeliveryStatus's monotonic guard to only advance it.
|
|
225
|
+
* Returns the number of rows advanced.
|
|
226
|
+
*/
|
|
227
|
+
consumeSessionDelivery(meshId: string, sessionId: string, status: 'acked' | 'completed', taskId?: string): number;
|
|
228
|
+
/**
|
|
229
|
+
* DELIVERED-NOT-CONSUMED-REDRIVE terminal path. Mark every OPEN delivery for a session
|
|
230
|
+
* (queued/delivering/delivered/acked) terminal on task completion/failure. The prior
|
|
231
|
+
* markSessionDeliveriesTerminal() routed through getActiveSessionDeliveries(), whose SQL
|
|
232
|
+
* EXCLUDES 'delivered'/'completed' — so a 'delivered' row (the common case, since the
|
|
233
|
+
* transport confirm flips it before the completion event) was never marked terminal and
|
|
234
|
+
* stayed 'delivered', keeping taskDeliveryConsumed() false and feeding the false re-drive.
|
|
235
|
+
* We match rows in OPEN states directly here. 'completed' advances monotonically (it is the
|
|
236
|
+
* top progress rank); 'failed' is an absorbing outcome written unconditionally.
|
|
237
|
+
*/
|
|
238
|
+
markOpenSessionDeliveriesTerminal(meshId: string, sessionId: string, terminalStatus: 'completed' | 'failed'): number;
|
|
214
239
|
getActiveSessionDeliveries(meshId: string, sessionId?: string): Array<{
|
|
215
240
|
id: string;
|
|
216
241
|
meshId: string;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@adhdev/daemon-core",
|
|
3
|
-
"version": "0.9.82-rc.
|
|
3
|
+
"version": "0.9.82-rc.534",
|
|
4
4
|
"description": "ADHDev daemon core — CDP, IDE detection, providers, command execution",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -47,8 +47,8 @@
|
|
|
47
47
|
"author": "vilmire",
|
|
48
48
|
"license": "AGPL-3.0-or-later",
|
|
49
49
|
"dependencies": {
|
|
50
|
-
"@adhdev/mesh-shared": "0.9.82-rc.
|
|
51
|
-
"@adhdev/session-host-core": "0.9.82-rc.
|
|
50
|
+
"@adhdev/mesh-shared": "0.9.82-rc.534",
|
|
51
|
+
"@adhdev/session-host-core": "0.9.82-rc.534",
|
|
52
52
|
"@agentclientprotocol/sdk": "^0.16.1",
|
|
53
53
|
"ajv": "^8.20.0",
|
|
54
54
|
"ajv-formats": "^3.0.1",
|
|
@@ -787,8 +787,8 @@ export class ProviderCliAdapter implements CliAdapter {
|
|
|
787
787
|
if (stableMs < 2000) return;
|
|
788
788
|
|
|
789
789
|
const startupModal = this.runParseApproval(this.recentOutputBuffer);
|
|
790
|
-
const
|
|
791
|
-
if (!startupModal &&
|
|
790
|
+
const startupIdle = this.detectIdleHonoringOnNoMatch(screenText || this.recentOutputBuffer);
|
|
791
|
+
if (!startupModal && !startupIdle) {
|
|
792
792
|
this.scheduleStartupSettleCheck();
|
|
793
793
|
return;
|
|
794
794
|
}
|
|
@@ -937,6 +937,43 @@ export class ProviderCliAdapter implements CliAdapter {
|
|
|
937
937
|
});
|
|
938
938
|
}
|
|
939
939
|
|
|
940
|
+
/**
|
|
941
|
+
* WRITE-READINESS ONNOMATCH (opencode): resolve whether the session is
|
|
942
|
+
* genuinely idle *for the purpose of opening the PTY write gate*, honoring
|
|
943
|
+
* the manifest's `dispatchOrder.onNoMatch` policy.
|
|
944
|
+
*
|
|
945
|
+
* The split-brain this closes: the engine's settled evaluation runs
|
|
946
|
+
* detectStatus through parseSession, whose builder collapses a null verdict
|
|
947
|
+
* to `'idle'` (buildParseSessionFromTui: `detectStatus(input) ?? 'idle'`),
|
|
948
|
+
* so the dashboard status flips to idle via `script_detect`. But the
|
|
949
|
+
* write-readiness gates (resolveStartupState / sendMessage recovery) call
|
|
950
|
+
* `runDetectStatus` *directly* and require the literal `=== 'idle'` return —
|
|
951
|
+
* they never apply the `onNoMatch` policy. opencode's only idle cue is the
|
|
952
|
+
* `Ask anything` composer placeholder in the last-8-lines scope with
|
|
953
|
+
* `onNoMatch: preserve-last`; when that placeholder is momentarily out of
|
|
954
|
+
* frame the raw detector returns null, the gate stays shut, `this.ready`
|
|
955
|
+
* never flips, and the first queued prompt sits in `not_ready_pending_prompt`
|
|
956
|
+
* forever (no turn ever starts, so no turn-completion drain fires).
|
|
957
|
+
*
|
|
958
|
+
* The fix keeps the raw detector as the primary signal (unchanged behavior
|
|
959
|
+
* for providers whose detector returns a literal idle) and only falls back
|
|
960
|
+
* when BOTH (a) the manifest policy is idle-preserving (`preserve-last` or
|
|
961
|
+
* `idle`) AND (b) the engine has *durably* settled to idle (no in-flight
|
|
962
|
+
* turn, no modal, no parse error). That guard makes the fallback safe: it
|
|
963
|
+
* cannot open the gate mid-turn or while a modal is up.
|
|
964
|
+
*/
|
|
965
|
+
private detectIdleHonoringOnNoMatch(text: string): boolean {
|
|
966
|
+
if (this.runDetectStatus(text) === 'idle') return true;
|
|
967
|
+
const dispatchOrder = (this.provider.tui as { dispatchOrder?: { onNoMatch?: unknown } } | undefined)?.dispatchOrder;
|
|
968
|
+
const onNoMatch = dispatchOrder?.onNoMatch;
|
|
969
|
+
if (onNoMatch !== 'preserve-last' && onNoMatch !== 'idle') return false;
|
|
970
|
+
return this.engine.currentStatus === 'idle'
|
|
971
|
+
&& !this.engine.isWaitingForResponse
|
|
972
|
+
&& !this.engine.currentTurnScope
|
|
973
|
+
&& !this.engine.activeModal
|
|
974
|
+
&& !this.parseErrorMessage;
|
|
975
|
+
}
|
|
976
|
+
|
|
940
977
|
runParseApproval(tail: string): { message: string; buttons: string[] } | null {
|
|
941
978
|
const screenText = this.terminalScreen.getText();
|
|
942
979
|
const buffer = screenText || this.accumulatedBuffer;
|
|
@@ -1706,7 +1743,7 @@ export class ProviderCliAdapter implements CliAdapter {
|
|
|
1706
1743
|
}
|
|
1707
1744
|
if (!this.ready) {
|
|
1708
1745
|
this.resolveStartupState('send_precheck');
|
|
1709
|
-
if (this.
|
|
1746
|
+
if (!this.ready && this.detectIdleHonoringOnNoMatch(this.recentOutputBuffer)) {
|
|
1710
1747
|
this.ready = true;
|
|
1711
1748
|
this.startupParseGate = false;
|
|
1712
1749
|
this.engine.setStatus('idle', 'send_message_idle_prompt_recovery');
|
|
@@ -254,6 +254,25 @@ export function getActiveSessionDeliveries(meshId: string, sessionId?: string) {
|
|
|
254
254
|
}
|
|
255
255
|
}
|
|
256
256
|
|
|
257
|
+
/**
|
|
258
|
+
* DELIVERED-NOT-CONSUMED-REDRIVE consume path. Advance a task's delivery record(s) to a
|
|
259
|
+
* CONSUMED status ('acked'/'completed') by (mesh, session[, task]), INCLUDING rows already in
|
|
260
|
+
* 'delivered' — unlike getActiveSessionDeliveries which excludes 'delivered'. The store's
|
|
261
|
+
* monotonic guard only ever advances the row. Returns the number of rows advanced.
|
|
262
|
+
*/
|
|
263
|
+
export function consumeSessionDelivery(
|
|
264
|
+
meshId: string,
|
|
265
|
+
sessionId: string,
|
|
266
|
+
status: 'acked' | 'completed',
|
|
267
|
+
taskId?: string,
|
|
268
|
+
): number {
|
|
269
|
+
try {
|
|
270
|
+
return MeshRuntimeStore.getInstance().consumeSessionDelivery(meshId, sessionId, status, taskId);
|
|
271
|
+
} catch {
|
|
272
|
+
return 0;
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
|
|
257
276
|
// MESH-COMPLEXITY-AUDIT Part 8-2: the completion-conflict diagnostic
|
|
258
277
|
// (recordCompletionConflict / getRecentCompletionConflicts, backed by
|
|
259
278
|
// mesh_completion_conflicts) was dropped. It recorded WHICH task lost a
|
|
@@ -276,9 +295,10 @@ export function markSessionDeliveriesTerminal(
|
|
|
276
295
|
terminalStatus: 'completed' | 'failed',
|
|
277
296
|
): void {
|
|
278
297
|
try {
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
298
|
+
// Route through markOpenSessionDeliveriesTerminal, which matches OPEN rows including
|
|
299
|
+
// 'delivered' — getActiveSessionDeliveries EXCLUDES 'delivered' and would silently
|
|
300
|
+
// leave the common (already-delivered) row un-terminated, keeping taskDeliveryConsumed()
|
|
301
|
+
// false and feeding the delivered_not_consumed_redrive false re-drive.
|
|
302
|
+
MeshRuntimeStore.getInstance().markOpenSessionDeliveriesTerminal(meshId, sessionId, terminalStatus);
|
|
283
303
|
} catch { /* best-effort */ }
|
|
284
304
|
}
|
|
@@ -5,7 +5,7 @@ import { LOG } from '../logging/logger.js';
|
|
|
5
5
|
import { appendLedgerEntry, buildTaskCompletionEvidence, getSessionRecoveryContext, isIntentionalCleanupStopEntry, readLedgerEntries } from './mesh-ledger.js';
|
|
6
6
|
import type { SessionRecoveryContext } from './mesh-ledger.js';
|
|
7
7
|
import { updateSessionTaskStatus, enqueueTask, updateDirectDispatchStatus, cleanupTerminalDirectDispatches, getActiveDirectDispatches, hasPendingDependents, getQueue } from './mesh-work-queue.js';
|
|
8
|
-
import { markSessionDeliveriesTerminal, updateSessionDeliveryStatus } from './mesh-delivery-policy.js';
|
|
8
|
+
import { markSessionDeliveriesTerminal, updateSessionDeliveryStatus, consumeSessionDelivery } from './mesh-delivery-policy.js';
|
|
9
9
|
import { MeshRuntimeStore, pruneMeshRuntimeRetention } from './mesh-runtime-store.js';
|
|
10
10
|
import { maybeInjectIdleActiveMissionReminder } from './mesh-idle-reminder.js';
|
|
11
11
|
import { queuePendingMeshCoordinatorEvent, drainPendingMeshCoordinatorEvents, prunePendingMeshCoordinatorEventsRetention, readV2EnvelopeFromWire, type PendingMeshCoordinatorEvent } from './mesh-events-pending.js';
|
|
@@ -1273,16 +1273,14 @@ function injectMeshSystemMessage(components: DaemonComponents, args: {
|
|
|
1273
1273
|
updateDirectDispatchStatus(args.meshId, sessionId, 'acked', soleTaskId);
|
|
1274
1274
|
}
|
|
1275
1275
|
}
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
updateSessionDeliveryStatus(d.id, 'acked');
|
|
1285
|
-
}
|
|
1276
|
+
// DELIVERED-NOT-CONSUMED-REDRIVE: ack the delivery via consumeSessionDelivery, which
|
|
1277
|
+
// matches rows INCLUDING 'delivered'. The prior path filtered getActiveSessionDeliveries
|
|
1278
|
+
// — whose SQL EXCLUDES 'delivered' — so in the normal event order (transport confirm
|
|
1279
|
+
// flips 'delivered' BEFORE generating_started fires) the ack matched zero rows and the
|
|
1280
|
+
// delivery was stranded 'delivered', never reaching the 'acked' consume signal that
|
|
1281
|
+
// taskDeliveryConsumed() keys on. The store's monotonic guard only advances the row.
|
|
1282
|
+
// With a named taskId, key on (mesh, task); otherwise fall back to the whole session.
|
|
1283
|
+
consumeSessionDelivery(args.meshId, sessionId, 'acked', startedTaskId);
|
|
1286
1284
|
}
|
|
1287
1285
|
} else if (args.event === 'agent:stopped') {
|
|
1288
1286
|
const sessionId = resolveEventSessionId(args.metadataEvent, args.sourceInstanceId);
|
|
@@ -710,9 +710,19 @@ const RECLAIM_UNKNOWN_GRACE_TICKS = 3;
|
|
|
710
710
|
// completed/reclaimed/claimed-elsewhere row's counter is dropped (no unbounded growth).
|
|
711
711
|
const deliveredNoTurnUnknownStreak = new Map<string, number>();
|
|
712
712
|
|
|
713
|
-
//
|
|
713
|
+
// DELIVERED-NOT-CONSUMED-REDRIVE (fix d): the SHORT-grace re-drive (delivered-but-unconsumed,
|
|
714
|
+
// 25s window) previously reclaimed on a SINGLE non-GENERATING tick — and a REMOTE worker's local
|
|
715
|
+
// busy verdict is UNKNOWN, not GENERATING, so a genuinely-mid-turn remote worker whose ack merely
|
|
716
|
+
// hadn't propagated yet was torn off its task and the SAME prompt re-injected. Give the short path
|
|
717
|
+
// the same bounded consecutive-UNKNOWN grace the long delivered-no-turn path uses: only an
|
|
718
|
+
// IDLE_CONFIRMED verdict (positive LOCAL evidence the session is present-and-idle) re-drives
|
|
719
|
+
// immediately; UNKNOWN accrues a streak and re-drives only after RECLAIM_UNKNOWN_GRACE_TICKS.
|
|
720
|
+
const deliveredUnconsumedUnknownStreak = new Map<string, number>();
|
|
721
|
+
|
|
722
|
+
// Test hook: clear the delivered-no-turn UNKNOWN streaks between cases.
|
|
714
723
|
export function __resetReclaimUnknownStreakForTests(): void {
|
|
715
724
|
deliveredNoTurnUnknownStreak.clear();
|
|
725
|
+
deliveredUnconsumedUnknownStreak.clear();
|
|
716
726
|
}
|
|
717
727
|
|
|
718
728
|
// PHASE 2.5 — assigned-stranded dispatch watchdog (Bug B). claimNextTask atomically
|
|
@@ -739,6 +749,9 @@ function recoverStrandedAssignedDispatches(components: DaemonComponents, meshId:
|
|
|
739
749
|
for (const key of [...deliveredNoTurnUnknownStreak.keys()]) {
|
|
740
750
|
if (key.startsWith(meshKeyPrefix) && !assignedKeys.has(key)) deliveredNoTurnUnknownStreak.delete(key);
|
|
741
751
|
}
|
|
752
|
+
for (const key of [...deliveredUnconsumedUnknownStreak.keys()]) {
|
|
753
|
+
if (key.startsWith(meshKeyPrefix) && !assignedKeys.has(key)) deliveredUnconsumedUnknownStreak.delete(key);
|
|
754
|
+
}
|
|
742
755
|
for (const row of assigned) {
|
|
743
756
|
const dispatchedAtMs = Date.parse(row.dispatchTimestamp ?? '');
|
|
744
757
|
if (!Number.isFinite(dispatchedAtMs)) continue; // no dispatch ts → can't age it
|
|
@@ -774,15 +787,42 @@ function recoverStrandedAssignedDispatches(components: DaemonComponents, meshId:
|
|
|
774
787
|
updateTaskStatus(meshId, row.id, status);
|
|
775
788
|
continue;
|
|
776
789
|
}
|
|
790
|
+
const shortStreakKey = `${meshId}::${row.id}`;
|
|
777
791
|
const verdict = row.assignedSessionId
|
|
778
792
|
? resolveSessionBusyVerdict(components, row.assignedSessionId)
|
|
779
793
|
: 'IDLE_CONFIRMED'; // no session bound → nothing live generating to protect
|
|
780
|
-
|
|
794
|
+
// GENERATING → demonstrably alive: never re-drive, reset the grace.
|
|
795
|
+
// IDLE_CONFIRMED → positive LOCAL evidence the session is present-and-idle: re-drive now.
|
|
796
|
+
// UNKNOWN → remote / gone / id-form-skewed session: DEFER. A remote worker whose ack
|
|
797
|
+
// merely hasn't propagated reads UNKNOWN here — reclaiming on a single UNKNOWN tick
|
|
798
|
+
// tears a live remote worker off its task and re-injects the same prompt (the exact
|
|
799
|
+
// delivered_not_consumed_redrive symptom). Accrue a bounded consecutive-UNKNOWN streak
|
|
800
|
+
// and only re-drive after RECLAIM_UNKNOWN_GRACE_TICKS, matching the long path.
|
|
801
|
+
if (verdict === 'GENERATING') {
|
|
802
|
+
deliveredUnconsumedUnknownStreak.delete(shortStreakKey);
|
|
803
|
+
} else {
|
|
804
|
+
if (verdict === 'IDLE_CONFIRMED') {
|
|
805
|
+
deliveredUnconsumedUnknownStreak.delete(shortStreakKey);
|
|
806
|
+
} else {
|
|
807
|
+
const streak = (deliveredUnconsumedUnknownStreak.get(shortStreakKey) ?? 0) + 1;
|
|
808
|
+
deliveredUnconsumedUnknownStreak.set(shortStreakKey, streak);
|
|
809
|
+
if (streak < RECLAIM_UNKNOWN_GRACE_TICKS) {
|
|
810
|
+
traceMeshEventDrop('short_redrive_deferred_unknown_verdict', {
|
|
811
|
+
taskId: row.id,
|
|
812
|
+
sessionId: row.assignedSessionId,
|
|
813
|
+
nodeId: row.assignedNodeId,
|
|
814
|
+
meshId,
|
|
815
|
+
event: 'agent:generating_started',
|
|
816
|
+
}, `unknown ${streak}/${RECLAIM_UNKNOWN_GRACE_TICKS}`);
|
|
817
|
+
continue;
|
|
818
|
+
}
|
|
819
|
+
}
|
|
781
820
|
const redriven = reclaimStrandedAssignedTask(meshId, row.id, {
|
|
782
821
|
reason: 'delivered_not_consumed_redrive',
|
|
783
822
|
ageMs,
|
|
784
823
|
});
|
|
785
824
|
if (redriven) {
|
|
825
|
+
deliveredUnconsumedUnknownStreak.delete(shortStreakKey);
|
|
786
826
|
LOG.warn('MeshReconcile', `Re-drove delivered-but-unconsumed task ${row.id} on mesh ${meshId} `
|
|
787
827
|
+ `(node=${row.assignedNodeId ?? '?'} session=${row.assignedSessionId ?? '?'}, delivered but no `
|
|
788
828
|
+ `generating_started in ${Math.round(ageMs / 1000)}s, verdict ${verdict} → ${redriven.status})`);
|
|
@@ -1429,21 +1429,123 @@ export class MeshRuntimeStore {
|
|
|
1429
1429
|
this.maybeCheckpointWal();
|
|
1430
1430
|
}
|
|
1431
1431
|
|
|
1432
|
+
// DELIVERED-NOT-CONSUMED-REDRIVE monotonic FSM: the forward-progress lifecycle of a
|
|
1433
|
+
// delivery is a strictly increasing rank — a status may only advance, never regress.
|
|
1434
|
+
// The redrive bug was a NON-monotonic FSM: the transport-confirm callback
|
|
1435
|
+
// (mesh-queue-assignment :384) writes 'delivered' unconditionally by PK, so when the
|
|
1436
|
+
// worker's agent:generating_started raced AHEAD of the confirm and already flipped the
|
|
1437
|
+
// row 'delivering'→'acked', the late confirm CLOBBERED 'acked' back to 'delivered'.
|
|
1438
|
+
// taskDeliveryConsumed() (which keys on 'acked'/'completed') then read false forever,
|
|
1439
|
+
// and the short-grace re-drive re-opened an already-consumed task. Enforcing the rank
|
|
1440
|
+
// ordering here makes the two event orders converge on the same monotone terminal state
|
|
1441
|
+
// regardless of arrival order, so a late confirm can never demote a consumed delivery.
|
|
1442
|
+
// 'failed'/'expired'/'cancelled' are absorbing OUTCOMES, not progress ranks — they are
|
|
1443
|
+
// always allowed (a genuine dispatch failure must be recordable even from 'acked').
|
|
1444
|
+
private static readonly DELIVERY_PROGRESS_RANK: Record<string, number> = {
|
|
1445
|
+
queued: 0,
|
|
1446
|
+
delivering: 1,
|
|
1447
|
+
delivered: 2,
|
|
1448
|
+
acked: 3,
|
|
1449
|
+
completed: 4,
|
|
1450
|
+
};
|
|
1451
|
+
|
|
1432
1452
|
updateSessionDeliveryStatus(id: string, status: string, opts?: { lastError?: string; incrementAttempt?: boolean }): void {
|
|
1433
1453
|
const now = new Date().toISOString();
|
|
1434
1454
|
if (opts?.incrementAttempt) {
|
|
1455
|
+
// Retry/requeue path (transport failure → 'failed', or an explicit re-queue): this is
|
|
1456
|
+
// the deliberate reset signal, NOT the racing progress writes that cause the clobber, so
|
|
1457
|
+
// it is exempt from the monotonic guard and always applies (preserves attempt_count
|
|
1458
|
+
// bookkeeping and the failure ledger). The clobber bug lives only in the plain
|
|
1459
|
+
// progress write below.
|
|
1435
1460
|
this.db.prepare(`
|
|
1436
1461
|
UPDATE mesh_session_delivery
|
|
1437
1462
|
SET status = @status, last_error = @lastError, attempt_count = attempt_count + 1, updated_at = @updatedAt
|
|
1438
1463
|
WHERE id = @id
|
|
1439
1464
|
`).run({ id, status, lastError: opts?.lastError ?? null, updatedAt: now });
|
|
1440
|
-
|
|
1465
|
+
return;
|
|
1466
|
+
}
|
|
1467
|
+
// Monotonic guard for forward-progress statuses: a plain status write may ADVANCE or
|
|
1468
|
+
// rewrite the SAME rank, but NEVER regress to a strictly-lower rank. This is what stops the
|
|
1469
|
+
// late transport-confirm ('delivered', rank 2) from clobbering an already-consumed row
|
|
1470
|
+
// ('acked', rank 3): the `@targetRank >= current` predicate fetches zero rows for 3→2, so
|
|
1471
|
+
// 'acked' survives. Absorbing failure outcomes (failed/expired/cancelled) have no rank and
|
|
1472
|
+
// are written unconditionally.
|
|
1473
|
+
const targetRank = MeshRuntimeStore.DELIVERY_PROGRESS_RANK[status];
|
|
1474
|
+
if (targetRank === undefined) {
|
|
1441
1475
|
this.db.prepare(`
|
|
1442
1476
|
UPDATE mesh_session_delivery
|
|
1443
1477
|
SET status = @status, last_error = @lastError, updated_at = @updatedAt
|
|
1444
1478
|
WHERE id = @id
|
|
1445
1479
|
`).run({ id, status, lastError: opts?.lastError ?? null, updatedAt: now });
|
|
1480
|
+
return;
|
|
1481
|
+
}
|
|
1482
|
+
// Absorbing failure states (failed/expired/cancelled) map to rank 99 so no progress write
|
|
1483
|
+
// (max rank 4) can ever resurrect a dead delivery.
|
|
1484
|
+
this.db.prepare(`
|
|
1485
|
+
UPDATE mesh_session_delivery
|
|
1486
|
+
SET status = @status, last_error = @lastError, updated_at = @updatedAt
|
|
1487
|
+
WHERE id = @id AND (@targetRank >= CASE status
|
|
1488
|
+
WHEN 'queued' THEN 0 WHEN 'delivering' THEN 1 WHEN 'delivered' THEN 2
|
|
1489
|
+
WHEN 'acked' THEN 3 WHEN 'completed' THEN 4 ELSE 99 END)
|
|
1490
|
+
`).run({ id, status, lastError: opts?.lastError ?? null, updatedAt: now, targetRank });
|
|
1491
|
+
}
|
|
1492
|
+
|
|
1493
|
+
/**
|
|
1494
|
+
* DELIVERED-NOT-CONSUMED-REDRIVE consume path. Advance a task's delivery record(s) to a
|
|
1495
|
+
* CONSUMED status ('acked' or 'completed'), matching on mesh + session (+ taskId when the
|
|
1496
|
+
* event names one) and INCLUDING rows already in 'delivered'/'acked'/'delivering'.
|
|
1497
|
+
*
|
|
1498
|
+
* The ack/terminal callers previously routed through getActiveSessionDeliveries(), whose SQL
|
|
1499
|
+
* EXCLUDES 'delivered' — so in the normal event order (transport confirm flips 'delivered'
|
|
1500
|
+
* BEFORE the worker's generating_started fires) the ack matched zero rows and the delivery
|
|
1501
|
+
* was stranded 'delivered', never 'acked'. This finds the row by (mesh, session[, task])
|
|
1502
|
+
* directly and relies on updateSessionDeliveryStatus's monotonic guard to only advance it.
|
|
1503
|
+
* Returns the number of rows advanced.
|
|
1504
|
+
*/
|
|
1505
|
+
consumeSessionDelivery(meshId: string, sessionId: string, status: 'acked' | 'completed', taskId?: string): number {
|
|
1506
|
+
const rows = this.db.prepare(
|
|
1507
|
+
taskId
|
|
1508
|
+
? `SELECT id, session_id FROM mesh_session_delivery
|
|
1509
|
+
WHERE mesh_id = ? AND task_id = ?
|
|
1510
|
+
AND status IN ('queued','delivering','delivered','acked')`
|
|
1511
|
+
: `SELECT id, session_id FROM mesh_session_delivery
|
|
1512
|
+
WHERE mesh_id = ? AND session_id = ?
|
|
1513
|
+
AND status IN ('queued','delivering','delivered','acked')`,
|
|
1514
|
+
).all(meshId, taskId ?? sessionId) as Array<{ id: string; session_id: string | null }>;
|
|
1515
|
+
// Filter session membership in JS with the trimming equivalence predicate (mirrors
|
|
1516
|
+
// findAssignedBySession): a taskId match must still belong to this session, and the
|
|
1517
|
+
// session-only match already selected by column may carry serialization skew.
|
|
1518
|
+
let advanced = 0;
|
|
1519
|
+
for (const r of rows) {
|
|
1520
|
+
if (!sessionIdsEquivalent(r.session_id ?? undefined, sessionId)) continue;
|
|
1521
|
+
this.updateSessionDeliveryStatus(r.id, status);
|
|
1522
|
+
advanced++;
|
|
1523
|
+
}
|
|
1524
|
+
return advanced;
|
|
1525
|
+
}
|
|
1526
|
+
|
|
1527
|
+
/**
|
|
1528
|
+
* DELIVERED-NOT-CONSUMED-REDRIVE terminal path. Mark every OPEN delivery for a session
|
|
1529
|
+
* (queued/delivering/delivered/acked) terminal on task completion/failure. The prior
|
|
1530
|
+
* markSessionDeliveriesTerminal() routed through getActiveSessionDeliveries(), whose SQL
|
|
1531
|
+
* EXCLUDES 'delivered'/'completed' — so a 'delivered' row (the common case, since the
|
|
1532
|
+
* transport confirm flips it before the completion event) was never marked terminal and
|
|
1533
|
+
* stayed 'delivered', keeping taskDeliveryConsumed() false and feeding the false re-drive.
|
|
1534
|
+
* We match rows in OPEN states directly here. 'completed' advances monotonically (it is the
|
|
1535
|
+
* top progress rank); 'failed' is an absorbing outcome written unconditionally.
|
|
1536
|
+
*/
|
|
1537
|
+
markOpenSessionDeliveriesTerminal(meshId: string, sessionId: string, terminalStatus: 'completed' | 'failed'): number {
|
|
1538
|
+
const rows = this.db.prepare(
|
|
1539
|
+
`SELECT id, session_id FROM mesh_session_delivery
|
|
1540
|
+
WHERE mesh_id = ? AND status IN ('queued','delivering','delivered','acked')`,
|
|
1541
|
+
).all(meshId) as Array<{ id: string; session_id: string | null }>;
|
|
1542
|
+
let marked = 0;
|
|
1543
|
+
for (const r of rows) {
|
|
1544
|
+
if (!sessionIdsEquivalent(r.session_id ?? undefined, sessionId)) continue;
|
|
1545
|
+
this.updateSessionDeliveryStatus(r.id, terminalStatus);
|
|
1546
|
+
marked++;
|
|
1446
1547
|
}
|
|
1548
|
+
return marked;
|
|
1447
1549
|
}
|
|
1448
1550
|
|
|
1449
1551
|
getActiveSessionDeliveries(meshId: string, sessionId?: string): Array<{
|