@adhdev/daemon-standalone 1.0.28-rc.19 → 1.0.28-rc.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +558 -28
- package/dist/index.js.map +1 -1
- package/package.json +2 -2
- package/public/assets/{index-CWWODSEE.js → index-C3DSJEo9.js} +54 -54
- package/public/index.html +1 -1
package/dist/index.js
CHANGED
|
@@ -33311,10 +33311,10 @@ var require_dist3 = __commonJS({
|
|
|
33311
33311
|
}
|
|
33312
33312
|
function getDaemonBuildInfo() {
|
|
33313
33313
|
if (cached2) return cached2;
|
|
33314
|
-
const commit = readInjected(true ? "
|
|
33315
|
-
const commitShort = readInjected(true ? "
|
|
33316
|
-
const version2 = readInjected(true ? "1.0.28-rc.
|
|
33317
|
-
const builtAt = readInjected(true ? "2026-07-
|
|
33314
|
+
const commit = readInjected(true ? "284a88d08b7bb93b250385f4da9c523fb4d8fd68" : void 0) ?? "unknown";
|
|
33315
|
+
const commitShort = readInjected(true ? "284a88d0" : void 0) ?? (commit !== "unknown" ? commit.slice(0, 7) : "unknown");
|
|
33316
|
+
const version2 = readInjected(true ? "1.0.28-rc.20" : void 0) ?? readInjected(typeof process !== "undefined" ? process.env?.ADHDEV_PKG_VERSION : void 0) ?? "unknown";
|
|
33317
|
+
const builtAt = readInjected(true ? "2026-07-28T17:20:58.778Z" : void 0);
|
|
33318
33318
|
cached2 = builtAt ? { commit, commitShort, version: version2, builtAt } : { commit, commitShort, version: version2 };
|
|
33319
33319
|
return cached2;
|
|
33320
33320
|
}
|
|
@@ -37340,6 +37340,14 @@ ${error48.message || ""}`;
|
|
|
37340
37340
|
function noteRejectedProposal(reason) {
|
|
37341
37341
|
metrics.completionProposalsRejected[reason] = (metrics.completionProposalsRejected[reason] ?? 0) + 1;
|
|
37342
37342
|
}
|
|
37343
|
+
function noteDroppedSuspension(reason) {
|
|
37344
|
+
metrics.suspensionsDropped[reason] = (metrics.suspensionsDropped[reason] ?? 0) + 1;
|
|
37345
|
+
}
|
|
37346
|
+
function logSuspensionOnce(level, key2, message) {
|
|
37347
|
+
if (suspensionLogKeys.has(key2)) return;
|
|
37348
|
+
if (suspensionLogKeys.size < MAX_SUSPENSION_LOG_KEYS) suspensionLogKeys.add(key2);
|
|
37349
|
+
LOG2[level]("TurnLedger", message);
|
|
37350
|
+
}
|
|
37343
37351
|
function noteAckLatency(fromIso, nowMs) {
|
|
37344
37352
|
if (!fromIso) return;
|
|
37345
37353
|
const parsed = Date.parse(fromIso);
|
|
@@ -37481,10 +37489,16 @@ ${error48.message || ""}`;
|
|
|
37481
37489
|
if (args.kind === "accepted") {
|
|
37482
37490
|
return { attemptId: attempt.attemptId, stage: attempt.stage, applied: false, staleAttempt: false };
|
|
37483
37491
|
}
|
|
37484
|
-
const stageAfter =
|
|
37492
|
+
const stageAfter = args.kind === "consumed" ? store.transaction(() => {
|
|
37493
|
+
const after = store.advanceTurnAttemptStage(attempt.attemptId, args.kind, allowedFromStages(args.kind).join(","), {
|
|
37494
|
+
updatedAt: nowIso,
|
|
37495
|
+
consumedAt: nowIso
|
|
37496
|
+
});
|
|
37497
|
+
drainHeldSuspensionsForAttempt(store, attempt.attemptId, nowMs, nowIso);
|
|
37498
|
+
return after;
|
|
37499
|
+
}) : store.advanceTurnAttemptStage(attempt.attemptId, args.kind, allowedFromStages(args.kind).join(","), {
|
|
37485
37500
|
updatedAt: nowIso,
|
|
37486
|
-
deliveredAt: args.kind === "delivered" ? nowIso : void 0
|
|
37487
|
-
consumedAt: args.kind === "consumed" ? nowIso : void 0
|
|
37501
|
+
deliveredAt: args.kind === "delivered" ? nowIso : void 0
|
|
37488
37502
|
});
|
|
37489
37503
|
const applied = stageAfter === args.kind && attempt.stage !== args.kind;
|
|
37490
37504
|
if (applied) noteAckLatency(fromIso, nowMs);
|
|
@@ -37525,9 +37539,15 @@ ${error48.message || ""}`;
|
|
|
37525
37539
|
occurredAtMs: args.occurredAtMs ?? nowMs,
|
|
37526
37540
|
recordedAt: nowIso
|
|
37527
37541
|
});
|
|
37542
|
+
if (SUSPENDED_STAGES.has(args.stage)) {
|
|
37543
|
+
dropHeldSuspensionsForAttempt(store, attempt.attemptId, "stale_attempt", nowIso);
|
|
37544
|
+
}
|
|
37528
37545
|
return { attemptId: attempt.attemptId, stage: attempt.stage, applied: false, staleAttempt: true };
|
|
37529
37546
|
}
|
|
37530
37547
|
if (isTerminalTurnStage(attempt.stage)) {
|
|
37548
|
+
if (SUSPENDED_STAGES.has(args.stage)) {
|
|
37549
|
+
dropHeldSuspensionsForAttempt(store, attempt.attemptId, "attempt_terminal", nowIso);
|
|
37550
|
+
}
|
|
37531
37551
|
return { attemptId: attempt.attemptId, stage: attempt.stage, applied: false, staleAttempt: false };
|
|
37532
37552
|
}
|
|
37533
37553
|
const inserted = store.insertTurnEvent({
|
|
@@ -37542,9 +37562,45 @@ ${error48.message || ""}`;
|
|
|
37542
37562
|
recordedAt: nowIso
|
|
37543
37563
|
});
|
|
37544
37564
|
if (!inserted) metrics.duplicateTurnEvents += 1;
|
|
37565
|
+
if (args.stage === "generating" && SUSPENDED_STAGES.has(attempt.stage) && typeof args.occurredAtMs === "number") {
|
|
37566
|
+
const appliedHold = store.getHeldTurnSuspension(attempt.attemptId, attempt.stage);
|
|
37567
|
+
if (appliedHold && appliedHold.status === "applied" && typeof appliedHold.occurredAtMs === "number" && args.occurredAtMs < appliedHold.occurredAtMs) {
|
|
37568
|
+
metrics.reorderedGeneratingSuppressed += 1;
|
|
37569
|
+
logSuspensionOnce(
|
|
37570
|
+
"info",
|
|
37571
|
+
`reordered:${attempt.attemptId}`,
|
|
37572
|
+
`Suppressed reordered generating for task ${args.taskId} attempt ${attempt.attemptId}: event occurred ${args.occurredAtMs} < applied ${attempt.stage} suspension at ${appliedHold.occurredAtMs} \u2014 the suspension stage stands`
|
|
37573
|
+
);
|
|
37574
|
+
return { attemptId: attempt.attemptId, stage: attempt.stage, applied: false, staleAttempt: false };
|
|
37575
|
+
}
|
|
37576
|
+
}
|
|
37545
37577
|
const stageAfter = store.advanceTurnAttemptStage(attempt.attemptId, args.stage, allowedFromStages(args.stage).join(","), {
|
|
37546
37578
|
updatedAt: nowIso
|
|
37547
37579
|
});
|
|
37580
|
+
if (SUSPENDED_STAGES.has(args.stage) && stageAfter !== null && stageAfter !== args.stage && STAGE_RANK[stageAfter] < STAGE_RANK.consumed) {
|
|
37581
|
+
const held = store.insertHeldTurnSuspension({
|
|
37582
|
+
holdId: `${attempt.attemptId}:${args.stage}`,
|
|
37583
|
+
meshId: args.meshId,
|
|
37584
|
+
attemptId: attempt.attemptId,
|
|
37585
|
+
taskId: args.taskId,
|
|
37586
|
+
stage: args.stage,
|
|
37587
|
+
sessionId: args.sessionId,
|
|
37588
|
+
dispatchNonce: attempt.dispatchNonce,
|
|
37589
|
+
occurredAtMs: args.occurredAtMs ?? nowMs,
|
|
37590
|
+
recordedAt: nowIso
|
|
37591
|
+
});
|
|
37592
|
+
if (held) {
|
|
37593
|
+
metrics.suspensionsHeld += 1;
|
|
37594
|
+
logSuspensionOnce(
|
|
37595
|
+
"info",
|
|
37596
|
+
`held:${attempt.attemptId}:${args.stage}`,
|
|
37597
|
+
`Held ${args.stage} suspension for task ${args.taskId} attempt ${attempt.attemptId} (stage ${stageAfter} \u2014 consumed not yet durable); applying after the consumed ACK`
|
|
37598
|
+
);
|
|
37599
|
+
} else {
|
|
37600
|
+
metrics.duplicateTurnEvents += 1;
|
|
37601
|
+
}
|
|
37602
|
+
return { attemptId: attempt.attemptId, stage: stageAfter, applied: false, staleAttempt: false, deferred: true };
|
|
37603
|
+
}
|
|
37548
37604
|
return {
|
|
37549
37605
|
attemptId: attempt.attemptId,
|
|
37550
37606
|
stage: stageAfter ?? attempt.stage,
|
|
@@ -37552,6 +37608,188 @@ ${error48.message || ""}`;
|
|
|
37552
37608
|
staleAttempt: false
|
|
37553
37609
|
};
|
|
37554
37610
|
}
|
|
37611
|
+
function dropHeldSuspensionsForAttempt(store, attemptId, reason, nowIso) {
|
|
37612
|
+
const held = store.listHeldTurnSuspensionsForAttempt(attemptId, "held");
|
|
37613
|
+
let dropped = 0;
|
|
37614
|
+
for (const hold of held) {
|
|
37615
|
+
if (store.resolveHeldTurnSuspension(hold.holdId, "dropped", reason, nowIso)) {
|
|
37616
|
+
noteDroppedSuspension(reason);
|
|
37617
|
+
logSuspensionOnce(
|
|
37618
|
+
"info",
|
|
37619
|
+
`dropped:${reason}:${attemptId}:${hold.stage}`,
|
|
37620
|
+
`Dropped held ${hold.stage} suspension for task ${hold.taskId} attempt ${attemptId}: ${reason}`
|
|
37621
|
+
);
|
|
37622
|
+
dropped += 1;
|
|
37623
|
+
}
|
|
37624
|
+
}
|
|
37625
|
+
return dropped;
|
|
37626
|
+
}
|
|
37627
|
+
function drainHeldSuspensionsForAttempt(store, attemptId, nowMs, nowIso) {
|
|
37628
|
+
const held = store.listHeldTurnSuspensionsForAttempt(attemptId, "held");
|
|
37629
|
+
if (held.length === 0) return { applied: 0, dropped: 0 };
|
|
37630
|
+
let attempt = store.getTurnAttempt(attemptId);
|
|
37631
|
+
let applied = 0;
|
|
37632
|
+
let dropped = 0;
|
|
37633
|
+
for (const hold of held) {
|
|
37634
|
+
if (!attempt) {
|
|
37635
|
+
if (store.resolveHeldTurnSuspension(hold.holdId, "dropped", "attempt_missing", nowIso)) {
|
|
37636
|
+
noteDroppedSuspension("attempt_missing");
|
|
37637
|
+
dropped += 1;
|
|
37638
|
+
}
|
|
37639
|
+
continue;
|
|
37640
|
+
}
|
|
37641
|
+
if (attempt.terminalOutcome || isTerminalTurnStage(attempt.stage)) {
|
|
37642
|
+
if (store.resolveHeldTurnSuspension(hold.holdId, "dropped", "attempt_terminal", nowIso)) {
|
|
37643
|
+
noteDroppedSuspension("attempt_terminal");
|
|
37644
|
+
logSuspensionOnce(
|
|
37645
|
+
"info",
|
|
37646
|
+
`dropped:attempt_terminal:${attemptId}:${hold.stage}`,
|
|
37647
|
+
`Dropped held ${hold.stage} suspension for task ${hold.taskId} attempt ${attemptId}: attempt terminal (${attempt.terminalOutcome ?? attempt.stage})`
|
|
37648
|
+
);
|
|
37649
|
+
dropped += 1;
|
|
37650
|
+
}
|
|
37651
|
+
continue;
|
|
37652
|
+
}
|
|
37653
|
+
if (attempt.stage === "finalizing") {
|
|
37654
|
+
if (store.resolveHeldTurnSuspension(hold.holdId, "dropped", "finalizing", nowIso)) {
|
|
37655
|
+
noteDroppedSuspension("finalizing");
|
|
37656
|
+
dropped += 1;
|
|
37657
|
+
}
|
|
37658
|
+
continue;
|
|
37659
|
+
}
|
|
37660
|
+
if (STAGE_RANK[attempt.stage] < STAGE_RANK.consumed) {
|
|
37661
|
+
continue;
|
|
37662
|
+
}
|
|
37663
|
+
const stageAfter = store.advanceTurnAttemptStage(attemptId, hold.stage, allowedFromStages(hold.stage).join(","), {
|
|
37664
|
+
updatedAt: nowIso
|
|
37665
|
+
});
|
|
37666
|
+
if (stageAfter === hold.stage) {
|
|
37667
|
+
if (store.resolveHeldTurnSuspension(hold.holdId, "applied", "applied", nowIso)) {
|
|
37668
|
+
store.insertTurnEvent({
|
|
37669
|
+
eventId: (0, import_crypto5.randomUUID)(),
|
|
37670
|
+
meshId: hold.meshId,
|
|
37671
|
+
attemptId,
|
|
37672
|
+
taskId: hold.taskId,
|
|
37673
|
+
kind: `held_${hold.stage}_applied`,
|
|
37674
|
+
dedupeKey: hold.holdId,
|
|
37675
|
+
occurredAtMs: hold.occurredAtMs ?? nowMs,
|
|
37676
|
+
recordedAt: nowIso
|
|
37677
|
+
});
|
|
37678
|
+
metrics.suspensionsApplied += 1;
|
|
37679
|
+
logSuspensionOnce(
|
|
37680
|
+
"info",
|
|
37681
|
+
`applied:${attemptId}:${hold.stage}`,
|
|
37682
|
+
`Applied held ${hold.stage} suspension for task ${hold.taskId} attempt ${attemptId} after consumed became durable`
|
|
37683
|
+
);
|
|
37684
|
+
applied += 1;
|
|
37685
|
+
}
|
|
37686
|
+
attempt = { ...attempt, stage: stageAfter };
|
|
37687
|
+
} else {
|
|
37688
|
+
const reason = stageAfter === "finalizing" ? "finalizing" : "stage_advanced";
|
|
37689
|
+
if (store.resolveHeldTurnSuspension(hold.holdId, "dropped", reason, nowIso)) {
|
|
37690
|
+
noteDroppedSuspension(reason);
|
|
37691
|
+
logSuspensionOnce(
|
|
37692
|
+
"warn",
|
|
37693
|
+
`dropped:${reason}:${attemptId}:${hold.stage}`,
|
|
37694
|
+
`Held ${hold.stage} suspension for task ${hold.taskId} attempt ${attemptId} could not apply (stage now ${stageAfter ?? "missing"}) \u2014 dropped:${reason}`
|
|
37695
|
+
);
|
|
37696
|
+
dropped += 1;
|
|
37697
|
+
}
|
|
37698
|
+
}
|
|
37699
|
+
}
|
|
37700
|
+
return { applied, dropped };
|
|
37701
|
+
}
|
|
37702
|
+
function drainHeldTurnSuspensionsForMesh(meshId, nowMs = Date.now()) {
|
|
37703
|
+
try {
|
|
37704
|
+
const store = MeshRuntimeStore.getInstance();
|
|
37705
|
+
const nowIso = new Date(nowMs).toISOString();
|
|
37706
|
+
const held = store.listHeldTurnSuspensionsForMesh(meshId, "held");
|
|
37707
|
+
let applied = 0;
|
|
37708
|
+
let dropped = 0;
|
|
37709
|
+
for (const attemptId of new Set(held.map((h) => h.attemptId))) {
|
|
37710
|
+
const res = drainHeldSuspensionsForAttempt(store, attemptId, nowMs, nowIso);
|
|
37711
|
+
applied += res.applied;
|
|
37712
|
+
dropped += res.dropped;
|
|
37713
|
+
}
|
|
37714
|
+
const stillHeld = store.listHeldTurnSuspensionsForMesh(meshId, "held").length;
|
|
37715
|
+
if (applied + dropped > 0 || held.length > 0) {
|
|
37716
|
+
LOG2.info("TurnLedger", `Held-suspension drain for mesh ${meshId}: applied=${applied} dropped=${dropped} stillHeld=${stillHeld}`);
|
|
37717
|
+
}
|
|
37718
|
+
return { applied, dropped, stillHeld };
|
|
37719
|
+
} catch (e) {
|
|
37720
|
+
LOG2.warn("TurnLedger", `Held-suspension drain failed for mesh ${meshId} (rows stay held; retried on next boot/consumed): ${e?.message || e}`);
|
|
37721
|
+
return { applied: 0, dropped: 0, stillHeld: 0 };
|
|
37722
|
+
}
|
|
37723
|
+
}
|
|
37724
|
+
function heldSuspensionBlocksAttempt(hold, attempt) {
|
|
37725
|
+
if (hold.attemptId !== attempt.attemptId) return false;
|
|
37726
|
+
if (attempt.terminalOutcome || isTerminalTurnStage(attempt.stage)) return false;
|
|
37727
|
+
if (STAGE_RANK[attempt.stage] >= STAGE_RANK.consumed) return false;
|
|
37728
|
+
if (!hold.sessionId || !attempt.sessionId || !sessionIdsEquivalent(hold.sessionId, attempt.sessionId)) return false;
|
|
37729
|
+
if (typeof hold.dispatchNonce !== "number" || typeof attempt.dispatchNonce !== "number" || hold.dispatchNonce !== attempt.dispatchNonce) return false;
|
|
37730
|
+
return true;
|
|
37731
|
+
}
|
|
37732
|
+
function gateRedriveForHeldSuspension(args) {
|
|
37733
|
+
const store = MeshRuntimeStore.getInstance();
|
|
37734
|
+
const nowMs = args.nowMs ?? Date.now();
|
|
37735
|
+
const nowIso = new Date(nowMs).toISOString();
|
|
37736
|
+
const attempt = store.getCurrentTurnAttempt(args.meshId, args.taskId);
|
|
37737
|
+
if (!attempt) return { kind: "none" };
|
|
37738
|
+
const holds = store.listHeldTurnSuspensionsForAttempt(attempt.attemptId, "held").filter((h) => heldSuspensionBlocksAttempt(h, attempt));
|
|
37739
|
+
if (holds.length === 0) return { kind: "none" };
|
|
37740
|
+
if (args.sessionLiveness === "alive") {
|
|
37741
|
+
let promoted = false;
|
|
37742
|
+
store.transaction(() => {
|
|
37743
|
+
if (STAGE_RANK[attempt.stage] < STAGE_RANK.consumed) {
|
|
37744
|
+
store.insertTurnEvent({
|
|
37745
|
+
eventId: (0, import_crypto5.randomUUID)(),
|
|
37746
|
+
meshId: args.meshId,
|
|
37747
|
+
attemptId: attempt.attemptId,
|
|
37748
|
+
taskId: args.taskId,
|
|
37749
|
+
kind: "consumed",
|
|
37750
|
+
dedupeKey: "",
|
|
37751
|
+
payload: safeEvidenceJson({ source: "held_suspension_recovery" }),
|
|
37752
|
+
occurredAtMs: holds[0].occurredAtMs ?? nowMs,
|
|
37753
|
+
recordedAt: nowIso
|
|
37754
|
+
});
|
|
37755
|
+
store.advanceTurnAttemptStage(attempt.attemptId, "consumed", allowedFromStages("consumed").join(","), {
|
|
37756
|
+
updatedAt: nowIso,
|
|
37757
|
+
consumedAt: nowIso
|
|
37758
|
+
});
|
|
37759
|
+
promoted = true;
|
|
37760
|
+
}
|
|
37761
|
+
drainHeldSuspensionsForAttempt(store, attempt.attemptId, nowMs, nowIso);
|
|
37762
|
+
});
|
|
37763
|
+
const stage = store.getTurnAttempt(attempt.attemptId)?.stage ?? attempt.stage;
|
|
37764
|
+
if (promoted) {
|
|
37765
|
+
metrics.suspensionConsumedRecovered += 1;
|
|
37766
|
+
logSuspensionOnce(
|
|
37767
|
+
"info",
|
|
37768
|
+
`recovered:${attempt.attemptId}`,
|
|
37769
|
+
`Recovered consumed evidence for task ${args.taskId} attempt ${attempt.attemptId} from held ${holds.map((h) => h.stage).join("/")} suspension(s): surviving session confirmed rebound (audit source held_suspension_recovery) \u2014 the SAME attempt continues`
|
|
37770
|
+
);
|
|
37771
|
+
}
|
|
37772
|
+
return { kind: "recovered", attemptId: attempt.attemptId, stage };
|
|
37773
|
+
}
|
|
37774
|
+
if (args.sessionProvenDead) {
|
|
37775
|
+
const dropped = dropHeldSuspensionsForAttempt(store, attempt.attemptId, "session_dead", nowIso);
|
|
37776
|
+
if (dropped > 0) {
|
|
37777
|
+
logSuspensionOnce(
|
|
37778
|
+
"info",
|
|
37779
|
+
`released:${attempt.attemptId}`,
|
|
37780
|
+
`Released ${dropped} held suspension(s) for task ${args.taskId} attempt ${attempt.attemptId}: worker session demonstrably dead \u2014 redrive/reassign may proceed to a NEW attempt`
|
|
37781
|
+
);
|
|
37782
|
+
}
|
|
37783
|
+
return { kind: "released", attemptId: attempt.attemptId, dropped };
|
|
37784
|
+
}
|
|
37785
|
+
metrics.redriveBlockedBySuspension += 1;
|
|
37786
|
+
logSuspensionOnce(
|
|
37787
|
+
"info",
|
|
37788
|
+
`blocked:${attempt.attemptId}`,
|
|
37789
|
+
`Redrive blocked by unresolved held ${holds.map((h) => h.stage).join("/")} suspension(s) for task ${args.taskId} attempt ${attempt.attemptId}: the prompt reached the worker (session liveness unconfirmed \u2014 awaiting rebind or the dead-detection grace)`
|
|
37790
|
+
);
|
|
37791
|
+
return { kind: "blocked", attemptId: attempt.attemptId, stages: holds.map((h) => h.stage) };
|
|
37792
|
+
}
|
|
37555
37793
|
function proposeTurnCompletion(proposal) {
|
|
37556
37794
|
const store = MeshRuntimeStore.getInstance();
|
|
37557
37795
|
const nowMs = proposal.nowMs ?? Date.now();
|
|
@@ -37585,6 +37823,7 @@ ${error48.message || ""}`;
|
|
|
37585
37823
|
return { committed: false, reason: "epoch_mismatch", attemptId: attempt.attemptId };
|
|
37586
37824
|
}
|
|
37587
37825
|
if (attempt.terminalOutcome) {
|
|
37826
|
+
dropHeldSuspensionsForAttempt(store, attempt.attemptId, "attempt_terminal", nowIso);
|
|
37588
37827
|
if (attempt.terminalOutcome === proposal.outcome) {
|
|
37589
37828
|
metrics.duplicateTurnEvents += 1;
|
|
37590
37829
|
return { committed: true, attemptId: attempt.attemptId, outcome: proposal.outcome, duplicate: true };
|
|
@@ -37611,6 +37850,7 @@ ${error48.message || ""}`;
|
|
|
37611
37850
|
return { committed: false, reason: "already_terminal", attemptId: attempt.attemptId, existingOutcome: winner };
|
|
37612
37851
|
}
|
|
37613
37852
|
metrics.completionProposalsCommitted += 1;
|
|
37853
|
+
dropHeldSuspensionsForAttempt(store, attempt.attemptId, "attempt_terminal", nowIso);
|
|
37614
37854
|
LOG2.info("TurnLedger", `Committed terminal ${proposal.outcome} for task ${proposal.taskId} attempt ${attempt.attemptId} (source=${proposal.source}, stage was ${attempt.stage})`);
|
|
37615
37855
|
return { committed: true, attemptId: attempt.attemptId, outcome: proposal.outcome, duplicate: false };
|
|
37616
37856
|
}
|
|
@@ -37788,7 +38028,10 @@ ${error48.message || ""}`;
|
|
|
37788
38028
|
var import_crypto5;
|
|
37789
38029
|
var TURN_TERMINAL_OUTCOMES;
|
|
37790
38030
|
var STAGE_RANK;
|
|
38031
|
+
var SUSPENDED_STAGES;
|
|
37791
38032
|
var metrics;
|
|
38033
|
+
var MAX_SUSPENSION_LOG_KEYS;
|
|
38034
|
+
var suspensionLogKeys;
|
|
37792
38035
|
var MAX_REDRIVES_PER_ATTEMPT;
|
|
37793
38036
|
var sessionOrderChains;
|
|
37794
38037
|
var init_mesh_turn_ledger = __esm2({
|
|
@@ -37811,6 +38054,7 @@ ${error48.message || ""}`;
|
|
|
37811
38054
|
failed: 60,
|
|
37812
38055
|
cancelled: 60
|
|
37813
38056
|
};
|
|
38057
|
+
SUSPENDED_STAGES = /* @__PURE__ */ new Set(["waiting_approval", "waiting_choice"]);
|
|
37814
38058
|
metrics = {
|
|
37815
38059
|
duplicatePromptPrevented: 0,
|
|
37816
38060
|
ackLatencyCount: 0,
|
|
@@ -37820,8 +38064,16 @@ ${error48.message || ""}`;
|
|
|
37820
38064
|
staleAttemptEvents: 0,
|
|
37821
38065
|
transactionConflicts: 0,
|
|
37822
38066
|
sameSessionStaleNonceCompatAccepted: 0,
|
|
37823
|
-
duplicateTurnEvents: 0
|
|
38067
|
+
duplicateTurnEvents: 0,
|
|
38068
|
+
suspensionsHeld: 0,
|
|
38069
|
+
suspensionsApplied: 0,
|
|
38070
|
+
suspensionsDropped: {},
|
|
38071
|
+
reorderedGeneratingSuppressed: 0,
|
|
38072
|
+
redriveBlockedBySuspension: 0,
|
|
38073
|
+
suspensionConsumedRecovered: 0
|
|
37824
38074
|
};
|
|
38075
|
+
MAX_SUSPENSION_LOG_KEYS = 200;
|
|
38076
|
+
suspensionLogKeys = /* @__PURE__ */ new Set();
|
|
37825
38077
|
MAX_REDRIVES_PER_ATTEMPT = 1;
|
|
37826
38078
|
sessionOrderChains = /* @__PURE__ */ new Map();
|
|
37827
38079
|
}
|
|
@@ -41011,6 +41263,22 @@ Next step: ${nextStep}`;
|
|
|
41011
41263
|
updatedAt: r.updated_at
|
|
41012
41264
|
};
|
|
41013
41265
|
}
|
|
41266
|
+
function meshTurnHeldSuspensionFromRow(r) {
|
|
41267
|
+
return {
|
|
41268
|
+
holdId: r.hold_id,
|
|
41269
|
+
meshId: r.mesh_id,
|
|
41270
|
+
attemptId: r.attempt_id,
|
|
41271
|
+
taskId: r.task_id,
|
|
41272
|
+
stage: r.stage,
|
|
41273
|
+
sessionId: r.session_id,
|
|
41274
|
+
dispatchNonce: r.dispatch_nonce,
|
|
41275
|
+
occurredAtMs: r.occurred_at_ms,
|
|
41276
|
+
recordedAt: r.recorded_at,
|
|
41277
|
+
status: r.status,
|
|
41278
|
+
resolution: r.resolution,
|
|
41279
|
+
resolvedAt: r.resolved_at
|
|
41280
|
+
};
|
|
41281
|
+
}
|
|
41014
41282
|
function pruneMeshRuntimeRetention() {
|
|
41015
41283
|
try {
|
|
41016
41284
|
const store = MeshRuntimeStore.getInstance();
|
|
@@ -41450,6 +41718,37 @@ Next step: ${nextStep}`;
|
|
|
41450
41718
|
ON mesh_turn_outbox(status, next_attempt_at_ms);
|
|
41451
41719
|
CREATE INDEX IF NOT EXISTS idx_mesh_turn_outbox_mesh
|
|
41452
41720
|
ON mesh_turn_outbox(mesh_id, status);
|
|
41721
|
+
|
|
41722
|
+
-- TURN-LEDGER (Stage 5): durable HELD SUSPENSIONS. A waiting_approval /
|
|
41723
|
+
-- waiting_choice edge can legitimately arrive BEFORE the consumed ACK
|
|
41724
|
+
-- (a fast picker fires ahead of the generating_started processing, whose
|
|
41725
|
+
-- attempt-resolution preamble defers the consumed write). The causal FSM
|
|
41726
|
+
-- rightly refuses accepted/delivered \u2192 waiting_*; instead of dropping the
|
|
41727
|
+
-- edge, the reducer holds it here \u2014 attempt/session/epoch-scoped and
|
|
41728
|
+
-- content-free \u2014 insert-once via hold_id (<attempt_id>:<stage>). The
|
|
41729
|
+
-- consumed commit applies the hold through the SAME FSM in the same
|
|
41730
|
+
-- transaction; the restart reconcile drain covers a crash between hold
|
|
41731
|
+
-- and consumed; terminal commits resolve held rows as dropped so a held
|
|
41732
|
+
-- picker can never resurrect a finished/reassigned attempt.
|
|
41733
|
+
CREATE TABLE IF NOT EXISTS mesh_turn_held_suspensions (
|
|
41734
|
+
hold_id TEXT PRIMARY KEY,
|
|
41735
|
+
mesh_id TEXT NOT NULL,
|
|
41736
|
+
attempt_id TEXT NOT NULL,
|
|
41737
|
+
task_id TEXT NOT NULL,
|
|
41738
|
+
stage TEXT NOT NULL,
|
|
41739
|
+
session_id TEXT,
|
|
41740
|
+
dispatch_nonce INTEGER,
|
|
41741
|
+
occurred_at_ms INTEGER,
|
|
41742
|
+
recorded_at TEXT NOT NULL,
|
|
41743
|
+
status TEXT NOT NULL DEFAULT 'held',
|
|
41744
|
+
resolution TEXT,
|
|
41745
|
+
resolved_at TEXT
|
|
41746
|
+
);
|
|
41747
|
+
|
|
41748
|
+
CREATE INDEX IF NOT EXISTS idx_mesh_turn_held_suspensions_mesh
|
|
41749
|
+
ON mesh_turn_held_suspensions(mesh_id, status);
|
|
41750
|
+
CREATE INDEX IF NOT EXISTS idx_mesh_turn_held_suspensions_attempt
|
|
41751
|
+
ON mesh_turn_held_suspensions(attempt_id, status);
|
|
41453
41752
|
`);
|
|
41454
41753
|
this.migrateMeshIsolationColumns();
|
|
41455
41754
|
}
|
|
@@ -43234,6 +43533,74 @@ Next step: ${nextStep}`;
|
|
|
43234
43533
|
for (const r of rows) out[r.status] = r.n;
|
|
43235
43534
|
return out;
|
|
43236
43535
|
}
|
|
43536
|
+
// ── TURN-LEDGER (Stage 5): held suspensions (pre-consumed waiting_*) ─────
|
|
43537
|
+
/**
|
|
43538
|
+
* Hold a pre-consumed suspension edge. INSERT OR IGNORE on the hold id
|
|
43539
|
+
* (`<attemptId>:<stage>`) makes duplicate/reordered suspension arrivals
|
|
43540
|
+
* insert-once. Returns true when this call inserted (first hold).
|
|
43541
|
+
*/
|
|
43542
|
+
insertHeldTurnSuspension(row) {
|
|
43543
|
+
const res = this.db.prepare(`
|
|
43544
|
+
INSERT OR IGNORE INTO mesh_turn_held_suspensions (
|
|
43545
|
+
hold_id, mesh_id, attempt_id, task_id, stage, session_id, dispatch_nonce, occurred_at_ms, recorded_at, status
|
|
43546
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'held')
|
|
43547
|
+
`).run(
|
|
43548
|
+
row.holdId,
|
|
43549
|
+
row.meshId,
|
|
43550
|
+
row.attemptId,
|
|
43551
|
+
row.taskId,
|
|
43552
|
+
row.stage,
|
|
43553
|
+
row.sessionId ?? null,
|
|
43554
|
+
row.dispatchNonce ?? null,
|
|
43555
|
+
row.occurredAtMs ?? null,
|
|
43556
|
+
row.recordedAt
|
|
43557
|
+
);
|
|
43558
|
+
this.maybeCheckpointWal();
|
|
43559
|
+
return res.changes > 0;
|
|
43560
|
+
}
|
|
43561
|
+
/** The hold row for one (attempt, stage) pair, any status (held/applied/dropped). */
|
|
43562
|
+
getHeldTurnSuspension(attemptId, stage) {
|
|
43563
|
+
const row = this.db.prepare(
|
|
43564
|
+
"SELECT * FROM mesh_turn_held_suspensions WHERE hold_id = ? LIMIT 1"
|
|
43565
|
+
).get(`${attemptId}:${stage}`);
|
|
43566
|
+
return row ? meshTurnHeldSuspensionFromRow(row) : null;
|
|
43567
|
+
}
|
|
43568
|
+
/** Hold rows for an attempt, oldest occurrence first (drain order). */
|
|
43569
|
+
listHeldTurnSuspensionsForAttempt(attemptId, status) {
|
|
43570
|
+
const rows = status ? this.db.prepare(`
|
|
43571
|
+
SELECT * FROM mesh_turn_held_suspensions
|
|
43572
|
+
WHERE attempt_id = ? AND status = ?
|
|
43573
|
+
ORDER BY occurred_at_ms ASC, hold_id ASC
|
|
43574
|
+
`).all(attemptId, status) : this.db.prepare(`
|
|
43575
|
+
SELECT * FROM mesh_turn_held_suspensions
|
|
43576
|
+
WHERE attempt_id = ?
|
|
43577
|
+
ORDER BY occurred_at_ms ASC, hold_id ASC
|
|
43578
|
+
`).all(attemptId);
|
|
43579
|
+
return rows.map(meshTurnHeldSuspensionFromRow);
|
|
43580
|
+
}
|
|
43581
|
+
/** Hold rows for a mesh by status (the restart-reconcile drain set). */
|
|
43582
|
+
listHeldTurnSuspensionsForMesh(meshId, status) {
|
|
43583
|
+
const rows = this.db.prepare(`
|
|
43584
|
+
SELECT * FROM mesh_turn_held_suspensions
|
|
43585
|
+
WHERE mesh_id = ? AND status = ?
|
|
43586
|
+
ORDER BY occurred_at_ms ASC, hold_id ASC
|
|
43587
|
+
`).all(meshId, status);
|
|
43588
|
+
return rows.map(meshTurnHeldSuspensionFromRow);
|
|
43589
|
+
}
|
|
43590
|
+
/**
|
|
43591
|
+
* Resolve a hold exactly once: the status='held' guard makes a concurrent
|
|
43592
|
+
* drain/terminal resolution converge on a single winner. Returns true when
|
|
43593
|
+
* this call flipped the row.
|
|
43594
|
+
*/
|
|
43595
|
+
resolveHeldTurnSuspension(holdId, status, resolution, resolvedAt) {
|
|
43596
|
+
const res = this.db.prepare(`
|
|
43597
|
+
UPDATE mesh_turn_held_suspensions
|
|
43598
|
+
SET status = ?, resolution = ?, resolved_at = ?
|
|
43599
|
+
WHERE hold_id = ? AND status = 'held'
|
|
43600
|
+
`).run(status, resolution, resolvedAt, holdId);
|
|
43601
|
+
this.maybeCheckpointWal();
|
|
43602
|
+
return res.changes > 0;
|
|
43603
|
+
}
|
|
43237
43604
|
};
|
|
43238
43605
|
MESH_EVENT_LEDGER_RETENTION_MS = 30 * 24 * 60 * 60 * 1e3;
|
|
43239
43606
|
MESH_TOOL_CALL_LOG_RETENTION_MS = 14 * 24 * 60 * 60 * 1e3;
|
|
@@ -49679,8 +50046,17 @@ ${rendered}`, "utf-8");
|
|
|
49679
50046
|
waitingMs: record2.elapsedMs
|
|
49680
50047
|
});
|
|
49681
50048
|
}
|
|
49682
|
-
|
|
49683
|
-
|
|
50049
|
+
const bySession = /* @__PURE__ */ new Map();
|
|
50050
|
+
for (const approval of approvals) {
|
|
50051
|
+
const key2 = `${approval.nodeId}::${approval.sessionId}`;
|
|
50052
|
+
const existing = bySession.get(key2);
|
|
50053
|
+
if (!existing || approval.waitingMs > existing.waitingMs) {
|
|
50054
|
+
bySession.set(key2, approval);
|
|
50055
|
+
}
|
|
50056
|
+
}
|
|
50057
|
+
const deduped = [...bySession.values()];
|
|
50058
|
+
deduped.sort((a, b) => b.waitingMs - a.waitingMs);
|
|
50059
|
+
return deduped;
|
|
49684
50060
|
}
|
|
49685
50061
|
function buildMeshActiveWorkSummary(activeWork) {
|
|
49686
50062
|
const statusCounts = {
|
|
@@ -57418,7 +57794,7 @@ ${cleanBody}`;
|
|
|
57418
57794
|
const suspendedTaskId = readNonEmptyString(args.metadataEvent.taskId);
|
|
57419
57795
|
if (suspendedTaskId) {
|
|
57420
57796
|
try {
|
|
57421
|
-
recordTurnStage({
|
|
57797
|
+
const recorded = recordTurnStage({
|
|
57422
57798
|
meshId: args.meshId,
|
|
57423
57799
|
taskId: suspendedTaskId,
|
|
57424
57800
|
stage: args.event === "agent:waiting_approval" ? "waiting_approval" : "waiting_choice",
|
|
@@ -57426,7 +57802,12 @@ ${cleanBody}`;
|
|
|
57426
57802
|
sessionId: eventSessionId,
|
|
57427
57803
|
occurredAtMs: eventTimestamp ?? void 0
|
|
57428
57804
|
});
|
|
57429
|
-
|
|
57805
|
+
if (recorded?.deferred) {
|
|
57806
|
+
traceMeshEventStage("turn_suspension_held", traceCtx, `${args.event} preceded the consumed ACK \u2014 held for post-consumed drain`);
|
|
57807
|
+
}
|
|
57808
|
+
} catch (e) {
|
|
57809
|
+
LOG2.warn("TurnLedger", `Failed to record ${args.event} suspension for task ${suspendedTaskId} (session ${eventSessionId}): ${e?.message || e}`);
|
|
57810
|
+
traceMeshEventDrop("turn_suspension_record_error", traceCtx, e?.message ? String(e.message) : void 0);
|
|
57430
57811
|
}
|
|
57431
57812
|
}
|
|
57432
57813
|
}
|
|
@@ -59649,6 +60030,51 @@ ${cleanBody}`;
|
|
|
59649
60030
|
}
|
|
59650
60031
|
return true;
|
|
59651
60032
|
}
|
|
60033
|
+
function restampReboundMeshWorkerAssignment(components, store, meshId, row) {
|
|
60034
|
+
const sessionId = readNonEmptyString(row.assignedSessionId);
|
|
60035
|
+
if (!sessionId) return false;
|
|
60036
|
+
const inst = components.instanceManager?.getInstance?.(sessionId);
|
|
60037
|
+
if (!inst || typeof inst.attachMeshAssignment !== "function") return false;
|
|
60038
|
+
let settings = {};
|
|
60039
|
+
try {
|
|
60040
|
+
settings = inst.getState?.()?.settings || {};
|
|
60041
|
+
} catch {
|
|
60042
|
+
return false;
|
|
60043
|
+
}
|
|
60044
|
+
if (readNonEmptyString(settings.meshActiveTaskId)) return false;
|
|
60045
|
+
const attempt = (() => {
|
|
60046
|
+
try {
|
|
60047
|
+
return store.getCurrentTurnAttempt(meshId, row.id);
|
|
60048
|
+
} catch {
|
|
60049
|
+
return null;
|
|
60050
|
+
}
|
|
60051
|
+
})();
|
|
60052
|
+
if (attempt) {
|
|
60053
|
+
if (attempt.terminalOutcome) return false;
|
|
60054
|
+
const attemptSessionId = readNonEmptyString(attempt.sessionId);
|
|
60055
|
+
if (attemptSessionId && !sessionIdsEquivalent(attemptSessionId, sessionId)) return false;
|
|
60056
|
+
}
|
|
60057
|
+
const nodeId = readNonEmptyString(row.assignedNodeId);
|
|
60058
|
+
const coordinatorDaemonId = readNonEmptyString(attempt?.coordinatorDaemonId);
|
|
60059
|
+
const coordinatorSessionId = readNonEmptyString(attempt?.coordinatorSessionId);
|
|
60060
|
+
try {
|
|
60061
|
+
const result = components.instanceManager?.attachMeshAssignmentToInstance?.(sessionId, {
|
|
60062
|
+
meshId,
|
|
60063
|
+
...nodeId ? { nodeId } : {},
|
|
60064
|
+
taskId: row.id,
|
|
60065
|
+
...typeof attempt?.dispatchNonce === "number" ? { dispatchNonce: attempt.dispatchNonce } : typeof row.dispatchNonce === "number" ? { dispatchNonce: row.dispatchNonce } : {},
|
|
60066
|
+
...attempt?.attemptId ? { attemptId: attempt.attemptId } : {},
|
|
60067
|
+
...coordinatorDaemonId ? { coordinatorDaemonId } : {},
|
|
60068
|
+
...coordinatorSessionId ? { coordinatorSessionId } : {}
|
|
60069
|
+
});
|
|
60070
|
+
if (result?.stamped) {
|
|
60071
|
+
LOG2.info("MeshReconcile", `Re-stamped rebound mesh worker ${sessionId} from the durable ledger: task ${row.id} on mesh ${meshId} (attempt=${attempt?.attemptId ?? "n/a"} nonce=${attempt?.dispatchNonce ?? row.dispatchNonce ?? "n/a"}) \u2014 in-memory envelope lost (daemon restart)`);
|
|
60072
|
+
return true;
|
|
60073
|
+
}
|
|
60074
|
+
} catch {
|
|
60075
|
+
}
|
|
60076
|
+
return false;
|
|
60077
|
+
}
|
|
59652
60078
|
async function recoverStrandedAssignedDispatches(components, mesh, store, localDaemonId, selfIds = []) {
|
|
59653
60079
|
const meshId = mesh.id;
|
|
59654
60080
|
const assigned = getQueue(meshId, { status: ["assigned"] });
|
|
@@ -59666,6 +60092,7 @@ ${cleanBody}`;
|
|
|
59666
60092
|
if (key2.startsWith(meshKeyPrefix) && !assignedKeys.has(key2)) assignedIdleFinalAssistantSince.delete(key2);
|
|
59667
60093
|
}
|
|
59668
60094
|
for (const row of assigned) {
|
|
60095
|
+
restampReboundMeshWorkerAssignment(components, store, meshId, row);
|
|
59669
60096
|
const dispatchedAtMs = Date.parse(row.dispatchTimestamp ?? "");
|
|
59670
60097
|
if (!Number.isFinite(dispatchedAtMs)) continue;
|
|
59671
60098
|
const ageMs2 = nowMs - dispatchedAtMs;
|
|
@@ -59764,6 +60191,16 @@ ${cleanBody}`;
|
|
|
59764
60191
|
const verdict = row.assignedSessionId ? resolveSessionBusyVerdict(components, row.assignedSessionId) : "IDLE_CONFIRMED";
|
|
59765
60192
|
if (verdict === "GENERATING") {
|
|
59766
60193
|
deliveredUnconsumedUnknownStreak.delete(shortStreakKey);
|
|
60194
|
+
const gate = gateRedriveForHeldSuspension({ meshId, taskId: row.id, sessionLiveness: "alive", nowMs });
|
|
60195
|
+
if (gate.kind === "recovered") {
|
|
60196
|
+
traceMeshEventDrop("suspension_consumed_recovered", {
|
|
60197
|
+
taskId: row.id,
|
|
60198
|
+
sessionId: row.assignedSessionId,
|
|
60199
|
+
nodeId: row.assignedNodeId,
|
|
60200
|
+
meshId,
|
|
60201
|
+
event: "agent:generating_started"
|
|
60202
|
+
}, `held suspension recovered \u2192 ${gate.stage} (verdict GENERATING)`);
|
|
60203
|
+
}
|
|
59767
60204
|
} else {
|
|
59768
60205
|
const redriveProfile = row.assignedSessionId ? resolveAssignedTranscriptProfile(components, row) : void 0;
|
|
59769
60206
|
if (redriveProfile?.emitsPtyTurnEvents === false && await pollAssignedTaskInTurnProgress(components, { id: meshId, nodes: mesh.nodes }, row)) {
|
|
@@ -59792,16 +60229,39 @@ ${cleanBody}`;
|
|
|
59792
60229
|
const streak = (deliveredUnconsumedUnknownStreak.get(shortStreakKey) ?? 0) + 1;
|
|
59793
60230
|
deliveredUnconsumedUnknownStreak.set(shortStreakKey, streak);
|
|
59794
60231
|
if (streak < RECLAIM_UNKNOWN_GRACE_TICKS) {
|
|
59795
|
-
|
|
60232
|
+
const gate = gateRedriveForHeldSuspension({ meshId, taskId: row.id, sessionLiveness: "unknown", sessionProvenDead: false, nowMs });
|
|
60233
|
+
traceMeshEventDrop(gate.kind === "blocked" ? "redrive_blocked_by_suspension" : "short_redrive_deferred_unknown_verdict", {
|
|
59796
60234
|
taskId: row.id,
|
|
59797
60235
|
sessionId: row.assignedSessionId,
|
|
59798
60236
|
nodeId: row.assignedNodeId,
|
|
59799
60237
|
meshId,
|
|
59800
60238
|
event: "agent:generating_started"
|
|
59801
|
-
}, `unknown ${streak}/${RECLAIM_UNKNOWN_GRACE_TICKS}`);
|
|
60239
|
+
}, gate.kind === "blocked" ? `held ${gate.stages.join("/")} unresolved; unknown ${streak}/${RECLAIM_UNKNOWN_GRACE_TICKS}` : `unknown ${streak}/${RECLAIM_UNKNOWN_GRACE_TICKS}`);
|
|
59802
60240
|
continue;
|
|
59803
60241
|
}
|
|
59804
60242
|
}
|
|
60243
|
+
const suspensionGate = gateRedriveForHeldSuspension({
|
|
60244
|
+
meshId,
|
|
60245
|
+
taskId: row.id,
|
|
60246
|
+
sessionLiveness: verdict === "UNKNOWN" ? "unknown" : "alive",
|
|
60247
|
+
sessionProvenDead: verdict === "UNKNOWN",
|
|
60248
|
+
// the bounded UNKNOWN grace just exhausted above
|
|
60249
|
+
nowMs
|
|
60250
|
+
});
|
|
60251
|
+
if (suspensionGate.kind === "recovered" || suspensionGate.kind === "blocked") {
|
|
60252
|
+
deliveredUnconsumedUnknownStreak.delete(shortStreakKey);
|
|
60253
|
+
traceMeshEventDrop(suspensionGate.kind === "recovered" ? "suspension_consumed_recovered" : "redrive_blocked_by_suspension", {
|
|
60254
|
+
taskId: row.id,
|
|
60255
|
+
sessionId: row.assignedSessionId,
|
|
60256
|
+
nodeId: row.assignedNodeId,
|
|
60257
|
+
meshId,
|
|
60258
|
+
event: "agent:generating_started"
|
|
60259
|
+
}, suspensionGate.kind === "recovered" ? `consumed recovered from held suspension \u2192 ${suspensionGate.stage}; same attempt continues` : "held suspension unresolved \u2014 redrive suppressed");
|
|
60260
|
+
continue;
|
|
60261
|
+
}
|
|
60262
|
+
if (suspensionGate.kind === "released" && suspensionGate.dropped > 0) {
|
|
60263
|
+
LOG2.info("MeshReconcile", `Dropped ${suspensionGate.dropped} held suspension(s) for task ${row.id} on mesh ${meshId}: worker session ${row.assignedSessionId ?? "?"} demonstrably dead \u2014 delivered_not_consumed_redrive proceeds to a new attempt`);
|
|
60264
|
+
}
|
|
59805
60265
|
const redriveEval = (() => {
|
|
59806
60266
|
try {
|
|
59807
60267
|
return evaluateRedrive(meshId, row.id, nowMs);
|
|
@@ -59868,6 +60328,16 @@ ${cleanBody}`;
|
|
|
59868
60328
|
const verdict = row.assignedSessionId ? resolveSessionBusyVerdict(components, row.assignedSessionId) : "IDLE_CONFIRMED";
|
|
59869
60329
|
if (verdict === "GENERATING") {
|
|
59870
60330
|
deliveredNoTurnUnknownStreak.delete(streakKey);
|
|
60331
|
+
const gate = gateRedriveForHeldSuspension({ meshId, taskId: row.id, sessionLiveness: "alive", nowMs });
|
|
60332
|
+
if (gate.kind === "recovered") {
|
|
60333
|
+
traceMeshEventDrop("suspension_consumed_recovered", {
|
|
60334
|
+
taskId: row.id,
|
|
60335
|
+
sessionId: row.assignedSessionId,
|
|
60336
|
+
nodeId: row.assignedNodeId,
|
|
60337
|
+
meshId,
|
|
60338
|
+
event: "agent:generating_completed"
|
|
60339
|
+
}, `held suspension recovered \u2192 ${gate.stage} (verdict GENERATING)`);
|
|
60340
|
+
}
|
|
59871
60341
|
continue;
|
|
59872
60342
|
}
|
|
59873
60343
|
let reclaimReason;
|
|
@@ -59887,17 +60357,40 @@ ${cleanBody}`;
|
|
|
59887
60357
|
const streak = (deliveredNoTurnUnknownStreak.get(streakKey) ?? 0) + 1;
|
|
59888
60358
|
deliveredNoTurnUnknownStreak.set(streakKey, streak);
|
|
59889
60359
|
if (streak < RECLAIM_UNKNOWN_GRACE_TICKS) {
|
|
59890
|
-
|
|
60360
|
+
const gate = gateRedriveForHeldSuspension({ meshId, taskId: row.id, sessionLiveness: "unknown", sessionProvenDead: false, nowMs });
|
|
60361
|
+
traceMeshEventDrop(gate.kind === "blocked" ? "redrive_blocked_by_suspension" : "reclaim_deferred_unknown_verdict", {
|
|
59891
60362
|
taskId: row.id,
|
|
59892
60363
|
sessionId: row.assignedSessionId,
|
|
59893
60364
|
nodeId: row.assignedNodeId,
|
|
59894
60365
|
meshId,
|
|
59895
60366
|
event: "agent:generating_completed"
|
|
59896
|
-
}, `unknown ${streak}/${RECLAIM_UNKNOWN_GRACE_TICKS}`);
|
|
60367
|
+
}, gate.kind === "blocked" ? `held ${gate.stages.join("/")} unresolved; unknown ${streak}/${RECLAIM_UNKNOWN_GRACE_TICKS}` : `unknown ${streak}/${RECLAIM_UNKNOWN_GRACE_TICKS}`);
|
|
59897
60368
|
continue;
|
|
59898
60369
|
}
|
|
59899
60370
|
reclaimReason = "reclaim_after_unknown_grace";
|
|
59900
60371
|
}
|
|
60372
|
+
const suspensionGate = gateRedriveForHeldSuspension({
|
|
60373
|
+
meshId,
|
|
60374
|
+
taskId: row.id,
|
|
60375
|
+
sessionLiveness: verdict === "UNKNOWN" ? "unknown" : "alive",
|
|
60376
|
+
sessionProvenDead: verdict === "UNKNOWN",
|
|
60377
|
+
// the bounded UNKNOWN grace just exhausted above
|
|
60378
|
+
nowMs
|
|
60379
|
+
});
|
|
60380
|
+
if (suspensionGate.kind === "recovered" || suspensionGate.kind === "blocked") {
|
|
60381
|
+
deliveredNoTurnUnknownStreak.delete(streakKey);
|
|
60382
|
+
traceMeshEventDrop(suspensionGate.kind === "recovered" ? "suspension_consumed_recovered" : "redrive_blocked_by_suspension", {
|
|
60383
|
+
taskId: row.id,
|
|
60384
|
+
sessionId: row.assignedSessionId,
|
|
60385
|
+
nodeId: row.assignedNodeId,
|
|
60386
|
+
meshId,
|
|
60387
|
+
event: "agent:generating_completed"
|
|
60388
|
+
}, suspensionGate.kind === "recovered" ? `consumed recovered from held suspension \u2192 ${suspensionGate.stage}; same attempt continues` : "held suspension unresolved \u2014 reclaim suppressed");
|
|
60389
|
+
continue;
|
|
60390
|
+
}
|
|
60391
|
+
if (suspensionGate.kind === "released" && suspensionGate.dropped > 0) {
|
|
60392
|
+
LOG2.info("MeshReconcile", `Dropped ${suspensionGate.dropped} held suspension(s) for task ${row.id} on mesh ${meshId}: worker session ${row.assignedSessionId ?? "?"} demonstrably dead \u2014 ${reclaimReason} proceeds to a new attempt`);
|
|
60393
|
+
}
|
|
59901
60394
|
const terminalEvidence = await pollAssignedTaskTerminalEvidence(components, mesh, row);
|
|
59902
60395
|
if (terminalEvidence) {
|
|
59903
60396
|
deliveredNoTurnUnknownStreak.delete(streakKey);
|
|
@@ -59985,6 +60478,7 @@ ${cleanBody}`;
|
|
|
59985
60478
|
const assigned = getQueue(meshId, { status: ["assigned"] });
|
|
59986
60479
|
if (!assigned.length) return;
|
|
59987
60480
|
const nowMs = Date.now();
|
|
60481
|
+
const store = MeshRuntimeStore.getInstance();
|
|
59988
60482
|
const assignedNodeIsLocal = (assignedNodeId) => {
|
|
59989
60483
|
if (!assignedNodeId) return true;
|
|
59990
60484
|
if (selfIds.some((id) => daemonIdsEquivalent(id, assignedNodeId))) return true;
|
|
@@ -59995,6 +60489,7 @@ ${cleanBody}`;
|
|
|
59995
60489
|
};
|
|
59996
60490
|
for (const row of assigned) {
|
|
59997
60491
|
if (Number.isFinite(Date.parse(row.dispatchTimestamp ?? ""))) continue;
|
|
60492
|
+
restampReboundMeshWorkerAssignment(components, store, meshId, row);
|
|
59998
60493
|
const updatedMs = Date.parse(row.updatedAt ?? "");
|
|
59999
60494
|
const createdMs = Date.parse(row.createdAt ?? "");
|
|
60000
60495
|
const anchorMs = Number.isFinite(updatedMs) ? updatedMs : createdMs;
|
|
@@ -60470,6 +60965,7 @@ ${cleanBody}`;
|
|
|
60470
60965
|
if (recovered.length > 0) {
|
|
60471
60966
|
LOG2.info("TurnLedger", `Restart recovery: reconstructed ${recovered.length} active turn attempt(s) for mesh ${mesh.id} (${recovered.map((a) => `${a.taskId.slice(0, 8)}@${a.stage}`).join(", ")})`);
|
|
60472
60967
|
}
|
|
60968
|
+
drainHeldTurnSuspensionsForMesh(mesh.id);
|
|
60473
60969
|
}
|
|
60474
60970
|
} catch (e) {
|
|
60475
60971
|
LOG2.warn("TurnLedger", `Restart attempt reconstruction failed (reconcile continues on row state): ${e?.message || e}`);
|
|
@@ -62467,7 +62963,7 @@ ${cleanBody}`;
|
|
|
62467
62963
|
return out;
|
|
62468
62964
|
}
|
|
62469
62965
|
function buttonBlockApprovalCue(spec, text) {
|
|
62470
|
-
if (/Enter to select/i.test(text)
|
|
62966
|
+
if (/Enter to select/i.test(text)) return false;
|
|
62471
62967
|
const labels = extractButtonLabels(spec, text);
|
|
62472
62968
|
if (labels.length < 2) return false;
|
|
62473
62969
|
if (pickApprovalButton(labels).index < 0) return false;
|
|
@@ -62475,8 +62971,7 @@ ${cleanBody}`;
|
|
|
62475
62971
|
}
|
|
62476
62972
|
function isAskUserQuestionPickerSignature(text) {
|
|
62477
62973
|
if (!text) return false;
|
|
62478
|
-
|
|
62479
|
-
if (!hasSelectFooter) return false;
|
|
62974
|
+
if (!/Enter to select/i.test(text)) return false;
|
|
62480
62975
|
return PICKER_OPTION_ROW.test(text);
|
|
62481
62976
|
}
|
|
62482
62977
|
function modalMatches(spec, input) {
|
|
@@ -62625,8 +63120,7 @@ ${cleanBody}`;
|
|
|
62625
63120
|
}
|
|
62626
63121
|
function isAskUserQuestionPickerSignature2(text) {
|
|
62627
63122
|
if (!text) return false;
|
|
62628
|
-
|
|
62629
|
-
if (!hasSelectFooter) return false;
|
|
63123
|
+
if (!/Enter to select/i.test(text)) return false;
|
|
62630
63124
|
return PICKER_OPTION_ROW2.test(text);
|
|
62631
63125
|
}
|
|
62632
63126
|
function findQuestionLineIndex(spec, lines) {
|
|
@@ -67745,7 +68239,7 @@ ${lastSnapshot}`;
|
|
|
67745
68239
|
return headers;
|
|
67746
68240
|
}
|
|
67747
68241
|
function isClaudeTuiSelectFooter(text) {
|
|
67748
|
-
return /Enter to select/i.test(text)
|
|
68242
|
+
return /Enter to select/i.test(text);
|
|
67749
68243
|
}
|
|
67750
68244
|
function detectClaudeTuiMultiSelect(screenText) {
|
|
67751
68245
|
if (/Space to (?:select|toggle)|toggle selection|select multiple|select all that apply/i.test(screenText)) {
|
|
@@ -75803,6 +76297,16 @@ ${effect.notification.body || ""}`.trim();
|
|
|
75803
76297
|
const status = adapter.getStatus();
|
|
75804
76298
|
const targetInstance = getTargetInstance(h, args);
|
|
75805
76299
|
const targetState = targetInstance?.getState?.();
|
|
76300
|
+
const heldPrompt = status?.activeInteractivePrompt ?? targetState?.activeChat?.activeInteractivePrompt ?? null;
|
|
76301
|
+
if (heldPrompt && typeof heldPrompt.promptId === "string" && heldPrompt.promptId) {
|
|
76302
|
+
LOG2.info("Command", `[resolveAction] refused \u2014 session holds interactive prompt ${heldPrompt.promptId} (waiting_choice, not an approval)`);
|
|
76303
|
+
return {
|
|
76304
|
+
success: false,
|
|
76305
|
+
error: `Session is awaiting a multi-choice answer (waiting_choice), not an approval. Use mesh_answer_question with promptId "${heldPrompt.promptId}" instead of mesh_approve.`,
|
|
76306
|
+
waitingChoice: true,
|
|
76307
|
+
promptId: heldPrompt.promptId
|
|
76308
|
+
};
|
|
76309
|
+
}
|
|
75806
76310
|
const surfacedModal = targetState?.activeChat?.activeModal && Array.isArray(targetState.activeChat.activeModal.buttons) && targetState.activeChat.activeModal.buttons.some((candidate) => typeof candidate === "string" && candidate.trim()) ? targetState.activeChat.activeModal : null;
|
|
75807
76311
|
const statusModal = status?.activeModal && Array.isArray(status.activeModal.buttons) && status.activeModal.buttons.some((candidate) => typeof candidate === "string" && candidate.trim()) ? status.activeModal : null;
|
|
75808
76312
|
const parsedStatus = !statusModal && !surfacedModal && typeof adapter.getScriptParsedStatus === "function" ? (() => {
|
|
@@ -88106,9 +88610,13 @@ ${buttons.join("\n")}`;
|
|
|
88106
88610
|
}
|
|
88107
88611
|
const parsedStatus = null;
|
|
88108
88612
|
const rawStatus = adapterStatus.status;
|
|
88613
|
+
const interactivePrompt = adapterStatus.activeInteractivePrompt ?? null;
|
|
88614
|
+
const activeModal = adapterStatus.activeModal ?? null;
|
|
88615
|
+
const activeModalKind = activeModal && typeof activeModal.kind === "string" ? activeModal.kind : null;
|
|
88616
|
+
const isQuestionPicker = rawStatus === "waiting_approval" && !!interactivePrompt && (!activeModal || activeModalKind === "picker");
|
|
88109
88617
|
const autoApproveActive = this.maybeAutoApproveStatus(adapterStatus, now);
|
|
88110
88618
|
const autoApproveHoldIdle = this.autoApproveBusy && rawStatus === "idle";
|
|
88111
|
-
const newStatus = autoApproveActive || autoApproveHoldIdle ? "generating" : rawStatus;
|
|
88619
|
+
const newStatus = isQuestionPicker ? "waiting_choice" : autoApproveActive || autoApproveHoldIdle ? "generating" : rawStatus;
|
|
88112
88620
|
const dirName = workingDirBasename(this.workingDir);
|
|
88113
88621
|
const chatTitle = `${this.provider.name} \xB7 ${dirName}`;
|
|
88114
88622
|
const partial2 = this.adapter.getPartialResponse();
|
|
@@ -88155,7 +88663,7 @@ ${buttons.join("\n")}`;
|
|
|
88155
88663
|
}
|
|
88156
88664
|
this.generatingDebounceTimer = null;
|
|
88157
88665
|
}, 3e3);
|
|
88158
|
-
} else if (newStatus === "waiting_approval") {
|
|
88666
|
+
} else if (newStatus === "waiting_approval" && !(previousStatus === "waiting_choice" && !adapterStatus.activeModal)) {
|
|
88159
88667
|
this.suppressIdleHistoryReplay = false;
|
|
88160
88668
|
if (this.generatingDebouncePending) {
|
|
88161
88669
|
if (this.generatingDebounceTimer) {
|
|
@@ -88195,6 +88703,23 @@ ${buttons.join("\n")}`;
|
|
|
88195
88703
|
modalButtons: modal?.buttons
|
|
88196
88704
|
});
|
|
88197
88705
|
}
|
|
88706
|
+
} else if (newStatus === "waiting_choice") {
|
|
88707
|
+
this.suppressIdleHistoryReplay = false;
|
|
88708
|
+
if (this.generatingDebouncePending) {
|
|
88709
|
+
if (this.generatingDebounceTimer) {
|
|
88710
|
+
clearTimeout(this.generatingDebounceTimer);
|
|
88711
|
+
this.generatingDebounceTimer = null;
|
|
88712
|
+
}
|
|
88713
|
+
this.pushEvent({ event: "agent:generating_started", ...this.generatingDebouncePending });
|
|
88714
|
+
this.generatingDebouncePending = null;
|
|
88715
|
+
}
|
|
88716
|
+
if (this.completedDebounceTimer) {
|
|
88717
|
+
clearTimeout(this.completedDebounceTimer);
|
|
88718
|
+
this.completedDebounceTimer = null;
|
|
88719
|
+
}
|
|
88720
|
+
this.completedDebouncePending = null;
|
|
88721
|
+
if (!this.generatingStartedAt) this.generatingStartedAt = now;
|
|
88722
|
+
this.busyEpoch++;
|
|
88198
88723
|
} else if (newStatus === "generating" && this.lastStatus === "waiting_approval") {
|
|
88199
88724
|
this.lastApprovalEventFingerprint = "";
|
|
88200
88725
|
} else if (newStatus === "idle" && (this.lastStatus === "generating" || this.lastStatus === "waiting_approval")) {
|
|
@@ -88367,7 +88892,6 @@ ${buttons.join("\n")}`;
|
|
|
88367
88892
|
}
|
|
88368
88893
|
this.lastStatus = newStatus;
|
|
88369
88894
|
}
|
|
88370
|
-
const interactivePrompt = adapterStatus.activeInteractivePrompt ?? null;
|
|
88371
88895
|
if (interactivePrompt && newStatus !== "waiting_approval") {
|
|
88372
88896
|
const firstQuestion = interactivePrompt.questions?.[0];
|
|
88373
88897
|
const promptKey = `${interactivePrompt.promptId}::${firstQuestion?.question ?? ""}`;
|
|
@@ -88477,9 +89001,12 @@ ${buttons.join("\n")}`;
|
|
|
88477
89001
|
this.events.push(enrichedEvent);
|
|
88478
89002
|
}
|
|
88479
89003
|
if (TERMINAL_MESH_EVENTS.has(event.event) && this.settings.meshActiveTaskId) {
|
|
88480
|
-
|
|
88481
|
-
|
|
88482
|
-
|
|
89004
|
+
const readyWithTurnInFlight = event.event === "agent:ready" && (this.generatingStartedAt !== 0 || this.completedDebouncePending !== null || this.generatingDebouncePending !== null);
|
|
89005
|
+
if (!readyWithTurnInFlight) {
|
|
89006
|
+
try {
|
|
89007
|
+
this.detachMeshAssignment();
|
|
89008
|
+
} catch {
|
|
89009
|
+
}
|
|
88483
89010
|
}
|
|
88484
89011
|
}
|
|
88485
89012
|
}
|
|
@@ -88651,6 +89178,9 @@ ${buttons.join("\n")}`;
|
|
|
88651
89178
|
if (!this.shouldUsePtyAutoApprove()) return;
|
|
88652
89179
|
if (!this.isMeshWorkerSession()) return;
|
|
88653
89180
|
if (adapterStatus?.status !== "waiting_approval") return;
|
|
89181
|
+
const nudgeModal = adapterStatus.activeModal;
|
|
89182
|
+
const nudgeModalKind = nudgeModal && typeof nudgeModal.kind === "string" ? nudgeModal.kind : null;
|
|
89183
|
+
if (adapterStatus.activeInteractivePrompt && (!nudgeModal || nudgeModalKind === "picker")) return;
|
|
88654
89184
|
if (!this.autoApproveMaskStalled(now)) return;
|
|
88655
89185
|
const currentSignature = this.approvableModalSignature(adapterStatus.activeModal);
|
|
88656
89186
|
const settleProgressing = !!currentSignature && this.pendingAutoApprovalSince > 0 && currentSignature === this.pendingAutoApprovalSignature;
|