botmux 3.12.1 → 3.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (123) hide show
  1. package/dist/.runtime-build-id +1 -1
  2. package/dist/bot-registry.d.ts +7 -0
  3. package/dist/bot-registry.d.ts.map +1 -1
  4. package/dist/bot-registry.js +46 -13
  5. package/dist/bot-registry.js.map +1 -1
  6. package/dist/core/config-dir.d.ts +167 -0
  7. package/dist/core/config-dir.d.ts.map +1 -0
  8. package/dist/core/config-dir.js +152 -0
  9. package/dist/core/config-dir.js.map +1 -0
  10. package/dist/core/dashboard-ipc-server.d.ts.map +1 -1
  11. package/dist/core/dashboard-ipc-server.js +48 -0
  12. package/dist/core/dashboard-ipc-server.js.map +1 -1
  13. package/dist/core/per-bot-env.d.ts.map +1 -1
  14. package/dist/core/per-bot-env.js +9 -0
  15. package/dist/core/per-bot-env.js.map +1 -1
  16. package/dist/core/session-discovery.d.ts.map +1 -1
  17. package/dist/core/session-discovery.js +19 -2
  18. package/dist/core/session-discovery.js.map +1 -1
  19. package/dist/core/session-manager.d.ts.map +1 -1
  20. package/dist/core/session-manager.js +44 -3
  21. package/dist/core/session-manager.js.map +1 -1
  22. package/dist/core/trigger-session.d.ts.map +1 -1
  23. package/dist/core/trigger-session.js +499 -63
  24. package/dist/core/trigger-session.js.map +1 -1
  25. package/dist/core/types.d.ts +26 -4
  26. package/dist/core/types.d.ts.map +1 -1
  27. package/dist/core/types.js.map +1 -1
  28. package/dist/core/worker-pool.d.ts +64 -0
  29. package/dist/core/worker-pool.d.ts.map +1 -1
  30. package/dist/core/worker-pool.js +301 -10
  31. package/dist/core/worker-pool.js.map +1 -1
  32. package/dist/daemon.d.ts.map +1 -1
  33. package/dist/daemon.js +13 -6
  34. package/dist/daemon.js.map +1 -1
  35. package/dist/dashboard/auth.d.ts +8 -0
  36. package/dist/dashboard/auth.d.ts.map +1 -1
  37. package/dist/dashboard/auth.js +16 -11
  38. package/dist/dashboard/auth.js.map +1 -1
  39. package/dist/dashboard/web/bot-multi-select.d.ts +33 -0
  40. package/dist/dashboard/web/bot-multi-select.d.ts.map +1 -0
  41. package/dist/dashboard/web/bot-multi-select.js +34 -0
  42. package/dist/dashboard/web/bot-multi-select.js.map +1 -0
  43. package/dist/dashboard/web/groups-page.d.ts +13 -0
  44. package/dist/dashboard/web/groups-page.d.ts.map +1 -1
  45. package/dist/dashboard/web/groups-page.js +25 -7
  46. package/dist/dashboard/web/groups-page.js.map +1 -1
  47. package/dist/dashboard/web/i18n.d.ts.map +1 -1
  48. package/dist/dashboard/web/i18n.js +8 -4
  49. package/dist/dashboard/web/i18n.js.map +1 -1
  50. package/dist/dashboard/web/sessions-page.d.ts.map +1 -1
  51. package/dist/dashboard/web/sessions-page.js +15 -23
  52. package/dist/dashboard/web/sessions-page.js.map +1 -1
  53. package/dist/dashboard/web/sessions.d.ts +13 -0
  54. package/dist/dashboard/web/sessions.d.ts.map +1 -1
  55. package/dist/dashboard/web/sessions.js +25 -0
  56. package/dist/dashboard/web/sessions.js.map +1 -1
  57. package/dist/dashboard-web/app.js +1 -1
  58. package/dist/dashboard-web/chunks/{bot-defaults-page-HL655CNR.js → bot-defaults-page-NQJFJ45Z.js} +1 -1
  59. package/dist/dashboard-web/chunks/{chunk-BFUTTYVK.js → chunk-5QJB24I2.js} +1 -1
  60. package/dist/dashboard-web/chunks/{chunk-ELL53D62.js → chunk-6JTZJRQJ.js} +2 -2
  61. package/dist/dashboard-web/chunks/{chunk-5ZL7NLR5.js → chunk-CU4SUKK3.js} +1 -1
  62. package/dist/dashboard-web/chunks/{chunk-5PAYQ3KT.js → chunk-KEEYQ3WF.js} +1 -1
  63. package/dist/dashboard-web/chunks/chunk-WLIVBPJC.js +1 -0
  64. package/dist/dashboard-web/chunks/{connectors-page-3QQACF23.js → connectors-page-HADHZXXZ.js} +1 -1
  65. package/dist/dashboard-web/chunks/groups-page-5TYF46RS.js +8 -0
  66. package/dist/dashboard-web/chunks/{insights-page-ZZ3JEJ6L.js → insights-page-EVGGJI5L.js} +1 -1
  67. package/dist/dashboard-web/chunks/{monitor-room-ROJ5GZ4N.js → monitor-room-HRGR2ENP.js} +1 -1
  68. package/dist/dashboard-web/chunks/{monitoring-page-IATV7KJF.js → monitoring-page-LBU7TTTW.js} +1 -1
  69. package/dist/dashboard-web/chunks/{office-page-TYO5OIIT.js → office-page-DF6RRBMN.js} +1 -1
  70. package/dist/dashboard-web/chunks/{overview-page-VJUULZF2.js → overview-page-ZB6QH2EX.js} +1 -1
  71. package/dist/dashboard-web/chunks/{roles-page-7RJJKZGK.js → roles-page-ZBZGHIIF.js} +1 -1
  72. package/dist/dashboard-web/chunks/{schedules-page-JTEJ5IXW.js → schedules-page-NQVQC3QY.js} +1 -1
  73. package/dist/dashboard-web/chunks/sessions-page-DXBCDWCS.js +8 -0
  74. package/dist/dashboard-web/chunks/{settings-page-7CTTFBRS.js → settings-page-E7NN4NVQ.js} +1 -1
  75. package/dist/dashboard-web/chunks/{skills-page-57NZECJ3.js → skills-page-2QAVB2SA.js} +1 -1
  76. package/dist/dashboard-web/chunks/{team-federation-page-6YYIMJG3.js → team-federation-page-VGWL53AE.js} +1 -1
  77. package/dist/dashboard-web/chunks/{v3-page-ZGKBSBK3.js → v3-page-AY35LKCX.js} +1 -1
  78. package/dist/dashboard-web/chunks/{whiteboards-page-AACJF6DB.js → whiteboards-page-BNWUU4KG.js} +1 -1
  79. package/dist/dashboard-web/style.css +61 -53
  80. package/dist/features/codex-notifier/internal-turn.d.ts +1 -2
  81. package/dist/features/codex-notifier/internal-turn.d.ts.map +1 -1
  82. package/dist/features/codex-notifier/internal-turn.js +1 -13
  83. package/dist/features/codex-notifier/internal-turn.js.map +1 -1
  84. package/dist/i18n/en.js +2 -2
  85. package/dist/i18n/en.js.map +1 -1
  86. package/dist/i18n/zh.js +2 -2
  87. package/dist/i18n/zh.js.map +1 -1
  88. package/dist/platform/secure-host-file.d.ts +63 -0
  89. package/dist/platform/secure-host-file.d.ts.map +1 -1
  90. package/dist/platform/secure-host-file.js +90 -25
  91. package/dist/platform/secure-host-file.js.map +1 -1
  92. package/dist/services/async-trigger-store.d.ts +24 -4
  93. package/dist/services/async-trigger-store.d.ts.map +1 -1
  94. package/dist/services/async-trigger-store.js +81 -19
  95. package/dist/services/async-trigger-store.js.map +1 -1
  96. package/dist/services/codex-session-meta.d.ts +4 -0
  97. package/dist/services/codex-session-meta.d.ts.map +1 -0
  98. package/dist/services/codex-session-meta.js +15 -0
  99. package/dist/services/codex-session-meta.js.map +1 -0
  100. package/dist/services/idempotency-store.d.ts +18 -3
  101. package/dist/services/idempotency-store.d.ts.map +1 -1
  102. package/dist/services/idempotency-store.js +30 -12
  103. package/dist/services/idempotency-store.js.map +1 -1
  104. package/dist/services/traex-transcript.d.ts +7 -5
  105. package/dist/services/traex-transcript.d.ts.map +1 -1
  106. package/dist/services/traex-transcript.js +139 -33
  107. package/dist/services/traex-transcript.js.map +1 -1
  108. package/dist/services/trigger-types.d.ts +14 -1
  109. package/dist/services/trigger-types.d.ts.map +1 -1
  110. package/dist/services/trigger-types.js +35 -0
  111. package/dist/services/trigger-types.js.map +1 -1
  112. package/dist/types.d.ts +2 -0
  113. package/dist/types.d.ts.map +1 -1
  114. package/dist/utils/child-env.d.ts +1 -1
  115. package/dist/utils/child-env.d.ts.map +1 -1
  116. package/dist/utils/child-env.js +18 -0
  117. package/dist/utils/child-env.js.map +1 -1
  118. package/dist/worker.js +35 -2
  119. package/dist/worker.js.map +1 -1
  120. package/package.json +1 -1
  121. package/dist/dashboard-web/chunks/chunk-6ALR252E.js +0 -1
  122. package/dist/dashboard-web/chunks/groups-page-SDZMJ7UV.js +0 -8
  123. package/dist/dashboard-web/chunks/sessions-page-KD7HDT3U.js +0 -8
@@ -256,6 +256,46 @@ export async function reconcileIdempotencyLeasesOnBoot(ownerLarkAppId, currentBo
256
256
  continue;
257
257
  if (record.ownerBootId === currentBootId)
258
258
  continue;
259
+ // TURN leases govern a follow-up turn on a SHARED, long-lived session — NOT a
260
+ // throwaway fresh-session. The fresh-session reconcile below quarantines +
261
+ // closeSession(record.sessionId) to stop restore reviving an orphaned session;
262
+ // doing that to a turn lease would destroy a healthy shared session that other
263
+ // (completed or future) turns depend on (codex #818 P1-3). So reconcile a turn
264
+ // lease by fencing ONLY the exact turn: terminalize an attempting lease into
265
+ // the async store (so the caller polls `failed` at-most-once) or CAS-remove a
266
+ // provably-pre-dispatch reserved lease — and NEVER touch the session. A
267
+ // genuine I/O failure still throws → fail-closed (aggregated below).
268
+ if (record.kind === 'turn') {
269
+ try {
270
+ const asyncRec = asyncTriggerStore.lookup(record.sessionId, record.triggerId);
271
+ const outcome = (asyncRec && asyncRec.ownerLarkAppId === ownerLarkAppId) ? asyncRec.result.status : undefined;
272
+ if (outcome === 'completed' || outcome === 'failed')
273
+ continue; // already durable-terminal
274
+ if (record.state === 'attempting') {
275
+ terminalizeAttempting(record); // durable dispatch_unknown; throws on real I/O failure
276
+ continue;
277
+ }
278
+ // reserved: provably never dispatched → fenced remove (only if still this
279
+ // exact snapshot). changed-under-us / EIO throws → fail-closed. We do NOT
280
+ // reclassify a different-identity winner here: turn leases never close a
281
+ // session, and the winner (if any) is a live/attempting turn the
282
+ // worker-exit convergence + retry path already handle at-most-once.
283
+ const rm = idempotencyStore.compareAndRemoveByPath(file, record);
284
+ if (rm.kind === 'changed' && rm.current.state === 'attempting' && rm.current.ownerBootId !== currentBootId) {
285
+ // A previous-boot crossed the commit-unknown barrier for this same turn
286
+ // → terminalize by the current record so the caller polls `failed`.
287
+ terminalizeAttempting(rm.current);
288
+ }
289
+ continue;
290
+ }
291
+ catch (err) {
292
+ const e = err;
293
+ logger.error(`[idempotency] reconcile could not converge TURN lease for ${record.sessionId}/${record.triggerId}: ${e.message}`);
294
+ if (!hardFailure)
295
+ hardFailure = e;
296
+ continue;
297
+ }
298
+ }
259
299
  try {
260
300
  // Trust async terminal evidence ONLY when it was written by THIS owner. A
261
301
  // foreign completed/failed on the same sessionId/triggerId must NOT let us
@@ -390,53 +430,63 @@ export async function reconcileIdempotencyLeasesOnBoot(ownerLarkAppId, currentBo
390
430
  return quarantined;
391
431
  }
392
432
  export function convergeIdempotentAsyncTurnOnWorkerExit(ds, exitingWorkerGeneration) {
393
- const turn = ds.idempotentAsyncTurn;
394
- if (!turn)
433
+ const turns = ds.idempotentAsyncTurns;
434
+ if (!turns || turns.size === 0)
395
435
  return 'noop';
396
- // Only converge the generation this turn was dispatched on. A later worker
397
- // generation (post-completion re-fork, takeover) exiting must not retro-fail a
398
- // turn that already produced output on an earlier generation.
399
- if (turn.workerGeneration !== exitingWorkerGeneration)
400
- return 'noop';
401
- // Already completed by THIS owner (final_output cleared the stamp, or a durable
402
- // owner-matched completed exists)? Then clear the stamp and skip the write.
403
- // CRITICAL: only OUR OWN completed counts. async-trigger-store is keyed by
404
- // sessionId, so a foreign/unstamped completed on the same sessionId/triggerId
405
- // must NOT clear our only exit-convergence stamp (codex #776 round-7 finding
406
- // #3): if it did, onCliExit (Node worker still live) would leave no stamp, then
407
- // resolveIdempotencyHit ignores the foreign outcome yet reuses the attempting
408
- // lease via liveWorker, and the later onWorkerExit now stampless — can never
409
- // converge it → permanent running. Owner positive-proof, mirroring the
410
- // resolveIdempotencyHit / reconcile evidence gates.
411
- const existing = asyncTriggerStore.lookup(ds.session.sessionId, turn.triggerId);
412
- const ownedCompleted = existing?.result.status === 'completed'
413
- && existing.ownerLarkAppId === turn.ownerLarkAppId;
414
- if (ownedCompleted) {
415
- ds.idempotentAsyncTurn = undefined;
416
- return 'converged';
417
- }
418
- try {
419
- // recordFailedStrict is itself owner-proofed (throws on owner mismatch) and
420
- // completed-wins, so a foreign-occupied slot throws here → caught below →
421
- // stamp intact, caller fail-closes. Never clobbers another bot's evidence.
422
- asyncTriggerStore.recordFailedStrict(ds.session.sessionId, turn.triggerId, Date.now(), turn.ownerLarkAppId, 'dispatch_unknown');
423
- logger.warn(`[idempotency] worker exit converged incomplete idempotent async turn ${ds.session.sessionId}/${turn.triggerId} (gen ${exitingWorkerGeneration}) → durable dispatch_unknown`);
424
- // The turn is now durably terminal; clear the stamp so a subsequent exit of a
425
- // replacement generation is a no-op.
426
- ds.idempotentAsyncTurn = undefined;
427
- return 'converged';
436
+ // Converge EVERY still-pending keyed turn dispatched on the exiting generation
437
+ // a shared session can hold multiple concurrent keyed turns, and a single
438
+ // slot would only converge the last one, stranding the rest `pending` forever
439
+ // (codex #818 P1-1). Aggregate outcome: if ANY turn's durable write fails the
440
+ // caller must fail-close the whole session (some turn is unconvergeable); else
441
+ // `converged` if we settled/observed-completed at least one, `noop` if none of
442
+ // the entries belonged to this generation.
443
+ let convergedAny = false;
444
+ let anyWriteFailed = false;
445
+ for (const [triggerId, turn] of [...turns.entries()]) {
446
+ // Only converge the generation each turn was dispatched on. A later worker
447
+ // generation (post-completion re-fork, takeover) exiting must not retro-fail
448
+ // a turn that already produced output on an earlier generation.
449
+ if (turn.workerGeneration !== exitingWorkerGeneration)
450
+ continue;
451
+ // Already completed by THIS owner (final_output cleared the entry, or a durable
452
+ // owner-matched completed exists)? Then drop the entry and skip the write.
453
+ // CRITICAL: only OUR OWN completed counts. async-trigger-store is keyed by
454
+ // sessionId, so a foreign/unstamped completed on the same sessionId/triggerId
455
+ // must NOT clear our convergence stamp (codex #776 round-7 finding #3).
456
+ const existing = asyncTriggerStore.lookup(ds.session.sessionId, triggerId);
457
+ const ownedCompleted = existing?.result.status === 'completed'
458
+ && existing.ownerLarkAppId === turn.ownerLarkAppId;
459
+ if (ownedCompleted) {
460
+ turns.delete(triggerId);
461
+ convergedAny = true;
462
+ continue;
463
+ }
464
+ try {
465
+ // recordFailedStrict is itself owner-proofed (throws on owner mismatch) and
466
+ // completed-wins, so a foreign-occupied slot throws here → caught below →
467
+ // entry intact, caller fail-closes. Never clobbers another bot's evidence.
468
+ asyncTriggerStore.recordFailedStrict(ds.session.sessionId, triggerId, Date.now(), turn.ownerLarkAppId, 'dispatch_unknown');
469
+ logger.warn(`[idempotency] worker exit converged incomplete idempotent async turn ${ds.session.sessionId}/${triggerId} (gen ${exitingWorkerGeneration}) → durable dispatch_unknown`);
470
+ // The turn is now durably terminal; drop the entry so a subsequent exit of a
471
+ // replacement generation is a no-op for it.
472
+ turns.delete(triggerId);
473
+ convergedAny = true;
474
+ }
475
+ catch (err) {
476
+ // Durable write failed (EIO/ENOSPC, or a foreign-occupied slot). Keep the
477
+ // entry and report write_failed so the caller takes this session to an
478
+ // observable fail-closed terminal (close it). We must NOT merely wait for a
479
+ // next-boot reconcile: the same Node worker auto-restarts to a healthy idle
480
+ // CLI, per-item noReplay stops the keyed input from re-running, and the live
481
+ // ds would otherwise strand the poller on `running` forever (codex #776
482
+ // round-8 finding #2).
483
+ logger.error(`[idempotency] worker-exit convergence write failed for ${ds.session.sessionId}/${triggerId}; caller must fail-close: ${err.message}`);
484
+ anyWriteFailed = true;
485
+ }
428
486
  }
429
- catch (err) {
430
- // Durable write failed (EIO/ENOSPC, or a foreign-occupied slot). Keep the
431
- // stamp and report write_failed so the caller takes this session to an
432
- // observable fail-closed terminal (close it). We must NOT merely wait for a
433
- // next-boot reconcile: the same Node worker auto-restarts to a healthy idle
434
- // CLI, per-item noReplay stops the keyed input from re-running, and the live
435
- // ds would otherwise strand the poller on `running` forever (codex #776
436
- // round-8 finding #2).
437
- logger.error(`[idempotency] worker-exit convergence write failed for ${ds.session.sessionId}/${turn.triggerId}; caller must fail-close: ${err.message}`);
487
+ if (anyWriteFailed)
438
488
  return 'write_failed';
439
- }
489
+ return convergedAny ? 'converged' : 'noop';
440
490
  }
441
491
  function waitForSessionFinalOutput(ds, triggerId, timeoutMs, buildCompletedResponse, dispatchTurn) {
442
492
  ds.pendingWaitPromises ??= new Map();
@@ -729,6 +779,143 @@ async function triggerSessionTurnAdmitted(req, deps, internal) {
729
779
  idempotencyTakeover = hit;
730
780
  }
731
781
  }
782
+ // ── Turn-level idempotency (follow-up async append; validator guarantees the
783
+ // shape: target.sessionId set, asyncReturnSessionId, no wait/dryRun, and
784
+ // mutually exclusive with the fresh-session idempotencyKey above) ──
785
+ // Same at-most-once dispatch LEASE as the fresh-session key, but scoped to
786
+ // (sessionId, turnIdempotencyKey) so a retried append to an existing session
787
+ // resolves to the SAME turn instead of injecting twice. UNFORGEABLE domain
788
+ // separation from fresh-session keys comes from the store's `kind: 'turn'`
789
+ // (baked into the on-disk key digest) — NOT a user-constructable string prefix
790
+ // (codex #818 P1-2). The lease key still embeds sessionId so two different
791
+ // sessions with the same turnIdempotencyKey never share a lease. The heavy
792
+ // lifting (claim → reserved→attempting barrier → convergence) reuses
793
+ // idempotencyStore and is woven into deliverToExisting's async branch below;
794
+ // here we only do the fast pre-check (reuse/terminal short-circuit).
795
+ const turnIdempotencyKey = req.options?.turnIdempotencyKey?.trim();
796
+ const turnLeaseKey = turnIdempotencyKey && req.target.sessionId
797
+ ? `${req.target.sessionId}\u0000${turnIdempotencyKey}`
798
+ : undefined;
799
+ // requestHash binds the key to its full business payload (same rule as the
800
+ // fresh-session key): a same-key retry with a DIFFERENT payload is a caller bug
801
+ // (409), not a silent join. Excludes both idempotency keys (lookup keys, not
802
+ // payload). Also binds the target sessionId so a turn hash can never be
803
+ // mistaken for (or collide with) any other session's/seam's hash (codex #818
804
+ // P1-2: the fresh hash omits sessionId, so without this a same-payload fresh
805
+ // and turn request could hash-match).
806
+ const { idempotencyKey: _omitK1, turnIdempotencyKey: _omitK2, ...turnOptionsForHash } = (req.options ?? {});
807
+ const turnRequestHash = turnLeaseKey
808
+ ? computeInputHash({
809
+ seam: 'turn',
810
+ sessionId: req.target.sessionId ?? null,
811
+ instruction: req.instruction ?? null,
812
+ envelope: req.envelope,
813
+ source: req.source,
814
+ presentation: req.presentation ?? null,
815
+ options: turnOptionsForHash,
816
+ })
817
+ : '';
818
+ let turnIdempotencyTakeover;
819
+ if (turnLeaseKey && !dryRun) {
820
+ // A device-isolation freeze defers spawns; refuse the keyed dispatch up-front
821
+ // so nothing is dispatched and the caller can retry cleanly (mirrors the
822
+ // fresh-session guard — a follow-up fork can be deferred the same way).
823
+ if (currentDeviceIsolationFreezeLease()) {
824
+ return {
825
+ ok: false,
826
+ errorCode: 'trigger_failed',
827
+ error: 'device credential activation in progress; retry the idempotent trigger shortly',
828
+ turnIdempotencyKey,
829
+ };
830
+ }
831
+ let hit;
832
+ try {
833
+ hit = idempotencyStore.lookup(larkAppId, turnLeaseKey, 'turn');
834
+ }
835
+ catch (err) {
836
+ return { ok: false, errorCode: 'trigger_failed', error: `idempotency lease unreadable: ${err.message}` };
837
+ }
838
+ if (hit) {
839
+ if (hit.requestHash !== turnRequestHash) {
840
+ return {
841
+ ok: false,
842
+ errorCode: 'idempotency_conflict',
843
+ error: 'turnIdempotencyKey already used with a different request payload',
844
+ turnIdempotencyKey,
845
+ };
846
+ }
847
+ const decision = resolveIdempotencyHit(hit, ownerBootId, deps.activeSessions);
848
+ if (decision.kind === 'reuse') {
849
+ // Recovery for the double-fault case (P1-8): a live shared-session turn
850
+ // whose post-barrier terminalize failed is `attempting` + live-worker, so
851
+ // resolveIdempotencyHit says `reuse` — but nothing dispatched and no
852
+ // durable result exists, so reusing would hang `running` forever. If this
853
+ // turn is flagged postBarrierFault, RE-ATTEMPT the strict terminalize now.
854
+ const faultDs = activeBySessionId(deps.activeSessions, hit.sessionId);
855
+ const faultEntry = faultDs?.idempotentAsyncTurns?.get(hit.triggerId);
856
+ if (faultDs && faultEntry?.postBarrierFault) {
857
+ // COMPLETED-WINS (codex #818 P1-8 race): the turn may have actually
858
+ // finished between the fault and this retry. The AUTHORITATIVE decision
859
+ // is recordFailedStrict's IN-LOCK outcome — no TOCTOU: if a completed is
860
+ // on disk when the lock is held it returns `already_completed` (no write),
861
+ // else `written_failed`. A pre-read fast-path avoids the lock when already
862
+ // visibly completed, but correctness rests on the in-lock return.
863
+ const preRead = asyncTriggerStore.lookup(hit.sessionId, hit.triggerId);
864
+ const reuseCompleted = () => {
865
+ faultDs.idempotentAsyncTurns?.delete(hit.triggerId);
866
+ return {
867
+ ...buildAsyncQueuedResponse(hit.triggerId, hit.sessionId, decision.chatId, 'idempotency key already completed; reuse the session (poll trigger-result)'),
868
+ turnIdempotencyKey, idempotent: true,
869
+ };
870
+ };
871
+ if (preRead?.result.status === 'completed' && preRead.ownerLarkAppId === hit.ownerLarkAppId) {
872
+ return reuseCompleted();
873
+ }
874
+ try {
875
+ const outcome = asyncTriggerStore.recordFailedStrict(hit.sessionId, hit.triggerId, Date.now(), larkAppId, 'dispatch_unknown');
876
+ if (outcome === 'already_completed') {
877
+ // A completion landed AFTER our pre-read but was seen under the lock —
878
+ // completed wins, nothing was written. Reuse it, never report failed.
879
+ return reuseCompleted();
880
+ }
881
+ faultDs.idempotentAsyncTurns?.delete(hit.triggerId);
882
+ faultDs.asyncTriggerResults?.delete(hit.triggerId);
883
+ return {
884
+ ok: false, state: 'failed', triggerId: hit.triggerId,
885
+ errorCode: 'no_output', error: 'previous dispatch was interrupted with unknown outcome; not re-run (at-most-once)',
886
+ target: { kind: 'turn', sessionId: hit.sessionId, chatId: decision.chatId },
887
+ turnIdempotencyKey, idempotent: true,
888
+ };
889
+ }
890
+ catch (e) {
891
+ // Genuine I/O failure (owner mismatch / EIO) — keep the flag for the
892
+ // next retry / worker-exit / boot reconcile; do NOT report a phantom.
893
+ return { ok: false, errorCode: 'trigger_failed', error: `idempotent turn terminal outcome could not be persisted: ${e.message}`, turnIdempotencyKey };
894
+ }
895
+ }
896
+ return {
897
+ ...buildAsyncQueuedResponse(hit.triggerId, hit.sessionId, decision.chatId, decision.message),
898
+ turnIdempotencyKey,
899
+ idempotent: true,
900
+ };
901
+ }
902
+ if (decision.kind === 'terminal') {
903
+ return {
904
+ ok: false,
905
+ state: 'failed',
906
+ triggerId: hit.triggerId,
907
+ errorCode: 'no_output',
908
+ error: decision.message,
909
+ target: { kind: 'turn', sessionId: hit.sessionId, chatId: decision.chatId },
910
+ turnIdempotencyKey,
911
+ idempotent: true,
912
+ };
913
+ }
914
+ // decision.kind === 'takeover' — an older-boot pre-dispatch reserved lease
915
+ // for this same turn key. Re-claim via takeover at dispatch time below.
916
+ turnIdempotencyTakeover = hit;
917
+ }
918
+ }
732
919
  const rootMessageId = typeof req.target.rootMessageId === 'string' ? req.target.rootMessageId.trim() : '';
733
920
  let ds = req.target.sessionId ? activeBySessionId(deps.activeSessions, req.target.sessionId) : undefined;
734
921
  if (req.target.sessionId && !ds) {
@@ -828,6 +1015,138 @@ async function triggerSessionTurnAdmitted(req, deps, internal) {
828
1015
  markSessionActivity(target);
829
1016
  rememberInput(target, prompt, content);
830
1017
  };
1018
+ // Turn-level idempotency lease for a follow-up async append. Claims (or takes
1019
+ // over an older-boot reserved) lease keyed by (sessionId, turnIdempotencyKey),
1020
+ // then CASes reserved→attempting as the commit-unknown barrier BEFORE the
1021
+ // worker side effect — exactly the fresh-session discipline, minus the
1022
+ // create/close-session dance (the session already exists and is never torn
1023
+ // down here). Returns:
1024
+ // - { reuse } → a concurrent same-key winner already holds the lease; do NOT
1025
+ // dispatch, return its queued/terminal response.
1026
+ // - { lease } → we hold an `attempting` lease; proceed to dispatch. Caller
1027
+ // stamps target.idempotentAsyncTurn so a worker that dies with no
1028
+ // final_output converges to a durable dispatch_unknown (at-most-once).
1029
+ // Only meaningful when turnLeaseKey is set (async branch); a no-key turn gets
1030
+ // { lease: undefined } and dispatches exactly as before.
1031
+ const claimTurnLeaseForDispatch = () => {
1032
+ if (!turnLeaseKey)
1033
+ return { lease: undefined };
1034
+ // At-most-once cannot be held through the queued-activation durable tail:
1035
+ // that staging path (admitQueuedActivationTail → promotion → queuedActivation
1036
+ // fork) does not carry atMostOnce/noReplay, so a turn durably staged there
1037
+ // could execute on a replacement CLI AFTER the caller already saw
1038
+ // dispatch_unknown (codex #818 P1-4). Rather than thread the lease through
1039
+ // that whole subsystem, refuse the keyed dispatch RETRYABLY while the
1040
+ // admission gate owns submission order — nothing is claimed or dispatched, so
1041
+ // the caller simply retries once the opening activation drains. (A no-key
1042
+ // follow-up is unaffected: it just queues as before.)
1043
+ if (queuedBehindActivation) {
1044
+ return {
1045
+ retry: {
1046
+ ok: false,
1047
+ triggerId,
1048
+ errorCode: 'trigger_failed',
1049
+ error: 'session activation in progress; retry the idempotent follow-up shortly',
1050
+ turnIdempotencyKey,
1051
+ },
1052
+ };
1053
+ }
1054
+ const reuseWinner = (winner) => {
1055
+ // No session to close (existing-session append): just resolve from the
1056
+ // winner's terminal evidence, never dispatching a second time.
1057
+ const decision = resolveIdempotencyHit(winner, ownerBootId, deps.activeSessions);
1058
+ const winnerChatId = (decision.kind !== 'takeover' && decision.chatId) ? decision.chatId : chatId;
1059
+ if (decision.kind === 'terminal') {
1060
+ return {
1061
+ ok: false, state: 'failed', triggerId: winner.triggerId,
1062
+ errorCode: 'no_output', error: decision.message,
1063
+ target: { kind: 'turn', sessionId: winner.sessionId, chatId: winnerChatId },
1064
+ turnIdempotencyKey, idempotent: true,
1065
+ };
1066
+ }
1067
+ return {
1068
+ ...buildAsyncQueuedResponse(winner.triggerId, winner.sessionId, winnerChatId, 'turnIdempotencyKey already claimed; reusing the in-flight turn (no new dispatch)'),
1069
+ turnIdempotencyKey, idempotent: true,
1070
+ };
1071
+ };
1072
+ const res = turnIdempotencyTakeover
1073
+ ? idempotencyStore.takeover({
1074
+ ownerLarkAppId: larkAppId, key: turnLeaseKey, expect: turnIdempotencyTakeover,
1075
+ sessionId: target.session.sessionId, triggerId, requestHash: turnRequestHash, ownerBootId, now: Date.now(), kind: 'turn',
1076
+ })
1077
+ : idempotencyStore.claim({
1078
+ ownerLarkAppId: larkAppId, sessionId: target.session.sessionId, triggerId,
1079
+ requestHash: turnRequestHash, ownerBootId, key: turnLeaseKey, now: Date.now(), kind: 'turn',
1080
+ });
1081
+ if (res.kind === 'existing')
1082
+ return { reuse: reuseWinner(res.record) };
1083
+ // Commit-unknown barrier: CAS reserved→attempting durably BEFORE dispatch.
1084
+ const attempting = idempotencyStore.transition(larkAppId, turnLeaseKey, res.record, {
1085
+ state: 'attempting', now: Date.now(),
1086
+ }, 'turn');
1087
+ return { lease: attempting };
1088
+ };
1089
+ // Stamp the idempotent-async-turn descriptor so a worker that dies with no
1090
+ // final_output converges to a durable dispatch_unknown (at-most-once) instead
1091
+ // of polling `running` forever — mirrors the fresh-session path. The
1092
+ // generation is the one this dispatch runs on.
1093
+ const stampTurnConvergence = () => {
1094
+ if (!turnLeaseKey)
1095
+ return;
1096
+ const dispatchedGeneration = Math.max(target.workerGeneration ?? 0, target.session.workerGeneration ?? 0) + (workerIsLive ? 0 : 1);
1097
+ (target.idempotentAsyncTurns ??= new Map()).set(triggerId, {
1098
+ ownerLarkAppId: larkAppId,
1099
+ key: turnLeaseKey,
1100
+ kind: 'turn',
1101
+ workerGeneration: dispatchedGeneration,
1102
+ });
1103
+ };
1104
+ // Durably terminalize a keyed turn lease after a post-barrier fault (a throw or
1105
+ // synchronous refusal between the reserved→attempting barrier and a proven
1106
+ // dispatch). Returns an observable terminal `failed` response when the durable
1107
+ // write succeeds (and drops the convergence entry); returns null when the write
1108
+ // itself throws — the caller returns a 5xx and the entry is KEPT so worker-exit
1109
+ // / retry / next-boot reconcile can still converge the still-attempting lease
1110
+ // (codex #818 P1-6/P1-7). No session close (shared session).
1111
+ const terminalizeTurnLeaseOnPostBarrierFault = (target, triggerId, leaseKey, lease) => {
1112
+ target.asyncTriggerResults?.delete(triggerId);
1113
+ if (target.latestAsyncTriggerId === triggerId)
1114
+ target.latestAsyncTriggerId = undefined;
1115
+ if (!leaseKey || !lease) {
1116
+ target.idempotentAsyncTurns?.delete(triggerId);
1117
+ return null;
1118
+ }
1119
+ try {
1120
+ const outcome = asyncTriggerStore.recordFailedStrict(target.session.sessionId, triggerId, Date.now(), larkAppId, 'dispatch_unknown');
1121
+ target.idempotentAsyncTurns?.delete(triggerId);
1122
+ if (outcome === 'already_completed') {
1123
+ // Defensive: a completed somehow already exists (completed-wins) — reuse
1124
+ // it instead of reporting failed over a real completion.
1125
+ return {
1126
+ ...buildAsyncQueuedResponse(triggerId, target.session.sessionId, chatId, 'idempotency key already completed; reuse the session (poll trigger-result)'),
1127
+ turnIdempotencyKey, idempotent: false,
1128
+ };
1129
+ }
1130
+ return {
1131
+ ok: false, state: 'failed', triggerId,
1132
+ errorCode: 'no_output', error: 'previous dispatch was interrupted with unknown outcome; not re-run (at-most-once)',
1133
+ target: { kind: 'turn', sessionId: target.session.sessionId, chatId },
1134
+ turnIdempotencyKey, idempotent: false,
1135
+ };
1136
+ }
1137
+ catch (e) {
1138
+ // Double fault: the dispatch-prep threw AND the durable terminalize threw.
1139
+ // Keep the convergence entry but MARK it postBarrierFault so a same-key
1140
+ // retry (or poll) re-attempts the strict terminalize — for a LIVE shared
1141
+ // worker the exit handler never fires, and resolveIdempotencyHit would
1142
+ // otherwise `reuse` the attempting lease forever (codex #818 P1-8).
1143
+ const entry = target.idempotentAsyncTurns?.get(triggerId);
1144
+ if (entry)
1145
+ entry.postBarrierFault = true;
1146
+ logger.error(`[idempotency] turn post-barrier fault AND recordFailedStrict failed — entry marked postBarrierFault for retry/exit convergence: ${e.message}`);
1147
+ return null;
1148
+ }
1149
+ };
831
1150
  if (workerIsLive) {
832
1151
  if (req.options?.waitForFinalOutput) {
833
1152
  return waitForSessionFinalOutput(target, triggerId, req.options?.timeoutMs ?? 120_000, (text) => ({
@@ -849,25 +1168,91 @@ async function triggerSessionTurnAdmitted(req, deps, internal) {
849
1168
  });
850
1169
  }
851
1170
  if (req.options?.asyncReturnSessionId) {
852
- beginAsyncTrigger(target, triggerId);
853
- const dispatchAttempt = prepareStableDispatch(target, false);
854
- armFinalOutputSuppression(target, dispatchAttempt);
855
- const accepted = sendWorkerInput(target, content, triggerId, {
856
- ...(dispatchAttempt !== undefined ? { dispatchAttempt } : {}),
857
- });
1171
+ // Turn-level idempotency: claim + reserved→attempting barrier BEFORE the
1172
+ // worker send. A same-key winner short-circuits (no second inject).
1173
+ let turnLease;
1174
+ if (turnLeaseKey) {
1175
+ try {
1176
+ const claimed = claimTurnLeaseForDispatch();
1177
+ if ('reuse' in claimed)
1178
+ return claimed.reuse;
1179
+ if ('retry' in claimed)
1180
+ return claimed.retry;
1181
+ turnLease = claimed.lease;
1182
+ }
1183
+ catch (err) {
1184
+ if (err instanceof idempotencyStore.IdempotencyConflictError) {
1185
+ return { ok: false, errorCode: 'idempotency_conflict', error: 'turnIdempotencyKey already used with a different request payload', turnIdempotencyKey };
1186
+ }
1187
+ return { ok: false, errorCode: 'trigger_failed', error: `turn idempotency claim failed: ${err.message}` };
1188
+ }
1189
+ }
1190
+ // Post-barrier convergence: the lease is `attempting` from here on, so ANY
1191
+ // throw between the barrier and a proven dispatch (beginAsyncTrigger /
1192
+ // prepareStableDispatch / arm / sendWorkerInput) must durably terminalize
1193
+ // the lease — else it stays `attempting` with no convergence entry and a
1194
+ // same-key retry reuses it forever (codex #818 P1-7). Mirrors the
1195
+ // fresh-session path's single wrapping try. Stamp convergence FIRST (before
1196
+ // beginAsyncTrigger) so even an early throw leaves a Map entry that
1197
+ // worker-exit can converge as a backstop.
1198
+ if (turnLease)
1199
+ stampTurnConvergence();
1200
+ let accepted;
1201
+ try {
1202
+ beginAsyncTrigger(target, triggerId);
1203
+ const dispatchAttempt = prepareStableDispatch(target, false);
1204
+ armFinalOutputSuppression(target, dispatchAttempt);
1205
+ accepted = sendWorkerInput(target, content, triggerId, {
1206
+ ...(dispatchAttempt !== undefined ? { dispatchAttempt } : {}),
1207
+ // At-most-once: a keyed follow-up delivered to a LIVE worker must be
1208
+ // tagged so a CLI crash never replays it onto the auto-restarted CLI
1209
+ // after the daemon has already terminalized it (dispatch_unknown). The
1210
+ // dormant-fork branch rides atMostOnce on the fork init instead.
1211
+ ...(turnLease ? { atMostOnce: true } : {}),
1212
+ });
1213
+ }
1214
+ catch (err) {
1215
+ // A throw AFTER the barrier (begin/prepare/arm/send). Nothing is proven
1216
+ // dispatched; terminalize the attempting lease durably so a same-key
1217
+ // retry resolves `failed` at-most-once instead of reusing it forever.
1218
+ const settled = terminalizeTurnLeaseOnPostBarrierFault(target, triggerId, turnLeaseKey, turnLease);
1219
+ return settled ?? {
1220
+ ok: false, triggerId, errorCode: 'trigger_failed',
1221
+ error: `dispatch failed and terminal outcome could not be persisted: ${err.message}`,
1222
+ target: { kind: 'turn', sessionId: target.session.sessionId, chatId },
1223
+ ...(turnIdempotencyKey ? { turnIdempotencyKey } : {}),
1224
+ };
1225
+ }
858
1226
  if (!accepted) {
859
1227
  target.asyncTriggerResults?.delete(triggerId);
860
1228
  if (target.latestAsyncTriggerId === triggerId)
861
1229
  target.latestAsyncTriggerId = undefined;
1230
+ // The worker refused the input synchronously — nothing was dispatched.
1231
+ // Terminalize the attempting lease durably so a same-key retry resolves
1232
+ // `failed` (at-most-once) rather than seeing a live-looking attempting
1233
+ // lease with no worker and hanging. Only DROP the convergence entry AFTER
1234
+ // the durable terminal write succeeds — if it throws (EIO), the lease is
1235
+ // still attempting + async still pending, so the entry MUST survive so a
1236
+ // later worker-exit / retry can still converge it (codex #818 P1-6).
1237
+ if (turnLeaseKey && turnLease) {
1238
+ terminalizeTurnLeaseOnPostBarrierFault(target, triggerId, turnLeaseKey, turnLease);
1239
+ }
1240
+ else {
1241
+ target.idempotentAsyncTurns?.delete(triggerId);
1242
+ }
862
1243
  return {
863
1244
  ok: false,
864
1245
  triggerId,
865
1246
  errorCode: 'trigger_failed',
866
1247
  error: 'worker refused async trigger input before acceptance',
1248
+ ...(turnIdempotencyKey ? { turnIdempotencyKey } : {}),
867
1249
  };
868
1250
  }
869
1251
  recordAcceptedInput();
870
- return buildAsyncQueuedResponse(triggerId, target.session.sessionId, chatId, 'delivered to existing session; poll by sessionId or triggerId for final output');
1252
+ return {
1253
+ ...buildAsyncQueuedResponse(triggerId, target.session.sessionId, chatId, 'delivered to existing session; poll by sessionId or triggerId for final output'),
1254
+ ...(turnIdempotencyKey ? { turnIdempotencyKey } : {}),
1255
+ };
871
1256
  }
872
1257
  const dispatchAttempt = prepareStableDispatch(target, false);
873
1258
  armFinalOutputSuppression(target, dispatchAttempt);
@@ -919,15 +1304,66 @@ async function triggerSessionTurnAdmitted(req, deps, internal) {
919
1304
  });
920
1305
  }
921
1306
  if (req.options?.asyncReturnSessionId) {
922
- beginAsyncTrigger(target, triggerId);
923
- const dispatchAttempt = prepareStableDispatch(target, true);
924
- armFinalOutputSuppression(target, dispatchAttempt);
925
- forkWorker(target, content, {
926
- resume: target.hasHistory,
927
- turnId: triggerId,
928
- ...(dispatchAttempt !== undefined ? { dispatchAttempt } : {}),
929
- });
930
- return buildAsyncQueuedResponse(triggerId, target.session.sessionId, chatId, 'delivered to existing session; poll by sessionId or triggerId for final output');
1307
+ // Turn-level idempotency on the dormant-worker fork path: same claim +
1308
+ // reserved→attempting barrier before the fork side effect.
1309
+ let turnLease;
1310
+ if (turnLeaseKey) {
1311
+ try {
1312
+ const claimed = claimTurnLeaseForDispatch();
1313
+ if ('reuse' in claimed)
1314
+ return claimed.reuse;
1315
+ if ('retry' in claimed)
1316
+ return claimed.retry;
1317
+ turnLease = claimed.lease;
1318
+ }
1319
+ catch (err) {
1320
+ if (err instanceof idempotencyStore.IdempotencyConflictError) {
1321
+ return { ok: false, errorCode: 'idempotency_conflict', error: 'turnIdempotencyKey already used with a different request payload', turnIdempotencyKey };
1322
+ }
1323
+ return { ok: false, errorCode: 'trigger_failed', error: `turn idempotency claim failed: ${err.message}` };
1324
+ }
1325
+ }
1326
+ // Post-barrier convergence (same as the live branch — codex #818 P1-7): the
1327
+ // lease is `attempting`, so a throw anywhere between the barrier and a proven
1328
+ // fork (beginAsyncTrigger / prepareStableDispatch / forkWorker) must durably
1329
+ // terminalize it. Stamp FIRST so an early throw still has a Map backstop.
1330
+ if (turnLease)
1331
+ stampTurnConvergence();
1332
+ // Keyed turns are at-most-once: once terminalized on worker exit, the input
1333
+ // must NEVER replay onto an auto-restarted CLI. Ride `atMostOnce` on the
1334
+ // fork init (mirrors the fresh-session path).
1335
+ const atMostOnce = !!(turnLeaseKey && turnLease);
1336
+ try {
1337
+ beginAsyncTrigger(target, triggerId);
1338
+ const dispatchAttempt = prepareStableDispatch(target, true);
1339
+ armFinalOutputSuppression(target, dispatchAttempt);
1340
+ forkWorker(target, content, {
1341
+ resume: target.hasHistory,
1342
+ turnId: triggerId,
1343
+ ...(dispatchAttempt !== undefined ? { dispatchAttempt } : {}),
1344
+ ...(atMostOnce ? { atMostOnce: true } : {}),
1345
+ });
1346
+ }
1347
+ catch (err) {
1348
+ // A throw AFTER the attempting barrier (begin/prepare/fork). Terminalize
1349
+ // the lease durably so the caller polls a terminal instead of `running`
1350
+ // forever; at-most-once, never re-dispatched. Keyed turns route through the
1351
+ // shared helper (session is never closed); a non-keyed turn rethrows.
1352
+ if (turnLeaseKey && turnLease) {
1353
+ const settled = terminalizeTurnLeaseOnPostBarrierFault(target, triggerId, turnLeaseKey, turnLease);
1354
+ return settled ?? {
1355
+ ok: false, triggerId, errorCode: 'trigger_failed',
1356
+ error: `dispatch failed and terminal outcome could not be persisted: ${err.message}`,
1357
+ target: { kind: 'turn', sessionId: target.session.sessionId, chatId },
1358
+ turnIdempotencyKey,
1359
+ };
1360
+ }
1361
+ throw err;
1362
+ }
1363
+ return {
1364
+ ...buildAsyncQueuedResponse(triggerId, target.session.sessionId, chatId, 'delivered to existing session; poll by sessionId or triggerId for final output'),
1365
+ ...(turnIdempotencyKey ? { turnIdempotencyKey } : {}),
1366
+ };
931
1367
  }
932
1368
  const dispatchAttempt = prepareStableDispatch(target, true);
933
1369
  armFinalOutputSuppression(target, dispatchAttempt);
@@ -1319,12 +1755,12 @@ async function triggerSessionTurnAdmitted(req, deps, internal) {
1319
1755
  // ignore a later generation that already moved on.
1320
1756
  if (idempotencyKey && idempotencyLease) {
1321
1757
  const dispatchedGeneration = Math.max(newDs.workerGeneration ?? 0, newDs.session.workerGeneration ?? 0) + 1;
1322
- newDs.idempotentAsyncTurn = {
1758
+ (newDs.idempotentAsyncTurns ??= new Map()).set(triggerId, {
1323
1759
  ownerLarkAppId: larkAppId,
1324
1760
  key: idempotencyKey,
1325
- triggerId,
1761
+ kind: 'fresh',
1326
1762
  workerGeneration: dispatchedGeneration,
1327
- };
1763
+ });
1328
1764
  }
1329
1765
  const dispatchAttempt = prepareStableDispatch(newDs, true);
1330
1766
  armFinalOutputSuppression(newDs, dispatchAttempt);