@dsh-enhanced/assistant-delivery 0.1.62 → 0.1.64

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/README.md +32 -10
  2. package/bin/dsh-restart-supervisor.js +76 -0
  3. package/lib/agent-runtime.d.ts +21 -1
  4. package/lib/agent-runtime.d.ts.map +1 -1
  5. package/lib/agent-runtime.js +192 -23
  6. package/lib/agent-runtime.js.map +1 -1
  7. package/lib/coordinator.d.ts.map +1 -1
  8. package/lib/coordinator.js +10 -0
  9. package/lib/coordinator.js.map +1 -1
  10. package/lib/operator-snapshot.d.ts +1 -1
  11. package/lib/operator-snapshot.d.ts.map +1 -1
  12. package/lib/operator-snapshot.js +5 -5
  13. package/lib/operator-snapshot.js.map +1 -1
  14. package/lib/restart-native.d.ts +47 -0
  15. package/lib/restart-native.d.ts.map +1 -0
  16. package/lib/restart-native.js +170 -0
  17. package/lib/restart-native.js.map +1 -0
  18. package/lib/restart-runtime.d.ts +38 -0
  19. package/lib/restart-runtime.d.ts.map +1 -0
  20. package/lib/restart-runtime.js +335 -0
  21. package/lib/restart-runtime.js.map +1 -0
  22. package/lib/restart-store.d.ts +132 -0
  23. package/lib/restart-store.d.ts.map +1 -0
  24. package/lib/restart-store.js +757 -0
  25. package/lib/restart-store.js.map +1 -0
  26. package/lib/restart-supervisor.d.ts +129 -0
  27. package/lib/restart-supervisor.d.ts.map +1 -0
  28. package/lib/restart-supervisor.js +615 -0
  29. package/lib/restart-supervisor.js.map +1 -0
  30. package/lib/service.d.ts +45 -1
  31. package/lib/service.d.ts.map +1 -1
  32. package/lib/service.js +488 -6
  33. package/lib/service.js.map +1 -1
  34. package/lib/sqlite.d.ts +1 -1
  35. package/lib/sqlite.d.ts.map +1 -1
  36. package/lib/sqlite.js +76 -2
  37. package/lib/sqlite.js.map +1 -1
  38. package/lib/store.d.ts +41 -1
  39. package/lib/store.d.ts.map +1 -1
  40. package/lib/store.js +433 -85
  41. package/lib/store.js.map +1 -1
  42. package/lib/tools.d.ts.map +1 -1
  43. package/lib/tools.js +21 -2
  44. package/lib/tools.js.map +1 -1
  45. package/lib/types.d.ts +33 -0
  46. package/lib/types.d.ts.map +1 -1
  47. package/lib/version.d.ts +1 -1
  48. package/lib/version.js +1 -1
  49. package/package.json +11 -10
package/lib/store.js CHANGED
@@ -1366,6 +1366,8 @@ function leaseTargetJson(target) {
1366
1366
  }
1367
1367
  export class DeliveryStore {
1368
1368
  database;
1369
+ /** Scan position only; every candidate is authorized again before a provider retry. */
1370
+ progressRecoveryCursors = new Map();
1369
1371
  databaseInstanceId;
1370
1372
  now;
1371
1373
  codeGenerator;
@@ -2606,6 +2608,214 @@ export class DeliveryStore {
2606
2608
  .get(channel, account, eventId);
2607
2609
  return row === undefined ? undefined : inboxFromRow(row);
2608
2610
  }
2611
+ progressAuthority(intent, credentialFingerprint) {
2612
+ if (!/^[0-9a-f]{64}$/u.test(credentialFingerprint))
2613
+ return undefined;
2614
+ const target = canonicalTarget(intent.target);
2615
+ const inbox = this.getInboxByProviderEvent(target.conversation.channel, target.conversation.account, intent.eventId);
2616
+ const binding = this.getBinding(intent.bindingId);
2617
+ const activeBinding = this.getActiveBinding(target.conversation);
2618
+ const principal = this.getPrincipal(target.principal);
2619
+ if (inbox?.bindingId !== intent.bindingId || binding?.status !== 'active'
2620
+ || activeBinding?.id !== binding.id || principal?.status !== 'active'
2621
+ || JSON.stringify(canonicalTarget({ conversation: binding.conversation, principal: binding.principal })) !== JSON.stringify(target)
2622
+ || JSON.stringify(canonicalTarget({ conversation: inbox.envelope.conversation,
2623
+ principal: inbox.envelope.principal })) !== JSON.stringify(target))
2624
+ return undefined;
2625
+ return { inbox, binding, targetHash: createHash('sha256').update(JSON.stringify(target)).digest('hex') };
2626
+ }
2627
+ progressRecord(row, inbox) {
2628
+ return {
2629
+ bindingId: row.binding_id,
2630
+ eventId: inbox.eventId,
2631
+ target: { conversation: inbox.envelope.conversation, principal: inbox.envelope.principal },
2632
+ ...(row.provider_handle === null ? {} : { providerHandle: row.provider_handle }),
2633
+ sequence: row.sequence,
2634
+ step: row.step,
2635
+ showDetails: row.show_details === 1,
2636
+ state: row.state,
2637
+ ...(row.terminal_kind === null ? {} : { terminal: row.terminal_kind === 'failed'
2638
+ ? { kind: 'failed', ...(row.terminal_code === null ? {} : { code: row.terminal_code }) }
2639
+ : { kind: row.terminal_kind } }),
2640
+ };
2641
+ }
2642
+ progressRow(inboxId) {
2643
+ return this.database.prepare('SELECT * FROM delivery_progress WHERE inbox_id = ?').get(inboxId);
2644
+ }
2645
+ /** Reserve before provider I/O; a stranded creating row must never trigger another create. */
2646
+ beginProgress(intent, credentialFingerprint, sequence, showDetails) {
2647
+ this.assertOpen();
2648
+ if (intent.update.kind !== 'started' || !Number.isSafeInteger(sequence) || sequence < 1
2649
+ || typeof showDetails !== 'boolean')
2650
+ return undefined;
2651
+ return this.transaction(() => {
2652
+ const authority = this.progressAuthority(intent, credentialFingerprint);
2653
+ if (authority === undefined || authority.inbox.status !== 'claimed')
2654
+ return undefined;
2655
+ const previous = this.progressRow(authority.inbox.id);
2656
+ if (previous !== undefined) {
2657
+ return previous.binding_id === intent.bindingId && previous.binding_version === authority.binding.version
2658
+ && previous.binding_generation === authority.binding.generation
2659
+ && previous.target_hash === authority.targetHash && previous.credential_hash === credentialFingerprint
2660
+ ? { record: this.progressRecord(previous, authority.inbox), created: false } : undefined;
2661
+ }
2662
+ this.database.prepare(`INSERT INTO delivery_progress(
2663
+ inbox_id, binding_id, binding_version, binding_generation, target_hash, credential_hash,
2664
+ sequence, step, show_details, state, created_at, updated_at
2665
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, 0, ?, 'creating', ?, ?)`).run(authority.inbox.id, intent.bindingId, authority.binding.version, authority.binding.generation, authority.targetHash, credentialFingerprint, sequence, showDetails ? 1 : 0, this.now(), this.now());
2666
+ return { record: this.progressRecord(this.progressRow(authority.inbox.id), authority.inbox), created: true };
2667
+ });
2668
+ }
2669
+ getProgress(intent, credentialFingerprint) {
2670
+ this.assertOpen();
2671
+ const authority = this.progressAuthority(intent, credentialFingerprint);
2672
+ if (authority === undefined)
2673
+ return undefined;
2674
+ const row = this.progressRow(authority.inbox.id);
2675
+ if (row === undefined || row.binding_id !== intent.bindingId || row.binding_version !== authority.binding.version
2676
+ || row.binding_generation !== authority.binding.generation || row.target_hash !== authority.targetHash
2677
+ || row.credential_hash !== credentialFingerprint)
2678
+ return undefined;
2679
+ return this.progressRecord(row, authority.inbox);
2680
+ }
2681
+ attachProgressHandle(intent, credentialFingerprint, providerHandle) {
2682
+ this.assertOpen();
2683
+ if (typeof providerHandle !== 'string' || providerHandle.length < 2 || Buffer.byteLength(providerHandle) > 2_048)
2684
+ return undefined;
2685
+ return this.transaction(() => {
2686
+ const current = this.getProgress(intent, credentialFingerprint);
2687
+ if (current?.state !== 'creating')
2688
+ return undefined;
2689
+ const inbox = this.getInboxByProviderEvent(intent.target.conversation.channel, intent.target.conversation.account, intent.eventId);
2690
+ this.database.prepare(`UPDATE delivery_progress SET provider_handle = ?,
2691
+ state = CASE WHEN terminal_kind IS NULL THEN 'active' ELSE 'pending' END,
2692
+ retry_attempts = CASE WHEN terminal_kind IS NULL THEN 0 ELSE 1 END,
2693
+ next_retry_at = CASE WHEN terminal_kind IS NULL THEN 0 ELSE ? END, updated_at = ?
2694
+ WHERE inbox_id = ? AND state = 'creating' AND provider_handle IS NULL`).run(providerHandle, this.now() + 10_000, this.now(), inbox.id);
2695
+ return this.getProgress(intent, credentialFingerprint);
2696
+ });
2697
+ }
2698
+ advanceProgress(intent, credentialFingerprint) {
2699
+ this.assertOpen();
2700
+ return this.transaction(() => {
2701
+ const current = this.getProgress(intent, credentialFingerprint);
2702
+ if (current?.state !== 'active' || current.providerHandle === undefined)
2703
+ return undefined;
2704
+ const inbox = this.getInboxByProviderEvent(intent.target.conversation.channel, intent.target.conversation.account, intent.eventId);
2705
+ this.database.prepare(`UPDATE delivery_progress SET step = step + 1, updated_at = ?
2706
+ WHERE inbox_id = ? AND state = 'active' AND step < 9007199254740991`).run(this.now(), inbox.id);
2707
+ return this.getProgress(intent, credentialFingerprint);
2708
+ });
2709
+ }
2710
+ stageProgressTerminal(intent, credentialFingerprint) {
2711
+ this.assertOpen();
2712
+ if (!['completed', 'failed', 'paused'].includes(intent.update.kind))
2713
+ return undefined;
2714
+ return this.transaction(() => {
2715
+ const authority = this.progressAuthority(intent, credentialFingerprint);
2716
+ if (authority === undefined)
2717
+ return undefined;
2718
+ const inbox = authority.inbox;
2719
+ const code = intent.update.kind === 'failed' ? intent.update.code?.slice(0, 80) ?? null : null;
2720
+ const current = this.getProgress(intent, credentialFingerprint);
2721
+ if (current === undefined) {
2722
+ if (this.progressRow(inbox.id) !== undefined)
2723
+ return undefined;
2724
+ // A terminal may bypass queued provider steps before `started` reserved a row. Fence the
2725
+ // would-be create instead of constructing an untracked progress message after completion.
2726
+ this.database.prepare(`INSERT INTO delivery_progress(
2727
+ inbox_id, binding_id, binding_version, binding_generation, target_hash, credential_hash,
2728
+ sequence, step, show_details, state, terminal_kind, terminal_code, created_at, updated_at
2729
+ ) VALUES (?, ?, ?, ?, ?, ?, 1, 0, 0, 'creating', ?, ?, ?, ?)`).run(inbox.id, intent.bindingId, authority.binding.version, authority.binding.generation, authority.targetHash, credentialFingerprint, intent.update.kind, code, this.now(), this.now());
2730
+ return this.getProgress(intent, credentialFingerprint);
2731
+ }
2732
+ if (current.state === 'creating') {
2733
+ this.database.prepare(`UPDATE delivery_progress SET terminal_kind = ?, terminal_code = ?, updated_at = ?
2734
+ WHERE inbox_id = ? AND state = 'creating'`).run(intent.update.kind, code, this.now(), inbox.id);
2735
+ return this.getProgress(intent, credentialFingerprint);
2736
+ }
2737
+ if (current.state !== 'active')
2738
+ return current;
2739
+ this.database.prepare(`UPDATE delivery_progress SET state = 'pending', terminal_kind = ?, terminal_code = ?,
2740
+ step = step + 1, retry_attempts = 1, next_retry_at = ?, updated_at = ?
2741
+ WHERE inbox_id = ? AND state = 'active'`).run(intent.update.kind, code, this.now() + 10_000, this.now(), inbox.id);
2742
+ return this.getProgress(intent, credentialFingerprint);
2743
+ });
2744
+ }
2745
+ confirmProgressTerminal(intent, credentialFingerprint) {
2746
+ this.assertOpen();
2747
+ const current = this.getProgress(intent, credentialFingerprint);
2748
+ if (current?.state !== 'pending')
2749
+ return;
2750
+ const inbox = this.getInboxByProviderEvent(intent.target.conversation.channel, intent.target.conversation.account, intent.eventId);
2751
+ this.database.prepare("UPDATE delivery_progress SET state = 'delivered', updated_at = ? WHERE inbox_id = ? AND state = 'pending'")
2752
+ .run(this.now(), inbox.id);
2753
+ }
2754
+ /** Only authoritative settled Inbox rows are eligible for restart repair. */
2755
+ recoverProgress(channel, account, credentialFingerprint, limit, authorize) {
2756
+ this.assertOpen();
2757
+ if (!/^[0-9a-f]{64}$/u.test(credentialFingerprint) || !Number.isSafeInteger(limit) || limit < 1 || limit > 1_000)
2758
+ return [];
2759
+ const cursorKey = JSON.stringify([channel, account]);
2760
+ const previous = this.progressRecoveryCursors.get(cursorKey);
2761
+ const cursor = previous?.credentialFingerprint === credentialFingerprint ? previous : undefined;
2762
+ let nextCursor;
2763
+ const result = this.transaction(() => {
2764
+ const rows = this.database.prepare(`SELECT progress.* FROM delivery_progress AS progress
2765
+ JOIN inbox_messages AS inbox ON inbox.id = progress.inbox_id
2766
+ WHERE inbox.channel = ? AND inbox.account = ? AND progress.credential_hash = ?
2767
+ AND inbox.status IN ('processed', 'dead_letter')
2768
+ AND progress.state IN ('active', 'pending') AND progress.provider_handle IS NOT NULL
2769
+ AND progress.retry_attempts < 8 AND progress.next_retry_at <= ?
2770
+ AND (? = 0 OR progress.created_at > ?
2771
+ OR (progress.created_at = ? AND progress.inbox_id > ?))
2772
+ ORDER BY progress.created_at, progress.inbox_id LIMIT ?`).all(channel, account, credentialFingerprint, this.now(), cursor === undefined ? 0 : 1, cursor?.createdAt ?? 0, cursor?.createdAt ?? 0, cursor?.inboxId ?? '', limit);
2773
+ if (rows.length === 0)
2774
+ return [];
2775
+ const recovered = [];
2776
+ for (const row of rows) {
2777
+ nextCursor = { credentialFingerprint, createdAt: row.created_at, inboxId: row.inbox_id };
2778
+ const inbox = this.getInbox(row.inbox_id);
2779
+ if (inbox === undefined || inbox.bindingId !== row.binding_id)
2780
+ continue;
2781
+ const intent = { bindingId: row.binding_id, eventId: inbox.eventId,
2782
+ target: { conversation: inbox.envelope.conversation, principal: inbox.envelope.principal }, update: { kind: 'started' } };
2783
+ const current = this.getProgress(intent, credentialFingerprint);
2784
+ if (current === undefined)
2785
+ continue;
2786
+ const binding = this.getBinding(row.binding_id);
2787
+ if (authorize !== undefined && !authorize(binding, inbox))
2788
+ continue;
2789
+ if (this.database.prepare(`SELECT 1 FROM delivery_session_leases
2790
+ WHERE session_id = ? AND state = 'dispatched' AND lease_until > ?`).get(binding.sessionId, this.now()) !== undefined)
2791
+ continue;
2792
+ const execution = this.database.prepare('SELECT status, quiescent FROM delivery_foreground_executions WHERE inbox_id = ?')
2793
+ .get(inbox.id);
2794
+ if (inbox.status === 'dead_letter' && inbox.failureCode === 'dispatch-ambiguous' && execution?.status === 'pending') {
2795
+ this.database.prepare(`UPDATE delivery_foreground_executions SET status = 'unknown', quiescent = 0,
2796
+ completed_at = MAX(dispatched_at, ?), execution_ref = inbox_id WHERE inbox_id = ? AND status = 'pending'`)
2797
+ .run(this.now(), inbox.id);
2798
+ }
2799
+ const succeeded = inbox.status === 'processed' && execution?.status === 'succeeded' && execution.quiescent === 1;
2800
+ const terminal = current.terminal?.kind === 'failed' || current.terminal?.kind === 'paused'
2801
+ ? current.terminal
2802
+ : succeeded ? { kind: 'completed' }
2803
+ : { kind: 'failed', code: 'INTERRUPTED_RESULT_UNKNOWN' };
2804
+ this.database.prepare(`UPDATE delivery_progress SET state = 'pending', terminal_kind = ?, terminal_code = ?,
2805
+ step = step + ?, retry_attempts = retry_attempts + 1, next_retry_at = ?, updated_at = ?
2806
+ WHERE inbox_id = ? AND state IN ('active', 'pending') AND retry_attempts < 8`).run(terminal.kind, terminal.kind === 'failed' ? terminal.code ?? null : null, row.state === 'active' || row.terminal_kind !== terminal.kind ? 1 : 0, this.now() + Math.min(300_000, 10_000 * (2 ** row.retry_attempts)), this.now(), inbox.id);
2807
+ recovered.push(this.getProgress(intent, credentialFingerprint));
2808
+ // Claim one retry per sweep so stopping cannot consume a later record's retry budget.
2809
+ break;
2810
+ }
2811
+ return recovered;
2812
+ });
2813
+ if (nextCursor === undefined)
2814
+ this.progressRecoveryCursors.delete(cursorKey);
2815
+ else
2816
+ this.progressRecoveryCursors.set(cursorKey, nextCursor);
2817
+ return result;
2818
+ }
2609
2819
  /** Return the oldest durable admission that did not finish before another Inbox. */
2610
2820
  findPendingInboundBefore(input) {
2611
2821
  this.assertOpen();
@@ -2649,6 +2859,95 @@ export class DeliveryStore {
2649
2859
  `).all(afterSequence, limit);
2650
2860
  return rows.map(inboxFromRow);
2651
2861
  }
2862
+ /** Scan durable, unacknowledged dispatched turns without replaying their Agent work. */
2863
+ listInterruptedInboxNotices(input = {}) {
2864
+ this.assertOpen();
2865
+ const afterSequence = input.afterSequence ?? 0;
2866
+ if (!Number.isSafeInteger(afterSequence) || afterSequence < 0) {
2867
+ throw new DeliveryStoreError('conflict', 'invalid interrupted Inbox cursor');
2868
+ }
2869
+ const limit = boundedMaintenanceLimit(input.limit, 100, 'interrupted Inbox');
2870
+ const rows = this.database.prepare(`${inboxSelect}
2871
+ WHERE inbox_messages.status = 'dead_letter'
2872
+ AND inbox_messages.failure_code IN ('dispatch-ambiguous', 'agent-task-timeout')
2873
+ AND json_extract(inbox_messages.envelope_json, '$.kind') = 'text'
2874
+ AND inbox_messages.channel != 'web'
2875
+ AND inbox_messages.binding_id IS NOT NULL
2876
+ AND EXISTS (
2877
+ SELECT 1 FROM delivery_foreground_executions AS execution
2878
+ WHERE execution.inbox_id = inbox_messages.id
2879
+ )
2880
+ AND admission_sequence > ?
2881
+ AND NOT EXISTS (
2882
+ SELECT 1 FROM dead_letter_resolutions AS resolution
2883
+ WHERE resolution.kind = 'inbox' AND resolution.message_id = inbox_messages.id
2884
+ AND resolution.attempt_count = inbox_messages.attempt_count
2885
+ )
2886
+ AND NOT EXISTS (
2887
+ SELECT 1 FROM outbox_messages AS reply
2888
+ WHERE reply.idempotency_key IN (
2889
+ 'inbound:' || inbox_messages.id || ':reply',
2890
+ 'inbound:' || inbox_messages.event_id || ':reply',
2891
+ 'inbound:' || inbox_messages.id || ':interrupted'
2892
+ )
2893
+ )
2894
+ ORDER BY admission_sequence LIMIT ?
2895
+ `).all(afterSequence, limit);
2896
+ return rows.map(inboxFromRow);
2897
+ }
2898
+ /** The final notice decision and Outbox insert share one SQLite writer fence. */
2899
+ enqueueInterruptedInboxNotice(input) {
2900
+ this.assertOpen();
2901
+ return this.transaction(() => {
2902
+ if (!this.interruptedInboxNoticeCurrent(input)) {
2903
+ throw new DeliveryStoreError('version-conflict', 'interrupted Inbox notice lost its exact owner or reply fence');
2904
+ }
2905
+ return this.enqueueIntent(input.intent, false);
2906
+ });
2907
+ }
2908
+ /** Recheck a queued interruption notice immediately before an adapter send. */
2909
+ canSendInterruptedInboxNotice(record) {
2910
+ this.assertOpen();
2911
+ const key = record.intent.idempotencyKey;
2912
+ if (!key.startsWith('inbound:') || !key.endsWith(':interrupted'))
2913
+ return false;
2914
+ const inboxId = key.slice('inbound:'.length, -':interrupted'.length);
2915
+ const binding = this.getBinding(record.intent.bindingId);
2916
+ return binding !== undefined
2917
+ && this.getOutboxByIdempotencyKey(key)?.id === record.id
2918
+ && this.interruptedInboxNoticeCurrent({ inboxId, binding, intent: record.intent });
2919
+ }
2920
+ interruptedInboxNoticeCurrent(input) {
2921
+ const inbox = this.getInbox(input.inboxId);
2922
+ const binding = inbox?.bindingId === undefined ? undefined : this.getBinding(inbox.bindingId);
2923
+ const owner = binding === undefined ? undefined : this.getBindingPrincipal(binding.id);
2924
+ const execution = this.database.prepare(`
2925
+ SELECT principal_record_id, principal_version, binding_id, binding_version,
2926
+ binding_generation, workspace, preset
2927
+ FROM delivery_foreground_executions WHERE inbox_id = ?
2928
+ `).get(input.inboxId);
2929
+ return inbox?.status === 'dead_letter'
2930
+ && (inbox.failureCode === 'dispatch-ambiguous' || inbox.failureCode === 'agent-task-timeout')
2931
+ && inbox.envelope.kind === 'text' && inbox.channel !== 'web'
2932
+ && binding?.status === 'active' && binding.id === input.binding.id
2933
+ && binding.version === input.binding.version && binding.generation === input.binding.generation
2934
+ && binding.sessionId === input.binding.sessionId
2935
+ && this.getActiveBinding(inbox.envelope.conversation)?.id === binding.id
2936
+ && owner?.status === 'active' && owner.role === 'owner'
2937
+ && execution?.principal_record_id === owner.id && execution.principal_version === owner.version
2938
+ && execution.binding_id === binding.id && execution.binding_version === binding.version
2939
+ && execution.binding_generation === binding.generation
2940
+ && execution.workspace === binding.workspace && execution.preset === binding.agentPreset
2941
+ && principalHash(inbox.envelope.principal) === principalHash(binding.principal)
2942
+ && conversationHash(inbox.envelope.conversation) === conversationHash(binding.conversation)
2943
+ && input.intent.bindingId === binding.id
2944
+ && input.intent.replyToEventId === inbox.eventId
2945
+ && input.intent.idempotencyKey === `inbound:${inbox.id}:interrupted`
2946
+ && this.getDeadLetterResolution({ kind: 'inbox', id: inbox.id,
2947
+ attemptCount: inbox.attemptCount }) === undefined
2948
+ && this.getOutboxByIdempotencyKey(`inbound:${inbox.id}:reply`) === undefined
2949
+ && this.getOutboxByIdempotencyKey(`inbound:${inbox.eventId}:reply`) === undefined;
2950
+ }
2652
2951
  listInbox(input = {}) {
2653
2952
  this.assertOpen();
2654
2953
  const limit = Math.max(1, Math.min(100, input.limit ?? 20));
@@ -2804,6 +3103,40 @@ export class DeliveryStore {
2804
3103
  }
2805
3104
  return this.getInbox(inboxId);
2806
3105
  }
3106
+ /** A durable control message wins even while its async binding transition is waiting. */
3107
+ hasRestartCancellationAfter(inboxId, binding) {
3108
+ this.assertOpen();
3109
+ const rows = this.database.prepare(`SELECT message.envelope_json FROM inbox_messages AS message
3110
+ JOIN delivery_inbox_admissions AS admission ON admission.inbox_id = message.id
3111
+ WHERE admission.admission_sequence > (SELECT admission_sequence FROM delivery_inbox_admissions WHERE inbox_id = ?)
3112
+ AND message.status != 'dead_letter'
3113
+ AND json_extract(message.envelope_json, '$.kind') = 'command'
3114
+ AND json_extract(message.envelope_json, '$.conversation') = json(?)
3115
+ AND json_extract(message.envelope_json, '$.principal') = json(?)`)
3116
+ .all(inboxId, conversationJson(binding.conversation), principalJson(binding.principal));
3117
+ return rows.some(row => isExactDeliveryCommand(parseDeliveryCommand(JSON.parse(row.envelope_json)), 'stop', 'new', 'clear'));
3118
+ }
3119
+ /** The fence and Inbox claims share this database's write transaction. */
3120
+ setRestartFence(requestId, sourceInvocationId) {
3121
+ this.assertOpen();
3122
+ this.transaction(() => {
3123
+ const current = this.getRestartFence();
3124
+ if (current !== undefined && (current.requestId !== requestId || current.sourceInvocationId !== sourceInvocationId)) {
3125
+ throw new DeliveryStoreError('conflict', 'another restart owns the delivery drain fence');
3126
+ }
3127
+ this.database.prepare('INSERT OR IGNORE INTO delivery_restart_fence VALUES (1, ?, ?)').run(requestId, sourceInvocationId);
3128
+ });
3129
+ }
3130
+ getRestartFence() {
3131
+ this.assertOpen();
3132
+ const row = this.database.prepare('SELECT request_id, source_invocation_id FROM delivery_restart_fence WHERE singleton = 1')
3133
+ .get();
3134
+ return row === undefined ? undefined : { requestId: row.request_id, sourceInvocationId: row.source_invocation_id };
3135
+ }
3136
+ clearRestartFence(requestId) {
3137
+ this.assertOpen();
3138
+ this.database.prepare('DELETE FROM delivery_restart_fence WHERE singleton = 1 AND request_id = ?').run(requestId);
3139
+ }
2807
3140
  claimInbox(input) {
2808
3141
  this.assertOpen();
2809
3142
  const ownerId = validateBindingText(input.ownerId, 'ownerId', 256);
@@ -2819,6 +3152,8 @@ export class DeliveryStore {
2819
3152
  const now = this.now();
2820
3153
  const claims = [];
2821
3154
  this.transaction(() => {
3155
+ if (this.getRestartFence() !== undefined)
3156
+ return;
2822
3157
  this.database.prepare(`
2823
3158
  UPDATE inbox_messages SET status = 'dead_letter', failure_code = 'attempts-exhausted',
2824
3159
  next_attempt_at = NULL, updated_at = ?
@@ -4397,93 +4732,98 @@ export class DeliveryStore {
4397
4732
  const claims = [];
4398
4733
  const validatedOwnerRouteIds = [];
4399
4734
  this.transaction(() => {
4400
- this.database.prepare(`
4401
- UPDATE outbox_messages SET status = 'dead', failure_code = 'attempts-exhausted',
4402
- next_attempt_at = NULL, updated_at = ?
4403
- WHERE rowid IN (
4404
- SELECT candidate.rowid FROM outbox_messages AS candidate
4405
- WHERE candidate.status = 'retry_wait' AND candidate.next_attempt_at <= ?
4406
- AND (
4407
- SELECT COUNT(*) FROM outbox_attempts AS history
4408
- WHERE history.outbox_id = candidate.id AND history.operation = 'send'
4409
- ) >= ?
4410
- ORDER BY candidate.rowid LIMIT ?
4411
- )
4412
- `).run(now, now, input.maxAttempts, maintenanceLimit);
4413
- this.database.prepare(`
4414
- UPDATE outbox_messages SET status = 'dead', failure_code = 'reconcile-attempts-exhausted',
4415
- next_attempt_at = NULL, updated_at = ?
4416
- WHERE rowid IN (
4417
- SELECT candidate.rowid FROM outbox_messages AS candidate
4418
- WHERE candidate.status = 'unknown_after_send'
4419
- AND NOT EXISTS (
4420
- SELECT 1 FROM dead_letter_resolutions AS resolution
4421
- WHERE resolution.kind = 'outbox' AND resolution.message_id = candidate.id
4422
- AND resolution.attempt_count = candidate.attempt_count AND resolution.resolution = 'cancel'
4423
- )
4424
- AND (
4425
- SELECT COUNT(*) FROM outbox_attempts AS history
4426
- WHERE history.outbox_id = candidate.id AND history.operation = 'reconcile'
4427
- ) >= ?
4428
- ORDER BY candidate.rowid LIMIT ?
4429
- )
4430
- `).run(now, input.maxAttempts, maintenanceLimit);
4431
- const ownerRouteRows = this.database.prepare(`
4432
- SELECT id FROM outbox_messages
4433
- WHERE status IN ('pending', 'retry_wait') AND claimed_by IS NULL
4434
- AND (next_attempt_at IS NULL OR next_attempt_at <= ?)
4435
- AND json_type(intent_json, '$.metadata."dsh.route.authority"') IS NOT NULL
4436
- ORDER BY rowid LIMIT ?
4437
- `).all(now, maintenanceLimit);
4438
- const ownerRouteGuard = input.ownerRouteGuard ?? { ownerRoutes: [], authorize: () => false };
4439
- for (const row of ownerRouteRows) {
4440
- const record = this.getOutbox(row.id);
4441
- if (record === undefined)
4442
- continue;
4443
- const inspected = this.inspectOwnerRouteDispatch(record, ownerRouteGuard);
4444
- if (inspected.kind === 'deferred') {
4445
- this.deferOwnerRouteBeforeClaim(record, inspected.failureCode, now, now + Math.max(1, Math.min(input.leaseMs, 30_000)));
4446
- continue;
4447
- }
4448
- if (inspected.kind !== 'authorized') {
4449
- const failureCode = inspected.kind === 'denied'
4450
- ? inspected.failureCode
4451
- : 'owner-route-receipt-invalid';
4452
- this.deadLetterOwnerRouteBeforeClaim(record, ownerId, failureCode, now);
4453
- continue;
4454
- }
4455
- if (!this.ownerRouteIsProvablyUnsent(record)) {
4456
- this.deadLetterOwnerRouteBeforeClaim(record, ownerId, 'owner-route-send-proof-missing', now);
4457
- continue;
4458
- }
4459
- if (!this.rebindOwnerRouteBeforeClaim(record, inspected)) {
4460
- this.deadLetterOwnerRouteBeforeClaim(record, ownerId, 'owner-route-rebind-conflict', now);
4461
- continue;
4735
+ const restartFenceActive = this.getRestartFence() !== undefined;
4736
+ // Drain without changing queued retries or unresolved sends. Only a
4737
+ // plain maintenance acknowledgement may start during this fence.
4738
+ if (!restartFenceActive) {
4739
+ this.database.prepare(`
4740
+ UPDATE outbox_messages SET status = 'dead', failure_code = 'attempts-exhausted',
4741
+ next_attempt_at = NULL, updated_at = ?
4742
+ WHERE rowid IN (
4743
+ SELECT candidate.rowid FROM outbox_messages AS candidate
4744
+ WHERE candidate.status = 'retry_wait' AND candidate.next_attempt_at <= ?
4745
+ AND (
4746
+ SELECT COUNT(*) FROM outbox_attempts AS history
4747
+ WHERE history.outbox_id = candidate.id AND history.operation = 'send'
4748
+ ) >= ?
4749
+ ORDER BY candidate.rowid LIMIT ?
4750
+ )
4751
+ `).run(now, now, input.maxAttempts, maintenanceLimit);
4752
+ this.database.prepare(`
4753
+ UPDATE outbox_messages SET status = 'dead', failure_code = 'reconcile-attempts-exhausted',
4754
+ next_attempt_at = NULL, updated_at = ?
4755
+ WHERE rowid IN (
4756
+ SELECT candidate.rowid FROM outbox_messages AS candidate
4757
+ WHERE candidate.status = 'unknown_after_send'
4758
+ AND NOT EXISTS (
4759
+ SELECT 1 FROM dead_letter_resolutions AS resolution
4760
+ WHERE resolution.kind = 'outbox' AND resolution.message_id = candidate.id
4761
+ AND resolution.attempt_count = candidate.attempt_count AND resolution.resolution = 'cancel'
4762
+ )
4763
+ AND (
4764
+ SELECT COUNT(*) FROM outbox_attempts AS history
4765
+ WHERE history.outbox_id = candidate.id AND history.operation = 'reconcile'
4766
+ ) >= ?
4767
+ ORDER BY candidate.rowid LIMIT ?
4768
+ )
4769
+ `).run(now, input.maxAttempts, maintenanceLimit);
4770
+ const ownerRouteRows = this.database.prepare(`
4771
+ SELECT id FROM outbox_messages
4772
+ WHERE status IN ('pending', 'retry_wait') AND claimed_by IS NULL
4773
+ AND (next_attempt_at IS NULL OR next_attempt_at <= ?)
4774
+ AND json_type(intent_json, '$.metadata."dsh.route.authority"') IS NOT NULL
4775
+ ORDER BY rowid LIMIT ?
4776
+ `).all(now, maintenanceLimit);
4777
+ const ownerRouteGuard = input.ownerRouteGuard ?? { ownerRoutes: [], authorize: () => false };
4778
+ for (const row of ownerRouteRows) {
4779
+ const record = this.getOutbox(row.id);
4780
+ if (record === undefined)
4781
+ continue;
4782
+ const inspected = this.inspectOwnerRouteDispatch(record, ownerRouteGuard);
4783
+ if (inspected.kind === 'deferred') {
4784
+ this.deferOwnerRouteBeforeClaim(record, inspected.failureCode, now, now + Math.max(1, Math.min(input.leaseMs, 30_000)));
4785
+ continue;
4786
+ }
4787
+ if (inspected.kind !== 'authorized') {
4788
+ const failureCode = inspected.kind === 'denied'
4789
+ ? inspected.failureCode
4790
+ : 'owner-route-receipt-invalid';
4791
+ this.deadLetterOwnerRouteBeforeClaim(record, ownerId, failureCode, now);
4792
+ continue;
4793
+ }
4794
+ if (!this.ownerRouteIsProvablyUnsent(record)) {
4795
+ this.deadLetterOwnerRouteBeforeClaim(record, ownerId, 'owner-route-send-proof-missing', now);
4796
+ continue;
4797
+ }
4798
+ if (!this.rebindOwnerRouteBeforeClaim(record, inspected)) {
4799
+ this.deadLetterOwnerRouteBeforeClaim(record, ownerId, 'owner-route-rebind-conflict', now);
4800
+ continue;
4801
+ }
4802
+ validatedOwnerRouteIds.push(record.id);
4462
4803
  }
4463
- validatedOwnerRouteIds.push(record.id);
4804
+ this.database.prepare(`
4805
+ UPDATE outbox_messages SET status = 'dead', failure_code = CASE
4806
+ WHEN status = 'unknown_after_send' THEN 'binding-revoked-unknown'
4807
+ ELSE 'binding-revoked'
4808
+ END,
4809
+ next_attempt_at = NULL, updated_at = ?
4810
+ WHERE rowid IN (
4811
+ SELECT candidate.rowid FROM outbox_messages AS candidate
4812
+ WHERE candidate.status IN ('pending', 'retry_wait', 'unknown_after_send')
4813
+ AND json_type(candidate.intent_json, '$.metadata."dsh.route.authority"') IS NULL
4814
+ AND NOT EXISTS (
4815
+ SELECT 1 FROM dead_letter_resolutions AS resolution
4816
+ WHERE resolution.kind = 'outbox' AND resolution.message_id = candidate.id
4817
+ AND resolution.attempt_count = candidate.attempt_count AND resolution.resolution = 'cancel'
4818
+ )
4819
+ AND NOT EXISTS (
4820
+ SELECT 1 FROM conversation_bindings AS binding
4821
+ WHERE binding.id = candidate.binding_id AND binding.status = 'active'
4822
+ )
4823
+ ORDER BY candidate.rowid LIMIT ?
4824
+ )
4825
+ `).run(now, maintenanceLimit);
4464
4826
  }
4465
- this.database.prepare(`
4466
- UPDATE outbox_messages SET status = 'dead', failure_code = CASE
4467
- WHEN status = 'unknown_after_send' THEN 'binding-revoked-unknown'
4468
- ELSE 'binding-revoked'
4469
- END,
4470
- next_attempt_at = NULL, updated_at = ?
4471
- WHERE rowid IN (
4472
- SELECT candidate.rowid FROM outbox_messages AS candidate
4473
- WHERE candidate.status IN ('pending', 'retry_wait', 'unknown_after_send')
4474
- AND json_type(candidate.intent_json, '$.metadata."dsh.route.authority"') IS NULL
4475
- AND NOT EXISTS (
4476
- SELECT 1 FROM dead_letter_resolutions AS resolution
4477
- WHERE resolution.kind = 'outbox' AND resolution.message_id = candidate.id
4478
- AND resolution.attempt_count = candidate.attempt_count AND resolution.resolution = 'cancel'
4479
- )
4480
- AND NOT EXISTS (
4481
- SELECT 1 FROM conversation_bindings AS binding
4482
- WHERE binding.id = candidate.binding_id AND binding.status = 'active'
4483
- )
4484
- ORDER BY candidate.rowid LIMIT ?
4485
- )
4486
- `).run(now, maintenanceLimit);
4487
4827
  const ownerRouteClaimClause = validatedOwnerRouteIds.length === 0
4488
4828
  ? `AND (
4489
4829
  json_type(candidate.intent_json, '$.metadata."dsh.route.authority"') IS NULL
@@ -4498,6 +4838,13 @@ export class DeliveryStore {
4498
4838
  SELECT candidate.id, candidate.status FROM outbox_messages AS candidate
4499
4839
  WHERE candidate.status IN ('pending', 'retry_wait', 'unknown_after_send')
4500
4840
  AND candidate.claimed_by IS NULL
4841
+ ${restartFenceActive ? `AND candidate.status IN ('pending', 'retry_wait')
4842
+ AND candidate.idempotency_key GLOB 'inbound:*:maintenance'
4843
+ AND json_extract(candidate.intent_json, '$.format') = 'plain'
4844
+ AND json_type(candidate.intent_json, '$.metadata') IS NULL
4845
+ AND json_type(candidate.intent_json, '$.approval') IS NULL
4846
+ AND json_type(candidate.intent_json, '$.modelPicker') IS NULL
4847
+ AND json_type(candidate.intent_json, '$.permissionPicker') IS NULL` : ''}
4501
4848
  AND NOT EXISTS (
4502
4849
  SELECT 1 FROM dead_letter_resolutions AS resolution
4503
4850
  WHERE resolution.kind = 'outbox' AND resolution.message_id = candidate.id
@@ -4532,6 +4879,7 @@ export class DeliveryStore {
4532
4879
  WHERE earlier.lane_hash = candidate.lane_hash
4533
4880
  AND earlier.rowid < candidate.rowid
4534
4881
  AND earlier.status NOT IN ('accepted', 'delivered', 'read', 'dead')
4882
+ ${restartFenceActive ? "AND earlier.idempotency_key GLOB 'inbound:*:maintenance'" : ''}
4535
4883
  AND NOT EXISTS (
4536
4884
  SELECT 1 FROM dead_letter_resolutions AS resolution
4537
4885
  WHERE resolution.kind = 'outbox' AND resolution.message_id = earlier.id