@dsh-enhanced/assistant-delivery 0.1.62 → 0.1.64
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -10
- package/bin/dsh-restart-supervisor.js +76 -0
- package/lib/agent-runtime.d.ts +21 -1
- package/lib/agent-runtime.d.ts.map +1 -1
- package/lib/agent-runtime.js +192 -23
- package/lib/agent-runtime.js.map +1 -1
- package/lib/coordinator.d.ts.map +1 -1
- package/lib/coordinator.js +10 -0
- package/lib/coordinator.js.map +1 -1
- package/lib/operator-snapshot.d.ts +1 -1
- package/lib/operator-snapshot.d.ts.map +1 -1
- package/lib/operator-snapshot.js +5 -5
- package/lib/operator-snapshot.js.map +1 -1
- package/lib/restart-native.d.ts +47 -0
- package/lib/restart-native.d.ts.map +1 -0
- package/lib/restart-native.js +170 -0
- package/lib/restart-native.js.map +1 -0
- package/lib/restart-runtime.d.ts +38 -0
- package/lib/restart-runtime.d.ts.map +1 -0
- package/lib/restart-runtime.js +335 -0
- package/lib/restart-runtime.js.map +1 -0
- package/lib/restart-store.d.ts +132 -0
- package/lib/restart-store.d.ts.map +1 -0
- package/lib/restart-store.js +757 -0
- package/lib/restart-store.js.map +1 -0
- package/lib/restart-supervisor.d.ts +129 -0
- package/lib/restart-supervisor.d.ts.map +1 -0
- package/lib/restart-supervisor.js +615 -0
- package/lib/restart-supervisor.js.map +1 -0
- package/lib/service.d.ts +45 -1
- package/lib/service.d.ts.map +1 -1
- package/lib/service.js +488 -6
- package/lib/service.js.map +1 -1
- package/lib/sqlite.d.ts +1 -1
- package/lib/sqlite.d.ts.map +1 -1
- package/lib/sqlite.js +76 -2
- package/lib/sqlite.js.map +1 -1
- package/lib/store.d.ts +41 -1
- package/lib/store.d.ts.map +1 -1
- package/lib/store.js +433 -85
- package/lib/store.js.map +1 -1
- package/lib/tools.d.ts.map +1 -1
- package/lib/tools.js +21 -2
- package/lib/tools.js.map +1 -1
- package/lib/types.d.ts +33 -0
- package/lib/types.d.ts.map +1 -1
- package/lib/version.d.ts +1 -1
- package/lib/version.js +1 -1
- package/package.json +11 -10
package/lib/store.js
CHANGED
|
@@ -1366,6 +1366,8 @@ function leaseTargetJson(target) {
|
|
|
1366
1366
|
}
|
|
1367
1367
|
export class DeliveryStore {
|
|
1368
1368
|
database;
|
|
1369
|
+
/** Scan position only; every candidate is authorized again before a provider retry. */
|
|
1370
|
+
progressRecoveryCursors = new Map();
|
|
1369
1371
|
databaseInstanceId;
|
|
1370
1372
|
now;
|
|
1371
1373
|
codeGenerator;
|
|
@@ -2606,6 +2608,214 @@ export class DeliveryStore {
|
|
|
2606
2608
|
.get(channel, account, eventId);
|
|
2607
2609
|
return row === undefined ? undefined : inboxFromRow(row);
|
|
2608
2610
|
}
|
|
2611
|
+
progressAuthority(intent, credentialFingerprint) {
|
|
2612
|
+
if (!/^[0-9a-f]{64}$/u.test(credentialFingerprint))
|
|
2613
|
+
return undefined;
|
|
2614
|
+
const target = canonicalTarget(intent.target);
|
|
2615
|
+
const inbox = this.getInboxByProviderEvent(target.conversation.channel, target.conversation.account, intent.eventId);
|
|
2616
|
+
const binding = this.getBinding(intent.bindingId);
|
|
2617
|
+
const activeBinding = this.getActiveBinding(target.conversation);
|
|
2618
|
+
const principal = this.getPrincipal(target.principal);
|
|
2619
|
+
if (inbox?.bindingId !== intent.bindingId || binding?.status !== 'active'
|
|
2620
|
+
|| activeBinding?.id !== binding.id || principal?.status !== 'active'
|
|
2621
|
+
|| JSON.stringify(canonicalTarget({ conversation: binding.conversation, principal: binding.principal })) !== JSON.stringify(target)
|
|
2622
|
+
|| JSON.stringify(canonicalTarget({ conversation: inbox.envelope.conversation,
|
|
2623
|
+
principal: inbox.envelope.principal })) !== JSON.stringify(target))
|
|
2624
|
+
return undefined;
|
|
2625
|
+
return { inbox, binding, targetHash: createHash('sha256').update(JSON.stringify(target)).digest('hex') };
|
|
2626
|
+
}
|
|
2627
|
+
progressRecord(row, inbox) {
|
|
2628
|
+
return {
|
|
2629
|
+
bindingId: row.binding_id,
|
|
2630
|
+
eventId: inbox.eventId,
|
|
2631
|
+
target: { conversation: inbox.envelope.conversation, principal: inbox.envelope.principal },
|
|
2632
|
+
...(row.provider_handle === null ? {} : { providerHandle: row.provider_handle }),
|
|
2633
|
+
sequence: row.sequence,
|
|
2634
|
+
step: row.step,
|
|
2635
|
+
showDetails: row.show_details === 1,
|
|
2636
|
+
state: row.state,
|
|
2637
|
+
...(row.terminal_kind === null ? {} : { terminal: row.terminal_kind === 'failed'
|
|
2638
|
+
? { kind: 'failed', ...(row.terminal_code === null ? {} : { code: row.terminal_code }) }
|
|
2639
|
+
: { kind: row.terminal_kind } }),
|
|
2640
|
+
};
|
|
2641
|
+
}
|
|
2642
|
+
progressRow(inboxId) {
|
|
2643
|
+
return this.database.prepare('SELECT * FROM delivery_progress WHERE inbox_id = ?').get(inboxId);
|
|
2644
|
+
}
|
|
2645
|
+
/** Reserve before provider I/O; a stranded creating row must never trigger another create. */
|
|
2646
|
+
beginProgress(intent, credentialFingerprint, sequence, showDetails) {
|
|
2647
|
+
this.assertOpen();
|
|
2648
|
+
if (intent.update.kind !== 'started' || !Number.isSafeInteger(sequence) || sequence < 1
|
|
2649
|
+
|| typeof showDetails !== 'boolean')
|
|
2650
|
+
return undefined;
|
|
2651
|
+
return this.transaction(() => {
|
|
2652
|
+
const authority = this.progressAuthority(intent, credentialFingerprint);
|
|
2653
|
+
if (authority === undefined || authority.inbox.status !== 'claimed')
|
|
2654
|
+
return undefined;
|
|
2655
|
+
const previous = this.progressRow(authority.inbox.id);
|
|
2656
|
+
if (previous !== undefined) {
|
|
2657
|
+
return previous.binding_id === intent.bindingId && previous.binding_version === authority.binding.version
|
|
2658
|
+
&& previous.binding_generation === authority.binding.generation
|
|
2659
|
+
&& previous.target_hash === authority.targetHash && previous.credential_hash === credentialFingerprint
|
|
2660
|
+
? { record: this.progressRecord(previous, authority.inbox), created: false } : undefined;
|
|
2661
|
+
}
|
|
2662
|
+
this.database.prepare(`INSERT INTO delivery_progress(
|
|
2663
|
+
inbox_id, binding_id, binding_version, binding_generation, target_hash, credential_hash,
|
|
2664
|
+
sequence, step, show_details, state, created_at, updated_at
|
|
2665
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, 0, ?, 'creating', ?, ?)`).run(authority.inbox.id, intent.bindingId, authority.binding.version, authority.binding.generation, authority.targetHash, credentialFingerprint, sequence, showDetails ? 1 : 0, this.now(), this.now());
|
|
2666
|
+
return { record: this.progressRecord(this.progressRow(authority.inbox.id), authority.inbox), created: true };
|
|
2667
|
+
});
|
|
2668
|
+
}
|
|
2669
|
+
getProgress(intent, credentialFingerprint) {
|
|
2670
|
+
this.assertOpen();
|
|
2671
|
+
const authority = this.progressAuthority(intent, credentialFingerprint);
|
|
2672
|
+
if (authority === undefined)
|
|
2673
|
+
return undefined;
|
|
2674
|
+
const row = this.progressRow(authority.inbox.id);
|
|
2675
|
+
if (row === undefined || row.binding_id !== intent.bindingId || row.binding_version !== authority.binding.version
|
|
2676
|
+
|| row.binding_generation !== authority.binding.generation || row.target_hash !== authority.targetHash
|
|
2677
|
+
|| row.credential_hash !== credentialFingerprint)
|
|
2678
|
+
return undefined;
|
|
2679
|
+
return this.progressRecord(row, authority.inbox);
|
|
2680
|
+
}
|
|
2681
|
+
attachProgressHandle(intent, credentialFingerprint, providerHandle) {
|
|
2682
|
+
this.assertOpen();
|
|
2683
|
+
if (typeof providerHandle !== 'string' || providerHandle.length < 2 || Buffer.byteLength(providerHandle) > 2_048)
|
|
2684
|
+
return undefined;
|
|
2685
|
+
return this.transaction(() => {
|
|
2686
|
+
const current = this.getProgress(intent, credentialFingerprint);
|
|
2687
|
+
if (current?.state !== 'creating')
|
|
2688
|
+
return undefined;
|
|
2689
|
+
const inbox = this.getInboxByProviderEvent(intent.target.conversation.channel, intent.target.conversation.account, intent.eventId);
|
|
2690
|
+
this.database.prepare(`UPDATE delivery_progress SET provider_handle = ?,
|
|
2691
|
+
state = CASE WHEN terminal_kind IS NULL THEN 'active' ELSE 'pending' END,
|
|
2692
|
+
retry_attempts = CASE WHEN terminal_kind IS NULL THEN 0 ELSE 1 END,
|
|
2693
|
+
next_retry_at = CASE WHEN terminal_kind IS NULL THEN 0 ELSE ? END, updated_at = ?
|
|
2694
|
+
WHERE inbox_id = ? AND state = 'creating' AND provider_handle IS NULL`).run(providerHandle, this.now() + 10_000, this.now(), inbox.id);
|
|
2695
|
+
return this.getProgress(intent, credentialFingerprint);
|
|
2696
|
+
});
|
|
2697
|
+
}
|
|
2698
|
+
advanceProgress(intent, credentialFingerprint) {
|
|
2699
|
+
this.assertOpen();
|
|
2700
|
+
return this.transaction(() => {
|
|
2701
|
+
const current = this.getProgress(intent, credentialFingerprint);
|
|
2702
|
+
if (current?.state !== 'active' || current.providerHandle === undefined)
|
|
2703
|
+
return undefined;
|
|
2704
|
+
const inbox = this.getInboxByProviderEvent(intent.target.conversation.channel, intent.target.conversation.account, intent.eventId);
|
|
2705
|
+
this.database.prepare(`UPDATE delivery_progress SET step = step + 1, updated_at = ?
|
|
2706
|
+
WHERE inbox_id = ? AND state = 'active' AND step < 9007199254740991`).run(this.now(), inbox.id);
|
|
2707
|
+
return this.getProgress(intent, credentialFingerprint);
|
|
2708
|
+
});
|
|
2709
|
+
}
|
|
2710
|
+
stageProgressTerminal(intent, credentialFingerprint) {
|
|
2711
|
+
this.assertOpen();
|
|
2712
|
+
if (!['completed', 'failed', 'paused'].includes(intent.update.kind))
|
|
2713
|
+
return undefined;
|
|
2714
|
+
return this.transaction(() => {
|
|
2715
|
+
const authority = this.progressAuthority(intent, credentialFingerprint);
|
|
2716
|
+
if (authority === undefined)
|
|
2717
|
+
return undefined;
|
|
2718
|
+
const inbox = authority.inbox;
|
|
2719
|
+
const code = intent.update.kind === 'failed' ? intent.update.code?.slice(0, 80) ?? null : null;
|
|
2720
|
+
const current = this.getProgress(intent, credentialFingerprint);
|
|
2721
|
+
if (current === undefined) {
|
|
2722
|
+
if (this.progressRow(inbox.id) !== undefined)
|
|
2723
|
+
return undefined;
|
|
2724
|
+
// A terminal may bypass queued provider steps before `started` reserved a row. Fence the
|
|
2725
|
+
// would-be create instead of constructing an untracked progress message after completion.
|
|
2726
|
+
this.database.prepare(`INSERT INTO delivery_progress(
|
|
2727
|
+
inbox_id, binding_id, binding_version, binding_generation, target_hash, credential_hash,
|
|
2728
|
+
sequence, step, show_details, state, terminal_kind, terminal_code, created_at, updated_at
|
|
2729
|
+
) VALUES (?, ?, ?, ?, ?, ?, 1, 0, 0, 'creating', ?, ?, ?, ?)`).run(inbox.id, intent.bindingId, authority.binding.version, authority.binding.generation, authority.targetHash, credentialFingerprint, intent.update.kind, code, this.now(), this.now());
|
|
2730
|
+
return this.getProgress(intent, credentialFingerprint);
|
|
2731
|
+
}
|
|
2732
|
+
if (current.state === 'creating') {
|
|
2733
|
+
this.database.prepare(`UPDATE delivery_progress SET terminal_kind = ?, terminal_code = ?, updated_at = ?
|
|
2734
|
+
WHERE inbox_id = ? AND state = 'creating'`).run(intent.update.kind, code, this.now(), inbox.id);
|
|
2735
|
+
return this.getProgress(intent, credentialFingerprint);
|
|
2736
|
+
}
|
|
2737
|
+
if (current.state !== 'active')
|
|
2738
|
+
return current;
|
|
2739
|
+
this.database.prepare(`UPDATE delivery_progress SET state = 'pending', terminal_kind = ?, terminal_code = ?,
|
|
2740
|
+
step = step + 1, retry_attempts = 1, next_retry_at = ?, updated_at = ?
|
|
2741
|
+
WHERE inbox_id = ? AND state = 'active'`).run(intent.update.kind, code, this.now() + 10_000, this.now(), inbox.id);
|
|
2742
|
+
return this.getProgress(intent, credentialFingerprint);
|
|
2743
|
+
});
|
|
2744
|
+
}
|
|
2745
|
+
confirmProgressTerminal(intent, credentialFingerprint) {
|
|
2746
|
+
this.assertOpen();
|
|
2747
|
+
const current = this.getProgress(intent, credentialFingerprint);
|
|
2748
|
+
if (current?.state !== 'pending')
|
|
2749
|
+
return;
|
|
2750
|
+
const inbox = this.getInboxByProviderEvent(intent.target.conversation.channel, intent.target.conversation.account, intent.eventId);
|
|
2751
|
+
this.database.prepare("UPDATE delivery_progress SET state = 'delivered', updated_at = ? WHERE inbox_id = ? AND state = 'pending'")
|
|
2752
|
+
.run(this.now(), inbox.id);
|
|
2753
|
+
}
|
|
2754
|
+
/** Only authoritative settled Inbox rows are eligible for restart repair. */
|
|
2755
|
+
recoverProgress(channel, account, credentialFingerprint, limit, authorize) {
|
|
2756
|
+
this.assertOpen();
|
|
2757
|
+
if (!/^[0-9a-f]{64}$/u.test(credentialFingerprint) || !Number.isSafeInteger(limit) || limit < 1 || limit > 1_000)
|
|
2758
|
+
return [];
|
|
2759
|
+
const cursorKey = JSON.stringify([channel, account]);
|
|
2760
|
+
const previous = this.progressRecoveryCursors.get(cursorKey);
|
|
2761
|
+
const cursor = previous?.credentialFingerprint === credentialFingerprint ? previous : undefined;
|
|
2762
|
+
let nextCursor;
|
|
2763
|
+
const result = this.transaction(() => {
|
|
2764
|
+
const rows = this.database.prepare(`SELECT progress.* FROM delivery_progress AS progress
|
|
2765
|
+
JOIN inbox_messages AS inbox ON inbox.id = progress.inbox_id
|
|
2766
|
+
WHERE inbox.channel = ? AND inbox.account = ? AND progress.credential_hash = ?
|
|
2767
|
+
AND inbox.status IN ('processed', 'dead_letter')
|
|
2768
|
+
AND progress.state IN ('active', 'pending') AND progress.provider_handle IS NOT NULL
|
|
2769
|
+
AND progress.retry_attempts < 8 AND progress.next_retry_at <= ?
|
|
2770
|
+
AND (? = 0 OR progress.created_at > ?
|
|
2771
|
+
OR (progress.created_at = ? AND progress.inbox_id > ?))
|
|
2772
|
+
ORDER BY progress.created_at, progress.inbox_id LIMIT ?`).all(channel, account, credentialFingerprint, this.now(), cursor === undefined ? 0 : 1, cursor?.createdAt ?? 0, cursor?.createdAt ?? 0, cursor?.inboxId ?? '', limit);
|
|
2773
|
+
if (rows.length === 0)
|
|
2774
|
+
return [];
|
|
2775
|
+
const recovered = [];
|
|
2776
|
+
for (const row of rows) {
|
|
2777
|
+
nextCursor = { credentialFingerprint, createdAt: row.created_at, inboxId: row.inbox_id };
|
|
2778
|
+
const inbox = this.getInbox(row.inbox_id);
|
|
2779
|
+
if (inbox === undefined || inbox.bindingId !== row.binding_id)
|
|
2780
|
+
continue;
|
|
2781
|
+
const intent = { bindingId: row.binding_id, eventId: inbox.eventId,
|
|
2782
|
+
target: { conversation: inbox.envelope.conversation, principal: inbox.envelope.principal }, update: { kind: 'started' } };
|
|
2783
|
+
const current = this.getProgress(intent, credentialFingerprint);
|
|
2784
|
+
if (current === undefined)
|
|
2785
|
+
continue;
|
|
2786
|
+
const binding = this.getBinding(row.binding_id);
|
|
2787
|
+
if (authorize !== undefined && !authorize(binding, inbox))
|
|
2788
|
+
continue;
|
|
2789
|
+
if (this.database.prepare(`SELECT 1 FROM delivery_session_leases
|
|
2790
|
+
WHERE session_id = ? AND state = 'dispatched' AND lease_until > ?`).get(binding.sessionId, this.now()) !== undefined)
|
|
2791
|
+
continue;
|
|
2792
|
+
const execution = this.database.prepare('SELECT status, quiescent FROM delivery_foreground_executions WHERE inbox_id = ?')
|
|
2793
|
+
.get(inbox.id);
|
|
2794
|
+
if (inbox.status === 'dead_letter' && inbox.failureCode === 'dispatch-ambiguous' && execution?.status === 'pending') {
|
|
2795
|
+
this.database.prepare(`UPDATE delivery_foreground_executions SET status = 'unknown', quiescent = 0,
|
|
2796
|
+
completed_at = MAX(dispatched_at, ?), execution_ref = inbox_id WHERE inbox_id = ? AND status = 'pending'`)
|
|
2797
|
+
.run(this.now(), inbox.id);
|
|
2798
|
+
}
|
|
2799
|
+
const succeeded = inbox.status === 'processed' && execution?.status === 'succeeded' && execution.quiescent === 1;
|
|
2800
|
+
const terminal = current.terminal?.kind === 'failed' || current.terminal?.kind === 'paused'
|
|
2801
|
+
? current.terminal
|
|
2802
|
+
: succeeded ? { kind: 'completed' }
|
|
2803
|
+
: { kind: 'failed', code: 'INTERRUPTED_RESULT_UNKNOWN' };
|
|
2804
|
+
this.database.prepare(`UPDATE delivery_progress SET state = 'pending', terminal_kind = ?, terminal_code = ?,
|
|
2805
|
+
step = step + ?, retry_attempts = retry_attempts + 1, next_retry_at = ?, updated_at = ?
|
|
2806
|
+
WHERE inbox_id = ? AND state IN ('active', 'pending') AND retry_attempts < 8`).run(terminal.kind, terminal.kind === 'failed' ? terminal.code ?? null : null, row.state === 'active' || row.terminal_kind !== terminal.kind ? 1 : 0, this.now() + Math.min(300_000, 10_000 * (2 ** row.retry_attempts)), this.now(), inbox.id);
|
|
2807
|
+
recovered.push(this.getProgress(intent, credentialFingerprint));
|
|
2808
|
+
// Claim one retry per sweep so stopping cannot consume a later record's retry budget.
|
|
2809
|
+
break;
|
|
2810
|
+
}
|
|
2811
|
+
return recovered;
|
|
2812
|
+
});
|
|
2813
|
+
if (nextCursor === undefined)
|
|
2814
|
+
this.progressRecoveryCursors.delete(cursorKey);
|
|
2815
|
+
else
|
|
2816
|
+
this.progressRecoveryCursors.set(cursorKey, nextCursor);
|
|
2817
|
+
return result;
|
|
2818
|
+
}
|
|
2609
2819
|
/** Return the oldest durable admission that did not finish before another Inbox. */
|
|
2610
2820
|
findPendingInboundBefore(input) {
|
|
2611
2821
|
this.assertOpen();
|
|
@@ -2649,6 +2859,95 @@ export class DeliveryStore {
|
|
|
2649
2859
|
`).all(afterSequence, limit);
|
|
2650
2860
|
return rows.map(inboxFromRow);
|
|
2651
2861
|
}
|
|
2862
|
+
/** Scan durable, unacknowledged dispatched turns without replaying their Agent work. */
|
|
2863
|
+
listInterruptedInboxNotices(input = {}) {
|
|
2864
|
+
this.assertOpen();
|
|
2865
|
+
const afterSequence = input.afterSequence ?? 0;
|
|
2866
|
+
if (!Number.isSafeInteger(afterSequence) || afterSequence < 0) {
|
|
2867
|
+
throw new DeliveryStoreError('conflict', 'invalid interrupted Inbox cursor');
|
|
2868
|
+
}
|
|
2869
|
+
const limit = boundedMaintenanceLimit(input.limit, 100, 'interrupted Inbox');
|
|
2870
|
+
const rows = this.database.prepare(`${inboxSelect}
|
|
2871
|
+
WHERE inbox_messages.status = 'dead_letter'
|
|
2872
|
+
AND inbox_messages.failure_code IN ('dispatch-ambiguous', 'agent-task-timeout')
|
|
2873
|
+
AND json_extract(inbox_messages.envelope_json, '$.kind') = 'text'
|
|
2874
|
+
AND inbox_messages.channel != 'web'
|
|
2875
|
+
AND inbox_messages.binding_id IS NOT NULL
|
|
2876
|
+
AND EXISTS (
|
|
2877
|
+
SELECT 1 FROM delivery_foreground_executions AS execution
|
|
2878
|
+
WHERE execution.inbox_id = inbox_messages.id
|
|
2879
|
+
)
|
|
2880
|
+
AND admission_sequence > ?
|
|
2881
|
+
AND NOT EXISTS (
|
|
2882
|
+
SELECT 1 FROM dead_letter_resolutions AS resolution
|
|
2883
|
+
WHERE resolution.kind = 'inbox' AND resolution.message_id = inbox_messages.id
|
|
2884
|
+
AND resolution.attempt_count = inbox_messages.attempt_count
|
|
2885
|
+
)
|
|
2886
|
+
AND NOT EXISTS (
|
|
2887
|
+
SELECT 1 FROM outbox_messages AS reply
|
|
2888
|
+
WHERE reply.idempotency_key IN (
|
|
2889
|
+
'inbound:' || inbox_messages.id || ':reply',
|
|
2890
|
+
'inbound:' || inbox_messages.event_id || ':reply',
|
|
2891
|
+
'inbound:' || inbox_messages.id || ':interrupted'
|
|
2892
|
+
)
|
|
2893
|
+
)
|
|
2894
|
+
ORDER BY admission_sequence LIMIT ?
|
|
2895
|
+
`).all(afterSequence, limit);
|
|
2896
|
+
return rows.map(inboxFromRow);
|
|
2897
|
+
}
|
|
2898
|
+
/** The final notice decision and Outbox insert share one SQLite writer fence. */
|
|
2899
|
+
enqueueInterruptedInboxNotice(input) {
|
|
2900
|
+
this.assertOpen();
|
|
2901
|
+
return this.transaction(() => {
|
|
2902
|
+
if (!this.interruptedInboxNoticeCurrent(input)) {
|
|
2903
|
+
throw new DeliveryStoreError('version-conflict', 'interrupted Inbox notice lost its exact owner or reply fence');
|
|
2904
|
+
}
|
|
2905
|
+
return this.enqueueIntent(input.intent, false);
|
|
2906
|
+
});
|
|
2907
|
+
}
|
|
2908
|
+
/** Recheck a queued interruption notice immediately before an adapter send. */
|
|
2909
|
+
canSendInterruptedInboxNotice(record) {
|
|
2910
|
+
this.assertOpen();
|
|
2911
|
+
const key = record.intent.idempotencyKey;
|
|
2912
|
+
if (!key.startsWith('inbound:') || !key.endsWith(':interrupted'))
|
|
2913
|
+
return false;
|
|
2914
|
+
const inboxId = key.slice('inbound:'.length, -':interrupted'.length);
|
|
2915
|
+
const binding = this.getBinding(record.intent.bindingId);
|
|
2916
|
+
return binding !== undefined
|
|
2917
|
+
&& this.getOutboxByIdempotencyKey(key)?.id === record.id
|
|
2918
|
+
&& this.interruptedInboxNoticeCurrent({ inboxId, binding, intent: record.intent });
|
|
2919
|
+
}
|
|
2920
|
+
interruptedInboxNoticeCurrent(input) {
|
|
2921
|
+
const inbox = this.getInbox(input.inboxId);
|
|
2922
|
+
const binding = inbox?.bindingId === undefined ? undefined : this.getBinding(inbox.bindingId);
|
|
2923
|
+
const owner = binding === undefined ? undefined : this.getBindingPrincipal(binding.id);
|
|
2924
|
+
const execution = this.database.prepare(`
|
|
2925
|
+
SELECT principal_record_id, principal_version, binding_id, binding_version,
|
|
2926
|
+
binding_generation, workspace, preset
|
|
2927
|
+
FROM delivery_foreground_executions WHERE inbox_id = ?
|
|
2928
|
+
`).get(input.inboxId);
|
|
2929
|
+
return inbox?.status === 'dead_letter'
|
|
2930
|
+
&& (inbox.failureCode === 'dispatch-ambiguous' || inbox.failureCode === 'agent-task-timeout')
|
|
2931
|
+
&& inbox.envelope.kind === 'text' && inbox.channel !== 'web'
|
|
2932
|
+
&& binding?.status === 'active' && binding.id === input.binding.id
|
|
2933
|
+
&& binding.version === input.binding.version && binding.generation === input.binding.generation
|
|
2934
|
+
&& binding.sessionId === input.binding.sessionId
|
|
2935
|
+
&& this.getActiveBinding(inbox.envelope.conversation)?.id === binding.id
|
|
2936
|
+
&& owner?.status === 'active' && owner.role === 'owner'
|
|
2937
|
+
&& execution?.principal_record_id === owner.id && execution.principal_version === owner.version
|
|
2938
|
+
&& execution.binding_id === binding.id && execution.binding_version === binding.version
|
|
2939
|
+
&& execution.binding_generation === binding.generation
|
|
2940
|
+
&& execution.workspace === binding.workspace && execution.preset === binding.agentPreset
|
|
2941
|
+
&& principalHash(inbox.envelope.principal) === principalHash(binding.principal)
|
|
2942
|
+
&& conversationHash(inbox.envelope.conversation) === conversationHash(binding.conversation)
|
|
2943
|
+
&& input.intent.bindingId === binding.id
|
|
2944
|
+
&& input.intent.replyToEventId === inbox.eventId
|
|
2945
|
+
&& input.intent.idempotencyKey === `inbound:${inbox.id}:interrupted`
|
|
2946
|
+
&& this.getDeadLetterResolution({ kind: 'inbox', id: inbox.id,
|
|
2947
|
+
attemptCount: inbox.attemptCount }) === undefined
|
|
2948
|
+
&& this.getOutboxByIdempotencyKey(`inbound:${inbox.id}:reply`) === undefined
|
|
2949
|
+
&& this.getOutboxByIdempotencyKey(`inbound:${inbox.eventId}:reply`) === undefined;
|
|
2950
|
+
}
|
|
2652
2951
|
listInbox(input = {}) {
|
|
2653
2952
|
this.assertOpen();
|
|
2654
2953
|
const limit = Math.max(1, Math.min(100, input.limit ?? 20));
|
|
@@ -2804,6 +3103,40 @@ export class DeliveryStore {
|
|
|
2804
3103
|
}
|
|
2805
3104
|
return this.getInbox(inboxId);
|
|
2806
3105
|
}
|
|
3106
|
+
/** A durable control message wins even while its async binding transition is waiting. */
|
|
3107
|
+
hasRestartCancellationAfter(inboxId, binding) {
|
|
3108
|
+
this.assertOpen();
|
|
3109
|
+
const rows = this.database.prepare(`SELECT message.envelope_json FROM inbox_messages AS message
|
|
3110
|
+
JOIN delivery_inbox_admissions AS admission ON admission.inbox_id = message.id
|
|
3111
|
+
WHERE admission.admission_sequence > (SELECT admission_sequence FROM delivery_inbox_admissions WHERE inbox_id = ?)
|
|
3112
|
+
AND message.status != 'dead_letter'
|
|
3113
|
+
AND json_extract(message.envelope_json, '$.kind') = 'command'
|
|
3114
|
+
AND json_extract(message.envelope_json, '$.conversation') = json(?)
|
|
3115
|
+
AND json_extract(message.envelope_json, '$.principal') = json(?)`)
|
|
3116
|
+
.all(inboxId, conversationJson(binding.conversation), principalJson(binding.principal));
|
|
3117
|
+
return rows.some(row => isExactDeliveryCommand(parseDeliveryCommand(JSON.parse(row.envelope_json)), 'stop', 'new', 'clear'));
|
|
3118
|
+
}
|
|
3119
|
+
/** The fence and Inbox claims share this database's write transaction. */
|
|
3120
|
+
setRestartFence(requestId, sourceInvocationId) {
|
|
3121
|
+
this.assertOpen();
|
|
3122
|
+
this.transaction(() => {
|
|
3123
|
+
const current = this.getRestartFence();
|
|
3124
|
+
if (current !== undefined && (current.requestId !== requestId || current.sourceInvocationId !== sourceInvocationId)) {
|
|
3125
|
+
throw new DeliveryStoreError('conflict', 'another restart owns the delivery drain fence');
|
|
3126
|
+
}
|
|
3127
|
+
this.database.prepare('INSERT OR IGNORE INTO delivery_restart_fence VALUES (1, ?, ?)').run(requestId, sourceInvocationId);
|
|
3128
|
+
});
|
|
3129
|
+
}
|
|
3130
|
+
getRestartFence() {
|
|
3131
|
+
this.assertOpen();
|
|
3132
|
+
const row = this.database.prepare('SELECT request_id, source_invocation_id FROM delivery_restart_fence WHERE singleton = 1')
|
|
3133
|
+
.get();
|
|
3134
|
+
return row === undefined ? undefined : { requestId: row.request_id, sourceInvocationId: row.source_invocation_id };
|
|
3135
|
+
}
|
|
3136
|
+
clearRestartFence(requestId) {
|
|
3137
|
+
this.assertOpen();
|
|
3138
|
+
this.database.prepare('DELETE FROM delivery_restart_fence WHERE singleton = 1 AND request_id = ?').run(requestId);
|
|
3139
|
+
}
|
|
2807
3140
|
claimInbox(input) {
|
|
2808
3141
|
this.assertOpen();
|
|
2809
3142
|
const ownerId = validateBindingText(input.ownerId, 'ownerId', 256);
|
|
@@ -2819,6 +3152,8 @@ export class DeliveryStore {
|
|
|
2819
3152
|
const now = this.now();
|
|
2820
3153
|
const claims = [];
|
|
2821
3154
|
this.transaction(() => {
|
|
3155
|
+
if (this.getRestartFence() !== undefined)
|
|
3156
|
+
return;
|
|
2822
3157
|
this.database.prepare(`
|
|
2823
3158
|
UPDATE inbox_messages SET status = 'dead_letter', failure_code = 'attempts-exhausted',
|
|
2824
3159
|
next_attempt_at = NULL, updated_at = ?
|
|
@@ -4397,93 +4732,98 @@ export class DeliveryStore {
|
|
|
4397
4732
|
const claims = [];
|
|
4398
4733
|
const validatedOwnerRouteIds = [];
|
|
4399
4734
|
this.transaction(() => {
|
|
4400
|
-
this.
|
|
4401
|
-
|
|
4402
|
-
|
|
4403
|
-
|
|
4404
|
-
|
|
4405
|
-
|
|
4406
|
-
|
|
4407
|
-
|
|
4408
|
-
|
|
4409
|
-
|
|
4410
|
-
|
|
4411
|
-
|
|
4412
|
-
|
|
4413
|
-
|
|
4414
|
-
|
|
4415
|
-
|
|
4416
|
-
|
|
4417
|
-
|
|
4418
|
-
|
|
4419
|
-
|
|
4420
|
-
|
|
4421
|
-
|
|
4422
|
-
|
|
4423
|
-
|
|
4424
|
-
|
|
4425
|
-
|
|
4426
|
-
|
|
4427
|
-
|
|
4428
|
-
|
|
4429
|
-
|
|
4430
|
-
|
|
4431
|
-
|
|
4432
|
-
|
|
4433
|
-
|
|
4434
|
-
|
|
4435
|
-
|
|
4436
|
-
|
|
4437
|
-
|
|
4438
|
-
|
|
4439
|
-
|
|
4440
|
-
|
|
4441
|
-
|
|
4442
|
-
|
|
4443
|
-
const
|
|
4444
|
-
|
|
4445
|
-
|
|
4446
|
-
|
|
4447
|
-
|
|
4448
|
-
|
|
4449
|
-
|
|
4450
|
-
|
|
4451
|
-
|
|
4452
|
-
|
|
4453
|
-
|
|
4454
|
-
|
|
4455
|
-
|
|
4456
|
-
|
|
4457
|
-
|
|
4458
|
-
|
|
4459
|
-
|
|
4460
|
-
|
|
4461
|
-
|
|
4735
|
+
const restartFenceActive = this.getRestartFence() !== undefined;
|
|
4736
|
+
// Drain without changing queued retries or unresolved sends. Only a
|
|
4737
|
+
// plain maintenance acknowledgement may start during this fence.
|
|
4738
|
+
if (!restartFenceActive) {
|
|
4739
|
+
this.database.prepare(`
|
|
4740
|
+
UPDATE outbox_messages SET status = 'dead', failure_code = 'attempts-exhausted',
|
|
4741
|
+
next_attempt_at = NULL, updated_at = ?
|
|
4742
|
+
WHERE rowid IN (
|
|
4743
|
+
SELECT candidate.rowid FROM outbox_messages AS candidate
|
|
4744
|
+
WHERE candidate.status = 'retry_wait' AND candidate.next_attempt_at <= ?
|
|
4745
|
+
AND (
|
|
4746
|
+
SELECT COUNT(*) FROM outbox_attempts AS history
|
|
4747
|
+
WHERE history.outbox_id = candidate.id AND history.operation = 'send'
|
|
4748
|
+
) >= ?
|
|
4749
|
+
ORDER BY candidate.rowid LIMIT ?
|
|
4750
|
+
)
|
|
4751
|
+
`).run(now, now, input.maxAttempts, maintenanceLimit);
|
|
4752
|
+
this.database.prepare(`
|
|
4753
|
+
UPDATE outbox_messages SET status = 'dead', failure_code = 'reconcile-attempts-exhausted',
|
|
4754
|
+
next_attempt_at = NULL, updated_at = ?
|
|
4755
|
+
WHERE rowid IN (
|
|
4756
|
+
SELECT candidate.rowid FROM outbox_messages AS candidate
|
|
4757
|
+
WHERE candidate.status = 'unknown_after_send'
|
|
4758
|
+
AND NOT EXISTS (
|
|
4759
|
+
SELECT 1 FROM dead_letter_resolutions AS resolution
|
|
4760
|
+
WHERE resolution.kind = 'outbox' AND resolution.message_id = candidate.id
|
|
4761
|
+
AND resolution.attempt_count = candidate.attempt_count AND resolution.resolution = 'cancel'
|
|
4762
|
+
)
|
|
4763
|
+
AND (
|
|
4764
|
+
SELECT COUNT(*) FROM outbox_attempts AS history
|
|
4765
|
+
WHERE history.outbox_id = candidate.id AND history.operation = 'reconcile'
|
|
4766
|
+
) >= ?
|
|
4767
|
+
ORDER BY candidate.rowid LIMIT ?
|
|
4768
|
+
)
|
|
4769
|
+
`).run(now, input.maxAttempts, maintenanceLimit);
|
|
4770
|
+
const ownerRouteRows = this.database.prepare(`
|
|
4771
|
+
SELECT id FROM outbox_messages
|
|
4772
|
+
WHERE status IN ('pending', 'retry_wait') AND claimed_by IS NULL
|
|
4773
|
+
AND (next_attempt_at IS NULL OR next_attempt_at <= ?)
|
|
4774
|
+
AND json_type(intent_json, '$.metadata."dsh.route.authority"') IS NOT NULL
|
|
4775
|
+
ORDER BY rowid LIMIT ?
|
|
4776
|
+
`).all(now, maintenanceLimit);
|
|
4777
|
+
const ownerRouteGuard = input.ownerRouteGuard ?? { ownerRoutes: [], authorize: () => false };
|
|
4778
|
+
for (const row of ownerRouteRows) {
|
|
4779
|
+
const record = this.getOutbox(row.id);
|
|
4780
|
+
if (record === undefined)
|
|
4781
|
+
continue;
|
|
4782
|
+
const inspected = this.inspectOwnerRouteDispatch(record, ownerRouteGuard);
|
|
4783
|
+
if (inspected.kind === 'deferred') {
|
|
4784
|
+
this.deferOwnerRouteBeforeClaim(record, inspected.failureCode, now, now + Math.max(1, Math.min(input.leaseMs, 30_000)));
|
|
4785
|
+
continue;
|
|
4786
|
+
}
|
|
4787
|
+
if (inspected.kind !== 'authorized') {
|
|
4788
|
+
const failureCode = inspected.kind === 'denied'
|
|
4789
|
+
? inspected.failureCode
|
|
4790
|
+
: 'owner-route-receipt-invalid';
|
|
4791
|
+
this.deadLetterOwnerRouteBeforeClaim(record, ownerId, failureCode, now);
|
|
4792
|
+
continue;
|
|
4793
|
+
}
|
|
4794
|
+
if (!this.ownerRouteIsProvablyUnsent(record)) {
|
|
4795
|
+
this.deadLetterOwnerRouteBeforeClaim(record, ownerId, 'owner-route-send-proof-missing', now);
|
|
4796
|
+
continue;
|
|
4797
|
+
}
|
|
4798
|
+
if (!this.rebindOwnerRouteBeforeClaim(record, inspected)) {
|
|
4799
|
+
this.deadLetterOwnerRouteBeforeClaim(record, ownerId, 'owner-route-rebind-conflict', now);
|
|
4800
|
+
continue;
|
|
4801
|
+
}
|
|
4802
|
+
validatedOwnerRouteIds.push(record.id);
|
|
4462
4803
|
}
|
|
4463
|
-
|
|
4804
|
+
this.database.prepare(`
|
|
4805
|
+
UPDATE outbox_messages SET status = 'dead', failure_code = CASE
|
|
4806
|
+
WHEN status = 'unknown_after_send' THEN 'binding-revoked-unknown'
|
|
4807
|
+
ELSE 'binding-revoked'
|
|
4808
|
+
END,
|
|
4809
|
+
next_attempt_at = NULL, updated_at = ?
|
|
4810
|
+
WHERE rowid IN (
|
|
4811
|
+
SELECT candidate.rowid FROM outbox_messages AS candidate
|
|
4812
|
+
WHERE candidate.status IN ('pending', 'retry_wait', 'unknown_after_send')
|
|
4813
|
+
AND json_type(candidate.intent_json, '$.metadata."dsh.route.authority"') IS NULL
|
|
4814
|
+
AND NOT EXISTS (
|
|
4815
|
+
SELECT 1 FROM dead_letter_resolutions AS resolution
|
|
4816
|
+
WHERE resolution.kind = 'outbox' AND resolution.message_id = candidate.id
|
|
4817
|
+
AND resolution.attempt_count = candidate.attempt_count AND resolution.resolution = 'cancel'
|
|
4818
|
+
)
|
|
4819
|
+
AND NOT EXISTS (
|
|
4820
|
+
SELECT 1 FROM conversation_bindings AS binding
|
|
4821
|
+
WHERE binding.id = candidate.binding_id AND binding.status = 'active'
|
|
4822
|
+
)
|
|
4823
|
+
ORDER BY candidate.rowid LIMIT ?
|
|
4824
|
+
)
|
|
4825
|
+
`).run(now, maintenanceLimit);
|
|
4464
4826
|
}
|
|
4465
|
-
this.database.prepare(`
|
|
4466
|
-
UPDATE outbox_messages SET status = 'dead', failure_code = CASE
|
|
4467
|
-
WHEN status = 'unknown_after_send' THEN 'binding-revoked-unknown'
|
|
4468
|
-
ELSE 'binding-revoked'
|
|
4469
|
-
END,
|
|
4470
|
-
next_attempt_at = NULL, updated_at = ?
|
|
4471
|
-
WHERE rowid IN (
|
|
4472
|
-
SELECT candidate.rowid FROM outbox_messages AS candidate
|
|
4473
|
-
WHERE candidate.status IN ('pending', 'retry_wait', 'unknown_after_send')
|
|
4474
|
-
AND json_type(candidate.intent_json, '$.metadata."dsh.route.authority"') IS NULL
|
|
4475
|
-
AND NOT EXISTS (
|
|
4476
|
-
SELECT 1 FROM dead_letter_resolutions AS resolution
|
|
4477
|
-
WHERE resolution.kind = 'outbox' AND resolution.message_id = candidate.id
|
|
4478
|
-
AND resolution.attempt_count = candidate.attempt_count AND resolution.resolution = 'cancel'
|
|
4479
|
-
)
|
|
4480
|
-
AND NOT EXISTS (
|
|
4481
|
-
SELECT 1 FROM conversation_bindings AS binding
|
|
4482
|
-
WHERE binding.id = candidate.binding_id AND binding.status = 'active'
|
|
4483
|
-
)
|
|
4484
|
-
ORDER BY candidate.rowid LIMIT ?
|
|
4485
|
-
)
|
|
4486
|
-
`).run(now, maintenanceLimit);
|
|
4487
4827
|
const ownerRouteClaimClause = validatedOwnerRouteIds.length === 0
|
|
4488
4828
|
? `AND (
|
|
4489
4829
|
json_type(candidate.intent_json, '$.metadata."dsh.route.authority"') IS NULL
|
|
@@ -4498,6 +4838,13 @@ export class DeliveryStore {
|
|
|
4498
4838
|
SELECT candidate.id, candidate.status FROM outbox_messages AS candidate
|
|
4499
4839
|
WHERE candidate.status IN ('pending', 'retry_wait', 'unknown_after_send')
|
|
4500
4840
|
AND candidate.claimed_by IS NULL
|
|
4841
|
+
${restartFenceActive ? `AND candidate.status IN ('pending', 'retry_wait')
|
|
4842
|
+
AND candidate.idempotency_key GLOB 'inbound:*:maintenance'
|
|
4843
|
+
AND json_extract(candidate.intent_json, '$.format') = 'plain'
|
|
4844
|
+
AND json_type(candidate.intent_json, '$.metadata') IS NULL
|
|
4845
|
+
AND json_type(candidate.intent_json, '$.approval') IS NULL
|
|
4846
|
+
AND json_type(candidate.intent_json, '$.modelPicker') IS NULL
|
|
4847
|
+
AND json_type(candidate.intent_json, '$.permissionPicker') IS NULL` : ''}
|
|
4501
4848
|
AND NOT EXISTS (
|
|
4502
4849
|
SELECT 1 FROM dead_letter_resolutions AS resolution
|
|
4503
4850
|
WHERE resolution.kind = 'outbox' AND resolution.message_id = candidate.id
|
|
@@ -4532,6 +4879,7 @@ export class DeliveryStore {
|
|
|
4532
4879
|
WHERE earlier.lane_hash = candidate.lane_hash
|
|
4533
4880
|
AND earlier.rowid < candidate.rowid
|
|
4534
4881
|
AND earlier.status NOT IN ('accepted', 'delivered', 'read', 'dead')
|
|
4882
|
+
${restartFenceActive ? "AND earlier.idempotency_key GLOB 'inbound:*:maintenance'" : ''}
|
|
4535
4883
|
AND NOT EXISTS (
|
|
4536
4884
|
SELECT 1 FROM dead_letter_resolutions AS resolution
|
|
4537
4885
|
WHERE resolution.kind = 'outbox' AND resolution.message_id = earlier.id
|