@dsh-enhanced/assistant-delivery 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +225 -0
- package/cordis.patch.yml +29 -0
- package/lib/agent-runtime.d.ts +112 -0
- package/lib/agent-runtime.d.ts.map +1 -0
- package/lib/agent-runtime.js +1866 -0
- package/lib/agent-runtime.js.map +1 -0
- package/lib/canonical.d.ts +18 -0
- package/lib/canonical.d.ts.map +1 -0
- package/lib/canonical.js +83 -0
- package/lib/canonical.js.map +1 -0
- package/lib/coordinator.d.ts +124 -0
- package/lib/coordinator.d.ts.map +1 -0
- package/lib/coordinator.js +676 -0
- package/lib/coordinator.js.map +1 -0
- package/lib/inbound-images.d.ts +38 -0
- package/lib/inbound-images.d.ts.map +1 -0
- package/lib/inbound-images.js +267 -0
- package/lib/inbound-images.js.map +1 -0
- package/lib/index.d.ts +12 -0
- package/lib/index.d.ts.map +1 -0
- package/lib/index.js +12 -0
- package/lib/index.js.map +1 -0
- package/lib/operator.d.ts +23 -0
- package/lib/operator.d.ts.map +1 -0
- package/lib/operator.js +38 -0
- package/lib/operator.js.map +1 -0
- package/lib/service.d.ts +269 -0
- package/lib/service.d.ts.map +1 -0
- package/lib/service.js +1698 -0
- package/lib/service.js.map +1 -0
- package/lib/session-commands.d.ts +17 -0
- package/lib/session-commands.d.ts.map +1 -0
- package/lib/session-commands.js +39 -0
- package/lib/session-commands.js.map +1 -0
- package/lib/sqlite.d.ts +8 -0
- package/lib/sqlite.d.ts.map +1 -0
- package/lib/sqlite.js +411 -0
- package/lib/sqlite.js.map +1 -0
- package/lib/store.d.ts +353 -0
- package/lib/store.d.ts.map +1 -0
- package/lib/store.js +2614 -0
- package/lib/store.js.map +1 -0
- package/lib/tools.d.ts +4 -0
- package/lib/tools.d.ts.map +1 -0
- package/lib/tools.js +54 -0
- package/lib/tools.js.map +1 -0
- package/lib/types.d.ts +329 -0
- package/lib/types.d.ts.map +1 -0
- package/lib/types.js +2 -0
- package/lib/types.js.map +1 -0
- package/lib/version.d.ts +2 -0
- package/lib/version.d.ts.map +1 -0
- package/lib/version.js +2 -0
- package/lib/version.js.map +1 -0
- package/package.json +122 -0
|
@@ -0,0 +1,676 @@
|
|
|
1
|
+
import { permissionDispatchRecoveryFromFailureCode, } from './session-commands.js';
|
|
2
|
+
import { DeliveryStoreError } from './store.js';
|
|
3
|
+
function adapterKey(channel, account) {
|
|
4
|
+
return JSON.stringify([channel, account]);
|
|
5
|
+
}
|
|
6
|
+
function samePrincipal(left, right) {
|
|
7
|
+
return left.channel === right.channel
|
|
8
|
+
&& left.account === right.account
|
|
9
|
+
&& left.tenant === right.tenant
|
|
10
|
+
&& left.user === right.user;
|
|
11
|
+
}
|
|
12
|
+
function sameConversation(left, right) {
|
|
13
|
+
return left.channel === right.channel
|
|
14
|
+
&& left.account === right.account
|
|
15
|
+
&& left.tenant === right.tenant
|
|
16
|
+
&& left.kind === right.kind
|
|
17
|
+
&& left.chat === right.chat
|
|
18
|
+
&& left.thread === right.thread;
|
|
19
|
+
}
|
|
20
|
+
function bindingChangedBeforeDispatch(claimed, current, envelope) {
|
|
21
|
+
return current.id !== claimed.id
|
|
22
|
+
|| current.version !== claimed.version
|
|
23
|
+
|| current.sessionId !== claimed.sessionId
|
|
24
|
+
|| current.generation !== claimed.generation
|
|
25
|
+
|| !samePrincipal(current.principal, claimed.principal)
|
|
26
|
+
|| !samePrincipal(current.principal, envelope.principal)
|
|
27
|
+
|| !sameConversation(current.conversation, claimed.conversation)
|
|
28
|
+
|| !sameConversation(current.conversation, envelope.conversation);
|
|
29
|
+
}
|
|
30
|
+
function dispatchAuthorizationFailure(error) {
|
|
31
|
+
if (!(error instanceof DeliveryStoreError))
|
|
32
|
+
return undefined;
|
|
33
|
+
if (error.code === 'unauthorized-principal')
|
|
34
|
+
return 'authorization-revoked-before-dispatch';
|
|
35
|
+
if (error.code === 'invalid-binding')
|
|
36
|
+
return 'binding-changed-before-dispatch';
|
|
37
|
+
return undefined;
|
|
38
|
+
}
|
|
39
|
+
const adapterDisposeGraceMs = 5_000;
|
|
40
|
+
async function boundedWait(operation, graceMs) {
|
|
41
|
+
let timeout;
|
|
42
|
+
try {
|
|
43
|
+
return await Promise.race([
|
|
44
|
+
operation,
|
|
45
|
+
new Promise(resolve => { timeout = setTimeout(() => resolve(undefined), graceMs); }),
|
|
46
|
+
]);
|
|
47
|
+
}
|
|
48
|
+
finally {
|
|
49
|
+
if (timeout !== undefined)
|
|
50
|
+
clearTimeout(timeout);
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
async function abortAndBoundedDrain(active, leaseMs, reason) {
|
|
54
|
+
const snapshot = [...active.entries()];
|
|
55
|
+
for (const [, entry] of snapshot)
|
|
56
|
+
entry.abort.abort(reason);
|
|
57
|
+
if (snapshot.length === 0)
|
|
58
|
+
return;
|
|
59
|
+
// Abort stops lease renewal immediately. One heartbeat interval gives a
|
|
60
|
+
// cooperative adapter time to unwind without allowing an uncooperative one
|
|
61
|
+
// to hold service disposal (and the SQLite handle) forever.
|
|
62
|
+
const graceMs = Math.max(1, Math.floor(leaseMs / 3));
|
|
63
|
+
let timeout;
|
|
64
|
+
try {
|
|
65
|
+
await Promise.race([
|
|
66
|
+
Promise.all(snapshot.map(([, entry]) => entry.promise)),
|
|
67
|
+
new Promise(resolve => { timeout = setTimeout(resolve, graceMs); }),
|
|
68
|
+
]);
|
|
69
|
+
}
|
|
70
|
+
finally {
|
|
71
|
+
if (timeout !== undefined)
|
|
72
|
+
clearTimeout(timeout);
|
|
73
|
+
for (const [id, entry] of snapshot) {
|
|
74
|
+
if (active.get(id) === entry)
|
|
75
|
+
active.delete(id);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
export class DeliveryAdapterRegistry {
|
|
80
|
+
context;
|
|
81
|
+
adapters = new Map();
|
|
82
|
+
starting = new Map();
|
|
83
|
+
stopSignal;
|
|
84
|
+
resolveStop;
|
|
85
|
+
stopped = false;
|
|
86
|
+
constructor(context) {
|
|
87
|
+
this.context = context;
|
|
88
|
+
this.stopSignal = new Promise(resolve => { this.resolveStop = resolve; });
|
|
89
|
+
}
|
|
90
|
+
async register(adapter) {
|
|
91
|
+
if (this.stopped)
|
|
92
|
+
throw new Error('assistant-delivery adapter registry is stopped');
|
|
93
|
+
const key = adapterKey(adapter.channel, adapter.account);
|
|
94
|
+
if (this.adapters.has(key) || this.starting.has(key)) {
|
|
95
|
+
throw new Error(`delivery adapter ${adapter.channel}/${adapter.account} is already registered`);
|
|
96
|
+
}
|
|
97
|
+
const result = Promise.resolve()
|
|
98
|
+
.then(() => adapter.start(this.context))
|
|
99
|
+
.then(dispose => ({ status: 'started', ...(dispose === undefined ? {} : { dispose }) }), error => ({ status: 'failed', error }));
|
|
100
|
+
const entry = { result };
|
|
101
|
+
this.starting.set(key, entry);
|
|
102
|
+
let dispose;
|
|
103
|
+
try {
|
|
104
|
+
const outcome = await Promise.race([
|
|
105
|
+
result,
|
|
106
|
+
this.stopSignal.then(() => ({ status: 'stopped' })),
|
|
107
|
+
]);
|
|
108
|
+
if (outcome.status === 'stopped') {
|
|
109
|
+
void this.cleanupStarting(entry);
|
|
110
|
+
throw new Error('assistant-delivery adapter registry is stopped');
|
|
111
|
+
}
|
|
112
|
+
if (outcome.status === 'failed')
|
|
113
|
+
throw outcome.error;
|
|
114
|
+
dispose = outcome.dispose;
|
|
115
|
+
if (this.stopped) {
|
|
116
|
+
await this.cleanupStarting(entry);
|
|
117
|
+
throw new Error('assistant-delivery adapter registry is stopped');
|
|
118
|
+
}
|
|
119
|
+
if (this.starting.get(key) === entry)
|
|
120
|
+
this.starting.delete(key);
|
|
121
|
+
this.adapters.set(key, { adapter, ...(dispose === undefined ? {} : { dispose }) });
|
|
122
|
+
}
|
|
123
|
+
finally {
|
|
124
|
+
if (this.starting.get(key) === entry)
|
|
125
|
+
this.starting.delete(key);
|
|
126
|
+
}
|
|
127
|
+
let active = true;
|
|
128
|
+
return async () => {
|
|
129
|
+
if (!active)
|
|
130
|
+
return;
|
|
131
|
+
active = false;
|
|
132
|
+
const current = this.adapters.get(key);
|
|
133
|
+
if (current?.adapter !== adapter)
|
|
134
|
+
return;
|
|
135
|
+
this.adapters.delete(key);
|
|
136
|
+
await boundedWait(Promise.resolve().then(() => current.dispose?.()), adapterDisposeGraceMs);
|
|
137
|
+
};
|
|
138
|
+
}
|
|
139
|
+
get(channel, account) {
|
|
140
|
+
return this.adapters.get(adapterKey(channel, account))?.adapter;
|
|
141
|
+
}
|
|
142
|
+
reconcilableUnknownRoutes() {
|
|
143
|
+
const routes = [];
|
|
144
|
+
for (const { adapter } of this.adapters.values()) {
|
|
145
|
+
if (adapter.capabilities.reconcileUnknownSend === true && adapter.reconcileUnknownSend !== undefined) {
|
|
146
|
+
routes.push({ channel: adapter.channel, account: adapter.account });
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
return routes;
|
|
150
|
+
}
|
|
151
|
+
size() { return this.adapters.size; }
|
|
152
|
+
async stop() {
|
|
153
|
+
if (this.stopped)
|
|
154
|
+
return;
|
|
155
|
+
this.stopped = true;
|
|
156
|
+
this.resolveStop();
|
|
157
|
+
const current = [...this.adapters.values()];
|
|
158
|
+
const pending = [...this.starting.values()];
|
|
159
|
+
this.adapters.clear();
|
|
160
|
+
await boundedWait(Promise.allSettled([
|
|
161
|
+
...current.map(entry => Promise.resolve().then(() => entry.dispose?.())),
|
|
162
|
+
...pending.map(entry => this.cleanupStarting(entry)),
|
|
163
|
+
]), adapterDisposeGraceMs);
|
|
164
|
+
}
|
|
165
|
+
cleanupStarting(entry) {
|
|
166
|
+
entry.cleanup ??= entry.result.then(async (outcome) => {
|
|
167
|
+
if (outcome.status !== 'started')
|
|
168
|
+
return;
|
|
169
|
+
const dispose = outcome.dispose;
|
|
170
|
+
if (dispose === undefined)
|
|
171
|
+
return;
|
|
172
|
+
await boundedWait(Promise.resolve().then(() => dispose()), adapterDisposeGraceMs)
|
|
173
|
+
.catch(() => undefined);
|
|
174
|
+
});
|
|
175
|
+
return entry.cleanup;
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
export class DeliveryCoordinator {
|
|
179
|
+
options;
|
|
180
|
+
active = new Map();
|
|
181
|
+
now;
|
|
182
|
+
random;
|
|
183
|
+
timer;
|
|
184
|
+
stopping = false;
|
|
185
|
+
constructor(options) {
|
|
186
|
+
this.options = options;
|
|
187
|
+
this.now = options.now ?? Date.now;
|
|
188
|
+
this.random = options.random ?? Math.random;
|
|
189
|
+
}
|
|
190
|
+
async tick() {
|
|
191
|
+
if (this.stopping)
|
|
192
|
+
return;
|
|
193
|
+
this.options.store.recoverOutbox({ maxAttempts: this.options.maxAttempts });
|
|
194
|
+
const available = this.options.maxConcurrency - this.active.size;
|
|
195
|
+
if (available <= 0)
|
|
196
|
+
return;
|
|
197
|
+
const claims = this.options.store.claimOutbox({
|
|
198
|
+
ownerId: this.options.ownerId,
|
|
199
|
+
leaseMs: this.options.leaseMs,
|
|
200
|
+
limit: available,
|
|
201
|
+
maxAttempts: this.options.maxAttempts,
|
|
202
|
+
unknownReconcileRoutes: this.options.registry.reconcilableUnknownRoutes(),
|
|
203
|
+
excludeIds: [...this.active.keys()],
|
|
204
|
+
});
|
|
205
|
+
for (const claim of claims) {
|
|
206
|
+
const abort = new AbortController();
|
|
207
|
+
const stopHeartbeat = this.startLeaseHeartbeat(claim.record.id, claim.fencingToken, abort);
|
|
208
|
+
let entry;
|
|
209
|
+
const promise = this.process(claim.record, claim.fencingToken, claim.mode, abort.signal)
|
|
210
|
+
.catch(() => { })
|
|
211
|
+
.finally(() => {
|
|
212
|
+
stopHeartbeat();
|
|
213
|
+
if (this.active.get(claim.record.id) === entry)
|
|
214
|
+
this.active.delete(claim.record.id);
|
|
215
|
+
});
|
|
216
|
+
entry = { abort, promise };
|
|
217
|
+
this.active.set(claim.record.id, entry);
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
start() {
|
|
221
|
+
if (this.timer !== undefined)
|
|
222
|
+
return;
|
|
223
|
+
this.stopping = false;
|
|
224
|
+
const interval = this.options.tickIntervalMs ?? 1_000;
|
|
225
|
+
this.timer = setInterval(() => void this.tick(), interval);
|
|
226
|
+
this.timer.unref?.();
|
|
227
|
+
void this.tick();
|
|
228
|
+
}
|
|
229
|
+
async stop() {
|
|
230
|
+
this.stopping = true;
|
|
231
|
+
if (this.timer !== undefined)
|
|
232
|
+
clearInterval(this.timer);
|
|
233
|
+
this.timer = undefined;
|
|
234
|
+
await abortAndBoundedDrain(this.active, this.options.leaseMs, new Error('assistant-delivery is stopping'));
|
|
235
|
+
}
|
|
236
|
+
async whenIdle() {
|
|
237
|
+
while (this.active.size > 0)
|
|
238
|
+
await Promise.all([...this.active.values()].map(entry => entry.promise));
|
|
239
|
+
}
|
|
240
|
+
startLeaseHeartbeat(outboxId, fencingToken, abort) {
|
|
241
|
+
let active = true;
|
|
242
|
+
const interval = Math.max(1, Math.floor(this.options.leaseMs / 3));
|
|
243
|
+
const heartbeat = setInterval(() => {
|
|
244
|
+
if (!active || abort.signal.aborted)
|
|
245
|
+
return;
|
|
246
|
+
try {
|
|
247
|
+
if (this.options.store.renewOutboxClaim({ outboxId, ownerId: this.options.ownerId, fencingToken,
|
|
248
|
+
leaseMs: this.options.leaseMs }))
|
|
249
|
+
return;
|
|
250
|
+
abort.abort(new Error('assistant-delivery outbox lease was lost'));
|
|
251
|
+
}
|
|
252
|
+
catch (error) {
|
|
253
|
+
abort.abort(error);
|
|
254
|
+
}
|
|
255
|
+
}, interval);
|
|
256
|
+
heartbeat.unref?.();
|
|
257
|
+
const stop = () => {
|
|
258
|
+
if (!active)
|
|
259
|
+
return;
|
|
260
|
+
active = false;
|
|
261
|
+
clearInterval(heartbeat);
|
|
262
|
+
abort.signal.removeEventListener('abort', stop);
|
|
263
|
+
};
|
|
264
|
+
abort.signal.addEventListener('abort', stop, { once: true });
|
|
265
|
+
return stop;
|
|
266
|
+
}
|
|
267
|
+
async process(record, fencingToken, mode, signal) {
|
|
268
|
+
const adapter = this.options.registry.get(record.intent.target.conversation.channel, record.intent.target.conversation.account);
|
|
269
|
+
if (adapter === undefined) {
|
|
270
|
+
this.finish(record, fencingToken, mode === 'reconcile'
|
|
271
|
+
? { outcome: 'unknown', failureCode: 'adapter-unavailable' }
|
|
272
|
+
: { outcome: 'not-sent', failureCode: 'adapter-unavailable', retryable: true });
|
|
273
|
+
return;
|
|
274
|
+
}
|
|
275
|
+
if (!adapter.capabilities.formats.includes(record.intent.format ?? 'plain')) {
|
|
276
|
+
this.finish(record, fencingToken, { outcome: 'not-sent', failureCode: 'unsupported-format', retryable: false });
|
|
277
|
+
return;
|
|
278
|
+
}
|
|
279
|
+
if (mode === 'reconcile') {
|
|
280
|
+
if (!adapter.capabilities.reconcileUnknownSend || adapter.reconcileUnknownSend === undefined) {
|
|
281
|
+
this.finish(record, fencingToken, { outcome: 'unknown', failureCode: 'reconcile-unsupported' });
|
|
282
|
+
return;
|
|
283
|
+
}
|
|
284
|
+
try {
|
|
285
|
+
const result = await adapter.reconcileUnknownSend(record, signal);
|
|
286
|
+
if (signal.aborted)
|
|
287
|
+
return;
|
|
288
|
+
this.finishReconcile(record, fencingToken, result);
|
|
289
|
+
}
|
|
290
|
+
catch {
|
|
291
|
+
if (signal.aborted)
|
|
292
|
+
return;
|
|
293
|
+
this.finish(record, fencingToken, { outcome: 'unknown', failureCode: 'reconcile-threw' });
|
|
294
|
+
}
|
|
295
|
+
return;
|
|
296
|
+
}
|
|
297
|
+
const binding = this.options.store.getBinding(record.intent.bindingId);
|
|
298
|
+
if (binding?.status !== 'active') {
|
|
299
|
+
this.finish(record, fencingToken, { outcome: 'not-sent', failureCode: 'binding-revoked', retryable: false });
|
|
300
|
+
return;
|
|
301
|
+
}
|
|
302
|
+
if (!sameConversation(binding.conversation, record.intent.target.conversation)
|
|
303
|
+
|| !samePrincipal(binding.principal, record.intent.target.principal)) {
|
|
304
|
+
this.finish(record, fencingToken, {
|
|
305
|
+
outcome: 'not-sent', failureCode: 'binding-target-mismatch', retryable: false,
|
|
306
|
+
});
|
|
307
|
+
return;
|
|
308
|
+
}
|
|
309
|
+
let authorized;
|
|
310
|
+
try {
|
|
311
|
+
authorized = this.options.store.isAuthorizedPrincipal(binding.principal);
|
|
312
|
+
}
|
|
313
|
+
catch {
|
|
314
|
+
this.finish(record, fencingToken, {
|
|
315
|
+
outcome: 'not-sent', failureCode: 'principal-authorization-check-failed', retryable: true,
|
|
316
|
+
});
|
|
317
|
+
return;
|
|
318
|
+
}
|
|
319
|
+
if (!authorized) {
|
|
320
|
+
this.finish(record, fencingToken, { outcome: 'not-sent', failureCode: 'principal-revoked', retryable: false });
|
|
321
|
+
return;
|
|
322
|
+
}
|
|
323
|
+
try {
|
|
324
|
+
const result = await adapter.send(record.intent, signal);
|
|
325
|
+
if (signal.aborted)
|
|
326
|
+
return;
|
|
327
|
+
this.finish(record, fencingToken, result);
|
|
328
|
+
}
|
|
329
|
+
catch {
|
|
330
|
+
if (signal.aborted)
|
|
331
|
+
return;
|
|
332
|
+
this.finish(record, fencingToken, { outcome: 'unknown', failureCode: 'adapter-threw' });
|
|
333
|
+
}
|
|
334
|
+
}
|
|
335
|
+
finish(record, fencingToken, result) {
|
|
336
|
+
if (result.outcome === 'accepted') {
|
|
337
|
+
this.options.store.finishOutbox({ outboxId: record.id, ownerId: this.options.ownerId, fencingToken,
|
|
338
|
+
outcome: 'accepted', providerMessageId: result.providerMessageId });
|
|
339
|
+
return;
|
|
340
|
+
}
|
|
341
|
+
if (result.outcome === 'unknown') {
|
|
342
|
+
this.options.store.finishOutbox({ outboxId: record.id, ownerId: this.options.ownerId, fencingToken,
|
|
343
|
+
outcome: 'unknown_after_send', failureCode: result.failureCode,
|
|
344
|
+
...(result.providerMessageId === undefined ? {} : { providerMessageId: result.providerMessageId }) });
|
|
345
|
+
return;
|
|
346
|
+
}
|
|
347
|
+
if (!result.retryable) {
|
|
348
|
+
this.options.store.finishOutbox({ outboxId: record.id, ownerId: this.options.ownerId, fencingToken,
|
|
349
|
+
outcome: 'dead', failureCode: result.failureCode });
|
|
350
|
+
return;
|
|
351
|
+
}
|
|
352
|
+
const exponential = Math.min(this.options.retryMaxMs, this.options.retryBaseMs * (2 ** Math.max(0, record.attemptCount - 1)) * (1 + this.random()));
|
|
353
|
+
const delay = Math.max(exponential, result.retryAfterMs ?? 0);
|
|
354
|
+
this.options.store.finishOutbox({ outboxId: record.id, ownerId: this.options.ownerId, fencingToken,
|
|
355
|
+
outcome: 'retry_wait', failureCode: result.failureCode, retryAt: this.now() + Math.ceil(delay) });
|
|
356
|
+
}
|
|
357
|
+
finishReconcile(record, fencingToken, result) {
|
|
358
|
+
if (result.outcome === 'unknown') {
|
|
359
|
+
this.finish(record, fencingToken, { outcome: 'unknown', failureCode: 'reconcile-inconclusive' });
|
|
360
|
+
return;
|
|
361
|
+
}
|
|
362
|
+
if (result.outcome === 'not-sent') {
|
|
363
|
+
this.finish(record, fencingToken, { outcome: 'not-sent', failureCode: 'reconciled-not-sent', retryable: true });
|
|
364
|
+
return;
|
|
365
|
+
}
|
|
366
|
+
this.options.store.finishOutbox({ outboxId: record.id, ownerId: this.options.ownerId, fencingToken,
|
|
367
|
+
outcome: 'accepted', providerMessageId: result.providerMessageId });
|
|
368
|
+
if (result.outcome !== 'accepted') {
|
|
369
|
+
this.options.store.recordReceipt({
|
|
370
|
+
channel: record.intent.target.conversation.channel,
|
|
371
|
+
account: record.intent.target.conversation.account,
|
|
372
|
+
providerMessageId: result.providerMessageId,
|
|
373
|
+
status: result.outcome,
|
|
374
|
+
occurredAt: this.now(),
|
|
375
|
+
});
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
}
|
|
379
|
+
export class InboundCoordinator {
|
|
380
|
+
options;
|
|
381
|
+
active = new Map();
|
|
382
|
+
now;
|
|
383
|
+
random;
|
|
384
|
+
stopping = false;
|
|
385
|
+
constructor(options) {
|
|
386
|
+
this.options = options;
|
|
387
|
+
this.now = options.now ?? Date.now;
|
|
388
|
+
this.random = options.random ?? Math.random;
|
|
389
|
+
}
|
|
390
|
+
async tick() {
|
|
391
|
+
if (this.stopping)
|
|
392
|
+
return;
|
|
393
|
+
this.options.store.recoverInbox({ maxAttempts: this.options.maxAttempts });
|
|
394
|
+
const available = this.options.maxConcurrency - this.active.size;
|
|
395
|
+
if (available <= 0)
|
|
396
|
+
return;
|
|
397
|
+
const claims = this.options.store.claimInbox({ ownerId: this.options.ownerId, leaseMs: this.options.leaseMs,
|
|
398
|
+
limit: available, maxAttempts: this.options.maxAttempts });
|
|
399
|
+
for (const claim of claims) {
|
|
400
|
+
const abort = new AbortController();
|
|
401
|
+
const stopHeartbeat = this.startLeaseHeartbeat(claim.record.id, claim.fencingToken, abort);
|
|
402
|
+
let entry;
|
|
403
|
+
const promise = this.process(claim.record.id, claim.record.bindingId, claim.record.envelope, claim.fencingToken, abort.signal, permissionDispatchRecoveryFromFailureCode(claim.record.failureCode))
|
|
404
|
+
.catch(() => { })
|
|
405
|
+
.finally(() => {
|
|
406
|
+
stopHeartbeat();
|
|
407
|
+
if (this.active.get(claim.record.id) === entry)
|
|
408
|
+
this.active.delete(claim.record.id);
|
|
409
|
+
});
|
|
410
|
+
entry = { abort, promise };
|
|
411
|
+
this.active.set(claim.record.id, entry);
|
|
412
|
+
}
|
|
413
|
+
}
|
|
414
|
+
async stop() {
|
|
415
|
+
this.stopping = true;
|
|
416
|
+
await abortAndBoundedDrain(this.active, this.options.leaseMs, new Error('assistant-delivery is stopping'));
|
|
417
|
+
}
|
|
418
|
+
/** Abort and drain exact claims that a durable command fence made terminal. */
|
|
419
|
+
async cancelUndispatchedClaims(inboxIds, command) {
|
|
420
|
+
const operations = [...new Set(inboxIds)]
|
|
421
|
+
.map(inboxId => this.active.get(inboxId))
|
|
422
|
+
.filter((entry) => entry !== undefined);
|
|
423
|
+
if (operations.length === 0)
|
|
424
|
+
return;
|
|
425
|
+
const reason = new Error(`assistant-delivery: inbox preparation cancelled by /${command}`);
|
|
426
|
+
for (const operation of operations) {
|
|
427
|
+
if (!operation.abort.signal.aborted)
|
|
428
|
+
operation.abort.abort(reason);
|
|
429
|
+
}
|
|
430
|
+
await Promise.all(operations.map(operation => operation.promise));
|
|
431
|
+
}
|
|
432
|
+
async whenIdle() {
|
|
433
|
+
while (this.active.size > 0)
|
|
434
|
+
await Promise.all([...this.active.values()].map(entry => entry.promise));
|
|
435
|
+
}
|
|
436
|
+
startLeaseHeartbeat(inboxId, fencingToken, abort) {
|
|
437
|
+
let active = true;
|
|
438
|
+
const interval = Math.max(1, Math.floor(this.options.leaseMs / 3));
|
|
439
|
+
const heartbeat = setInterval(() => {
|
|
440
|
+
if (!active || abort.signal.aborted)
|
|
441
|
+
return;
|
|
442
|
+
try {
|
|
443
|
+
if (this.options.store.renewInboxClaim({ inboxId, ownerId: this.options.ownerId, fencingToken,
|
|
444
|
+
leaseMs: this.options.leaseMs }))
|
|
445
|
+
return;
|
|
446
|
+
abort.abort(new Error('assistant-delivery inbox lease was lost'));
|
|
447
|
+
}
|
|
448
|
+
catch (error) {
|
|
449
|
+
abort.abort(error);
|
|
450
|
+
}
|
|
451
|
+
}, interval);
|
|
452
|
+
heartbeat.unref?.();
|
|
453
|
+
const stop = () => {
|
|
454
|
+
if (!active)
|
|
455
|
+
return;
|
|
456
|
+
active = false;
|
|
457
|
+
clearInterval(heartbeat);
|
|
458
|
+
abort.signal.removeEventListener('abort', stop);
|
|
459
|
+
};
|
|
460
|
+
abort.signal.addEventListener('abort', stop, { once: true });
|
|
461
|
+
return stop;
|
|
462
|
+
}
|
|
463
|
+
async process(inboxId, bindingId, envelope, fencingToken, signal, permissionDispatchRecovery) {
|
|
464
|
+
const binding = bindingId === undefined ? undefined : this.options.store.getBinding(bindingId);
|
|
465
|
+
if (binding?.status === 'revoked') {
|
|
466
|
+
this.options.store.finishInbox({ inboxId, ownerId: this.options.ownerId, fencingToken,
|
|
467
|
+
outcome: 'dead_letter', failureCode: 'binding-revoked' });
|
|
468
|
+
return;
|
|
469
|
+
}
|
|
470
|
+
if (binding === undefined) {
|
|
471
|
+
this.options.store.finishInbox({ inboxId, ownerId: this.options.ownerId, fencingToken,
|
|
472
|
+
outcome: 'retry_wait', failureCode: 'binding-unavailable',
|
|
473
|
+
retryAt: this.now() + this.options.retryBaseMs });
|
|
474
|
+
return;
|
|
475
|
+
}
|
|
476
|
+
const processor = this.options.processor();
|
|
477
|
+
if (processor === undefined) {
|
|
478
|
+
this.options.store.finishInbox({ inboxId, ownerId: this.options.ownerId, fencingToken,
|
|
479
|
+
outcome: 'retry_wait', failureCode: 'processor-unavailable', retryAt: this.now() + this.options.retryBaseMs });
|
|
480
|
+
return;
|
|
481
|
+
}
|
|
482
|
+
let prepared;
|
|
483
|
+
if (processor.prepare !== undefined) {
|
|
484
|
+
let preparation;
|
|
485
|
+
try {
|
|
486
|
+
preparation = await processor.prepare(binding, envelope, signal, {
|
|
487
|
+
inboxId,
|
|
488
|
+
ownerId: this.options.ownerId,
|
|
489
|
+
fencingToken,
|
|
490
|
+
...(permissionDispatchRecovery === undefined ? {} : { permissionDispatchRecovery }),
|
|
491
|
+
});
|
|
492
|
+
}
|
|
493
|
+
catch {
|
|
494
|
+
if (signal.aborted)
|
|
495
|
+
return;
|
|
496
|
+
this.finishInbound(inboxId, fencingToken, {
|
|
497
|
+
outcome: 'not-processed', failureCode: 'prepare-threw', retryable: true,
|
|
498
|
+
});
|
|
499
|
+
return;
|
|
500
|
+
}
|
|
501
|
+
if (signal.aborted)
|
|
502
|
+
return;
|
|
503
|
+
if (preparation.outcome !== 'prepared') {
|
|
504
|
+
this.finishInbound(inboxId, fencingToken, preparation);
|
|
505
|
+
return;
|
|
506
|
+
}
|
|
507
|
+
prepared = preparation.message;
|
|
508
|
+
}
|
|
509
|
+
if (signal.aborted)
|
|
510
|
+
return;
|
|
511
|
+
const currentBinding = this.options.store.getBinding(binding.id);
|
|
512
|
+
if (currentBinding === undefined || currentBinding.status !== 'active') {
|
|
513
|
+
this.options.store.finishInbox({ inboxId, ownerId: this.options.ownerId, fencingToken,
|
|
514
|
+
outcome: 'dead_letter', failureCode: 'binding-revoked' });
|
|
515
|
+
return;
|
|
516
|
+
}
|
|
517
|
+
if (bindingChangedBeforeDispatch(binding, currentBinding, envelope)) {
|
|
518
|
+
this.options.store.finishInbox({ inboxId, ownerId: this.options.ownerId, fencingToken,
|
|
519
|
+
outcome: 'dead_letter', failureCode: 'binding-changed-before-dispatch' });
|
|
520
|
+
return;
|
|
521
|
+
}
|
|
522
|
+
if (!this.options.store.isAuthorizedPrincipal(currentBinding.principal)) {
|
|
523
|
+
this.options.store.finishInbox({ inboxId, ownerId: this.options.ownerId, fencingToken,
|
|
524
|
+
outcome: 'dead_letter', failureCode: 'authorization-revoked-before-dispatch' });
|
|
525
|
+
return;
|
|
526
|
+
}
|
|
527
|
+
const explicitDispatch = processor.dispatchControl === 'explicit';
|
|
528
|
+
const dispatchBinding = {
|
|
529
|
+
id: currentBinding.id,
|
|
530
|
+
version: currentBinding.version,
|
|
531
|
+
sessionId: currentBinding.sessionId,
|
|
532
|
+
generation: currentBinding.generation,
|
|
533
|
+
conversation: { ...currentBinding.conversation },
|
|
534
|
+
principal: { ...currentBinding.principal },
|
|
535
|
+
};
|
|
536
|
+
let dispatchMarked = false;
|
|
537
|
+
let dispatchGateFailed = false;
|
|
538
|
+
let dispatchGateBindingFailure;
|
|
539
|
+
let dispatchGateOpen = explicitDispatch;
|
|
540
|
+
const markDispatching = explicitDispatch
|
|
541
|
+
? () => {
|
|
542
|
+
if (!dispatchGateOpen || dispatchGateFailed) {
|
|
543
|
+
throw new Error('assistant-delivery inbound dispatch gate is closed');
|
|
544
|
+
}
|
|
545
|
+
if (signal.aborted)
|
|
546
|
+
throw signal.reason;
|
|
547
|
+
if (dispatchMarked)
|
|
548
|
+
return;
|
|
549
|
+
let latestBinding;
|
|
550
|
+
try {
|
|
551
|
+
latestBinding = this.options.store.getBinding(currentBinding.id);
|
|
552
|
+
}
|
|
553
|
+
catch (error) {
|
|
554
|
+
dispatchGateFailed = true;
|
|
555
|
+
throw error;
|
|
556
|
+
}
|
|
557
|
+
let authorized;
|
|
558
|
+
try {
|
|
559
|
+
authorized = this.options.store.isAuthorizedPrincipal(dispatchBinding.principal);
|
|
560
|
+
}
|
|
561
|
+
catch (error) {
|
|
562
|
+
dispatchGateFailed = true;
|
|
563
|
+
throw error;
|
|
564
|
+
}
|
|
565
|
+
if (!authorized) {
|
|
566
|
+
dispatchGateFailed = true;
|
|
567
|
+
dispatchGateBindingFailure = 'authorization-revoked-before-dispatch';
|
|
568
|
+
throw new Error('assistant-delivery inbound authorization was revoked before dispatch');
|
|
569
|
+
}
|
|
570
|
+
if (latestBinding === undefined || latestBinding.status !== 'active') {
|
|
571
|
+
dispatchGateFailed = true;
|
|
572
|
+
dispatchGateBindingFailure = 'binding-revoked';
|
|
573
|
+
throw new Error('assistant-delivery inbound binding was revoked before dispatch');
|
|
574
|
+
}
|
|
575
|
+
if (bindingChangedBeforeDispatch(currentBinding, latestBinding, envelope)) {
|
|
576
|
+
dispatchGateFailed = true;
|
|
577
|
+
dispatchGateBindingFailure = 'binding-changed-before-dispatch';
|
|
578
|
+
throw new Error('assistant-delivery inbound binding changed before dispatch');
|
|
579
|
+
}
|
|
580
|
+
try {
|
|
581
|
+
this.options.store.markInboxDispatching({
|
|
582
|
+
inboxId, ownerId: this.options.ownerId, fencingToken, binding: dispatchBinding,
|
|
583
|
+
});
|
|
584
|
+
}
|
|
585
|
+
catch (error) {
|
|
586
|
+
dispatchGateFailed = true;
|
|
587
|
+
dispatchGateBindingFailure = dispatchAuthorizationFailure(error) ?? dispatchGateBindingFailure;
|
|
588
|
+
throw error;
|
|
589
|
+
}
|
|
590
|
+
dispatchMarked = true;
|
|
591
|
+
}
|
|
592
|
+
: undefined;
|
|
593
|
+
if (!explicitDispatch) {
|
|
594
|
+
try {
|
|
595
|
+
this.options.store.markInboxDispatching({
|
|
596
|
+
inboxId, ownerId: this.options.ownerId, fencingToken, binding: dispatchBinding,
|
|
597
|
+
});
|
|
598
|
+
dispatchMarked = true;
|
|
599
|
+
}
|
|
600
|
+
catch (error) {
|
|
601
|
+
const failureCode = dispatchAuthorizationFailure(error);
|
|
602
|
+
if (failureCode !== undefined) {
|
|
603
|
+
this.options.store.finishInbox({ inboxId, ownerId: this.options.ownerId, fencingToken,
|
|
604
|
+
outcome: 'dead_letter', failureCode });
|
|
605
|
+
}
|
|
606
|
+
// No processor call happened, so lease recovery remains the authority on
|
|
607
|
+
// whether a stale or failed marker is safe to retry.
|
|
608
|
+
return;
|
|
609
|
+
}
|
|
610
|
+
}
|
|
611
|
+
let result;
|
|
612
|
+
try {
|
|
613
|
+
result = await processor.process(currentBinding, envelope, signal, prepared, markDispatching);
|
|
614
|
+
}
|
|
615
|
+
catch {
|
|
616
|
+
dispatchGateOpen = false;
|
|
617
|
+
if (signal.aborted)
|
|
618
|
+
return;
|
|
619
|
+
if (explicitDispatch && !dispatchMarked) {
|
|
620
|
+
if (dispatchGateBindingFailure !== undefined) {
|
|
621
|
+
this.options.store.finishInbox({ inboxId, ownerId: this.options.ownerId, fencingToken,
|
|
622
|
+
outcome: 'dead_letter', failureCode: dispatchGateBindingFailure });
|
|
623
|
+
return;
|
|
624
|
+
}
|
|
625
|
+
this.finishInbound(inboxId, fencingToken, {
|
|
626
|
+
outcome: 'not-processed', failureCode: 'processor-threw-before-dispatch', retryable: true,
|
|
627
|
+
});
|
|
628
|
+
return;
|
|
629
|
+
}
|
|
630
|
+
this.options.store.finishInbox({ inboxId, ownerId: this.options.ownerId, fencingToken,
|
|
631
|
+
outcome: 'dead_letter', failureCode: 'processor-ambiguous' });
|
|
632
|
+
return;
|
|
633
|
+
}
|
|
634
|
+
dispatchGateOpen = false;
|
|
635
|
+
if (signal.aborted)
|
|
636
|
+
return;
|
|
637
|
+
if (dispatchGateBindingFailure !== undefined) {
|
|
638
|
+
this.options.store.finishInbox({ inboxId, ownerId: this.options.ownerId, fencingToken,
|
|
639
|
+
outcome: 'dead_letter', failureCode: dispatchGateBindingFailure });
|
|
640
|
+
return;
|
|
641
|
+
}
|
|
642
|
+
if (dispatchGateFailed) {
|
|
643
|
+
this.finishInbound(inboxId, fencingToken, {
|
|
644
|
+
outcome: 'not-processed', failureCode: 'processor-threw-before-dispatch', retryable: true,
|
|
645
|
+
});
|
|
646
|
+
return;
|
|
647
|
+
}
|
|
648
|
+
if (dispatchMarked && result.outcome !== 'processed') {
|
|
649
|
+
if (permissionDispatchRecoveryFromFailureCode(result.failureCode) !== undefined) {
|
|
650
|
+
this.finishInbound(inboxId, fencingToken, result);
|
|
651
|
+
return;
|
|
652
|
+
}
|
|
653
|
+
this.options.store.finishInbox({ inboxId, ownerId: this.options.ownerId, fencingToken,
|
|
654
|
+
outcome: 'dead_letter', failureCode: 'processor-ambiguous' });
|
|
655
|
+
return;
|
|
656
|
+
}
|
|
657
|
+
this.finishInbound(inboxId, fencingToken, result);
|
|
658
|
+
}
|
|
659
|
+
finishInbound(inboxId, fencingToken, result) {
|
|
660
|
+
if (result.outcome === 'processed') {
|
|
661
|
+
this.options.store.finishInbox({ inboxId, ownerId: this.options.ownerId, fencingToken, outcome: 'processed' });
|
|
662
|
+
return;
|
|
663
|
+
}
|
|
664
|
+
if (!result.retryable) {
|
|
665
|
+
this.options.store.finishInbox({ inboxId, ownerId: this.options.ownerId, fencingToken,
|
|
666
|
+
outcome: 'dead_letter', failureCode: result.failureCode });
|
|
667
|
+
return;
|
|
668
|
+
}
|
|
669
|
+
const current = this.options.store.getInbox(inboxId);
|
|
670
|
+
const exponential = Math.min(this.options.retryMaxMs, this.options.retryBaseMs * (2 ** Math.max(0, current.attemptCount - 1)) * (1 + this.random()));
|
|
671
|
+
this.options.store.finishInbox({ inboxId, ownerId: this.options.ownerId, fencingToken,
|
|
672
|
+
outcome: 'retry_wait', failureCode: result.failureCode,
|
|
673
|
+
retryAt: this.now() + Math.ceil(Math.max(exponential, result.retryAfterMs ?? 0)) });
|
|
674
|
+
}
|
|
675
|
+
}
|
|
676
|
+
//# sourceMappingURL=coordinator.js.map
|