@tinycloud/courier 0.2.0-beta.37 → 0.2.0-beta.39
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +369 -12
- package/dist/a2a.d.ts +6 -0
- package/dist/a2a.js +10 -0
- package/dist/a2a.js.map +1 -1
- package/dist/approval.d.ts +6 -5
- package/dist/approval.js +7 -16
- package/dist/approval.js.map +1 -1
- package/dist/args.js +1 -1
- package/dist/args.js.map +1 -1
- package/dist/broker-config.d.ts +24 -0
- package/dist/broker-config.js +99 -0
- package/dist/broker-config.js.map +1 -0
- package/dist/broker.d.ts +77 -0
- package/dist/broker.js +253 -0
- package/dist/broker.js.map +1 -0
- package/dist/commands.js +5 -2
- package/dist/commands.js.map +1 -1
- package/dist/consumed.d.ts +10 -0
- package/dist/consumed.js +54 -0
- package/dist/consumed.js.map +1 -0
- package/dist/guard/binaries.d.ts +17 -0
- package/dist/guard/binaries.js +89 -0
- package/dist/guard/binaries.js.map +1 -0
- package/dist/guard/cleanhome.d.ts +27 -0
- package/dist/guard/cleanhome.js +111 -0
- package/dist/guard/cleanhome.js.map +1 -0
- package/dist/guard/command.d.ts +13 -0
- package/dist/guard/command.js +158 -0
- package/dist/guard/command.js.map +1 -0
- package/dist/guard/expansion.d.ts +10 -0
- package/dist/guard/expansion.js +74 -0
- package/dist/guard/expansion.js.map +1 -0
- package/dist/guard/hook.d.ts +38 -0
- package/dist/guard/hook.js +124 -0
- package/dist/guard/hook.js.map +1 -0
- package/dist/guard/install.d.ts +67 -0
- package/dist/guard/install.js +193 -0
- package/dist/guard/install.js.map +1 -0
- package/dist/guard/network.d.ts +7 -0
- package/dist/guard/network.js +78 -0
- package/dist/guard/network.js.map +1 -0
- package/dist/guard/normalize.d.ts +28 -0
- package/dist/guard/normalize.js +257 -0
- package/dist/guard/normalize.js.map +1 -0
- package/dist/guard/paseo.d.ts +30 -0
- package/dist/guard/paseo.js +148 -0
- package/dist/guard/paseo.js.map +1 -0
- package/dist/guard/paths.d.ts +31 -0
- package/dist/guard/paths.js +180 -0
- package/dist/guard/paths.js.map +1 -0
- package/dist/guard/policy.d.ts +65 -0
- package/dist/guard/policy.js +1179 -0
- package/dist/guard/policy.js.map +1 -0
- package/dist/guard/shell.d.ts +55 -0
- package/dist/guard/shell.js +727 -0
- package/dist/guard/shell.js.map +1 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/keys.d.ts +4 -1
- package/dist/keys.js +7 -2
- package/dist/keys.js.map +1 -1
- package/dist/main.js +44 -4
- package/dist/main.js.map +1 -1
- package/dist/node.d.ts +1 -1
- package/dist/node.js +3 -3
- package/dist/node.js.map +1 -1
- package/dist/notify.d.ts +43 -0
- package/dist/notify.js +92 -0
- package/dist/notify.js.map +1 -0
- package/dist/pause.d.ts +32 -0
- package/dist/pause.js +111 -0
- package/dist/pause.js.map +1 -0
- package/dist/profile.d.ts +3 -3
- package/dist/profile.js +12 -12
- package/dist/profile.js.map +1 -1
- package/dist/wake.d.ts +17 -0
- package/dist/wake.js +74 -13
- package/dist/wake.js.map +1 -1
- package/dist/watch-cmd.d.ts +3 -0
- package/dist/watch-cmd.js +56 -2
- package/dist/watch-cmd.js.map +1 -1
- package/dist/watch.d.ts +134 -0
- package/dist/watch.js +697 -13
- package/dist/watch.js.map +1 -1
- package/package.json +1 -1
- package/skills/courier/SKILL.md +119 -8
package/dist/watch.js
CHANGED
|
@@ -5,6 +5,9 @@ import { k, PREFIX } from './keys.js';
|
|
|
5
5
|
import { isRecord, messageView, oneLine, taskView } from './a2a.js';
|
|
6
6
|
import { commitReceipts, selectUndelivered } from './inbox.js';
|
|
7
7
|
import { del, getJson, listAll, putCreate, putMatch, putOverwrite } from './store.js';
|
|
8
|
+
import { appendNotice } from './notify.js';
|
|
9
|
+
import { nextNoticeId, writePauseMarker } from './pause.js';
|
|
10
|
+
import { HANDLE, ULID } from './request.js';
|
|
8
11
|
/** Spec §8. inform and receipts never wake; a message with no valid intent is delivered but never wakes. */
|
|
9
12
|
const WAKE_INTENTS = new Set(['request', 'ask', 'answer', 'update']);
|
|
10
13
|
const TERMINAL_TASK_STATES = new Set([
|
|
@@ -23,13 +26,37 @@ const FOREIGN_CLAIM_LEASE_MS = 45 * 60_000;
|
|
|
23
26
|
/** A thread another watcher is waking is looked at again after this. */
|
|
24
27
|
const CLAIM_SKIP_MS = 30_000;
|
|
25
28
|
const DAY_MS = 24 * 3600 * 1000;
|
|
29
|
+
/** A peer can make this side's watcher notify its human at most this often per hour, and the outbox never grows past `MAX_OUTBOX`. */
|
|
30
|
+
const NOTICES_PER_PEER_PER_HOUR = 10;
|
|
31
|
+
const NOTICES_PER_HOUR = 10;
|
|
32
|
+
const NOTICE_RESERVE_PER_PEER = 3;
|
|
33
|
+
const MAX_COALESCED_PAUSES = 999999;
|
|
34
|
+
const MAX_PEER_CLOCK_SKEW_MS = 10 * 60_000;
|
|
35
|
+
const MAX_OUTBOX = 50;
|
|
36
|
+
const MAX_OUTBOX_TRIES = 8;
|
|
37
|
+
const MAX_NOTICE_ATTEMPTS = 3;
|
|
38
|
+
const MAX_HELD_PER_THREAD = 100;
|
|
26
39
|
/** The subject is peer-controlled text that reaches state, prompts and environments: one line, control characters removed, capped. */
|
|
27
40
|
const MAX_SUBJECT = 200;
|
|
28
41
|
/** Thread and message ids come from key names the peer controls; they reach prompts and argv, so only ULID-like ids pass. */
|
|
29
42
|
const SAFE_ID = /^[0-9A-Za-z_-]{1,64}$/;
|
|
30
|
-
|
|
43
|
+
const emptyNoticeRetry = () => ({ dispatchArrivals: '------', tries: 0, nextAttemptAt: 0, logged: false });
|
|
44
|
+
const ZERO_ULID = '0'.repeat(26);
|
|
45
|
+
const EPOCH = new Date(0).toISOString();
|
|
46
|
+
const needsKind = (kind) => kind === 'new-message' ? 'message' : kind === 'new-approval' ? 'approval' : undefined;
|
|
47
|
+
const emptyNeedsSlot = () => ({ threadId: ZERO_ULID, eventId: ZERO_ULID, count: '000000',
|
|
48
|
+
firstSeenAt: EPOCH, arrivalFirstSeenAt: EPOCH, ...emptyNoticeRetry() });
|
|
49
|
+
const needsKey = (peer, kind) => `needs:${peer}:${kind}`;
|
|
50
|
+
const emptyEpisode = () => ({ kind: '', eventId: ZERO_ULID, repeats: '000000', ...emptyNoticeRetry() });
|
|
51
|
+
const incrementCount = (count) => String(Math.min(MAX_COALESCED_PAUSES, Number(count) + 1)).padStart(6, '0');
|
|
52
|
+
const episodeKey = (key, slot) => `notice:${key}:${slot}`;
|
|
53
|
+
function threadNoticeState() {
|
|
54
|
+
return { ourPaused: false, peerPaused: false, lastPeerNoticeId: '0'.repeat(26), lastNoticeSentAt: new Date(0).toISOString(),
|
|
55
|
+
ourMarkerId: '', episodes: [emptyEpisode(), emptyEpisode()], informPending: false, informText: '' };
|
|
56
|
+
}
|
|
57
|
+
export const WATCH_DEFAULTS = { debounceMs: 30_000, pollMs: 60_000, ticketSeconds: 25, wakeBudget: 12 };
|
|
31
58
|
export function emptyWatchState() {
|
|
32
|
-
return { v: 1, pending: {}, woken: [], approvalsSeen: [] };
|
|
59
|
+
return { v: 1, pending: {}, woken: [], approvalsSeen: [], wakes: {}, paused: [], wakeRetries: {}, approvalRetries: {}, outbox: [], held: {}, threadNotices: {}, peerNotices: {}, noticeGlobalLog: [], noticeLog: {} };
|
|
33
60
|
}
|
|
34
61
|
/** Wake time: `debounce` after the latest message, but never later than twice `debounce` after the first. */
|
|
35
62
|
export function wakeDueAt(pending, debounceMs) {
|
|
@@ -76,6 +103,11 @@ export class Watcher {
|
|
|
76
103
|
retryAt = new Map();
|
|
77
104
|
waking = new Set();
|
|
78
105
|
approvalsRunning = false;
|
|
106
|
+
/** Thread keys currently paused: markers under me/state/paused/ plus those this watcher just wrote. */
|
|
107
|
+
pausedKeys = new Set();
|
|
108
|
+
/** Serialises everything that reads or writes pause markers, so a stale listing cannot drop a pause just written. */
|
|
109
|
+
pausedLock = Promise.resolve();
|
|
110
|
+
outboxBusy = new Set();
|
|
79
111
|
approvalsAgain = false;
|
|
80
112
|
/** Unique to this run: a claim with this id is this process's own. */
|
|
81
113
|
runId = ulid();
|
|
@@ -111,6 +143,67 @@ export class Watcher {
|
|
|
111
143
|
this.state = isRecord(value) && value.v === 1
|
|
112
144
|
? { ...emptyWatchState(), ...value }
|
|
113
145
|
: emptyWatchState();
|
|
146
|
+
// Discard the old per-peer-generation map; it was sized by untrusted ids.
|
|
147
|
+
delete this.state.peerPausedDay;
|
|
148
|
+
for (const key of [...Object.keys(this.state.wakes), ...this.state.paused]) {
|
|
149
|
+
const [peer, threadId] = key.split('/');
|
|
150
|
+
if (peer && threadId && SAFE_ID.test(peer) && SAFE_ID.test(threadId)) {
|
|
151
|
+
const record = this.state.threadNotices[key] ??= threadNoticeState();
|
|
152
|
+
record.ourPaused = this.state.paused.includes(key);
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
this.syncNoticePeers();
|
|
156
|
+
for (const record of Object.values(this.state.peerNotices))
|
|
157
|
+
for (const slot of Object.values(record))
|
|
158
|
+
slot.dispatchArrivals = '------';
|
|
159
|
+
// Upgrade the previous single-pending record while retaining its attempt/deadline metadata.
|
|
160
|
+
for (const [key, record] of Object.entries(this.state.threadNotices)) {
|
|
161
|
+
const old = record;
|
|
162
|
+
if (!record.episodes) {
|
|
163
|
+
record.episodes = [emptyEpisode(), emptyEpisode()];
|
|
164
|
+
for (const item of this.state.outbox) {
|
|
165
|
+
if (item.kind !== 'notify' || `${item.notice.peer}/${item.notice.threadId}` !== key
|
|
166
|
+
|| (item.notice.kind !== 'paused' && item.notice.kind !== 'peer-paused')
|
|
167
|
+
|| item.id !== `${item.notice.kind}:${key}` || !item.notice.eventId)
|
|
168
|
+
continue;
|
|
169
|
+
this.queueEpisode(record, item.notice.kind, item.notice.eventId);
|
|
170
|
+
}
|
|
171
|
+
if (old.ourNoticePending)
|
|
172
|
+
this.queueEpisode(record, 'paused', record.ourMarkerId);
|
|
173
|
+
if (old.peerNoticePending && old.peerNoticeId)
|
|
174
|
+
this.queueEpisode(record, 'peer-paused', old.peerNoticeId);
|
|
175
|
+
}
|
|
176
|
+
record.episodes = record.episodes.map(episode => ({ ...emptyEpisode(), ...episode, dispatchArrivals: '------' }));
|
|
177
|
+
delete old.ourNoticePending;
|
|
178
|
+
delete old.peerNoticePending;
|
|
179
|
+
delete old.peerNoticeId;
|
|
180
|
+
}
|
|
181
|
+
this.state.outbox = this.state.outbox.filter(item => {
|
|
182
|
+
if (item.kind === 'inform')
|
|
183
|
+
return true;
|
|
184
|
+
const kind = needsKind(item.notice.kind);
|
|
185
|
+
if (kind) {
|
|
186
|
+
const slot = this.noticeSlot(item);
|
|
187
|
+
if (!slot || item.id !== needsKey(item.notice.peer, kind))
|
|
188
|
+
return false;
|
|
189
|
+
this.rememberNotice(item);
|
|
190
|
+
return true;
|
|
191
|
+
}
|
|
192
|
+
const key = `${item.notice.peer}/${item.notice.threadId}`;
|
|
193
|
+
const record = this.state.threadNotices[key];
|
|
194
|
+
const slot = record?.episodes.findIndex(episode => episode.kind === item.notice.kind && episode.eventId === item.notice.eventId) ?? -1;
|
|
195
|
+
if (slot < 0 || (item.id !== `${item.notice.kind}:${key}` && item.id !== episodeKey(key, slot)))
|
|
196
|
+
return false;
|
|
197
|
+
item.id = episodeKey(key, slot);
|
|
198
|
+
item.notice.repeats ??= record.episodes[slot].repeats;
|
|
199
|
+
this.rememberNotice(item);
|
|
200
|
+
return true;
|
|
201
|
+
});
|
|
202
|
+
for (const item of [...this.state.outbox]) {
|
|
203
|
+
if ((item.tries ?? 0) >= (item.kind === 'notify' ? MAX_NOTICE_ATTEMPTS : MAX_OUTBOX_TRIES))
|
|
204
|
+
this.finishOutbox(item);
|
|
205
|
+
}
|
|
206
|
+
this.state.outbox = this.state.outbox.slice(-MAX_OUTBOX);
|
|
114
207
|
}
|
|
115
208
|
snapshot() {
|
|
116
209
|
return structuredClone(this.state);
|
|
@@ -271,7 +364,10 @@ export class Watcher {
|
|
|
271
364
|
const { selfKv, log } = this.deps;
|
|
272
365
|
const mailbox = peer.theirHandleForMe;
|
|
273
366
|
const peerKv = await this.deps.kvFor(peer);
|
|
274
|
-
|
|
367
|
+
// Pause markers are read with every listing, so a `courier resume` or `courier pause` takes effect on the very
|
|
368
|
+
// next message instead of at the next poll (a message delivered while the watcher still thought the thread was
|
|
369
|
+
// paused would never wake anyone).
|
|
370
|
+
const [keys] = await Promise.all([listAll(peerKv, `${PREFIX}/to/${mailbox}/threads/`), this.syncPaused(), this.syncKnownThreads(peer)]);
|
|
275
371
|
const receipts = (await getJson(selfKv, k.receipts(peer.handle)))?.value ?? { v: 1, threads: {} };
|
|
276
372
|
receipts.threads ??= {};
|
|
277
373
|
const delivered = [];
|
|
@@ -288,7 +384,8 @@ export class Watcher {
|
|
|
288
384
|
if (!isRecord(value) || value.messageId !== selected.messageId || value.contextId !== selected.threadId)
|
|
289
385
|
continue;
|
|
290
386
|
await this.mirrorCreate(k.mirrorMsg(peer.handle, selected.threadId, selected.messageId), value);
|
|
291
|
-
|
|
387
|
+
const view = messageView(value);
|
|
388
|
+
delivered.push({ threadId: selected.threadId, messageId: selected.messageId, intent: view.intent, notice: view.notice });
|
|
292
389
|
}
|
|
293
390
|
catch (error) {
|
|
294
391
|
// Not delivered, so no receipt: this message is retried on the next sync; the others go ahead.
|
|
@@ -309,15 +406,29 @@ export class Watcher {
|
|
|
309
406
|
const sentAt = sentAtOf(item.messageId);
|
|
310
407
|
log('delivered', {
|
|
311
408
|
via, peer: peer.handle, threadId: item.threadId, messageId: item.messageId, intent: item.intent ?? null,
|
|
312
|
-
wakes: Boolean(item.intent && WAKE_INTENTS.has(item.intent)),
|
|
409
|
+
wakes: !this.deps.notifyOnly && Boolean(item.intent && WAKE_INTENTS.has(item.intent)) && !this.pausedKeys.has(`${peer.handle}/${item.threadId}`),
|
|
410
|
+
...(this.pausedKeys.has(`${peer.handle}/${item.threadId}`) ? { paused: true } : {}),
|
|
313
411
|
sentAt, latencyMs: sentAt ? at - Date.parse(sentAt) : null,
|
|
314
412
|
});
|
|
315
413
|
}
|
|
316
414
|
await this.mutate(() => {
|
|
415
|
+
if (this.deps.notifyOnly) {
|
|
416
|
+
this.syncNoticePeers();
|
|
417
|
+
for (const item of [...delivered].sort((a, b) => a.messageId.localeCompare(b.messageId))) {
|
|
418
|
+
if (item.intent && item.notice !== 'thread-paused' && item.notice !== 'thread-resumed') {
|
|
419
|
+
this.queueNeeds(peer.handle, 'message', item.threadId);
|
|
420
|
+
}
|
|
421
|
+
}
|
|
422
|
+
}
|
|
317
423
|
for (const item of delivered) {
|
|
318
|
-
if (!item.intent || !WAKE_INTENTS.has(item.intent) || this.state.woken.includes(item.messageId))
|
|
424
|
+
if (this.deps.notifyOnly || !item.intent || !WAKE_INTENTS.has(item.intent) || this.state.woken.includes(item.messageId))
|
|
319
425
|
continue;
|
|
320
426
|
const key = `${peer.handle}/${item.threadId}`;
|
|
427
|
+
if (this.pausedKeys.has(key)) {
|
|
428
|
+
// Delivered and mirrored, but a paused thread never wakes: the message waits for `courier resume`.
|
|
429
|
+
this.hold(key, peer.handle, item.threadId, subjects.get(item.threadId), { messageId: item.messageId, intent: item.intent, sentAt: sentAtOf(item.messageId) });
|
|
430
|
+
continue;
|
|
431
|
+
}
|
|
321
432
|
const entry = this.state.pending[key] ??= {
|
|
322
433
|
peer: peer.handle, threadId: item.threadId, subject: subjects.get(item.threadId), messages: [], firstSeenAt: at, lastSeenAt: at,
|
|
323
434
|
};
|
|
@@ -326,10 +437,53 @@ export class Watcher {
|
|
|
326
437
|
entry.messages.push({ messageId: item.messageId, intent: item.intent, sentAt: sentAtOf(item.messageId) });
|
|
327
438
|
entry.lastSeenAt = at;
|
|
328
439
|
}
|
|
440
|
+
// Only our local listing establishes a known thread. The peer controls its own state, not allocation.
|
|
441
|
+
for (const item of delivered) {
|
|
442
|
+
if (item.intent !== 'inform' || (item.notice !== 'thread-paused' && item.notice !== 'thread-resumed'))
|
|
443
|
+
continue;
|
|
444
|
+
const record = this.state.threadNotices[`${peer.handle}/${item.threadId}`];
|
|
445
|
+
if (!record || !ULID.test(item.messageId) || item.messageId <= record.lastPeerNoticeId)
|
|
446
|
+
continue;
|
|
447
|
+
let sentAt;
|
|
448
|
+
try {
|
|
449
|
+
sentAt = decodeTime(item.messageId);
|
|
450
|
+
}
|
|
451
|
+
catch {
|
|
452
|
+
continue;
|
|
453
|
+
} // ULID's high bits must fit 128 bits
|
|
454
|
+
if (sentAt > at + MAX_PEER_CLOCK_SKEW_MS)
|
|
455
|
+
continue;
|
|
456
|
+
record.lastPeerNoticeId = item.messageId;
|
|
457
|
+
const paused = item.notice === 'thread-paused';
|
|
458
|
+
if (paused && !record.peerPaused)
|
|
459
|
+
this.queueEpisode(record, 'peer-paused', item.messageId);
|
|
460
|
+
record.peerPaused = paused;
|
|
461
|
+
}
|
|
462
|
+
this.fillOutbox();
|
|
329
463
|
});
|
|
330
464
|
await commitReceipts(selfKv, new Map([[peer.handle, delivered.map(({ threadId, messageId }) => ({ threadId, messageId }))]]));
|
|
331
465
|
await this.mirrorThreads(peer, peerKv, keys);
|
|
332
466
|
}
|
|
467
|
+
/** Only locally authored meta/messages/tasks, or a local wake/pause witness, establish a known thread. */
|
|
468
|
+
async syncKnownThreads(peer) {
|
|
469
|
+
const prefix = `${PREFIX}/to/${peer.handle}/threads/`;
|
|
470
|
+
const keys = await listAll(this.deps.selfKv, prefix);
|
|
471
|
+
const known = new Set();
|
|
472
|
+
for (const path of keys) {
|
|
473
|
+
const match = /^([^/]+)\/(?:meta\.json|(?:msg|task)\/[^/]+\.json)$/.exec(path.slice(prefix.length));
|
|
474
|
+
if (match && SAFE_ID.test(match[1]))
|
|
475
|
+
known.add(`${peer.handle}/${match[1]}`);
|
|
476
|
+
}
|
|
477
|
+
for (const key of Object.keys(this.state.wakes))
|
|
478
|
+
if (key.startsWith(`${peer.handle}/`))
|
|
479
|
+
known.add(key);
|
|
480
|
+
const missing = [...known].filter(key => !this.state.threadNotices[key]);
|
|
481
|
+
if (missing.length)
|
|
482
|
+
await this.mutate(() => {
|
|
483
|
+
for (const key of missing)
|
|
484
|
+
this.state.threadNotices[key] ??= threadNoticeState();
|
|
485
|
+
});
|
|
486
|
+
}
|
|
333
487
|
async mirrorCreate(key, value) {
|
|
334
488
|
try {
|
|
335
489
|
await putCreate(this.deps.selfKv, key, value);
|
|
@@ -421,7 +575,7 @@ export class Watcher {
|
|
|
421
575
|
const found = await this.approvalWake(id);
|
|
422
576
|
if ('wake' in found) {
|
|
423
577
|
wakes.push(found.wake);
|
|
424
|
-
log('approval-seen', { via, approvalId: id, wakes:
|
|
578
|
+
log('approval-seen', { via, approvalId: id, wakes: !this.deps.notifyOnly, notifies: Boolean(this.deps.notifyOnly), decision: found.wake.approval.decision });
|
|
425
579
|
}
|
|
426
580
|
else {
|
|
427
581
|
log('approval-seen', { via, approvalId: id, wakes: false, reason: found.skip });
|
|
@@ -438,7 +592,13 @@ export class Watcher {
|
|
|
438
592
|
const at = this.now();
|
|
439
593
|
await this.mutate(() => {
|
|
440
594
|
this.state.approvalsSeen = [...new Set([...this.state.approvalsSeen, ...handled])].sort();
|
|
595
|
+
if (this.deps.notifyOnly)
|
|
596
|
+
this.syncNoticePeers();
|
|
441
597
|
for (const { approval, peer, threadId, subject } of wakes) {
|
|
598
|
+
if (this.deps.notifyOnly) {
|
|
599
|
+
this.queueNeeds(peer, 'approval', threadId);
|
|
600
|
+
continue;
|
|
601
|
+
}
|
|
442
602
|
const key = `${peer}/${threadId}`;
|
|
443
603
|
const entry = this.state.pending[key] ??= { peer, threadId, subject, messages: [], firstSeenAt: at, lastSeenAt: at };
|
|
444
604
|
entry.approvals ??= [];
|
|
@@ -446,6 +606,8 @@ export class Watcher {
|
|
|
446
606
|
entry.approvals.push(approval);
|
|
447
607
|
entry.lastSeenAt = at;
|
|
448
608
|
}
|
|
609
|
+
if (this.deps.notifyOnly)
|
|
610
|
+
this.fillOutbox();
|
|
449
611
|
});
|
|
450
612
|
}
|
|
451
613
|
/** Reads one new approval and its request; `skip` says why it should not wake anyone. */
|
|
@@ -477,6 +639,23 @@ export class Watcher {
|
|
|
477
639
|
async tick(force = false) {
|
|
478
640
|
if (this.fatalError || this.stopping)
|
|
479
641
|
return;
|
|
642
|
+
if (this.deps.notifyOnly) {
|
|
643
|
+
// Switching to D mode converts the prior local backlog into the same fixed peer slots.
|
|
644
|
+
if (Object.keys(this.state.pending).length)
|
|
645
|
+
await this.mutate(() => {
|
|
646
|
+
this.syncNoticePeers();
|
|
647
|
+
for (const pending of Object.values(this.state.pending)) {
|
|
648
|
+
for (const _message of pending.messages)
|
|
649
|
+
this.queueNeeds(pending.peer, 'message', pending.threadId);
|
|
650
|
+
for (const _approval of pending.approvals ?? [])
|
|
651
|
+
this.queueNeeds(pending.peer, 'approval', pending.threadId);
|
|
652
|
+
}
|
|
653
|
+
this.state.pending = {};
|
|
654
|
+
this.fillOutbox();
|
|
655
|
+
});
|
|
656
|
+
await this.runOutbox();
|
|
657
|
+
return;
|
|
658
|
+
}
|
|
480
659
|
const at = this.now();
|
|
481
660
|
const runs = [];
|
|
482
661
|
for (const [key, pending] of Object.entries(this.state.pending)) {
|
|
@@ -484,19 +663,59 @@ export class Watcher {
|
|
|
484
663
|
continue;
|
|
485
664
|
if (!force && wakeDueAt(pending, this.opts.debounceMs) > at)
|
|
486
665
|
continue;
|
|
487
|
-
if (!force && (this.retryAt.get(key) ?? 0) > at)
|
|
666
|
+
if (!force && !pending.approvals?.length && (this.retryAt.get(key) ?? 0) > at)
|
|
488
667
|
continue;
|
|
489
668
|
this.waking.add(key);
|
|
490
669
|
runs.push(this.wakeThread(key, pending, at).finally(() => this.waking.delete(key)));
|
|
491
670
|
}
|
|
671
|
+
runs.push(this.runOutbox());
|
|
492
672
|
await Promise.all(runs);
|
|
493
673
|
}
|
|
674
|
+
/** Spec §8: the budget is per side, per thread, per day. A day is the UTC calendar day, so it is easy to state and to check. */
|
|
675
|
+
recentWakes(key) {
|
|
676
|
+
const today = Math.floor(this.now() / DAY_MS);
|
|
677
|
+
return (this.state.wakes[key] ?? []).filter((at) => Math.floor(at / DAY_MS) === today);
|
|
678
|
+
}
|
|
494
679
|
async wakeThread(key, pending, at) {
|
|
495
680
|
if (this.stopping)
|
|
496
681
|
return; // told to stop: claim nothing, start nothing
|
|
682
|
+
// A `courier pause` written since the last listing must stop a wake that is already queued.
|
|
683
|
+
await this.syncPaused();
|
|
497
684
|
// Messages that land while the wake runs stay pending for the next one, so wake exactly this snapshot.
|
|
498
|
-
|
|
499
|
-
|
|
685
|
+
let messages = [...pending.messages];
|
|
686
|
+
let approvals = [...(pending.approvals ?? [])];
|
|
687
|
+
// Spec §8: the loop stops here. A paused thread, or one over budget, wakes nobody for messages (approvals are
|
|
688
|
+
// a human decision, not a loop, and always wake).
|
|
689
|
+
if (messages.length) {
|
|
690
|
+
const used = this.recentWakes(key).length;
|
|
691
|
+
if (!this.pausedKeys.has(key) && used >= this.opts.wakeBudget)
|
|
692
|
+
await this.pauseThread(key, pending, used);
|
|
693
|
+
if (this.pausedKeys.has(key)) {
|
|
694
|
+
// Moved to the held backlog: they wake once, together, after `courier resume`.
|
|
695
|
+
const held = messages;
|
|
696
|
+
const heldIds = held.map((m) => m.messageId);
|
|
697
|
+
messages = [];
|
|
698
|
+
await this.mutate(() => {
|
|
699
|
+
for (const message of held)
|
|
700
|
+
this.hold(key, pending.peer, pending.threadId, pending.subject, message);
|
|
701
|
+
const current = this.state.pending[key];
|
|
702
|
+
if (!current)
|
|
703
|
+
return;
|
|
704
|
+
current.messages = current.messages.filter((m) => !heldIds.includes(m.messageId));
|
|
705
|
+
if (!current.messages.length && !(current.approvals ?? []).length)
|
|
706
|
+
delete this.state.pending[key];
|
|
707
|
+
});
|
|
708
|
+
if (!approvals.length)
|
|
709
|
+
return;
|
|
710
|
+
}
|
|
711
|
+
}
|
|
712
|
+
// Owner decisions have their own retry clock. A failed message adapter cannot delay an approval.
|
|
713
|
+
if ((this.state.wakeRetries[key]?.nextAttemptAt ?? 0) > at)
|
|
714
|
+
messages = [];
|
|
715
|
+
if ((this.state.approvalRetries[key]?.nextAttemptAt ?? 0) > at)
|
|
716
|
+
approvals = [];
|
|
717
|
+
if (!messages.length && !approvals.length)
|
|
718
|
+
return;
|
|
500
719
|
const woken = messages.map((m) => m.messageId);
|
|
501
720
|
const approvedIds = approvals.map((a) => a.approvalId);
|
|
502
721
|
const oldestSentAt = messages[0]?.sentAt ?? '';
|
|
@@ -509,7 +728,13 @@ export class Watcher {
|
|
|
509
728
|
return;
|
|
510
729
|
}
|
|
511
730
|
try {
|
|
512
|
-
|
|
731
|
+
// The attempt is counted against the budget BEFORE the adapter runs, and whether it then succeeds, fails or
|
|
732
|
+
// times out: an adapter that always throws must not be retried forever. (A crash mid-wake counts too.)
|
|
733
|
+
await this.mutate(() => {
|
|
734
|
+
this.state.threadNotices[key] ??= threadNoticeState();
|
|
735
|
+
if (woken.length)
|
|
736
|
+
this.state.wakes[key] = [...this.recentWakes(key), this.now()];
|
|
737
|
+
});
|
|
513
738
|
}
|
|
514
739
|
catch (error) {
|
|
515
740
|
await this.releaseClaim(pending.peer, pending.threadId, claimId);
|
|
@@ -531,8 +756,16 @@ export class Watcher {
|
|
|
531
756
|
await this.deps.wake({ peer: pending.peer, threadId: pending.threadId, subject: pending.subject, messages, approvals });
|
|
532
757
|
}
|
|
533
758
|
catch (error) {
|
|
534
|
-
|
|
535
|
-
|
|
759
|
+
// Failures back off: 1, 2, 4, ... minutes, at most 15.
|
|
760
|
+
let failures = 0;
|
|
761
|
+
await this.mutate(() => {
|
|
762
|
+
for (const retries of [...(woken.length ? [this.state.wakeRetries] : []), ...(approvedIds.length ? [this.state.approvalRetries] : [])]) {
|
|
763
|
+
const count = (retries[key]?.failures ?? 0) + 1;
|
|
764
|
+
retries[key] = { failures: count, nextAttemptAt: this.now() + Math.min(15 * 60_000, WAKE_RETRY_MS * 2 ** (count - 1)) };
|
|
765
|
+
failures = Math.max(failures, count);
|
|
766
|
+
}
|
|
767
|
+
});
|
|
768
|
+
this.deps.log('wake-error', { peer: pending.peer, threadId: pending.threadId, failures, ...errInfo(error) });
|
|
536
769
|
// The attempt failed, so the thread is not being woken: release the claim (a restart must not wait for it).
|
|
537
770
|
await this.releaseClaim(pending.peer, pending.threadId, claimId);
|
|
538
771
|
return;
|
|
@@ -546,6 +779,10 @@ export class Watcher {
|
|
|
546
779
|
delete this.state.pending[key];
|
|
547
780
|
}
|
|
548
781
|
this.state.woken = [...this.state.woken, ...woken].slice(-WAKEN_MEMORY);
|
|
782
|
+
if (woken.length)
|
|
783
|
+
delete this.state.wakeRetries[key];
|
|
784
|
+
if (approvedIds.length)
|
|
785
|
+
delete this.state.approvalRetries[key];
|
|
549
786
|
});
|
|
550
787
|
// Only now, with the wake recorded, is the retry hold-off cleared.
|
|
551
788
|
this.retryAt.delete(key);
|
|
@@ -554,8 +791,455 @@ export class Watcher {
|
|
|
554
791
|
runMs: this.now() - startedAt,
|
|
555
792
|
});
|
|
556
793
|
}
|
|
794
|
+
/** Budget reached: pause this side's wakes for the thread, tell the peer (inform), tell this side's human. */
|
|
795
|
+
async pauseThread(key, pending, used) {
|
|
796
|
+
const reason = `wake budget reached (${used} wakes today, UTC; limit ${this.opts.wakeBudget ?? WATCH_DEFAULTS.wakeBudget} per thread per day)`;
|
|
797
|
+
const at = new Date(this.now()).toISOString();
|
|
798
|
+
const pauseId = await nextNoticeId(this.deps.selfKv, pending.peer, pending.threadId, this.now());
|
|
799
|
+
const wrote = await this.exclusive(async () => {
|
|
800
|
+
const created = await writePauseMarker(this.deps.selfKv, { v: 1, peer: pending.peer, threadId: pending.threadId, by: 'budget', at, reason, informId: pauseId });
|
|
801
|
+
this.pausedKeys.add(key);
|
|
802
|
+
return created;
|
|
803
|
+
});
|
|
804
|
+
// The marker already existed (a human pause, or another run): the peer and the human were told then.
|
|
805
|
+
if (!wrote) {
|
|
806
|
+
this.deps.log('paused', { peer: pending.peer, threadId: pending.threadId, by: 'marker-existing' });
|
|
807
|
+
return;
|
|
808
|
+
}
|
|
809
|
+
const text = `Thread paused on this side: ${reason}. Its agent will not be woken for this thread until its owner resumes it. ` +
|
|
810
|
+
'This is an automatic stop for reply loops, not a refusal; messages you send are still stored and mirrored.';
|
|
811
|
+
await this.mutate(() => {
|
|
812
|
+
if (!this.state.paused.includes(key))
|
|
813
|
+
this.state.paused.push(key);
|
|
814
|
+
const record = this.state.threadNotices[key] ??= threadNoticeState();
|
|
815
|
+
Object.assign(record, { ourPaused: true, ourMarkerId: pauseId, informPending: true, informText: text });
|
|
816
|
+
this.queueEpisode(record, 'paused', pauseId);
|
|
817
|
+
this.fillOutbox();
|
|
818
|
+
});
|
|
819
|
+
this.deps.log('paused', { peer: pending.peer, threadId: pending.threadId, by: 'budget', reason });
|
|
820
|
+
}
|
|
821
|
+
exclusive(fn) {
|
|
822
|
+
const run = this.pausedLock.then(fn, fn);
|
|
823
|
+
this.pausedLock = run.catch(() => undefined);
|
|
824
|
+
return run;
|
|
825
|
+
}
|
|
826
|
+
/** Called inside `mutate`: parks a message that arrived while its thread is paused (bounded). */
|
|
827
|
+
hold(key, peer, threadId, subject, message) {
|
|
828
|
+
const entry = this.state.held[key] ??= { peer, threadId, subject, messages: [] };
|
|
829
|
+
if (entry.messages.some((m) => m.messageId === message.messageId))
|
|
830
|
+
return;
|
|
831
|
+
entry.messages = [...entry.messages, message].slice(-MAX_HELD_PER_THREAD);
|
|
832
|
+
}
|
|
833
|
+
/** Two episode values per known thread. Additional transitions coalesce, without allocating another slot. */
|
|
834
|
+
queueEpisode(record, kind, eventId) {
|
|
835
|
+
if (!ULID.test(eventId) || record.episodes.some(episode => episode.kind === kind && episode.eventId === eventId))
|
|
836
|
+
return;
|
|
837
|
+
const slot = record.episodes.find(episode => !episode.kind);
|
|
838
|
+
if (slot)
|
|
839
|
+
Object.assign(slot, emptyEpisode(), { kind, eventId });
|
|
840
|
+
else {
|
|
841
|
+
const last = record.episodes[1];
|
|
842
|
+
last.repeats = incrementCount(last.repeats);
|
|
843
|
+
if (last.dispatchArrivals !== '------')
|
|
844
|
+
last.dispatchArrivals = incrementCount(last.dispatchArrivals);
|
|
845
|
+
}
|
|
846
|
+
}
|
|
847
|
+
/** Only the paired handles in our local configuration allocate needs-you state. */
|
|
848
|
+
syncNoticePeers() {
|
|
849
|
+
const paired = new Set(this.deps.loadPeers().filter(peer => peer.status === 'paired' && HANDLE.test(peer.handle)).map(peer => peer.handle));
|
|
850
|
+
for (const peer of Object.keys(this.state.peerNotices))
|
|
851
|
+
if (!paired.has(peer))
|
|
852
|
+
delete this.state.peerNotices[peer];
|
|
853
|
+
if (this.deps.notifyOnly)
|
|
854
|
+
for (const peer of paired) {
|
|
855
|
+
if (!Object.hasOwn(this.state.peerNotices, peer))
|
|
856
|
+
this.state.peerNotices[peer] = { message: emptyNeedsSlot(), approval: emptyNeedsSlot() };
|
|
857
|
+
}
|
|
858
|
+
for (const record of Object.values(this.state.peerNotices))
|
|
859
|
+
for (const kind of ['message', 'approval']) {
|
|
860
|
+
record[kind] = { ...emptyNeedsSlot(), ...record[kind] };
|
|
861
|
+
}
|
|
862
|
+
}
|
|
863
|
+
/** Bounded values only: a later delivery updates the same paired-peer slot. Called inside mutate. */
|
|
864
|
+
queueNeeds(peer, kind, threadId) {
|
|
865
|
+
const slot = this.state.peerNotices[peer]?.[kind];
|
|
866
|
+
if (!slot || !ULID.test(threadId))
|
|
867
|
+
return;
|
|
868
|
+
try {
|
|
869
|
+
decodeTime(threadId);
|
|
870
|
+
}
|
|
871
|
+
catch {
|
|
872
|
+
return;
|
|
873
|
+
}
|
|
874
|
+
const at = new Date(this.now()).toISOString();
|
|
875
|
+
if (slot.count === '000000') {
|
|
876
|
+
slot.eventId = ulid(this.now());
|
|
877
|
+
slot.firstSeenAt = at;
|
|
878
|
+
}
|
|
879
|
+
slot.threadId = threadId;
|
|
880
|
+
slot.count = incrementCount(slot.count);
|
|
881
|
+
if (slot.dispatchArrivals !== '------') {
|
|
882
|
+
if (slot.dispatchArrivals === '000000')
|
|
883
|
+
slot.arrivalFirstSeenAt = at;
|
|
884
|
+
slot.dispatchArrivals = incrementCount(slot.dispatchArrivals);
|
|
885
|
+
}
|
|
886
|
+
}
|
|
887
|
+
noticeSlot(item) {
|
|
888
|
+
const kind = needsKind(item.notice.kind);
|
|
889
|
+
if (kind) {
|
|
890
|
+
const slot = this.state.peerNotices[item.notice.peer]?.[kind];
|
|
891
|
+
return slot?.count !== '000000' && slot?.eventId === item.notice.eventId ? slot : undefined;
|
|
892
|
+
}
|
|
893
|
+
return this.state.threadNotices[`${item.notice.peer}/${item.notice.threadId}`]?.episodes.find(episode => episode.kind === item.notice.kind && episode.eventId === item.notice.eventId);
|
|
894
|
+
}
|
|
895
|
+
rememberNotice(item) {
|
|
896
|
+
const episode = this.noticeSlot(item);
|
|
897
|
+
if (episode)
|
|
898
|
+
Object.assign(episode, { tries: item.tries ?? 0, nextAttemptAt: item.nextAttemptAt ?? 0, logged: item.logged ?? false });
|
|
899
|
+
}
|
|
900
|
+
/** Earliest rolling-hour retry under the global allowance and each paired peer's share. */
|
|
901
|
+
noticeRateDeadline(peer, now = this.now()) {
|
|
902
|
+
const recent = (this.state.noticeLog[peer] ?? []).filter(t => now - t < 3600_000);
|
|
903
|
+
const global = this.state.noticeGlobalLog.filter(t => now - t < 3600_000);
|
|
904
|
+
const peers = [...new Set([...this.deps.loadPeers().filter(peer => peer.status === 'paired').map(peer => peer.handle), peer])];
|
|
905
|
+
const share = Math.min(NOTICE_RESERVE_PER_PEER, Math.floor(NOTICES_PER_HOUR / peers.length));
|
|
906
|
+
const reservedForOthers = peers.filter(other => other !== peer).reduce((reserved, other) => reserved + Math.max(0, share - (this.state.noticeLog[other] ?? []).filter(t => now - t < 3600_000).length), 0);
|
|
907
|
+
if (recent.length < NOTICES_PER_PEER_PER_HOUR && global.length + reservedForOthers < NOTICES_PER_HOUR)
|
|
908
|
+
return 0;
|
|
909
|
+
return Math.max(recent.length >= NOTICES_PER_PEER_PER_HOUR ? Math.min(...recent) + 3600_000 : 0, global.length ? Math.min(...global) + 3600_000 : now + 3600_000);
|
|
910
|
+
}
|
|
911
|
+
/** -1 appends; a nonnegative index yields that place; undefined leaves the pending slot alone. */
|
|
912
|
+
admissionIndex(item) {
|
|
913
|
+
if (this.state.outbox.some(existing => existing.id === item.id))
|
|
914
|
+
return undefined;
|
|
915
|
+
if (this.state.outbox.length < MAX_OUTBOX)
|
|
916
|
+
return -1;
|
|
917
|
+
const now = this.now();
|
|
918
|
+
const eligible = item.kind === 'inform' || ((item.nextAttemptAt ?? 0) <= now && !this.noticeRateDeadline(item.notice.peer, now));
|
|
919
|
+
if (!eligible)
|
|
920
|
+
return undefined;
|
|
921
|
+
const deferred = this.state.outbox.findIndex(existing => existing.kind === 'notify' && !this.outboxBusy.has(existing.id)
|
|
922
|
+
&& ((existing.nextAttemptAt ?? 0) > now || this.noticeRateDeadline(existing.notice.peer, now) > now));
|
|
923
|
+
if (deferred >= 0)
|
|
924
|
+
return deferred;
|
|
925
|
+
if (item.kind === 'notify') {
|
|
926
|
+
const counts = new Map();
|
|
927
|
+
for (const existing of this.state.outbox)
|
|
928
|
+
if (existing.kind === 'notify')
|
|
929
|
+
counts.set(existing.notice.peer, (counts.get(existing.notice.peer) ?? 0) + 1);
|
|
930
|
+
const own = counts.get(item.notice.peer) ?? 0;
|
|
931
|
+
const overrepresented = this.state.outbox.findIndex(existing => existing.kind === 'notify'
|
|
932
|
+
&& existing.notice.peer !== item.notice.peer && (counts.get(existing.notice.peer) ?? 0) > own + 1
|
|
933
|
+
&& existing.notice.kind !== 'paused' && !existing.tries && !this.outboxBusy.has(existing.id));
|
|
934
|
+
if (overrepresented >= 0)
|
|
935
|
+
return overrepresented;
|
|
936
|
+
}
|
|
937
|
+
if (item.kind === 'inform' || item.notice.kind === 'paused') {
|
|
938
|
+
const unattempted = this.state.outbox.findIndex(existing => existing.kind === 'notify' && existing.notice.kind === 'peer-paused'
|
|
939
|
+
&& !existing.tries && !this.outboxBusy.has(existing.id));
|
|
940
|
+
if (unattempted >= 0)
|
|
941
|
+
return unattempted;
|
|
942
|
+
}
|
|
943
|
+
return undefined;
|
|
944
|
+
}
|
|
945
|
+
/** Queue places can yield; the two fixed episode records retain all retry metadata. */
|
|
946
|
+
enqueue(item) {
|
|
947
|
+
const index = this.admissionIndex(item);
|
|
948
|
+
if (index === undefined)
|
|
949
|
+
return;
|
|
950
|
+
if (index >= 0)
|
|
951
|
+
this.state.outbox.splice(index, 1);
|
|
952
|
+
this.state.outbox.push(item);
|
|
953
|
+
}
|
|
954
|
+
*outboxCandidates() {
|
|
955
|
+
for (const [key, record] of Object.entries(this.state.threadNotices)) {
|
|
956
|
+
const [peer, threadId] = key.split('/');
|
|
957
|
+
if (record.informPending)
|
|
958
|
+
yield { id: `inform:${key}`, kind: 'inform', peer, threadId, pauseId: record.ourMarkerId, text: record.informText };
|
|
959
|
+
}
|
|
960
|
+
for (const kind of ['paused', 'peer-paused']) {
|
|
961
|
+
for (const [key, record] of Object.entries(this.state.threadNotices)) {
|
|
962
|
+
const [peer, threadId] = key.split('/');
|
|
963
|
+
for (const [slot, episode] of record.episodes.entries()) {
|
|
964
|
+
if (episode.kind === kind)
|
|
965
|
+
yield { id: episodeKey(key, slot), kind: 'notify',
|
|
966
|
+
tries: episode.tries, nextAttemptAt: episode.nextAttemptAt, logged: episode.logged, notice: {
|
|
967
|
+
kind, peer, threadId, eventId: episode.eventId, repeats: episode.repeats,
|
|
968
|
+
detail: kind === 'paused' ? 'thread paused on this side' : "the peer's watcher reported the thread paused",
|
|
969
|
+
} };
|
|
970
|
+
}
|
|
971
|
+
}
|
|
972
|
+
}
|
|
973
|
+
for (const [peer, record] of Object.entries(this.state.peerNotices)) {
|
|
974
|
+
for (const kind of ['approval', 'message']) {
|
|
975
|
+
const slot = record[kind];
|
|
976
|
+
if (slot.count !== '000000')
|
|
977
|
+
yield { id: needsKey(peer, kind), kind: 'notify',
|
|
978
|
+
tries: slot.tries, nextAttemptAt: slot.nextAttemptAt, logged: slot.logged, notice: {
|
|
979
|
+
kind: kind === 'message' ? 'new-message' : 'new-approval', peer, threadId: slot.threadId,
|
|
980
|
+
eventId: slot.eventId, count: slot.count, detail: '',
|
|
981
|
+
} };
|
|
982
|
+
}
|
|
983
|
+
}
|
|
984
|
+
}
|
|
985
|
+
/** Admit local work first, allowing eligible peers to pass deferred entries in a full queue. */
|
|
986
|
+
fillOutbox() {
|
|
987
|
+
for (const item of this.fairOutbox([...this.outboxCandidates()]))
|
|
988
|
+
this.enqueue(item);
|
|
989
|
+
}
|
|
990
|
+
fairOutbox(items) {
|
|
991
|
+
const ordered = [];
|
|
992
|
+
for (const priority of ['inform', 'paused', 'other']) {
|
|
993
|
+
const peers = new Map();
|
|
994
|
+
for (const item of items) {
|
|
995
|
+
const group = item.kind === 'inform' ? 'inform' : item.notice.kind === 'paused' ? 'paused' : 'other';
|
|
996
|
+
if (group !== priority)
|
|
997
|
+
continue;
|
|
998
|
+
const peer = item.kind === 'inform' ? item.peer : item.notice.peer;
|
|
999
|
+
if (!peers.has(peer))
|
|
1000
|
+
peers.set(peer, []);
|
|
1001
|
+
peers.get(peer).push(item);
|
|
1002
|
+
}
|
|
1003
|
+
while (peers.size)
|
|
1004
|
+
for (const [peer, queue] of peers) {
|
|
1005
|
+
ordered.push(queue.shift());
|
|
1006
|
+
if (!queue.length)
|
|
1007
|
+
peers.delete(peer);
|
|
1008
|
+
}
|
|
1009
|
+
}
|
|
1010
|
+
return ordered;
|
|
1011
|
+
}
|
|
1012
|
+
/** Completion matches both the fixed slot and episode ULID; another queued episode is independent. */
|
|
1013
|
+
finishOutbox(item, delivered = false) {
|
|
1014
|
+
const peer = item.kind === 'inform' ? item.peer : item.notice.peer;
|
|
1015
|
+
const threadId = item.kind === 'inform' ? item.threadId : item.notice.threadId;
|
|
1016
|
+
const record = this.state.threadNotices[`${peer}/${threadId}`];
|
|
1017
|
+
if (item.kind === 'notify' && needsKind(item.notice.kind)) {
|
|
1018
|
+
const slot = this.noticeSlot(item);
|
|
1019
|
+
if (slot) {
|
|
1020
|
+
const arrivals = slot.dispatchArrivals === '------' ? '000000' : slot.dispatchArrivals;
|
|
1021
|
+
if (delivered && arrivals !== '000000')
|
|
1022
|
+
Object.assign(slot, emptyNeedsSlot(), {
|
|
1023
|
+
threadId: slot.threadId, eventId: ulid(this.now()), count: arrivals, firstSeenAt: slot.arrivalFirstSeenAt,
|
|
1024
|
+
});
|
|
1025
|
+
else
|
|
1026
|
+
Object.assign(slot, emptyNeedsSlot());
|
|
1027
|
+
}
|
|
1028
|
+
}
|
|
1029
|
+
else if (record) {
|
|
1030
|
+
if (item.kind === 'inform' && item.pauseId === record.ourMarkerId)
|
|
1031
|
+
record.informPending = false;
|
|
1032
|
+
if (item.kind === 'notify') {
|
|
1033
|
+
record.lastNoticeSentAt = new Date(this.now()).toISOString();
|
|
1034
|
+
const slot = record.episodes.find(episode => episode.kind === item.notice.kind && episode.eventId === item.notice.eventId);
|
|
1035
|
+
if (slot) {
|
|
1036
|
+
const arrivedDuringDelivery = slot.dispatchArrivals === '------' ? 0 : Number(slot.dispatchArrivals);
|
|
1037
|
+
if (delivered && arrivedDuringDelivery > 0)
|
|
1038
|
+
Object.assign(slot, emptyEpisode(), {
|
|
1039
|
+
kind: slot.kind, eventId: slot.eventId, repeats: String(arrivedDuringDelivery - 1).padStart(6, '0'),
|
|
1040
|
+
});
|
|
1041
|
+
else
|
|
1042
|
+
Object.assign(slot, emptyEpisode());
|
|
1043
|
+
}
|
|
1044
|
+
}
|
|
1045
|
+
}
|
|
1046
|
+
this.state.outbox = this.state.outbox.filter(o => o.id !== item.id);
|
|
1047
|
+
}
|
|
1048
|
+
/**
|
|
1049
|
+
* Markers under me/state/paused/ are the truth. A marker that disappeared means the human resumed: the budget starts
|
|
1050
|
+
* over and what arrived meanwhile wakes the agent once. Serialised, so overlapping calls never apply a stale listing.
|
|
1051
|
+
*/
|
|
1052
|
+
syncPaused() {
|
|
1053
|
+
return this.exclusive(async () => {
|
|
1054
|
+
try {
|
|
1055
|
+
const keys = await listAll(this.deps.selfKv, k.pausedPrefix);
|
|
1056
|
+
const listed = new Set();
|
|
1057
|
+
for (const key of keys) {
|
|
1058
|
+
const match = /\/paused\/([^/]+)\/([^/]+)\.json$/.exec(key);
|
|
1059
|
+
if (match && SAFE_ID.test(match[1]) && SAFE_ID.test(match[2]))
|
|
1060
|
+
listed.add(`${match[1]}/${match[2]}`);
|
|
1061
|
+
}
|
|
1062
|
+
const before = this.pausedKeys;
|
|
1063
|
+
const resumed = this.state.paused.filter((key) => !listed.has(key));
|
|
1064
|
+
const added = [...listed].filter((key) => !this.state.paused.includes(key));
|
|
1065
|
+
this.pausedKeys = listed;
|
|
1066
|
+
for (const key of added)
|
|
1067
|
+
if (!before.has(key))
|
|
1068
|
+
this.deps.log('paused', { thread: key, by: 'marker' });
|
|
1069
|
+
if (resumed.length || added.length) {
|
|
1070
|
+
const at = this.now();
|
|
1071
|
+
const backlog = {};
|
|
1072
|
+
await this.mutate(() => {
|
|
1073
|
+
for (const key of resumed) {
|
|
1074
|
+
if (this.state.threadNotices[key])
|
|
1075
|
+
this.state.threadNotices[key].ourPaused = false;
|
|
1076
|
+
delete this.state.wakes[key];
|
|
1077
|
+
delete this.state.wakeRetries[key];
|
|
1078
|
+
const held = this.state.held[key];
|
|
1079
|
+
if (held?.messages.length) {
|
|
1080
|
+
backlog[key] = held.messages.length;
|
|
1081
|
+
const entry = this.state.pending[key] ??= { peer: held.peer, threadId: held.threadId, subject: held.subject, messages: [], firstSeenAt: at, lastSeenAt: at };
|
|
1082
|
+
for (const message of held.messages)
|
|
1083
|
+
if (!entry.messages.some((m) => m.messageId === message.messageId))
|
|
1084
|
+
entry.messages.push(message);
|
|
1085
|
+
entry.lastSeenAt = at;
|
|
1086
|
+
}
|
|
1087
|
+
delete this.state.held[key];
|
|
1088
|
+
}
|
|
1089
|
+
for (const key of added) {
|
|
1090
|
+
const record = this.state.threadNotices[key] ??= threadNoticeState();
|
|
1091
|
+
record.ourPaused = true;
|
|
1092
|
+
}
|
|
1093
|
+
this.state.paused = [...new Set([...this.state.paused.filter((key) => !resumed.includes(key)), ...added])].sort();
|
|
1094
|
+
});
|
|
1095
|
+
for (const key of resumed)
|
|
1096
|
+
this.deps.log('resumed', { thread: key, backlog: backlog[key] ?? 0 });
|
|
1097
|
+
}
|
|
1098
|
+
}
|
|
1099
|
+
catch (error) {
|
|
1100
|
+
this.deps.log('sync-error', { source: 'paused', ...errInfo(error) });
|
|
1101
|
+
}
|
|
1102
|
+
});
|
|
1103
|
+
}
|
|
1104
|
+
/** State that only matters for a day or an hour is dropped, so it cannot grow without bound. */
|
|
1105
|
+
async pruneState() {
|
|
1106
|
+
const at = this.now();
|
|
1107
|
+
const stale = Object.keys(this.state.wakes).filter((key) => !this.recentWakes(key).length)
|
|
1108
|
+
.concat(Object.keys(this.state.approvalRetries).filter(key => !this.state.pending[key]?.approvals?.length))
|
|
1109
|
+
.concat(Object.keys(this.state.wakeRetries).filter(key => !this.state.pending[key]))
|
|
1110
|
+
.concat(Object.keys(this.state.noticeLog).filter((peer) => !this.state.noticeLog[peer].some((t) => at - t < 3600_000)));
|
|
1111
|
+
if (!stale.length && !this.state.noticeGlobalLog.some(t => at - t >= 3600_000))
|
|
1112
|
+
return;
|
|
1113
|
+
await this.mutate(() => {
|
|
1114
|
+
for (const key of Object.keys(this.state.wakeRetries))
|
|
1115
|
+
if (!this.state.pending[key])
|
|
1116
|
+
delete this.state.wakeRetries[key];
|
|
1117
|
+
for (const key of Object.keys(this.state.wakes))
|
|
1118
|
+
if (!this.recentWakes(key).length)
|
|
1119
|
+
delete this.state.wakes[key];
|
|
1120
|
+
for (const key of Object.keys(this.state.approvalRetries))
|
|
1121
|
+
if (!this.state.pending[key]?.approvals?.length)
|
|
1122
|
+
delete this.state.approvalRetries[key];
|
|
1123
|
+
this.state.noticeGlobalLog = this.state.noticeGlobalLog.filter(t => at - t < 3600_000);
|
|
1124
|
+
for (const peer of Object.keys(this.state.noticeLog))
|
|
1125
|
+
if (!this.state.noticeLog[peer].some((t) => at - t < 3600_000))
|
|
1126
|
+
delete this.state.noticeLog[peer];
|
|
1127
|
+
});
|
|
1128
|
+
}
|
|
1129
|
+
/** Informs and notifications queued in state; failures back off (1, 2, 4 ... 15 minutes) and give up after 3 notification attempts or 8 inform failures. */
|
|
1130
|
+
async runOutbox() {
|
|
1131
|
+
const at = this.now();
|
|
1132
|
+
const refill = [...this.outboxCandidates()].some(item => this.admissionIndex(item) !== undefined);
|
|
1133
|
+
if (refill)
|
|
1134
|
+
await this.mutate(() => this.fillOutbox());
|
|
1135
|
+
const runs = [];
|
|
1136
|
+
const items = this.fairOutbox([...this.state.outbox]);
|
|
1137
|
+
for (const item of items) {
|
|
1138
|
+
if (this.outboxBusy.has(item.id) || (item.nextAttemptAt ?? 0) > at)
|
|
1139
|
+
continue;
|
|
1140
|
+
this.outboxBusy.add(item.id);
|
|
1141
|
+
runs.push((async () => {
|
|
1142
|
+
try {
|
|
1143
|
+
if (item.kind === 'inform')
|
|
1144
|
+
await this.deps.postInform?.(item.peer, item.threadId, item.text, item.pauseId);
|
|
1145
|
+
else {
|
|
1146
|
+
const { notice } = item;
|
|
1147
|
+
let reserved = false;
|
|
1148
|
+
await this.mutate(() => {
|
|
1149
|
+
const stored = this.state.outbox.find(o => o.id === item.id);
|
|
1150
|
+
if (stored?.kind !== 'notify')
|
|
1151
|
+
return;
|
|
1152
|
+
const now = this.now();
|
|
1153
|
+
const recent = (this.state.noticeLog[notice.peer] ?? []).filter(t => now - t < 3600_000);
|
|
1154
|
+
if ((stored.tries ?? 0) >= MAX_NOTICE_ATTEMPTS) {
|
|
1155
|
+
this.finishOutbox(item);
|
|
1156
|
+
return;
|
|
1157
|
+
}
|
|
1158
|
+
const deadline = this.noticeRateDeadline(notice.peer, now);
|
|
1159
|
+
if (deadline) {
|
|
1160
|
+
stored.nextAttemptAt = deadline;
|
|
1161
|
+
this.rememberNotice(stored);
|
|
1162
|
+
this.deps.log('notice-rate-cap', { peer: notice.peer, id: item.id });
|
|
1163
|
+
return;
|
|
1164
|
+
}
|
|
1165
|
+
const episode = this.noticeSlot(stored);
|
|
1166
|
+
if (episode) {
|
|
1167
|
+
if ('repeats' in episode)
|
|
1168
|
+
item.notice.repeats = stored.notice.repeats = episode.repeats;
|
|
1169
|
+
else {
|
|
1170
|
+
item.notice.threadId = stored.notice.threadId = episode.threadId;
|
|
1171
|
+
item.notice.count = stored.notice.count = episode.count;
|
|
1172
|
+
episode.arrivalFirstSeenAt = EPOCH;
|
|
1173
|
+
}
|
|
1174
|
+
episode.dispatchArrivals = '000000';
|
|
1175
|
+
}
|
|
1176
|
+
this.state.noticeGlobalLog = [...this.state.noticeGlobalLog.filter(t => now - t < 3600_000), now];
|
|
1177
|
+
item.tries = stored.tries = (stored.tries ?? 0) + 1;
|
|
1178
|
+
this.rememberNotice(stored);
|
|
1179
|
+
this.state.noticeLog[notice.peer] = [...recent, now];
|
|
1180
|
+
reserved = true; // persisted BEFORE any channel operation, even one that delivers then fails
|
|
1181
|
+
});
|
|
1182
|
+
if (!reserved)
|
|
1183
|
+
return;
|
|
1184
|
+
this.deps.log('notify', { kind: notice.kind, peer: notice.peer, threadId: notice.threadId, detail: notice.detail });
|
|
1185
|
+
if (!item.logged && this.deps.notificationHome) {
|
|
1186
|
+
appendNotice(this.deps.notificationHome, notice);
|
|
1187
|
+
await this.mutate(() => {
|
|
1188
|
+
const stored = this.state.outbox.find(o => o.id === item.id);
|
|
1189
|
+
if (stored?.kind === 'notify') {
|
|
1190
|
+
stored.logged = true;
|
|
1191
|
+
this.rememberNotice(stored);
|
|
1192
|
+
}
|
|
1193
|
+
});
|
|
1194
|
+
}
|
|
1195
|
+
await this.deps.notify?.(notice);
|
|
1196
|
+
}
|
|
1197
|
+
await this.mutate(() => this.finishOutbox(item, true));
|
|
1198
|
+
}
|
|
1199
|
+
catch (error) {
|
|
1200
|
+
const tries = item.kind === 'notify' ? (item.tries ?? 1) : (item.tries ?? 0) + 1;
|
|
1201
|
+
this.deps.log(item.kind === 'inform' ? 'inform-error' : 'notify-error', { id: item.id, tries, ...errInfo(error) });
|
|
1202
|
+
if (tries >= (item.kind === 'notify' ? MAX_NOTICE_ATTEMPTS : MAX_OUTBOX_TRIES)) {
|
|
1203
|
+
this.deps.log('outbox-dropped', { id: item.id, reason: 'too-many-failures' });
|
|
1204
|
+
await this.mutate(() => this.finishOutbox(item));
|
|
1205
|
+
}
|
|
1206
|
+
else {
|
|
1207
|
+
await this.mutate(() => {
|
|
1208
|
+
const stored = this.state.outbox.find(o => o.id === item.id);
|
|
1209
|
+
if (!stored)
|
|
1210
|
+
return;
|
|
1211
|
+
stored.tries = tries;
|
|
1212
|
+
stored.nextAttemptAt = this.now() + Math.min(15 * 60_000, WAKE_RETRY_MS * 2 ** (tries - 1));
|
|
1213
|
+
if (stored.kind === 'notify') {
|
|
1214
|
+
this.rememberNotice(stored);
|
|
1215
|
+
const episode = this.noticeSlot(stored);
|
|
1216
|
+
if (episode)
|
|
1217
|
+
episode.dispatchArrivals = '------';
|
|
1218
|
+
}
|
|
1219
|
+
});
|
|
1220
|
+
}
|
|
1221
|
+
}
|
|
1222
|
+
finally {
|
|
1223
|
+
this.outboxBusy.delete(item.id);
|
|
1224
|
+
}
|
|
1225
|
+
})());
|
|
1226
|
+
}
|
|
1227
|
+
await Promise.all(runs);
|
|
1228
|
+
}
|
|
557
1229
|
// ---------- run loop ----------
|
|
558
1230
|
async pollOnce(via) {
|
|
1231
|
+
if (this.deps.notifyOnly) {
|
|
1232
|
+
const paired = this.deps.loadPeers().filter(peer => peer.status === 'paired' && HANDLE.test(peer.handle)).map(peer => peer.handle);
|
|
1233
|
+
if (paired.some(peer => !Object.hasOwn(this.state.peerNotices, peer))
|
|
1234
|
+
|| Object.keys(this.state.peerNotices).some(peer => !paired.includes(peer))) {
|
|
1235
|
+
await this.mutate(() => {
|
|
1236
|
+
this.syncNoticePeers();
|
|
1237
|
+
this.state.outbox = this.state.outbox.filter(item => item.kind !== 'notify' || !needsKind(item.notice.kind) || paired.includes(item.notice.peer));
|
|
1238
|
+
});
|
|
1239
|
+
}
|
|
1240
|
+
}
|
|
1241
|
+
await this.syncPaused();
|
|
1242
|
+
await this.pruneState();
|
|
559
1243
|
await Promise.all(this.deps.loadPeers().map((peer) => this.sync(peer, via)));
|
|
560
1244
|
await this.syncApprovals(via);
|
|
561
1245
|
}
|