@tinycloud/courier 0.2.0-beta.36 → 0.2.0-beta.38
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +336 -17
- package/dist/a2a.d.ts +6 -0
- package/dist/a2a.js +10 -0
- package/dist/a2a.js.map +1 -1
- package/dist/approval.d.ts +6 -5
- package/dist/approval.js +7 -16
- package/dist/approval.js.map +1 -1
- package/dist/args.js +1 -1
- package/dist/args.js.map +1 -1
- package/dist/broker-config.d.ts +24 -0
- package/dist/broker-config.js +99 -0
- package/dist/broker-config.js.map +1 -0
- package/dist/broker.d.ts +77 -0
- package/dist/broker.js +253 -0
- package/dist/broker.js.map +1 -0
- package/dist/commands.js +5 -2
- package/dist/commands.js.map +1 -1
- package/dist/consumed.d.ts +10 -0
- package/dist/consumed.js +54 -0
- package/dist/consumed.js.map +1 -0
- package/dist/guard/binaries.d.ts +17 -0
- package/dist/guard/binaries.js +89 -0
- package/dist/guard/binaries.js.map +1 -0
- package/dist/guard/cleanhome.d.ts +27 -0
- package/dist/guard/cleanhome.js +111 -0
- package/dist/guard/cleanhome.js.map +1 -0
- package/dist/guard/command.d.ts +13 -0
- package/dist/guard/command.js +158 -0
- package/dist/guard/command.js.map +1 -0
- package/dist/guard/expansion.d.ts +10 -0
- package/dist/guard/expansion.js +74 -0
- package/dist/guard/expansion.js.map +1 -0
- package/dist/guard/hook.d.ts +38 -0
- package/dist/guard/hook.js +124 -0
- package/dist/guard/hook.js.map +1 -0
- package/dist/guard/install.d.ts +67 -0
- package/dist/guard/install.js +193 -0
- package/dist/guard/install.js.map +1 -0
- package/dist/guard/network.d.ts +7 -0
- package/dist/guard/network.js +78 -0
- package/dist/guard/network.js.map +1 -0
- package/dist/guard/normalize.d.ts +28 -0
- package/dist/guard/normalize.js +257 -0
- package/dist/guard/normalize.js.map +1 -0
- package/dist/guard/paseo.d.ts +30 -0
- package/dist/guard/paseo.js +148 -0
- package/dist/guard/paseo.js.map +1 -0
- package/dist/guard/paths.d.ts +31 -0
- package/dist/guard/paths.js +180 -0
- package/dist/guard/paths.js.map +1 -0
- package/dist/guard/policy.d.ts +65 -0
- package/dist/guard/policy.js +1179 -0
- package/dist/guard/policy.js.map +1 -0
- package/dist/guard/shell.d.ts +55 -0
- package/dist/guard/shell.js +727 -0
- package/dist/guard/shell.js.map +1 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/keys.d.ts +4 -1
- package/dist/keys.js +7 -2
- package/dist/keys.js.map +1 -1
- package/dist/main.js +44 -4
- package/dist/main.js.map +1 -1
- package/dist/node.d.ts +1 -1
- package/dist/node.js +3 -3
- package/dist/node.js.map +1 -1
- package/dist/notify.d.ts +39 -0
- package/dist/notify.js +88 -0
- package/dist/notify.js.map +1 -0
- package/dist/pause.d.ts +26 -0
- package/dist/pause.js +99 -0
- package/dist/pause.js.map +1 -0
- package/dist/profile.d.ts +3 -3
- package/dist/profile.js +14 -14
- package/dist/profile.js.map +1 -1
- package/dist/wake.d.ts +17 -0
- package/dist/wake.js +74 -13
- package/dist/wake.js.map +1 -1
- package/dist/watch-cmd.d.ts +3 -0
- package/dist/watch-cmd.js +56 -2
- package/dist/watch-cmd.js.map +1 -1
- package/dist/watch.d.ts +70 -0
- package/dist/watch.js +292 -11
- package/dist/watch.js.map +1 -1
- package/package.json +1 -1
- package/skills/courier/SKILL.md +95 -9
package/dist/watch.d.ts
CHANGED
|
@@ -2,6 +2,7 @@ import type { HookSubscription } from '@tinycloud/node-sdk';
|
|
|
2
2
|
import type { StoredPeer } from './profile.js';
|
|
3
3
|
import { type KV } from './inbox.js';
|
|
4
4
|
import type { WakeApproval, WakeMessage, Waker, WatchLog } from './wake.js';
|
|
5
|
+
import { type Notice, type Notifier } from './notify.js';
|
|
5
6
|
export interface PendingWake {
|
|
6
7
|
peer: string;
|
|
7
8
|
threadId: string;
|
|
@@ -19,11 +20,48 @@ export interface WatchState {
|
|
|
19
20
|
/** Recently woken message ids, so a receipt write that failed cannot cause a second wake. */
|
|
20
21
|
woken: string[];
|
|
21
22
|
approvalsSeen: string[];
|
|
23
|
+
/** Message wake attempts today (UTC), per `<peer>/<threadId>`, as epoch ms (approval wakes do not count). */
|
|
24
|
+
wakes: Record<string, number[]>;
|
|
25
|
+
/** Threads this watcher has seen paused; a marker that vanishes from this list means the human resumed it. */
|
|
26
|
+
paused: string[];
|
|
27
|
+
/** Posts and notifications that must happen exactly-ish once and survive a restart (retried until they succeed). */
|
|
28
|
+
outbox: OutboxItem[];
|
|
29
|
+
held: Record<string, HeldBacklog>;
|
|
30
|
+
/** UTC day of the last notification per kind/peer/thread: one of each kind per day. */
|
|
31
|
+
peerPausedDay: Record<string, number>;
|
|
32
|
+
/** Times of recent notifications per peer, for the shared per-hour cap. */
|
|
33
|
+
noticeLog: Record<string, number[]>;
|
|
34
|
+
}
|
|
35
|
+
/** `tries` counts failed attempts; an item that keeps failing is dropped (see `MAX_OUTBOX_TRIES`). */
|
|
36
|
+
export type OutboxItem = {
|
|
37
|
+
id: string;
|
|
38
|
+
kind: 'inform';
|
|
39
|
+
peer: string;
|
|
40
|
+
threadId: string;
|
|
41
|
+
text: string;
|
|
42
|
+
tries?: number;
|
|
43
|
+
nextAttemptAt?: number;
|
|
44
|
+
} | {
|
|
45
|
+
id: string;
|
|
46
|
+
kind: 'notify';
|
|
47
|
+
notice: Notice;
|
|
48
|
+
tries?: number;
|
|
49
|
+
nextAttemptAt?: number;
|
|
50
|
+
logged?: boolean;
|
|
51
|
+
};
|
|
52
|
+
/** Messages that arrived while a thread was paused; they wake the agent once, together, after `courier resume`. */
|
|
53
|
+
export interface HeldBacklog {
|
|
54
|
+
peer: string;
|
|
55
|
+
threadId: string;
|
|
56
|
+
subject: string;
|
|
57
|
+
messages: WakeMessage[];
|
|
22
58
|
}
|
|
23
59
|
export interface WatchOptions {
|
|
24
60
|
debounceMs: number;
|
|
25
61
|
pollMs: number;
|
|
26
62
|
ticketSeconds: number;
|
|
63
|
+
/** Spec §8: at most this many wakes per side per thread per 24 h, then the thread pauses. */
|
|
64
|
+
wakeBudget: number;
|
|
27
65
|
}
|
|
28
66
|
export declare const WATCH_DEFAULTS: WatchOptions;
|
|
29
67
|
interface HookEventLike {
|
|
@@ -48,6 +86,14 @@ export interface WatchDeps {
|
|
|
48
86
|
kvFor(peer: StoredPeer): Promise<KV>;
|
|
49
87
|
wake: Waker;
|
|
50
88
|
log: WatchLog;
|
|
89
|
+
/** Posts the `thread-paused` inform to a peer (own space, `to/<peer>/threads/<id>/msg/`). Default: nothing. */
|
|
90
|
+
postInform?(peer: string, threadId: string, text: string): Promise<void>;
|
|
91
|
+
/** Tells this side's human without a model (desktop notification or operator command). Default: log only. */
|
|
92
|
+
notify?: Notifier;
|
|
93
|
+
/** Local bounded notification log; omitted by library callers that want no disk output. */
|
|
94
|
+
notificationHome?: string;
|
|
95
|
+
/** D mode: mirror/receipt and notify the human, never invoke the wake adapter or spend its budget. */
|
|
96
|
+
notifyOnly?: boolean;
|
|
51
97
|
/**
|
|
52
98
|
* Who this watcher is, for wake claims. `profileId` is stable for this profile on this machine (a restart keeps it,
|
|
53
99
|
* a second profile or machine differs); `pid` and `isAlive` tell a restarted watcher its predecessor is dead.
|
|
@@ -71,6 +117,12 @@ export declare class Watcher {
|
|
|
71
117
|
private readonly retryAt;
|
|
72
118
|
private readonly waking;
|
|
73
119
|
private approvalsRunning;
|
|
120
|
+
/** Thread keys currently paused: markers under me/state/paused/ plus those this watcher just wrote. */
|
|
121
|
+
private pausedKeys;
|
|
122
|
+
/** Serialises everything that reads or writes pause markers, so a stale listing cannot drop a pause just written. */
|
|
123
|
+
private pausedLock;
|
|
124
|
+
private readonly wakeFailures;
|
|
125
|
+
private readonly outboxBusy;
|
|
74
126
|
private approvalsAgain;
|
|
75
127
|
/** Unique to this run: a claim with this id is this process's own. */
|
|
76
128
|
private readonly runId;
|
|
@@ -137,7 +189,25 @@ export declare class Watcher {
|
|
|
137
189
|
private approvalWake;
|
|
138
190
|
/** Wake every thread whose debounce has elapsed (`force`: all of them now). Threads wake in parallel, one wake per thread at a time. */
|
|
139
191
|
tick(force?: boolean): Promise<void>;
|
|
192
|
+
/** Spec §8: the budget is per side, per thread, per day. A day is the UTC calendar day, so it is easy to state and to check. */
|
|
193
|
+
private recentWakes;
|
|
140
194
|
private wakeThread;
|
|
195
|
+
/** Budget reached: pause this side's wakes for the thread, tell the peer (inform), tell this side's human. */
|
|
196
|
+
private pauseThread;
|
|
197
|
+
private exclusive;
|
|
198
|
+
/** Called inside `mutate`: parks a message that arrived while its thread is paused (bounded). */
|
|
199
|
+
private hold;
|
|
200
|
+
/** Called inside `mutate`: queues an inform or notification; the outbox is bounded, the oldest item goes first. */
|
|
201
|
+
private enqueue;
|
|
202
|
+
/**
|
|
203
|
+
* Markers under me/state/paused/ are the truth. A marker that disappeared means the human resumed: the budget starts
|
|
204
|
+
* over and what arrived meanwhile wakes the agent once. Serialised, so overlapping calls never apply a stale listing.
|
|
205
|
+
*/
|
|
206
|
+
syncPaused(): Promise<void>;
|
|
207
|
+
/** State that only matters for a day or an hour is dropped, so it cannot grow without bound. */
|
|
208
|
+
private pruneState;
|
|
209
|
+
/** Informs and notifications queued in state; failures back off (1, 2, 4 ... 15 minutes) and give up after MAX_OUTBOX_TRIES. */
|
|
210
|
+
private runOutbox;
|
|
141
211
|
pollOnce(via: Via): Promise<void>;
|
|
142
212
|
/**
|
|
143
213
|
* Runs until `signal` aborts. With `once`, does one catch-up and wakes everything immediately. Throws
|
package/dist/watch.js
CHANGED
|
@@ -5,6 +5,8 @@ import { k, PREFIX } from './keys.js';
|
|
|
5
5
|
import { isRecord, messageView, oneLine, taskView } from './a2a.js';
|
|
6
6
|
import { commitReceipts, selectUndelivered } from './inbox.js';
|
|
7
7
|
import { del, getJson, listAll, putCreate, putMatch, putOverwrite } from './store.js';
|
|
8
|
+
import { appendNotice } from './notify.js';
|
|
9
|
+
import { writePauseMarker } from './pause.js';
|
|
8
10
|
/** Spec §8. inform and receipts never wake; a message with no valid intent is delivered but never wakes. */
|
|
9
11
|
const WAKE_INTENTS = new Set(['request', 'ask', 'answer', 'update']);
|
|
10
12
|
const TERMINAL_TASK_STATES = new Set([
|
|
@@ -23,13 +25,18 @@ const FOREIGN_CLAIM_LEASE_MS = 45 * 60_000;
|
|
|
23
25
|
/** A thread another watcher is waking is looked at again after this. */
|
|
24
26
|
const CLAIM_SKIP_MS = 30_000;
|
|
25
27
|
const DAY_MS = 24 * 3600 * 1000;
|
|
28
|
+
/** A peer can make this side's watcher notify its human at most this often per hour, and the outbox never grows past `MAX_OUTBOX`. */
|
|
29
|
+
const NOTICES_PER_PEER_PER_HOUR = 10;
|
|
30
|
+
const MAX_OUTBOX = 50;
|
|
31
|
+
const MAX_OUTBOX_TRIES = 8;
|
|
32
|
+
const MAX_HELD_PER_THREAD = 100;
|
|
26
33
|
/** The subject is peer-controlled text that reaches state, prompts and environments: one line, control characters removed, capped. */
|
|
27
34
|
const MAX_SUBJECT = 200;
|
|
28
35
|
/** Thread and message ids come from key names the peer controls; they reach prompts and argv, so only ULID-like ids pass. */
|
|
29
36
|
const SAFE_ID = /^[0-9A-Za-z_-]{1,64}$/;
|
|
30
|
-
export const WATCH_DEFAULTS = { debounceMs: 30_000, pollMs: 60_000, ticketSeconds: 25 };
|
|
37
|
+
export const WATCH_DEFAULTS = { debounceMs: 30_000, pollMs: 60_000, ticketSeconds: 25, wakeBudget: 12 };
|
|
31
38
|
export function emptyWatchState() {
|
|
32
|
-
return { v: 1, pending: {}, woken: [], approvalsSeen: [] };
|
|
39
|
+
return { v: 1, pending: {}, woken: [], approvalsSeen: [], wakes: {}, paused: [], outbox: [], held: {}, peerPausedDay: {}, noticeLog: {} };
|
|
33
40
|
}
|
|
34
41
|
/** Wake time: `debounce` after the latest message, but never later than twice `debounce` after the first. */
|
|
35
42
|
export function wakeDueAt(pending, debounceMs) {
|
|
@@ -76,6 +83,12 @@ export class Watcher {
|
|
|
76
83
|
retryAt = new Map();
|
|
77
84
|
waking = new Set();
|
|
78
85
|
approvalsRunning = false;
|
|
86
|
+
/** Thread keys currently paused: markers under me/state/paused/ plus those this watcher just wrote. */
|
|
87
|
+
pausedKeys = new Set();
|
|
88
|
+
/** Serialises everything that reads or writes pause markers, so a stale listing cannot drop a pause just written. */
|
|
89
|
+
pausedLock = Promise.resolve();
|
|
90
|
+
wakeFailures = new Map();
|
|
91
|
+
outboxBusy = new Set();
|
|
79
92
|
approvalsAgain = false;
|
|
80
93
|
/** Unique to this run: a claim with this id is this process's own. */
|
|
81
94
|
runId = ulid();
|
|
@@ -111,6 +124,7 @@ export class Watcher {
|
|
|
111
124
|
this.state = isRecord(value) && value.v === 1
|
|
112
125
|
? { ...emptyWatchState(), ...value }
|
|
113
126
|
: emptyWatchState();
|
|
127
|
+
this.state.outbox = this.state.outbox.slice(-MAX_OUTBOX).filter(item => (item.tries ?? 0) < MAX_OUTBOX_TRIES);
|
|
114
128
|
}
|
|
115
129
|
snapshot() {
|
|
116
130
|
return structuredClone(this.state);
|
|
@@ -271,7 +285,10 @@ export class Watcher {
|
|
|
271
285
|
const { selfKv, log } = this.deps;
|
|
272
286
|
const mailbox = peer.theirHandleForMe;
|
|
273
287
|
const peerKv = await this.deps.kvFor(peer);
|
|
274
|
-
|
|
288
|
+
// Pause markers are read with every listing, so a `courier resume` or `courier pause` takes effect on the very
|
|
289
|
+
// next message instead of at the next poll (a message delivered while the watcher still thought the thread was
|
|
290
|
+
// paused would never wake anyone).
|
|
291
|
+
const [keys] = await Promise.all([listAll(peerKv, `${PREFIX}/to/${mailbox}/threads/`), this.syncPaused()]);
|
|
275
292
|
const receipts = (await getJson(selfKv, k.receipts(peer.handle)))?.value ?? { v: 1, threads: {} };
|
|
276
293
|
receipts.threads ??= {};
|
|
277
294
|
const delivered = [];
|
|
@@ -288,7 +305,8 @@ export class Watcher {
|
|
|
288
305
|
if (!isRecord(value) || value.messageId !== selected.messageId || value.contextId !== selected.threadId)
|
|
289
306
|
continue;
|
|
290
307
|
await this.mirrorCreate(k.mirrorMsg(peer.handle, selected.threadId, selected.messageId), value);
|
|
291
|
-
|
|
308
|
+
const view = messageView(value);
|
|
309
|
+
delivered.push({ threadId: selected.threadId, messageId: selected.messageId, intent: view.intent, notice: view.notice });
|
|
292
310
|
}
|
|
293
311
|
catch (error) {
|
|
294
312
|
// Not delivered, so no receipt: this message is retried on the next sync; the others go ahead.
|
|
@@ -309,15 +327,27 @@ export class Watcher {
|
|
|
309
327
|
const sentAt = sentAtOf(item.messageId);
|
|
310
328
|
log('delivered', {
|
|
311
329
|
via, peer: peer.handle, threadId: item.threadId, messageId: item.messageId, intent: item.intent ?? null,
|
|
312
|
-
wakes: Boolean(item.intent && WAKE_INTENTS.has(item.intent)),
|
|
330
|
+
wakes: !this.deps.notifyOnly && Boolean(item.intent && WAKE_INTENTS.has(item.intent)) && !this.pausedKeys.has(`${peer.handle}/${item.threadId}`),
|
|
331
|
+
...(this.pausedKeys.has(`${peer.handle}/${item.threadId}`) ? { paused: true } : {}),
|
|
313
332
|
sentAt, latencyMs: sentAt ? at - Date.parse(sentAt) : null,
|
|
314
333
|
});
|
|
315
334
|
}
|
|
316
335
|
await this.mutate(() => {
|
|
336
|
+
if (this.deps.notifyOnly) {
|
|
337
|
+
for (const threadId of new Set(delivered.filter(item => item.intent && item.notice !== 'thread-paused').map(item => item.threadId))) {
|
|
338
|
+
const first = delivered.find(item => item.threadId === threadId);
|
|
339
|
+
this.enqueue({ id: `message:${peer.handle}/${threadId}:${first.messageId}`, kind: 'notify', notice: { kind: 'new-message', peer: peer.handle, threadId, eventId: first.messageId, detail: '' } });
|
|
340
|
+
}
|
|
341
|
+
}
|
|
317
342
|
for (const item of delivered) {
|
|
318
|
-
if (!item.intent || !WAKE_INTENTS.has(item.intent) || this.state.woken.includes(item.messageId))
|
|
343
|
+
if (this.deps.notifyOnly || !item.intent || !WAKE_INTENTS.has(item.intent) || this.state.woken.includes(item.messageId))
|
|
319
344
|
continue;
|
|
320
345
|
const key = `${peer.handle}/${item.threadId}`;
|
|
346
|
+
if (this.pausedKeys.has(key)) {
|
|
347
|
+
// Delivered and mirrored, but a paused thread never wakes: the message waits for `courier resume`.
|
|
348
|
+
this.hold(key, peer.handle, item.threadId, subjects.get(item.threadId), { messageId: item.messageId, intent: item.intent, sentAt: sentAtOf(item.messageId) });
|
|
349
|
+
continue;
|
|
350
|
+
}
|
|
321
351
|
const entry = this.state.pending[key] ??= {
|
|
322
352
|
peer: peer.handle, threadId: item.threadId, subject: subjects.get(item.threadId), messages: [], firstSeenAt: at, lastSeenAt: at,
|
|
323
353
|
};
|
|
@@ -326,6 +356,19 @@ export class Watcher {
|
|
|
326
356
|
entry.messages.push({ messageId: item.messageId, intent: item.intent, sentAt: sentAtOf(item.messageId) });
|
|
327
357
|
entry.lastSeenAt = at;
|
|
328
358
|
}
|
|
359
|
+
// The peer's watcher paused the thread: tell this side's human (their agent is not coming back until resumed).
|
|
360
|
+
// Only a paired peer's `inform` carrying the notice counts, once per thread per UTC day, at most
|
|
361
|
+
// NOTICES_PER_PEER_PER_HOUR per peer: a peer cannot turn this into a stream of notifications or agent runs.
|
|
362
|
+
for (const item of delivered) {
|
|
363
|
+
if (item.notice !== 'thread-paused' || item.intent !== 'inform')
|
|
364
|
+
continue;
|
|
365
|
+
const key = `${peer.handle}/${item.threadId}`;
|
|
366
|
+
const day = Math.floor(at / DAY_MS);
|
|
367
|
+
this.enqueue({
|
|
368
|
+
id: `peer-paused:${key}:${day}`, kind: 'notify',
|
|
369
|
+
notice: { kind: 'peer-paused', peer: peer.handle, threadId: item.threadId, detail: "the peer's watcher reported the thread paused" },
|
|
370
|
+
});
|
|
371
|
+
}
|
|
329
372
|
});
|
|
330
373
|
await commitReceipts(selfKv, new Map([[peer.handle, delivered.map(({ threadId, messageId }) => ({ threadId, messageId }))]]));
|
|
331
374
|
await this.mirrorThreads(peer, peerKv, keys);
|
|
@@ -421,7 +464,7 @@ export class Watcher {
|
|
|
421
464
|
const found = await this.approvalWake(id);
|
|
422
465
|
if ('wake' in found) {
|
|
423
466
|
wakes.push(found.wake);
|
|
424
|
-
log('approval-seen', { via, approvalId: id, wakes:
|
|
467
|
+
log('approval-seen', { via, approvalId: id, wakes: !this.deps.notifyOnly, notifies: Boolean(this.deps.notifyOnly), decision: found.wake.approval.decision });
|
|
425
468
|
}
|
|
426
469
|
else {
|
|
427
470
|
log('approval-seen', { via, approvalId: id, wakes: false, reason: found.skip });
|
|
@@ -439,6 +482,10 @@ export class Watcher {
|
|
|
439
482
|
await this.mutate(() => {
|
|
440
483
|
this.state.approvalsSeen = [...new Set([...this.state.approvalsSeen, ...handled])].sort();
|
|
441
484
|
for (const { approval, peer, threadId, subject } of wakes) {
|
|
485
|
+
if (this.deps.notifyOnly) {
|
|
486
|
+
this.enqueue({ id: `approval:${approval.approvalId}`, kind: 'notify', notice: { kind: 'new-approval', peer, threadId, eventId: approval.approvalId, detail: '' } });
|
|
487
|
+
continue;
|
|
488
|
+
}
|
|
442
489
|
const key = `${peer}/${threadId}`;
|
|
443
490
|
const entry = this.state.pending[key] ??= { peer, threadId, subject, messages: [], firstSeenAt: at, lastSeenAt: at };
|
|
444
491
|
entry.approvals ??= [];
|
|
@@ -477,6 +524,21 @@ export class Watcher {
|
|
|
477
524
|
async tick(force = false) {
|
|
478
525
|
if (this.fatalError || this.stopping)
|
|
479
526
|
return;
|
|
527
|
+
if (this.deps.notifyOnly) {
|
|
528
|
+
// A watcher switched from auto-wake to D may have a pending backlog from its previous run.
|
|
529
|
+
if (Object.keys(this.state.pending).length)
|
|
530
|
+
await this.mutate(() => {
|
|
531
|
+
for (const pending of Object.values(this.state.pending)) {
|
|
532
|
+
if (pending.messages.length)
|
|
533
|
+
this.enqueue({ id: `message:${pending.peer}/${pending.threadId}:${pending.messages[0].messageId}`, kind: 'notify', notice: { kind: 'new-message', peer: pending.peer, threadId: pending.threadId, eventId: pending.messages[0].messageId, detail: '' } });
|
|
534
|
+
for (const approval of pending.approvals ?? [])
|
|
535
|
+
this.enqueue({ id: `approval:${approval.approvalId}`, kind: 'notify', notice: { kind: 'new-approval', peer: pending.peer, threadId: pending.threadId, eventId: approval.approvalId, detail: '' } });
|
|
536
|
+
}
|
|
537
|
+
this.state.pending = {};
|
|
538
|
+
});
|
|
539
|
+
await this.runOutbox();
|
|
540
|
+
return;
|
|
541
|
+
}
|
|
480
542
|
const at = this.now();
|
|
481
543
|
const runs = [];
|
|
482
544
|
for (const [key, pending] of Object.entries(this.state.pending)) {
|
|
@@ -489,14 +551,47 @@ export class Watcher {
|
|
|
489
551
|
this.waking.add(key);
|
|
490
552
|
runs.push(this.wakeThread(key, pending, at).finally(() => this.waking.delete(key)));
|
|
491
553
|
}
|
|
554
|
+
runs.push(this.runOutbox());
|
|
492
555
|
await Promise.all(runs);
|
|
493
556
|
}
|
|
557
|
+
/** Spec §8: the budget is per side, per thread, per day. A day is the UTC calendar day, so it is easy to state and to check. */
|
|
558
|
+
recentWakes(key) {
|
|
559
|
+
const today = Math.floor(this.now() / DAY_MS);
|
|
560
|
+
return (this.state.wakes[key] ?? []).filter((at) => Math.floor(at / DAY_MS) === today);
|
|
561
|
+
}
|
|
494
562
|
async wakeThread(key, pending, at) {
|
|
495
563
|
if (this.stopping)
|
|
496
564
|
return; // told to stop: claim nothing, start nothing
|
|
565
|
+
// A `courier pause` written since the last listing must stop a wake that is already queued.
|
|
566
|
+
await this.syncPaused();
|
|
497
567
|
// Messages that land while the wake runs stay pending for the next one, so wake exactly this snapshot.
|
|
498
|
-
|
|
568
|
+
let messages = [...pending.messages];
|
|
499
569
|
const approvals = [...(pending.approvals ?? [])];
|
|
570
|
+
// Spec §8: the loop stops here. A paused thread, or one over budget, wakes nobody for messages (approvals are
|
|
571
|
+
// a human decision, not a loop, and always wake).
|
|
572
|
+
if (messages.length) {
|
|
573
|
+
const used = this.recentWakes(key).length;
|
|
574
|
+
if (!this.pausedKeys.has(key) && used >= this.opts.wakeBudget)
|
|
575
|
+
await this.pauseThread(key, pending, used);
|
|
576
|
+
if (this.pausedKeys.has(key)) {
|
|
577
|
+
// Moved to the held backlog: they wake once, together, after `courier resume`.
|
|
578
|
+
const held = messages;
|
|
579
|
+
const heldIds = held.map((m) => m.messageId);
|
|
580
|
+
messages = [];
|
|
581
|
+
await this.mutate(() => {
|
|
582
|
+
for (const message of held)
|
|
583
|
+
this.hold(key, pending.peer, pending.threadId, pending.subject, message);
|
|
584
|
+
const current = this.state.pending[key];
|
|
585
|
+
if (!current)
|
|
586
|
+
return;
|
|
587
|
+
current.messages = current.messages.filter((m) => !heldIds.includes(m.messageId));
|
|
588
|
+
if (!current.messages.length && !(current.approvals ?? []).length)
|
|
589
|
+
delete this.state.pending[key];
|
|
590
|
+
});
|
|
591
|
+
if (!approvals.length)
|
|
592
|
+
return;
|
|
593
|
+
}
|
|
594
|
+
}
|
|
500
595
|
const woken = messages.map((m) => m.messageId);
|
|
501
596
|
const approvedIds = approvals.map((a) => a.approvalId);
|
|
502
597
|
const oldestSentAt = messages[0]?.sentAt ?? '';
|
|
@@ -509,7 +604,12 @@ export class Watcher {
|
|
|
509
604
|
return;
|
|
510
605
|
}
|
|
511
606
|
try {
|
|
512
|
-
|
|
607
|
+
// The attempt is counted against the budget BEFORE the adapter runs, and whether it then succeeds, fails or
|
|
608
|
+
// times out: an adapter that always throws must not be retried forever. (A crash mid-wake counts too.)
|
|
609
|
+
await this.mutate(() => {
|
|
610
|
+
if (woken.length)
|
|
611
|
+
this.state.wakes[key] = [...this.recentWakes(key), this.now()];
|
|
612
|
+
});
|
|
513
613
|
}
|
|
514
614
|
catch (error) {
|
|
515
615
|
await this.releaseClaim(pending.peer, pending.threadId, claimId);
|
|
@@ -531,8 +631,11 @@ export class Watcher {
|
|
|
531
631
|
await this.deps.wake({ peer: pending.peer, threadId: pending.threadId, subject: pending.subject, messages, approvals });
|
|
532
632
|
}
|
|
533
633
|
catch (error) {
|
|
534
|
-
|
|
535
|
-
|
|
634
|
+
// Failures back off: 1, 2, 4, ... minutes, at most 15.
|
|
635
|
+
const failures = (this.wakeFailures.get(key) ?? 0) + 1;
|
|
636
|
+
this.wakeFailures.set(key, failures);
|
|
637
|
+
this.retryAt.set(key, at + Math.min(15 * 60_000, WAKE_RETRY_MS * 2 ** (failures - 1)));
|
|
638
|
+
this.deps.log('wake-error', { peer: pending.peer, threadId: pending.threadId, failures, ...errInfo(error) });
|
|
536
639
|
// The attempt failed, so the thread is not being woken: release the claim (a restart must not wait for it).
|
|
537
640
|
await this.releaseClaim(pending.peer, pending.threadId, claimId);
|
|
538
641
|
return;
|
|
@@ -547,6 +650,7 @@ export class Watcher {
|
|
|
547
650
|
}
|
|
548
651
|
this.state.woken = [...this.state.woken, ...woken].slice(-WAKEN_MEMORY);
|
|
549
652
|
});
|
|
653
|
+
this.wakeFailures.delete(key);
|
|
550
654
|
// Only now, with the wake recorded, is the retry hold-off cleared.
|
|
551
655
|
this.retryAt.delete(key);
|
|
552
656
|
this.deps.log('woke', {
|
|
@@ -554,8 +658,185 @@ export class Watcher {
|
|
|
554
658
|
runMs: this.now() - startedAt,
|
|
555
659
|
});
|
|
556
660
|
}
|
|
661
|
+
/** Budget reached: pause this side's wakes for the thread, tell the peer (inform), tell this side's human. */
|
|
662
|
+
async pauseThread(key, pending, used) {
|
|
663
|
+
const reason = `wake budget reached (${used} wakes today, UTC; limit ${this.opts.wakeBudget ?? WATCH_DEFAULTS.wakeBudget} per thread per day)`;
|
|
664
|
+
const at = new Date(this.now()).toISOString();
|
|
665
|
+
const wrote = await this.exclusive(async () => {
|
|
666
|
+
const created = await writePauseMarker(this.deps.selfKv, { v: 1, peer: pending.peer, threadId: pending.threadId, by: 'budget', at, reason });
|
|
667
|
+
this.pausedKeys.add(key);
|
|
668
|
+
return created;
|
|
669
|
+
});
|
|
670
|
+
// The marker already existed (a human pause, or another run): the peer and the human were told then.
|
|
671
|
+
if (!wrote) {
|
|
672
|
+
this.deps.log('paused', { peer: pending.peer, threadId: pending.threadId, by: 'marker-existing' });
|
|
673
|
+
return;
|
|
674
|
+
}
|
|
675
|
+
const text = `Thread paused on this side: ${reason}. Its agent will not be woken for this thread until its owner resumes it. ` +
|
|
676
|
+
'This is an automatic stop for reply loops, not a refusal; messages you send are still stored and mirrored.';
|
|
677
|
+
await this.mutate(() => {
|
|
678
|
+
if (!this.state.paused.includes(key))
|
|
679
|
+
this.state.paused.push(key);
|
|
680
|
+
this.enqueue({ id: `inform:${key}:${at}`, kind: 'inform', peer: pending.peer, threadId: pending.threadId, text });
|
|
681
|
+
this.enqueue({ id: `paused:${key}:${at}`, kind: 'notify', notice: { kind: 'paused', peer: pending.peer, threadId: pending.threadId, detail: reason } });
|
|
682
|
+
});
|
|
683
|
+
this.deps.log('paused', { peer: pending.peer, threadId: pending.threadId, by: 'budget', reason });
|
|
684
|
+
}
|
|
685
|
+
exclusive(fn) {
|
|
686
|
+
const run = this.pausedLock.then(fn, fn);
|
|
687
|
+
this.pausedLock = run.catch(() => undefined);
|
|
688
|
+
return run;
|
|
689
|
+
}
|
|
690
|
+
/** Called inside `mutate`: parks a message that arrived while its thread is paused (bounded). */
|
|
691
|
+
hold(key, peer, threadId, subject, message) {
|
|
692
|
+
const entry = this.state.held[key] ??= { peer, threadId, subject, messages: [] };
|
|
693
|
+
if (entry.messages.some((m) => m.messageId === message.messageId))
|
|
694
|
+
return;
|
|
695
|
+
entry.messages = [...entry.messages, message].slice(-MAX_HELD_PER_THREAD);
|
|
696
|
+
}
|
|
697
|
+
/** Called inside `mutate`: queues an inform or notification; the outbox is bounded, the oldest item goes first. */
|
|
698
|
+
enqueue(item) {
|
|
699
|
+
if (item.kind === 'notify') {
|
|
700
|
+
const { notice } = item;
|
|
701
|
+
const at = this.now();
|
|
702
|
+
const day = Math.floor(at / DAY_MS);
|
|
703
|
+
const reference = notice.eventId && SAFE_ID.test(notice.eventId) ? `:${notice.eventId}` : '';
|
|
704
|
+
const key = `${notice.kind}:${notice.peer}/${notice.threadId}${reference}`;
|
|
705
|
+
if (this.state.peerPausedDay[key] === day)
|
|
706
|
+
return;
|
|
707
|
+
const recent = (this.state.noticeLog[notice.peer] ?? []).filter(t => at - t < 3600_000);
|
|
708
|
+
if (recent.length >= NOTICES_PER_PEER_PER_HOUR) {
|
|
709
|
+
this.deps.log('notice-dropped', { peer: notice.peer, reason: 'peer-rate-cap' });
|
|
710
|
+
return;
|
|
711
|
+
}
|
|
712
|
+
this.state.peerPausedDay[key] = day;
|
|
713
|
+
this.state.noticeLog[notice.peer] = [...recent, at];
|
|
714
|
+
}
|
|
715
|
+
this.state.outbox.push(item);
|
|
716
|
+
while (this.state.outbox.length > MAX_OUTBOX) {
|
|
717
|
+
const dropped = this.state.outbox.shift();
|
|
718
|
+
this.deps.log('outbox-dropped', { id: dropped.id, reason: 'full' });
|
|
719
|
+
}
|
|
720
|
+
}
|
|
721
|
+
/**
|
|
722
|
+
* Markers under me/state/paused/ are the truth. A marker that disappeared means the human resumed: the budget starts
|
|
723
|
+
* over and what arrived meanwhile wakes the agent once. Serialised, so overlapping calls never apply a stale listing.
|
|
724
|
+
*/
|
|
725
|
+
syncPaused() {
|
|
726
|
+
return this.exclusive(async () => {
|
|
727
|
+
try {
|
|
728
|
+
const keys = await listAll(this.deps.selfKv, k.pausedPrefix);
|
|
729
|
+
const listed = new Set();
|
|
730
|
+
for (const key of keys) {
|
|
731
|
+
const match = /\/paused\/([^/]+)\/([^/]+)\.json$/.exec(key);
|
|
732
|
+
if (match && SAFE_ID.test(match[1]) && SAFE_ID.test(match[2]))
|
|
733
|
+
listed.add(`${match[1]}/${match[2]}`);
|
|
734
|
+
}
|
|
735
|
+
const before = this.pausedKeys;
|
|
736
|
+
const resumed = this.state.paused.filter((key) => !listed.has(key));
|
|
737
|
+
const added = [...listed].filter((key) => !this.state.paused.includes(key));
|
|
738
|
+
this.pausedKeys = listed;
|
|
739
|
+
for (const key of added)
|
|
740
|
+
if (!before.has(key))
|
|
741
|
+
this.deps.log('paused', { thread: key, by: 'marker' });
|
|
742
|
+
if (resumed.length || added.length) {
|
|
743
|
+
const at = this.now();
|
|
744
|
+
const backlog = {};
|
|
745
|
+
await this.mutate(() => {
|
|
746
|
+
for (const key of resumed) {
|
|
747
|
+
delete this.state.wakes[key];
|
|
748
|
+
const held = this.state.held[key];
|
|
749
|
+
if (held?.messages.length) {
|
|
750
|
+
backlog[key] = held.messages.length;
|
|
751
|
+
const entry = this.state.pending[key] ??= { peer: held.peer, threadId: held.threadId, subject: held.subject, messages: [], firstSeenAt: at, lastSeenAt: at };
|
|
752
|
+
for (const message of held.messages)
|
|
753
|
+
if (!entry.messages.some((m) => m.messageId === message.messageId))
|
|
754
|
+
entry.messages.push(message);
|
|
755
|
+
entry.lastSeenAt = at;
|
|
756
|
+
}
|
|
757
|
+
delete this.state.held[key];
|
|
758
|
+
}
|
|
759
|
+
this.state.paused = [...new Set([...this.state.paused.filter((key) => !resumed.includes(key)), ...added])].sort();
|
|
760
|
+
});
|
|
761
|
+
for (const key of resumed)
|
|
762
|
+
this.deps.log('resumed', { thread: key, backlog: backlog[key] ?? 0 });
|
|
763
|
+
}
|
|
764
|
+
}
|
|
765
|
+
catch (error) {
|
|
766
|
+
this.deps.log('sync-error', { source: 'paused', ...errInfo(error) });
|
|
767
|
+
}
|
|
768
|
+
});
|
|
769
|
+
}
|
|
770
|
+
/** State that only matters for a day or an hour is dropped, so it cannot grow without bound. */
|
|
771
|
+
async pruneState() {
|
|
772
|
+
const at = this.now();
|
|
773
|
+
const today = Math.floor(at / DAY_MS);
|
|
774
|
+
const stale = Object.keys(this.state.wakes).filter((key) => !this.recentWakes(key).length)
|
|
775
|
+
.concat(Object.keys(this.state.peerPausedDay).filter((key) => today - this.state.peerPausedDay[key] > 7))
|
|
776
|
+
.concat(Object.keys(this.state.noticeLog).filter((peer) => !this.state.noticeLog[peer].some((t) => at - t < 3600_000)));
|
|
777
|
+
if (!stale.length)
|
|
778
|
+
return;
|
|
779
|
+
await this.mutate(() => {
|
|
780
|
+
for (const key of Object.keys(this.state.wakes))
|
|
781
|
+
if (!this.recentWakes(key).length)
|
|
782
|
+
delete this.state.wakes[key];
|
|
783
|
+
for (const key of Object.keys(this.state.peerPausedDay))
|
|
784
|
+
if (today - this.state.peerPausedDay[key] > 7)
|
|
785
|
+
delete this.state.peerPausedDay[key];
|
|
786
|
+
for (const peer of Object.keys(this.state.noticeLog))
|
|
787
|
+
if (!this.state.noticeLog[peer].some((t) => at - t < 3600_000))
|
|
788
|
+
delete this.state.noticeLog[peer];
|
|
789
|
+
});
|
|
790
|
+
}
|
|
791
|
+
/** Informs and notifications queued in state; failures back off (1, 2, 4 ... 15 minutes) and give up after MAX_OUTBOX_TRIES. */
|
|
792
|
+
async runOutbox() {
|
|
793
|
+
const at = this.now();
|
|
794
|
+
const runs = [];
|
|
795
|
+
for (const item of [...this.state.outbox]) {
|
|
796
|
+
if (this.outboxBusy.has(item.id) || (item.nextAttemptAt ?? 0) > at)
|
|
797
|
+
continue;
|
|
798
|
+
this.outboxBusy.add(item.id);
|
|
799
|
+
runs.push((async () => {
|
|
800
|
+
try {
|
|
801
|
+
if (item.kind === 'inform')
|
|
802
|
+
await this.deps.postInform?.(item.peer, item.threadId, item.text);
|
|
803
|
+
else {
|
|
804
|
+
const { notice } = item;
|
|
805
|
+
this.deps.log('notify', { kind: notice.kind, peer: notice.peer, threadId: notice.threadId, detail: notice.detail });
|
|
806
|
+
if (!item.logged && this.deps.notificationHome) {
|
|
807
|
+
appendNotice(this.deps.notificationHome, notice);
|
|
808
|
+
await this.mutate(() => { const stored = this.state.outbox.find(o => o.id === item.id); if (stored?.kind === 'notify')
|
|
809
|
+
stored.logged = true; });
|
|
810
|
+
}
|
|
811
|
+
await this.deps.notify?.(notice);
|
|
812
|
+
}
|
|
813
|
+
await this.mutate(() => { this.state.outbox = this.state.outbox.filter((o) => o.id !== item.id); });
|
|
814
|
+
}
|
|
815
|
+
catch (error) {
|
|
816
|
+
const tries = (item.tries ?? 0) + 1;
|
|
817
|
+
this.deps.log(item.kind === 'inform' ? 'inform-error' : 'notify-error', { id: item.id, tries, ...errInfo(error) });
|
|
818
|
+
if (tries >= MAX_OUTBOX_TRIES) {
|
|
819
|
+
this.deps.log('outbox-dropped', { id: item.id, reason: 'too-many-failures' });
|
|
820
|
+
await this.mutate(() => { this.state.outbox = this.state.outbox.filter((o) => o.id !== item.id); });
|
|
821
|
+
}
|
|
822
|
+
else {
|
|
823
|
+
await this.mutate(() => { const stored = this.state.outbox.find((o) => o.id === item.id); if (stored) {
|
|
824
|
+
stored.tries = tries;
|
|
825
|
+
stored.nextAttemptAt = at + Math.min(15 * 60_000, WAKE_RETRY_MS * 2 ** (tries - 1));
|
|
826
|
+
} });
|
|
827
|
+
}
|
|
828
|
+
}
|
|
829
|
+
finally {
|
|
830
|
+
this.outboxBusy.delete(item.id);
|
|
831
|
+
}
|
|
832
|
+
})());
|
|
833
|
+
}
|
|
834
|
+
await Promise.all(runs);
|
|
835
|
+
}
|
|
557
836
|
// ---------- run loop ----------
|
|
558
837
|
async pollOnce(via) {
|
|
838
|
+
await this.syncPaused();
|
|
839
|
+
await this.pruneState();
|
|
559
840
|
await Promise.all(this.deps.loadPeers().map((peer) => this.sync(peer, via)));
|
|
560
841
|
await this.syncApprovals(via);
|
|
561
842
|
}
|