@tinycloud/courier 0.2.0-beta.37 → 0.2.0-beta.38

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. package/README.md +331 -12
  2. package/dist/a2a.d.ts +6 -0
  3. package/dist/a2a.js +10 -0
  4. package/dist/a2a.js.map +1 -1
  5. package/dist/approval.d.ts +6 -5
  6. package/dist/approval.js +7 -16
  7. package/dist/approval.js.map +1 -1
  8. package/dist/args.js +1 -1
  9. package/dist/args.js.map +1 -1
  10. package/dist/broker-config.d.ts +24 -0
  11. package/dist/broker-config.js +99 -0
  12. package/dist/broker-config.js.map +1 -0
  13. package/dist/broker.d.ts +77 -0
  14. package/dist/broker.js +253 -0
  15. package/dist/broker.js.map +1 -0
  16. package/dist/commands.js +5 -2
  17. package/dist/commands.js.map +1 -1
  18. package/dist/consumed.d.ts +10 -0
  19. package/dist/consumed.js +54 -0
  20. package/dist/consumed.js.map +1 -0
  21. package/dist/guard/binaries.d.ts +17 -0
  22. package/dist/guard/binaries.js +89 -0
  23. package/dist/guard/binaries.js.map +1 -0
  24. package/dist/guard/cleanhome.d.ts +27 -0
  25. package/dist/guard/cleanhome.js +111 -0
  26. package/dist/guard/cleanhome.js.map +1 -0
  27. package/dist/guard/command.d.ts +13 -0
  28. package/dist/guard/command.js +158 -0
  29. package/dist/guard/command.js.map +1 -0
  30. package/dist/guard/expansion.d.ts +10 -0
  31. package/dist/guard/expansion.js +74 -0
  32. package/dist/guard/expansion.js.map +1 -0
  33. package/dist/guard/hook.d.ts +38 -0
  34. package/dist/guard/hook.js +124 -0
  35. package/dist/guard/hook.js.map +1 -0
  36. package/dist/guard/install.d.ts +67 -0
  37. package/dist/guard/install.js +193 -0
  38. package/dist/guard/install.js.map +1 -0
  39. package/dist/guard/network.d.ts +7 -0
  40. package/dist/guard/network.js +78 -0
  41. package/dist/guard/network.js.map +1 -0
  42. package/dist/guard/normalize.d.ts +28 -0
  43. package/dist/guard/normalize.js +257 -0
  44. package/dist/guard/normalize.js.map +1 -0
  45. package/dist/guard/paseo.d.ts +30 -0
  46. package/dist/guard/paseo.js +148 -0
  47. package/dist/guard/paseo.js.map +1 -0
  48. package/dist/guard/paths.d.ts +31 -0
  49. package/dist/guard/paths.js +180 -0
  50. package/dist/guard/paths.js.map +1 -0
  51. package/dist/guard/policy.d.ts +65 -0
  52. package/dist/guard/policy.js +1179 -0
  53. package/dist/guard/policy.js.map +1 -0
  54. package/dist/guard/shell.d.ts +55 -0
  55. package/dist/guard/shell.js +727 -0
  56. package/dist/guard/shell.js.map +1 -0
  57. package/dist/index.d.ts +2 -0
  58. package/dist/index.js +2 -0
  59. package/dist/index.js.map +1 -1
  60. package/dist/keys.d.ts +4 -1
  61. package/dist/keys.js +7 -2
  62. package/dist/keys.js.map +1 -1
  63. package/dist/main.js +44 -4
  64. package/dist/main.js.map +1 -1
  65. package/dist/node.d.ts +1 -1
  66. package/dist/node.js +3 -3
  67. package/dist/node.js.map +1 -1
  68. package/dist/notify.d.ts +39 -0
  69. package/dist/notify.js +88 -0
  70. package/dist/notify.js.map +1 -0
  71. package/dist/pause.d.ts +26 -0
  72. package/dist/pause.js +99 -0
  73. package/dist/pause.js.map +1 -0
  74. package/dist/profile.d.ts +3 -3
  75. package/dist/profile.js +12 -12
  76. package/dist/profile.js.map +1 -1
  77. package/dist/wake.d.ts +17 -0
  78. package/dist/wake.js +74 -13
  79. package/dist/wake.js.map +1 -1
  80. package/dist/watch-cmd.d.ts +3 -0
  81. package/dist/watch-cmd.js +56 -2
  82. package/dist/watch-cmd.js.map +1 -1
  83. package/dist/watch.d.ts +70 -0
  84. package/dist/watch.js +292 -11
  85. package/dist/watch.js.map +1 -1
  86. package/package.json +1 -1
  87. package/skills/courier/SKILL.md +94 -8
package/dist/watch.d.ts CHANGED
@@ -2,6 +2,7 @@ import type { HookSubscription } from '@tinycloud/node-sdk';
2
2
  import type { StoredPeer } from './profile.js';
3
3
  import { type KV } from './inbox.js';
4
4
  import type { WakeApproval, WakeMessage, Waker, WatchLog } from './wake.js';
5
+ import { type Notice, type Notifier } from './notify.js';
5
6
  export interface PendingWake {
6
7
  peer: string;
7
8
  threadId: string;
@@ -19,11 +20,48 @@ export interface WatchState {
19
20
  /** Recently woken message ids, so a receipt write that failed cannot cause a second wake. */
20
21
  woken: string[];
21
22
  approvalsSeen: string[];
23
+ /** Message wake attempts today (UTC), per `<peer>/<threadId>`, as epoch ms (approval wakes do not count). */
24
+ wakes: Record<string, number[]>;
25
+ /** Threads this watcher has seen paused; a marker that vanishes from this list means the human resumed it. */
26
+ paused: string[];
27
+ /** Posts and notifications that must happen exactly-ish once and survive a restart (retried until they succeed). */
28
+ outbox: OutboxItem[];
29
+ held: Record<string, HeldBacklog>;
30
+ /** UTC day of the last notification per kind/peer/thread: one of each kind per day. */
31
+ peerPausedDay: Record<string, number>;
32
+ /** Times of recent notifications per peer, for the shared per-hour cap. */
33
+ noticeLog: Record<string, number[]>;
34
+ }
35
+ /** `tries` counts failed attempts; an item that keeps failing is dropped (see `MAX_OUTBOX_TRIES`). */
36
+ export type OutboxItem = {
37
+ id: string;
38
+ kind: 'inform';
39
+ peer: string;
40
+ threadId: string;
41
+ text: string;
42
+ tries?: number;
43
+ nextAttemptAt?: number;
44
+ } | {
45
+ id: string;
46
+ kind: 'notify';
47
+ notice: Notice;
48
+ tries?: number;
49
+ nextAttemptAt?: number;
50
+ logged?: boolean;
51
+ };
52
+ /** Messages that arrived while a thread was paused; they wake the agent once, together, after `courier resume`. */
53
+ export interface HeldBacklog {
54
+ peer: string;
55
+ threadId: string;
56
+ subject: string;
57
+ messages: WakeMessage[];
22
58
  }
23
59
  export interface WatchOptions {
24
60
  debounceMs: number;
25
61
  pollMs: number;
26
62
  ticketSeconds: number;
63
+ /** Spec §8: at most this many wakes per side per thread per 24 h, then the thread pauses. */
64
+ wakeBudget: number;
27
65
  }
28
66
  export declare const WATCH_DEFAULTS: WatchOptions;
29
67
  interface HookEventLike {
@@ -48,6 +86,14 @@ export interface WatchDeps {
48
86
  kvFor(peer: StoredPeer): Promise<KV>;
49
87
  wake: Waker;
50
88
  log: WatchLog;
89
+ /** Posts the `thread-paused` inform to a peer (own space, `to/<peer>/threads/<id>/msg/`). Default: nothing. */
90
+ postInform?(peer: string, threadId: string, text: string): Promise<void>;
91
+ /** Tells this side's human without a model (desktop notification or operator command). Default: log only. */
92
+ notify?: Notifier;
93
+ /** Local bounded notification log; omitted by library callers that want no disk output. */
94
+ notificationHome?: string;
95
+ /** D mode: mirror/receipt and notify the human, never invoke the wake adapter or spend its budget. */
96
+ notifyOnly?: boolean;
51
97
  /**
52
98
  * Who this watcher is, for wake claims. `profileId` is stable for this profile on this machine (a restart keeps it,
53
99
  * a second profile or machine differs); `pid` and `isAlive` tell a restarted watcher its predecessor is dead.
@@ -71,6 +117,12 @@ export declare class Watcher {
71
117
  private readonly retryAt;
72
118
  private readonly waking;
73
119
  private approvalsRunning;
120
+ /** Thread keys currently paused: markers under me/state/paused/ plus those this watcher just wrote. */
121
+ private pausedKeys;
122
+ /** Serialises everything that reads or writes pause markers, so a stale listing cannot drop a pause just written. */
123
+ private pausedLock;
124
+ private readonly wakeFailures;
125
+ private readonly outboxBusy;
74
126
  private approvalsAgain;
75
127
  /** Unique to this run: a claim with this id is this process's own. */
76
128
  private readonly runId;
@@ -137,7 +189,25 @@ export declare class Watcher {
137
189
  private approvalWake;
138
190
  /** Wake every thread whose debounce has elapsed (`force`: all of them now). Threads wake in parallel, one wake per thread at a time. */
139
191
  tick(force?: boolean): Promise<void>;
192
+ /** Spec §8: the budget is per side, per thread, per day. A day is the UTC calendar day, so it is easy to state and to check. */
193
+ private recentWakes;
140
194
  private wakeThread;
195
+ /** Budget reached: pause this side's wakes for the thread, tell the peer (inform), tell this side's human. */
196
+ private pauseThread;
197
+ private exclusive;
198
+ /** Called inside `mutate`: parks a message that arrived while its thread is paused (bounded). */
199
+ private hold;
200
+ /** Called inside `mutate`: queues an inform or notification; the outbox is bounded, the oldest item goes first. */
201
+ private enqueue;
202
+ /**
203
+ * Markers under me/state/paused/ are the truth. A marker that disappeared means the human resumed: the budget starts
204
+ * over and what arrived meanwhile wakes the agent once. Serialised, so overlapping calls never apply a stale listing.
205
+ */
206
+ syncPaused(): Promise<void>;
207
+ /** State that only matters for a day or an hour is dropped, so it cannot grow without bound. */
208
+ private pruneState;
209
+ /** Informs and notifications queued in state; failures back off (1, 2, 4 ... 15 minutes) and give up after MAX_OUTBOX_TRIES. */
210
+ private runOutbox;
141
211
  pollOnce(via: Via): Promise<void>;
142
212
  /**
143
213
  * Runs until `signal` aborts. With `once`, does one catch-up and wakes everything immediately. Throws
package/dist/watch.js CHANGED
@@ -5,6 +5,8 @@ import { k, PREFIX } from './keys.js';
5
5
  import { isRecord, messageView, oneLine, taskView } from './a2a.js';
6
6
  import { commitReceipts, selectUndelivered } from './inbox.js';
7
7
  import { del, getJson, listAll, putCreate, putMatch, putOverwrite } from './store.js';
8
+ import { appendNotice } from './notify.js';
9
+ import { writePauseMarker } from './pause.js';
8
10
  /** Spec §8. inform and receipts never wake; a message with no valid intent is delivered but never wakes. */
9
11
  const WAKE_INTENTS = new Set(['request', 'ask', 'answer', 'update']);
10
12
  const TERMINAL_TASK_STATES = new Set([
@@ -23,13 +25,18 @@ const FOREIGN_CLAIM_LEASE_MS = 45 * 60_000;
23
25
  /** A thread another watcher is waking is looked at again after this. */
24
26
  const CLAIM_SKIP_MS = 30_000;
25
27
  const DAY_MS = 24 * 3600 * 1000;
28
+ /** A peer can make this side's watcher notify its human at most this often per hour, and the outbox never grows past `MAX_OUTBOX`. */
29
+ const NOTICES_PER_PEER_PER_HOUR = 10;
30
+ const MAX_OUTBOX = 50;
31
+ const MAX_OUTBOX_TRIES = 8;
32
+ const MAX_HELD_PER_THREAD = 100;
26
33
  /** The subject is peer-controlled text that reaches state, prompts and environments: one line, control characters removed, capped. */
27
34
  const MAX_SUBJECT = 200;
28
35
  /** Thread and message ids come from key names the peer controls; they reach prompts and argv, so only ULID-like ids pass. */
29
36
  const SAFE_ID = /^[0-9A-Za-z_-]{1,64}$/;
30
- export const WATCH_DEFAULTS = { debounceMs: 30_000, pollMs: 60_000, ticketSeconds: 25 };
37
+ export const WATCH_DEFAULTS = { debounceMs: 30_000, pollMs: 60_000, ticketSeconds: 25, wakeBudget: 12 };
31
38
  export function emptyWatchState() {
32
- return { v: 1, pending: {}, woken: [], approvalsSeen: [] };
39
+ return { v: 1, pending: {}, woken: [], approvalsSeen: [], wakes: {}, paused: [], outbox: [], held: {}, peerPausedDay: {}, noticeLog: {} };
33
40
  }
34
41
  /** Wake time: `debounce` after the latest message, but never later than twice `debounce` after the first. */
35
42
  export function wakeDueAt(pending, debounceMs) {
@@ -76,6 +83,12 @@ export class Watcher {
76
83
  retryAt = new Map();
77
84
  waking = new Set();
78
85
  approvalsRunning = false;
86
+ /** Thread keys currently paused: markers under me/state/paused/ plus those this watcher just wrote. */
87
+ pausedKeys = new Set();
88
+ /** Serialises everything that reads or writes pause markers, so a stale listing cannot drop a pause just written. */
89
+ pausedLock = Promise.resolve();
90
+ wakeFailures = new Map();
91
+ outboxBusy = new Set();
79
92
  approvalsAgain = false;
80
93
  /** Unique to this run: a claim with this id is this process's own. */
81
94
  runId = ulid();
@@ -111,6 +124,7 @@ export class Watcher {
111
124
  this.state = isRecord(value) && value.v === 1
112
125
  ? { ...emptyWatchState(), ...value }
113
126
  : emptyWatchState();
127
+ this.state.outbox = this.state.outbox.slice(-MAX_OUTBOX).filter(item => (item.tries ?? 0) < MAX_OUTBOX_TRIES);
114
128
  }
115
129
  snapshot() {
116
130
  return structuredClone(this.state);
@@ -271,7 +285,10 @@ export class Watcher {
271
285
  const { selfKv, log } = this.deps;
272
286
  const mailbox = peer.theirHandleForMe;
273
287
  const peerKv = await this.deps.kvFor(peer);
274
- const keys = await listAll(peerKv, `${PREFIX}/to/${mailbox}/threads/`);
288
+ // Pause markers are read with every listing, so a `courier resume` or `courier pause` takes effect on the very
289
+ // next message instead of at the next poll (a message delivered while the watcher still thought the thread was
290
+ // paused would never wake anyone).
291
+ const [keys] = await Promise.all([listAll(peerKv, `${PREFIX}/to/${mailbox}/threads/`), this.syncPaused()]);
275
292
  const receipts = (await getJson(selfKv, k.receipts(peer.handle)))?.value ?? { v: 1, threads: {} };
276
293
  receipts.threads ??= {};
277
294
  const delivered = [];
@@ -288,7 +305,8 @@ export class Watcher {
288
305
  if (!isRecord(value) || value.messageId !== selected.messageId || value.contextId !== selected.threadId)
289
306
  continue;
290
307
  await this.mirrorCreate(k.mirrorMsg(peer.handle, selected.threadId, selected.messageId), value);
291
- delivered.push({ threadId: selected.threadId, messageId: selected.messageId, intent: messageView(value).intent });
308
+ const view = messageView(value);
309
+ delivered.push({ threadId: selected.threadId, messageId: selected.messageId, intent: view.intent, notice: view.notice });
292
310
  }
293
311
  catch (error) {
294
312
  // Not delivered, so no receipt: this message is retried on the next sync; the others go ahead.
@@ -309,15 +327,27 @@ export class Watcher {
309
327
  const sentAt = sentAtOf(item.messageId);
310
328
  log('delivered', {
311
329
  via, peer: peer.handle, threadId: item.threadId, messageId: item.messageId, intent: item.intent ?? null,
312
- wakes: Boolean(item.intent && WAKE_INTENTS.has(item.intent)),
330
+ wakes: !this.deps.notifyOnly && Boolean(item.intent && WAKE_INTENTS.has(item.intent)) && !this.pausedKeys.has(`${peer.handle}/${item.threadId}`),
331
+ ...(this.pausedKeys.has(`${peer.handle}/${item.threadId}`) ? { paused: true } : {}),
313
332
  sentAt, latencyMs: sentAt ? at - Date.parse(sentAt) : null,
314
333
  });
315
334
  }
316
335
  await this.mutate(() => {
336
+ if (this.deps.notifyOnly) {
337
+ for (const threadId of new Set(delivered.filter(item => item.intent && item.notice !== 'thread-paused').map(item => item.threadId))) {
338
+ const first = delivered.find(item => item.threadId === threadId);
339
+ this.enqueue({ id: `message:${peer.handle}/${threadId}:${first.messageId}`, kind: 'notify', notice: { kind: 'new-message', peer: peer.handle, threadId, eventId: first.messageId, detail: '' } });
340
+ }
341
+ }
317
342
  for (const item of delivered) {
318
- if (!item.intent || !WAKE_INTENTS.has(item.intent) || this.state.woken.includes(item.messageId))
343
+ if (this.deps.notifyOnly || !item.intent || !WAKE_INTENTS.has(item.intent) || this.state.woken.includes(item.messageId))
319
344
  continue;
320
345
  const key = `${peer.handle}/${item.threadId}`;
346
+ if (this.pausedKeys.has(key)) {
347
+ // Delivered and mirrored, but a paused thread never wakes: the message waits for `courier resume`.
348
+ this.hold(key, peer.handle, item.threadId, subjects.get(item.threadId), { messageId: item.messageId, intent: item.intent, sentAt: sentAtOf(item.messageId) });
349
+ continue;
350
+ }
321
351
  const entry = this.state.pending[key] ??= {
322
352
  peer: peer.handle, threadId: item.threadId, subject: subjects.get(item.threadId), messages: [], firstSeenAt: at, lastSeenAt: at,
323
353
  };
@@ -326,6 +356,19 @@ export class Watcher {
326
356
  entry.messages.push({ messageId: item.messageId, intent: item.intent, sentAt: sentAtOf(item.messageId) });
327
357
  entry.lastSeenAt = at;
328
358
  }
359
+ // The peer's watcher paused the thread: tell this side's human (their agent is not coming back until resumed).
360
+ // Only a paired peer's `inform` carrying the notice counts, once per thread per UTC day, at most
361
+ // NOTICES_PER_PEER_PER_HOUR per peer: a peer cannot turn this into a stream of notifications or agent runs.
362
+ for (const item of delivered) {
363
+ if (item.notice !== 'thread-paused' || item.intent !== 'inform')
364
+ continue;
365
+ const key = `${peer.handle}/${item.threadId}`;
366
+ const day = Math.floor(at / DAY_MS);
367
+ this.enqueue({
368
+ id: `peer-paused:${key}:${day}`, kind: 'notify',
369
+ notice: { kind: 'peer-paused', peer: peer.handle, threadId: item.threadId, detail: "the peer's watcher reported the thread paused" },
370
+ });
371
+ }
329
372
  });
330
373
  await commitReceipts(selfKv, new Map([[peer.handle, delivered.map(({ threadId, messageId }) => ({ threadId, messageId }))]]));
331
374
  await this.mirrorThreads(peer, peerKv, keys);
@@ -421,7 +464,7 @@ export class Watcher {
421
464
  const found = await this.approvalWake(id);
422
465
  if ('wake' in found) {
423
466
  wakes.push(found.wake);
424
- log('approval-seen', { via, approvalId: id, wakes: true, decision: found.wake.approval.decision });
467
+ log('approval-seen', { via, approvalId: id, wakes: !this.deps.notifyOnly, notifies: Boolean(this.deps.notifyOnly), decision: found.wake.approval.decision });
425
468
  }
426
469
  else {
427
470
  log('approval-seen', { via, approvalId: id, wakes: false, reason: found.skip });
@@ -439,6 +482,10 @@ export class Watcher {
439
482
  await this.mutate(() => {
440
483
  this.state.approvalsSeen = [...new Set([...this.state.approvalsSeen, ...handled])].sort();
441
484
  for (const { approval, peer, threadId, subject } of wakes) {
485
+ if (this.deps.notifyOnly) {
486
+ this.enqueue({ id: `approval:${approval.approvalId}`, kind: 'notify', notice: { kind: 'new-approval', peer, threadId, eventId: approval.approvalId, detail: '' } });
487
+ continue;
488
+ }
442
489
  const key = `${peer}/${threadId}`;
443
490
  const entry = this.state.pending[key] ??= { peer, threadId, subject, messages: [], firstSeenAt: at, lastSeenAt: at };
444
491
  entry.approvals ??= [];
@@ -477,6 +524,21 @@ export class Watcher {
477
524
  async tick(force = false) {
478
525
  if (this.fatalError || this.stopping)
479
526
  return;
527
+ if (this.deps.notifyOnly) {
528
+ // A watcher switched from auto-wake to D may have a pending backlog from its previous run.
529
+ if (Object.keys(this.state.pending).length)
530
+ await this.mutate(() => {
531
+ for (const pending of Object.values(this.state.pending)) {
532
+ if (pending.messages.length)
533
+ this.enqueue({ id: `message:${pending.peer}/${pending.threadId}:${pending.messages[0].messageId}`, kind: 'notify', notice: { kind: 'new-message', peer: pending.peer, threadId: pending.threadId, eventId: pending.messages[0].messageId, detail: '' } });
534
+ for (const approval of pending.approvals ?? [])
535
+ this.enqueue({ id: `approval:${approval.approvalId}`, kind: 'notify', notice: { kind: 'new-approval', peer: pending.peer, threadId: pending.threadId, eventId: approval.approvalId, detail: '' } });
536
+ }
537
+ this.state.pending = {};
538
+ });
539
+ await this.runOutbox();
540
+ return;
541
+ }
480
542
  const at = this.now();
481
543
  const runs = [];
482
544
  for (const [key, pending] of Object.entries(this.state.pending)) {
@@ -489,14 +551,47 @@ export class Watcher {
489
551
  this.waking.add(key);
490
552
  runs.push(this.wakeThread(key, pending, at).finally(() => this.waking.delete(key)));
491
553
  }
554
+ runs.push(this.runOutbox());
492
555
  await Promise.all(runs);
493
556
  }
557
+ /** Spec §8: the budget is per side, per thread, per day. A day is the UTC calendar day, so it is easy to state and to check. */
558
+ recentWakes(key) {
559
+ const today = Math.floor(this.now() / DAY_MS);
560
+ return (this.state.wakes[key] ?? []).filter((at) => Math.floor(at / DAY_MS) === today);
561
+ }
494
562
  async wakeThread(key, pending, at) {
495
563
  if (this.stopping)
496
564
  return; // told to stop: claim nothing, start nothing
565
+ // A `courier pause` written since the last listing must stop a wake that is already queued.
566
+ await this.syncPaused();
497
567
  // Messages that land while the wake runs stay pending for the next one, so wake exactly this snapshot.
498
- const messages = [...pending.messages];
568
+ let messages = [...pending.messages];
499
569
  const approvals = [...(pending.approvals ?? [])];
570
+ // Spec §8: the loop stops here. A paused thread, or one over budget, wakes nobody for messages (approvals are
571
+ // a human decision, not a loop, and always wake).
572
+ if (messages.length) {
573
+ const used = this.recentWakes(key).length;
574
+ if (!this.pausedKeys.has(key) && used >= this.opts.wakeBudget)
575
+ await this.pauseThread(key, pending, used);
576
+ if (this.pausedKeys.has(key)) {
577
+ // Moved to the held backlog: they wake once, together, after `courier resume`.
578
+ const held = messages;
579
+ const heldIds = held.map((m) => m.messageId);
580
+ messages = [];
581
+ await this.mutate(() => {
582
+ for (const message of held)
583
+ this.hold(key, pending.peer, pending.threadId, pending.subject, message);
584
+ const current = this.state.pending[key];
585
+ if (!current)
586
+ return;
587
+ current.messages = current.messages.filter((m) => !heldIds.includes(m.messageId));
588
+ if (!current.messages.length && !(current.approvals ?? []).length)
589
+ delete this.state.pending[key];
590
+ });
591
+ if (!approvals.length)
592
+ return;
593
+ }
594
+ }
500
595
  const woken = messages.map((m) => m.messageId);
501
596
  const approvedIds = approvals.map((a) => a.approvalId);
502
597
  const oldestSentAt = messages[0]?.sentAt ?? '';
@@ -509,7 +604,12 @@ export class Watcher {
509
604
  return;
510
605
  }
511
606
  try {
512
- await this.mutate(() => { });
607
+ // The attempt is counted against the budget BEFORE the adapter runs, and whether it then succeeds, fails or
608
+ // times out: an adapter that always throws must not be retried forever. (A crash mid-wake counts too.)
609
+ await this.mutate(() => {
610
+ if (woken.length)
611
+ this.state.wakes[key] = [...this.recentWakes(key), this.now()];
612
+ });
513
613
  }
514
614
  catch (error) {
515
615
  await this.releaseClaim(pending.peer, pending.threadId, claimId);
@@ -531,8 +631,11 @@ export class Watcher {
531
631
  await this.deps.wake({ peer: pending.peer, threadId: pending.threadId, subject: pending.subject, messages, approvals });
532
632
  }
533
633
  catch (error) {
534
- this.retryAt.set(key, at + WAKE_RETRY_MS);
535
- this.deps.log('wake-error', { peer: pending.peer, threadId: pending.threadId, ...errInfo(error) });
634
+ // Failures back off: 1, 2, 4, ... minutes, at most 15.
635
+ const failures = (this.wakeFailures.get(key) ?? 0) + 1;
636
+ this.wakeFailures.set(key, failures);
637
+ this.retryAt.set(key, at + Math.min(15 * 60_000, WAKE_RETRY_MS * 2 ** (failures - 1)));
638
+ this.deps.log('wake-error', { peer: pending.peer, threadId: pending.threadId, failures, ...errInfo(error) });
536
639
  // The attempt failed, so the thread is not being woken: release the claim (a restart must not wait for it).
537
640
  await this.releaseClaim(pending.peer, pending.threadId, claimId);
538
641
  return;
@@ -547,6 +650,7 @@ export class Watcher {
547
650
  }
548
651
  this.state.woken = [...this.state.woken, ...woken].slice(-WAKEN_MEMORY);
549
652
  });
653
+ this.wakeFailures.delete(key);
550
654
  // Only now, with the wake recorded, is the retry hold-off cleared.
551
655
  this.retryAt.delete(key);
552
656
  this.deps.log('woke', {
@@ -554,8 +658,185 @@ export class Watcher {
554
658
  runMs: this.now() - startedAt,
555
659
  });
556
660
  }
661
+ /** Budget reached: pause this side's wakes for the thread, tell the peer (inform), tell this side's human. */
662
+ async pauseThread(key, pending, used) {
663
+ const reason = `wake budget reached (${used} wakes today, UTC; limit ${this.opts.wakeBudget ?? WATCH_DEFAULTS.wakeBudget} per thread per day)`;
664
+ const at = new Date(this.now()).toISOString();
665
+ const wrote = await this.exclusive(async () => {
666
+ const created = await writePauseMarker(this.deps.selfKv, { v: 1, peer: pending.peer, threadId: pending.threadId, by: 'budget', at, reason });
667
+ this.pausedKeys.add(key);
668
+ return created;
669
+ });
670
+ // The marker already existed (a human pause, or another run): the peer and the human were told then.
671
+ if (!wrote) {
672
+ this.deps.log('paused', { peer: pending.peer, threadId: pending.threadId, by: 'marker-existing' });
673
+ return;
674
+ }
675
+ const text = `Thread paused on this side: ${reason}. Its agent will not be woken for this thread until its owner resumes it. ` +
676
+ 'This is an automatic stop for reply loops, not a refusal; messages you send are still stored and mirrored.';
677
+ await this.mutate(() => {
678
+ if (!this.state.paused.includes(key))
679
+ this.state.paused.push(key);
680
+ this.enqueue({ id: `inform:${key}:${at}`, kind: 'inform', peer: pending.peer, threadId: pending.threadId, text });
681
+ this.enqueue({ id: `paused:${key}:${at}`, kind: 'notify', notice: { kind: 'paused', peer: pending.peer, threadId: pending.threadId, detail: reason } });
682
+ });
683
+ this.deps.log('paused', { peer: pending.peer, threadId: pending.threadId, by: 'budget', reason });
684
+ }
685
+ exclusive(fn) {
686
+ const run = this.pausedLock.then(fn, fn);
687
+ this.pausedLock = run.catch(() => undefined);
688
+ return run;
689
+ }
690
+ /** Called inside `mutate`: parks a message that arrived while its thread is paused (bounded). */
691
+ hold(key, peer, threadId, subject, message) {
692
+ const entry = this.state.held[key] ??= { peer, threadId, subject, messages: [] };
693
+ if (entry.messages.some((m) => m.messageId === message.messageId))
694
+ return;
695
+ entry.messages = [...entry.messages, message].slice(-MAX_HELD_PER_THREAD);
696
+ }
697
+ /** Called inside `mutate`: queues an inform or notification; the outbox is bounded, the oldest item goes first. */
698
+ enqueue(item) {
699
+ if (item.kind === 'notify') {
700
+ const { notice } = item;
701
+ const at = this.now();
702
+ const day = Math.floor(at / DAY_MS);
703
+ const reference = notice.eventId && SAFE_ID.test(notice.eventId) ? `:${notice.eventId}` : '';
704
+ const key = `${notice.kind}:${notice.peer}/${notice.threadId}${reference}`;
705
+ if (this.state.peerPausedDay[key] === day)
706
+ return;
707
+ const recent = (this.state.noticeLog[notice.peer] ?? []).filter(t => at - t < 3600_000);
708
+ if (recent.length >= NOTICES_PER_PEER_PER_HOUR) {
709
+ this.deps.log('notice-dropped', { peer: notice.peer, reason: 'peer-rate-cap' });
710
+ return;
711
+ }
712
+ this.state.peerPausedDay[key] = day;
713
+ this.state.noticeLog[notice.peer] = [...recent, at];
714
+ }
715
+ this.state.outbox.push(item);
716
+ while (this.state.outbox.length > MAX_OUTBOX) {
717
+ const dropped = this.state.outbox.shift();
718
+ this.deps.log('outbox-dropped', { id: dropped.id, reason: 'full' });
719
+ }
720
+ }
721
+ /**
722
+ * Markers under me/state/paused/ are the truth. A marker that disappeared means the human resumed: the budget starts
723
+ * over and what arrived meanwhile wakes the agent once. Serialised, so overlapping calls never apply a stale listing.
724
+ */
725
+ syncPaused() {
726
+ return this.exclusive(async () => {
727
+ try {
728
+ const keys = await listAll(this.deps.selfKv, k.pausedPrefix);
729
+ const listed = new Set();
730
+ for (const key of keys) {
731
+ const match = /\/paused\/([^/]+)\/([^/]+)\.json$/.exec(key);
732
+ if (match && SAFE_ID.test(match[1]) && SAFE_ID.test(match[2]))
733
+ listed.add(`${match[1]}/${match[2]}`);
734
+ }
735
+ const before = this.pausedKeys;
736
+ const resumed = this.state.paused.filter((key) => !listed.has(key));
737
+ const added = [...listed].filter((key) => !this.state.paused.includes(key));
738
+ this.pausedKeys = listed;
739
+ for (const key of added)
740
+ if (!before.has(key))
741
+ this.deps.log('paused', { thread: key, by: 'marker' });
742
+ if (resumed.length || added.length) {
743
+ const at = this.now();
744
+ const backlog = {};
745
+ await this.mutate(() => {
746
+ for (const key of resumed) {
747
+ delete this.state.wakes[key];
748
+ const held = this.state.held[key];
749
+ if (held?.messages.length) {
750
+ backlog[key] = held.messages.length;
751
+ const entry = this.state.pending[key] ??= { peer: held.peer, threadId: held.threadId, subject: held.subject, messages: [], firstSeenAt: at, lastSeenAt: at };
752
+ for (const message of held.messages)
753
+ if (!entry.messages.some((m) => m.messageId === message.messageId))
754
+ entry.messages.push(message);
755
+ entry.lastSeenAt = at;
756
+ }
757
+ delete this.state.held[key];
758
+ }
759
+ this.state.paused = [...new Set([...this.state.paused.filter((key) => !resumed.includes(key)), ...added])].sort();
760
+ });
761
+ for (const key of resumed)
762
+ this.deps.log('resumed', { thread: key, backlog: backlog[key] ?? 0 });
763
+ }
764
+ }
765
+ catch (error) {
766
+ this.deps.log('sync-error', { source: 'paused', ...errInfo(error) });
767
+ }
768
+ });
769
+ }
770
+ /** State that only matters for a day or an hour is dropped, so it cannot grow without bound. */
771
+ async pruneState() {
772
+ const at = this.now();
773
+ const today = Math.floor(at / DAY_MS);
774
+ const stale = Object.keys(this.state.wakes).filter((key) => !this.recentWakes(key).length)
775
+ .concat(Object.keys(this.state.peerPausedDay).filter((key) => today - this.state.peerPausedDay[key] > 7))
776
+ .concat(Object.keys(this.state.noticeLog).filter((peer) => !this.state.noticeLog[peer].some((t) => at - t < 3600_000)));
777
+ if (!stale.length)
778
+ return;
779
+ await this.mutate(() => {
780
+ for (const key of Object.keys(this.state.wakes))
781
+ if (!this.recentWakes(key).length)
782
+ delete this.state.wakes[key];
783
+ for (const key of Object.keys(this.state.peerPausedDay))
784
+ if (today - this.state.peerPausedDay[key] > 7)
785
+ delete this.state.peerPausedDay[key];
786
+ for (const peer of Object.keys(this.state.noticeLog))
787
+ if (!this.state.noticeLog[peer].some((t) => at - t < 3600_000))
788
+ delete this.state.noticeLog[peer];
789
+ });
790
+ }
791
+ /** Informs and notifications queued in state; failures back off (1, 2, 4 ... 15 minutes) and give up after MAX_OUTBOX_TRIES. */
792
+ async runOutbox() {
793
+ const at = this.now();
794
+ const runs = [];
795
+ for (const item of [...this.state.outbox]) {
796
+ if (this.outboxBusy.has(item.id) || (item.nextAttemptAt ?? 0) > at)
797
+ continue;
798
+ this.outboxBusy.add(item.id);
799
+ runs.push((async () => {
800
+ try {
801
+ if (item.kind === 'inform')
802
+ await this.deps.postInform?.(item.peer, item.threadId, item.text);
803
+ else {
804
+ const { notice } = item;
805
+ this.deps.log('notify', { kind: notice.kind, peer: notice.peer, threadId: notice.threadId, detail: notice.detail });
806
+ if (!item.logged && this.deps.notificationHome) {
807
+ appendNotice(this.deps.notificationHome, notice);
808
+ await this.mutate(() => { const stored = this.state.outbox.find(o => o.id === item.id); if (stored?.kind === 'notify')
809
+ stored.logged = true; });
810
+ }
811
+ await this.deps.notify?.(notice);
812
+ }
813
+ await this.mutate(() => { this.state.outbox = this.state.outbox.filter((o) => o.id !== item.id); });
814
+ }
815
+ catch (error) {
816
+ const tries = (item.tries ?? 0) + 1;
817
+ this.deps.log(item.kind === 'inform' ? 'inform-error' : 'notify-error', { id: item.id, tries, ...errInfo(error) });
818
+ if (tries >= MAX_OUTBOX_TRIES) {
819
+ this.deps.log('outbox-dropped', { id: item.id, reason: 'too-many-failures' });
820
+ await this.mutate(() => { this.state.outbox = this.state.outbox.filter((o) => o.id !== item.id); });
821
+ }
822
+ else {
823
+ await this.mutate(() => { const stored = this.state.outbox.find((o) => o.id === item.id); if (stored) {
824
+ stored.tries = tries;
825
+ stored.nextAttemptAt = at + Math.min(15 * 60_000, WAKE_RETRY_MS * 2 ** (tries - 1));
826
+ } });
827
+ }
828
+ }
829
+ finally {
830
+ this.outboxBusy.delete(item.id);
831
+ }
832
+ })());
833
+ }
834
+ await Promise.all(runs);
835
+ }
557
836
  // ---------- run loop ----------
558
837
  async pollOnce(via) {
838
+ await this.syncPaused();
839
+ await this.pruneState();
559
840
  await Promise.all(this.deps.loadPeers().map((peer) => this.sync(peer, via)));
560
841
  await this.syncApprovals(via);
561
842
  }