@tinycloud/courier 0.6.0-beta.33 → 0.6.0-beta.35
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +54 -13
- package/dist/commands.js +21 -5
- package/dist/commands.js.map +1 -1
- package/dist/flags.js +1 -1
- package/dist/flags.js.map +1 -1
- package/dist/guard/paseo.js +1 -0
- package/dist/guard/paseo.js.map +1 -1
- package/dist/inbox.js +13 -8
- package/dist/inbox.js.map +1 -1
- package/dist/intake/intake.js +32 -2
- package/dist/intake/intake.js.map +1 -1
- package/dist/interim-owner.js +2 -2
- package/dist/interim-owner.js.map +1 -1
- package/dist/interim-runtime.js +2 -2
- package/dist/interim-runtime.js.map +1 -1
- package/dist/invitation.js +2 -0
- package/dist/invitation.js.map +1 -1
- package/dist/node.d.ts +4 -0
- package/dist/node.js +15 -0
- package/dist/node.js.map +1 -1
- package/dist/notify.d.ts +1 -1
- package/dist/notify.js +3 -1
- package/dist/notify.js.map +1 -1
- package/dist/pause.js +4 -1
- package/dist/pause.js.map +1 -1
- package/dist/private-dir.d.ts +37 -0
- package/dist/private-dir.js +270 -0
- package/dist/private-dir.js.map +1 -0
- package/dist/state.js +2 -2
- package/dist/state.js.map +1 -1
- package/dist/store.d.ts +30 -0
- package/dist/store.js +74 -1
- package/dist/store.js.map +1 -1
- package/dist/thread.d.ts +26 -3
- package/dist/thread.js +215 -50
- package/dist/thread.js.map +1 -1
- package/dist/watch-cache.d.ts +16 -0
- package/dist/watch-cache.js +84 -0
- package/dist/watch-cache.js.map +1 -0
- package/dist/watch-cmd.js +3 -0
- package/dist/watch-cmd.js.map +1 -1
- package/dist/watch.d.ts +73 -4
- package/dist/watch.js +275 -58
- package/dist/watch.js.map +1 -1
- package/package.json +3 -3
package/dist/watch.js
CHANGED
|
@@ -5,7 +5,7 @@ import { errInfo, CourierError } from './errors.js';
|
|
|
5
5
|
import { k, PREFIX, stateKeys } from './keys.js';
|
|
6
6
|
import { isRecord, messageView, oneLine, taskView } from './a2a.js';
|
|
7
7
|
import { commitReceipts, selectUndelivered } from './inbox.js';
|
|
8
|
-
import { del, getJson, listAll, putCreate, putMatch, putOverwrite } from './store.js';
|
|
8
|
+
import { del, getJson, isTooLarge, listAll, putCreate, putMatch, putOverwrite, readJsonMany } from './store.js';
|
|
9
9
|
import { appendNotice } from './notify.js';
|
|
10
10
|
import { nextNoticeId, writePauseMarker } from './pause.js';
|
|
11
11
|
import { HANDLE, ULID } from './request.js';
|
|
@@ -44,6 +44,8 @@ const MAX_OUTBOX = 50;
|
|
|
44
44
|
const MAX_OUTBOX_TRIES = 8;
|
|
45
45
|
const MAX_NOTICE_ATTEMPTS = 3;
|
|
46
46
|
const MAX_HELD_PER_THREAD = 100;
|
|
47
|
+
/** An action put off because its authority check gave no verdict is dropped after this; its peer's human is told once. */
|
|
48
|
+
const DEFERRAL_LIMIT_MS = DAY_MS;
|
|
47
49
|
/** The subject is peer-controlled text that reaches state, prompts and environments: one line, control characters removed, capped. */
|
|
48
50
|
const MAX_SUBJECT = 200;
|
|
49
51
|
/** Thread and message ids come from key names the peer controls; they reach prompts and argv, so only ULID-like ids pass. */
|
|
@@ -58,13 +60,16 @@ const needsKey = (peer, kind) => `needs:${peer}:${kind}`;
|
|
|
58
60
|
const emptyEpisode = () => ({ kind: '', eventId: ZERO_ULID, repeats: '000000', ...emptyNoticeRetry() });
|
|
59
61
|
const incrementCount = (count) => String(Math.min(MAX_COALESCED_PAUSES, Number(count) + 1)).padStart(6, '0');
|
|
60
62
|
const episodeKey = (key, slot) => `notice:${key}:${slot}`;
|
|
63
|
+
const accessKey = (peer) => `access:${peer}`;
|
|
61
64
|
function threadNoticeState() {
|
|
62
65
|
return { ourPaused: false, peerPaused: false, lastPeerNoticeId: '0'.repeat(26), lastNoticeSentAt: new Date(0).toISOString(),
|
|
63
66
|
ourMarkerId: '', episodes: [emptyEpisode(), emptyEpisode()], informPending: false, informText: '' };
|
|
64
67
|
}
|
|
68
|
+
export const peerIdentity = (peer) => ({ did: peer.courierDid, space: peer.spaceId, session: peer.sessionId ?? null });
|
|
69
|
+
export const sameIdentity = (a, b) => a !== undefined && a.did === b.did && a.space === b.space && a.session === b.session;
|
|
65
70
|
export const WATCH_DEFAULTS = { debounceMs: 30_000, pollMs: 60_000, ticketSeconds: 25, wakeBudget: 12 };
|
|
66
71
|
export function emptyWatchState() {
|
|
67
|
-
return { v: 1, pending: {}, woken: [], approvalsSeen: [], wakes: {}, paused: [], wakeRetries: {}, approvalRetries: {}, outbox: [], held: {}, threadNotices: {}, peerNotices: {}, noticeGlobalLog: [], noticeLog: {} };
|
|
72
|
+
return { v: 1, pending: {}, woken: [], approvalsSeen: [], wakes: {}, paused: [], wakeRetries: {}, approvalRetries: {}, outbox: [], held: {}, threadNotices: {}, peerNotices: {}, noticeGlobalLog: [], noticeLog: {}, unverified: {} };
|
|
68
73
|
}
|
|
69
74
|
/** Wake time: `debounce` after the latest message, but never later than twice `debounce` after the first. */
|
|
70
75
|
export function wakeDueAt(pending, debounceMs) {
|
|
@@ -122,7 +127,6 @@ export class Watcher {
|
|
|
122
127
|
profileId;
|
|
123
128
|
pid;
|
|
124
129
|
isAlive;
|
|
125
|
-
lastClaimPrune = 0;
|
|
126
130
|
/** Set when another watcher is writing this profile's state: this one stops and `run` throws it. */
|
|
127
131
|
fatalError;
|
|
128
132
|
/** Set once `run` is told to stop: nothing new is woken after that. */
|
|
@@ -130,8 +134,17 @@ export class Watcher {
|
|
|
130
134
|
onFatal;
|
|
131
135
|
/** Etag of this agent's watch.json as this watcher last saw it: null = absent, undefined = unknown (fetch before writing). */
|
|
132
136
|
stateEtag;
|
|
137
|
+
/** Per `<peer handle>\n<peer task key>`: two peers may use the same name for us, so a key alone is ambiguous. */
|
|
133
138
|
taskCache = new Map();
|
|
134
|
-
|
|
139
|
+
/** Peer keys whose value is over the batch cap: reported once and not read again by this process. */
|
|
140
|
+
tooLarge = new Set();
|
|
141
|
+
metaMirrored;
|
|
142
|
+
markerReadAt;
|
|
143
|
+
claimsPrunedAt;
|
|
144
|
+
/** Per peer, the identity the facts above were learned under: a peer that changed identity starts with none. */
|
|
145
|
+
factIdentity;
|
|
146
|
+
/** The local cache changed since it was last saved. */
|
|
147
|
+
localDirty = false;
|
|
135
148
|
syncing = new Map();
|
|
136
149
|
/** Peers whose last sync did not finish: a cycle between full catch-ups lists only these again. */
|
|
137
150
|
dirtyPeers = new Set();
|
|
@@ -143,7 +156,7 @@ export class Watcher {
|
|
|
143
156
|
/** Open hook streams by subscription id; absent while one is failed, ended or not yet opened. */
|
|
144
157
|
hookStreams = new Map();
|
|
145
158
|
/** Message keys each peer's receipts already list; receipts only grow, so a listing within this set has nothing new. */
|
|
146
|
-
receipted
|
|
159
|
+
receipted;
|
|
147
160
|
stateQueue = Promise.resolve();
|
|
148
161
|
now;
|
|
149
162
|
keys;
|
|
@@ -155,6 +168,46 @@ export class Watcher {
|
|
|
155
168
|
this.pid = deps.holder?.pid ?? process.pid;
|
|
156
169
|
this.isAlive = deps.holder?.isAlive ?? pidAlive;
|
|
157
170
|
this.keys = stateKeys(deps.agent);
|
|
171
|
+
const cached = deps.localCache?.load();
|
|
172
|
+
this.metaMirrored = new Map(Object.entries(cached?.metaMirrored ?? {}).map(([peer, keys]) => [peer, new Set(keys)]));
|
|
173
|
+
this.markerReadAt = new Map(Object.entries(cached?.markerReadAt ?? {}));
|
|
174
|
+
this.receipted = new Map(Object.entries(cached?.receipted ?? {}).map(([peer, ids]) => [peer, new Set(ids)]));
|
|
175
|
+
this.claimsPrunedAt = cached?.claimsPrunedAt ?? 0;
|
|
176
|
+
this.factIdentity = new Map(Object.entries(cached?.identities ?? {}));
|
|
177
|
+
}
|
|
178
|
+
/**
|
|
179
|
+
* Before using or learning facts about a peer: if the peer is now a different agent, space or session than the one
|
|
180
|
+
* the facts were learned under, they are dropped, so a new peer's delivery is never skipped on an old peer's facts.
|
|
181
|
+
*/
|
|
182
|
+
bindFacts(peer) {
|
|
183
|
+
const identity = peerIdentity(peer);
|
|
184
|
+
if (sameIdentity(this.factIdentity.get(peer.handle), identity))
|
|
185
|
+
return;
|
|
186
|
+
this.receipted.delete(peer.handle);
|
|
187
|
+
this.markerReadAt.delete(peer.handle);
|
|
188
|
+
this.metaMirrored.delete(peer.handle);
|
|
189
|
+
for (const cache of [this.taskCache, this.tooLarge])
|
|
190
|
+
for (const key of [...cache.keys()])
|
|
191
|
+
if (key.startsWith(`${peer.handle}\n`))
|
|
192
|
+
cache.delete(key);
|
|
193
|
+
this.factIdentity.set(peer.handle, identity);
|
|
194
|
+
this.localDirty = true;
|
|
195
|
+
}
|
|
196
|
+
saveLocal() {
|
|
197
|
+
if (!this.localDirty || !this.deps.localCache)
|
|
198
|
+
return;
|
|
199
|
+
this.localDirty = false;
|
|
200
|
+
try {
|
|
201
|
+
this.deps.localCache.save({
|
|
202
|
+
v: 1, claimsPrunedAt: this.claimsPrunedAt, markerReadAt: Object.fromEntries(this.markerReadAt),
|
|
203
|
+
metaMirrored: Object.fromEntries([...this.metaMirrored].map(([peer, keys]) => [peer, [...keys]])),
|
|
204
|
+
receipted: Object.fromEntries([...this.receipted].map(([peer, ids]) => [peer, [...ids]])),
|
|
205
|
+
identities: Object.fromEntries(this.factIdentity),
|
|
206
|
+
});
|
|
207
|
+
}
|
|
208
|
+
catch (error) {
|
|
209
|
+
this.deps.log('sync-error', { source: 'local-cache', ...errInfo(error) });
|
|
210
|
+
}
|
|
158
211
|
}
|
|
159
212
|
/**
|
|
160
213
|
* Before TC-2917 every agent of the owner shared `me/state/`. On an agent's first start its own keys get a copy of
|
|
@@ -164,8 +217,12 @@ export class Watcher {
|
|
|
164
217
|
*/
|
|
165
218
|
async migrateLegacyState() {
|
|
166
219
|
const { selfKv, log } = this.deps;
|
|
167
|
-
if (!this.deps.agent
|
|
168
|
-
return;
|
|
220
|
+
if (!this.deps.agent)
|
|
221
|
+
return undefined;
|
|
222
|
+
// Already moved: this read is also the state load.
|
|
223
|
+
const own = await getJson(selfKv, this.keys.watch);
|
|
224
|
+
if (own)
|
|
225
|
+
return own;
|
|
169
226
|
const mine = new Set(this.deps.peerHandles?.() ?? this.deps.loadPeers().map(peer => peer.handle));
|
|
170
227
|
const ownKey = (key) => mine.has(key.split('/')[0]);
|
|
171
228
|
let markers = 0;
|
|
@@ -192,10 +249,12 @@ export class Watcher {
|
|
|
192
249
|
};
|
|
193
250
|
await this.mirrorCreate(this.keys.watch, copy);
|
|
194
251
|
log('state-migrated', { pending: Object.keys(copy.pending).length, paused: markers });
|
|
252
|
+
return undefined;
|
|
195
253
|
}
|
|
196
254
|
// ---------- state ----------
|
|
197
|
-
|
|
198
|
-
|
|
255
|
+
/** `preloaded`: this agent's watch.json as just read (saves reading it twice at startup). */
|
|
256
|
+
async loadState(preloaded) {
|
|
257
|
+
const got = preloaded ?? await getJson(this.deps.selfKv, this.keys.watch);
|
|
199
258
|
const value = got?.value;
|
|
200
259
|
this.stateEtag = got?.etag ?? null;
|
|
201
260
|
this.state = isRecord(value) && value.v === 1
|
|
@@ -239,6 +298,12 @@ export class Watcher {
|
|
|
239
298
|
this.state.outbox = this.state.outbox.filter(item => {
|
|
240
299
|
if (item.kind === 'inform')
|
|
241
300
|
return true;
|
|
301
|
+
if (item.notice.kind === 'access-unverified') {
|
|
302
|
+
if (!this.noticeSlot(item) || item.id !== accessKey(item.notice.peer))
|
|
303
|
+
return false;
|
|
304
|
+
this.rememberNotice(item);
|
|
305
|
+
return true;
|
|
306
|
+
}
|
|
242
307
|
const kind = needsKind(item.notice.kind);
|
|
243
308
|
if (kind) {
|
|
244
309
|
const slot = this.noticeSlot(item);
|
|
@@ -377,19 +442,25 @@ export class Watcher {
|
|
|
377
442
|
this.deps.log('sync-error', { source: 'claim-release', ...errInfo(error) });
|
|
378
443
|
}
|
|
379
444
|
}
|
|
380
|
-
/** Claim markers are only needed while a double wake is possible; drop those older than two days. */
|
|
381
|
-
async
|
|
445
|
+
/** Claim markers are only needed while a double wake is possible; once an hour, drop those older than two days. */
|
|
446
|
+
async pruneClaimsIfDue() {
|
|
447
|
+
const at = this.now();
|
|
448
|
+
if (at - this.claimsPrunedAt <= 3600_000)
|
|
449
|
+
return;
|
|
382
450
|
try {
|
|
383
451
|
for (const key of await listAll(this.deps.selfKv, this.keys.wakeClaimsPrefix)) {
|
|
384
452
|
const id = /\/([^/]+)\.json$/.exec(key)?.[1];
|
|
385
453
|
const sentAt = id ? Date.parse(sentAtOf(id)) : NaN;
|
|
386
|
-
if (Number.isFinite(sentAt) &&
|
|
454
|
+
if (Number.isFinite(sentAt) && at - sentAt > 2 * DAY_MS)
|
|
387
455
|
await del(this.deps.selfKv, key);
|
|
388
456
|
}
|
|
389
457
|
}
|
|
390
458
|
catch (error) {
|
|
391
459
|
this.deps.log('sync-error', { source: 'claims', ...errInfo(error) });
|
|
460
|
+
return;
|
|
392
461
|
}
|
|
462
|
+
this.claimsPrunedAt = at;
|
|
463
|
+
this.localDirty = true;
|
|
393
464
|
}
|
|
394
465
|
// ---------- catch-up and delivery ----------
|
|
395
466
|
/**
|
|
@@ -473,23 +544,66 @@ export class Watcher {
|
|
|
473
544
|
const handle = typeof peer === 'string' ? peer : peer.handle;
|
|
474
545
|
if (!this.deps.checkAuthority)
|
|
475
546
|
return true; // a library caller without an authority source
|
|
476
|
-
|
|
547
|
+
try {
|
|
548
|
+
await this.deps.checkAuthority(handle);
|
|
549
|
+
}
|
|
550
|
+
catch (error) {
|
|
551
|
+
await this.noteUnverified(handle);
|
|
552
|
+
throw error;
|
|
553
|
+
}
|
|
554
|
+
if (Object.hasOwn(this.state.unverified, handle))
|
|
555
|
+
await this.mutate(() => this.endUnverified(handle));
|
|
477
556
|
return this.deps.loadPeers().some(current => current.handle === handle);
|
|
478
557
|
}
|
|
558
|
+
/**
|
|
559
|
+
* A check gave no verdict for this peer: the outage starts now if it isn't already running. Once it has lasted
|
|
560
|
+
* `DEFERRAL_LIMIT_MS`, the peer's human is told once (Naomi's TC-2951 ruling: imports keep retrying; wakes and informs
|
|
561
|
+
* deferred that long are dropped). Only configured paired peers allocate a record.
|
|
562
|
+
*/
|
|
563
|
+
async noteUnverified(handle) {
|
|
564
|
+
if (!HANDLE.test(handle) || !this.deps.loadPeers().some(peer => peer.handle === handle && peer.status === 'paired'))
|
|
565
|
+
return;
|
|
566
|
+
const now = this.now();
|
|
567
|
+
const record = this.state.unverified[handle];
|
|
568
|
+
if (record && (record.eventId || now - record.since < DEFERRAL_LIMIT_MS))
|
|
569
|
+
return;
|
|
570
|
+
await this.mutate(() => {
|
|
571
|
+
const current = this.state.unverified[handle] ??= { since: now, eventId: '', notified: false, ...emptyNoticeRetry() };
|
|
572
|
+
if (current.eventId || now - current.since < DEFERRAL_LIMIT_MS)
|
|
573
|
+
return;
|
|
574
|
+
current.eventId = ulid(now);
|
|
575
|
+
this.deps.log('access-unverified', { peer: handle, since: new Date(current.since).toISOString() });
|
|
576
|
+
this.fillOutbox();
|
|
577
|
+
});
|
|
578
|
+
}
|
|
579
|
+
/** A verdict ends the outage; a notice not yet sent for it is withdrawn. Called inside mutate. */
|
|
580
|
+
endUnverified(handle) {
|
|
581
|
+
delete this.state.unverified[handle];
|
|
582
|
+
this.state.outbox = this.state.outbox.filter(item => item.id !== accessKey(handle) || this.outboxBusy.has(item.id));
|
|
583
|
+
}
|
|
479
584
|
/** The scheduled renewal and authority check for every peer; each peer's next sync also reads its renewal marker. */
|
|
480
585
|
async checkAuthority() {
|
|
481
586
|
await this.deps.checkAuthority?.();
|
|
482
|
-
|
|
483
|
-
|
|
587
|
+
// The renewal marker is read on the same schedule, across runs: a `--once` loop does not read it every run.
|
|
588
|
+
const at = this.now();
|
|
589
|
+
for (const peer of this.deps.loadPeers()) {
|
|
590
|
+
this.bindFacts(peer);
|
|
591
|
+
if (at - (this.markerReadAt.get(peer.handle) ?? -Infinity) >= AUTHORITY_CHECK_MS)
|
|
592
|
+
this.renewalDue.add(peer.handle);
|
|
593
|
+
}
|
|
484
594
|
}
|
|
485
595
|
async syncPeer(peer, via, listPaused, renew) {
|
|
486
596
|
if (peer.status !== 'paired' || !peer.inbound || !peer.theirHandleForMe)
|
|
487
597
|
return;
|
|
598
|
+
this.bindFacts(peer);
|
|
488
599
|
const { selfKv, log } = this.deps;
|
|
489
600
|
const mailbox = peer.theirHandleForMe;
|
|
490
601
|
const peerKv = await this.deps.kvFor(peer, renew);
|
|
491
|
-
if (renew)
|
|
602
|
+
if (renew) {
|
|
492
603
|
this.renewalDue.delete(peer.handle);
|
|
604
|
+
this.markerReadAt.set(peer.handle, this.now());
|
|
605
|
+
this.localDirty = true;
|
|
606
|
+
}
|
|
493
607
|
let allowed;
|
|
494
608
|
if (this.authorityDue.delete(peer.handle)) {
|
|
495
609
|
// The marker changed: check now, even with nothing to import; the verdict also serves this sync's imports.
|
|
@@ -518,17 +632,43 @@ export class Watcher {
|
|
|
518
632
|
for (const id of thread.delivered ?? [])
|
|
519
633
|
seen.add(`${threadId}/${id}`);
|
|
520
634
|
this.receipted.set(peer.handle, seen);
|
|
635
|
+
this.localDirty = true;
|
|
521
636
|
}
|
|
522
637
|
const delivered = [];
|
|
523
|
-
|
|
524
|
-
if (
|
|
525
|
-
|
|
638
|
+
const selectedItems = unseen.filter(item => !seen.has(`${item.threadId}/${item.messageId}`)).filter(selected => {
|
|
639
|
+
if (this.tooLarge.has(`${peer.handle}\n${selected.key}`))
|
|
640
|
+
return false;
|
|
641
|
+
if (SAFE_ID.test(selected.threadId) && SAFE_ID.test(selected.messageId))
|
|
642
|
+
return true;
|
|
643
|
+
log('ignored', { peer: peer.handle, reason: 'unsafe-id' });
|
|
644
|
+
return false;
|
|
645
|
+
});
|
|
646
|
+
if (selectedItems.length && !(await authorized()))
|
|
647
|
+
return;
|
|
648
|
+
// One batch read per 100 messages. A failed batch leaves them all for the next sync; one message's failure, only it.
|
|
649
|
+
let outcomes;
|
|
650
|
+
try {
|
|
651
|
+
outcomes = await readJsonMany(peerKv, selectedItems.map(item => item.key));
|
|
652
|
+
}
|
|
653
|
+
catch (error) {
|
|
654
|
+
this.dirtyPeers.add(peer.handle);
|
|
655
|
+
log('sync-error', { peer: peer.handle, messages: selectedItems.length, ...errInfo(error) });
|
|
656
|
+
outcomes = new Map();
|
|
657
|
+
}
|
|
658
|
+
for (const selected of selectedItems) {
|
|
659
|
+
const outcome = outcomes.get(selected.key);
|
|
660
|
+
if (!outcome)
|
|
526
661
|
continue;
|
|
527
|
-
}
|
|
528
|
-
if (!(await authorized()))
|
|
529
|
-
return;
|
|
530
662
|
try {
|
|
531
|
-
|
|
663
|
+
if ('error' in outcome && isTooLarge(outcome.error)) {
|
|
664
|
+
// Over the cap: reported once and not read again by this process; the others go ahead. No receipt.
|
|
665
|
+
this.tooLarge.add(`${peer.handle}\n${selected.key}`);
|
|
666
|
+
log('ignored', { peer: peer.handle, threadId: selected.threadId, messageId: selected.messageId, reason: 'too-large' });
|
|
667
|
+
continue;
|
|
668
|
+
}
|
|
669
|
+
if ('error' in outcome)
|
|
670
|
+
throw outcome.error;
|
|
671
|
+
const got = outcome.got;
|
|
532
672
|
if (!got)
|
|
533
673
|
continue;
|
|
534
674
|
const value = got.value;
|
|
@@ -549,8 +689,13 @@ export class Watcher {
|
|
|
549
689
|
return;
|
|
550
690
|
}
|
|
551
691
|
const subjects = new Map();
|
|
552
|
-
|
|
553
|
-
|
|
692
|
+
const threads = [...new Set(delivered.map((d) => d.threadId))];
|
|
693
|
+
const metaKey = (threadId) => `${PREFIX}/to/${mailbox}/threads/${threadId}/meta.json`;
|
|
694
|
+
// A missing or unreadable meta only costs the subject.
|
|
695
|
+
const metas = await readJsonMany(peerKv, threads.map(metaKey)).catch(() => new Map());
|
|
696
|
+
for (const threadId of threads) {
|
|
697
|
+
const outcome = metas.get(metaKey(threadId));
|
|
698
|
+
const meta = outcome && 'got' in outcome ? outcome.got : undefined;
|
|
554
699
|
subjects.set(threadId, isRecord(meta?.value) && typeof meta.value.subject === 'string' ? oneLine(meta.value.subject, MAX_SUBJECT) : '(no subject)');
|
|
555
700
|
}
|
|
556
701
|
const at = this.now();
|
|
@@ -620,6 +765,8 @@ export class Watcher {
|
|
|
620
765
|
await commitReceipts(selfKv, new Map([[peer.handle, delivered.map(({ threadId, messageId }) => ({ threadId, messageId }))]]));
|
|
621
766
|
for (const item of delivered)
|
|
622
767
|
seen.add(`${item.threadId}/${item.messageId}`);
|
|
768
|
+
if (delivered.length)
|
|
769
|
+
this.localDirty = true;
|
|
623
770
|
await this.mirrorThreads(peer, peerKv, keys, authorized);
|
|
624
771
|
}
|
|
625
772
|
/** Only locally authored meta/messages/tasks, or a local wake/pause witness, establish a known thread. */
|
|
@@ -656,9 +803,38 @@ export class Watcher {
|
|
|
656
803
|
* message delivery and each item is isolated: a bad or unwritable peer task never blocks receipts or wakes.
|
|
657
804
|
*/
|
|
658
805
|
async mirrorThreads(peer, peerKv, keys, authorized) {
|
|
659
|
-
|
|
806
|
+
// Only what may need copying is read: a meta not yet mirrored, a task not yet terminal, never a key found oversized.
|
|
807
|
+
const wanted = keys.filter(key => {
|
|
808
|
+
if (this.tooLarge.has(`${peer.handle}\n${key}`))
|
|
809
|
+
return false;
|
|
810
|
+
const meta = /\/threads\/([^/]+)\/meta\.json$/.exec(key);
|
|
811
|
+
if (meta)
|
|
812
|
+
return SAFE_ID.test(meta[1]) && !this.metaMirrored.get(peer.handle)?.has(key);
|
|
813
|
+
const task = /\/threads\/([^/]+)\/task\/([^/]+)\.json$/.exec(key);
|
|
814
|
+
return Boolean(task && SAFE_ID.test(task[1]) && SAFE_ID.test(task[2]) && !this.taskCache.get(`${peer.handle}\n${key}`)?.terminal);
|
|
815
|
+
});
|
|
816
|
+
let outcomes;
|
|
817
|
+
try {
|
|
818
|
+
outcomes = await readJsonMany(peerKv, wanted);
|
|
819
|
+
}
|
|
820
|
+
catch (error) {
|
|
821
|
+
this.dirtyPeers.add(peer.handle);
|
|
822
|
+
this.deps.log('mirror-error', { peer: peer.handle, ...errInfo(error) });
|
|
823
|
+
return;
|
|
824
|
+
}
|
|
825
|
+
for (const key of wanted) {
|
|
660
826
|
try {
|
|
661
|
-
|
|
827
|
+
const outcome = outcomes.get(key);
|
|
828
|
+
if ('error' in outcome && isTooLarge(outcome.error)) {
|
|
829
|
+
// Over the cap: reported once and not read again by this process, like an oversized message.
|
|
830
|
+
this.tooLarge.add(`${peer.handle}\n${key}`);
|
|
831
|
+
const [, threadId, item] = /\/threads\/([^/]+)\/(meta|task\/[^/]+)\.json$/.exec(key);
|
|
832
|
+
this.deps.log('ignored', { peer: peer.handle, threadId, item, reason: 'too-large' });
|
|
833
|
+
continue;
|
|
834
|
+
}
|
|
835
|
+
if ('error' in outcome)
|
|
836
|
+
throw outcome.error;
|
|
837
|
+
await this.mirrorThreadItem(peer, key, outcome.got, authorized);
|
|
662
838
|
}
|
|
663
839
|
catch (error) {
|
|
664
840
|
this.dirtyPeers.add(peer.handle);
|
|
@@ -666,38 +842,27 @@ export class Watcher {
|
|
|
666
842
|
}
|
|
667
843
|
}
|
|
668
844
|
}
|
|
669
|
-
async mirrorThreadItem(peer,
|
|
845
|
+
async mirrorThreadItem(peer, key, got, authorized) {
|
|
846
|
+
if (!got)
|
|
847
|
+
return;
|
|
670
848
|
const meta = /\/threads\/([^/]+)\/meta\.json$/.exec(key);
|
|
671
849
|
if (meta) {
|
|
672
|
-
if (!SAFE_ID.test(meta[1]))
|
|
673
|
-
return;
|
|
674
|
-
if (this.metaMirrored.has(key))
|
|
675
|
-
return;
|
|
676
|
-
const got = await getJson(peerKv, key);
|
|
677
|
-
if (!got)
|
|
678
|
-
return;
|
|
679
850
|
if (!(await authorized()))
|
|
680
851
|
return;
|
|
681
852
|
await this.mirrorCreate(k.mirrorMeta(peer.handle, meta[1]), got.value);
|
|
682
|
-
this.metaMirrored.add(key);
|
|
853
|
+
this.metaMirrored.set(peer.handle, (this.metaMirrored.get(peer.handle) ?? new Set()).add(key));
|
|
854
|
+
this.localDirty = true;
|
|
683
855
|
return;
|
|
684
856
|
}
|
|
685
857
|
const task = /\/threads\/([^/]+)\/task\/([^/]+)\.json$/.exec(key);
|
|
686
|
-
|
|
687
|
-
return;
|
|
688
|
-
const cached = this.taskCache.get(key);
|
|
689
|
-
if (cached?.terminal)
|
|
690
|
-
return;
|
|
691
|
-
const got = await getJson(peerKv, key);
|
|
692
|
-
if (!got)
|
|
693
|
-
return;
|
|
858
|
+
const cached = this.taskCache.get(`${peer.handle}\n${key}`);
|
|
694
859
|
if (cached && got.etag && cached.etag === got.etag)
|
|
695
860
|
return;
|
|
696
861
|
if (!(await authorized()))
|
|
697
862
|
return;
|
|
698
863
|
await putOverwrite(this.deps.selfKv, k.mirrorTask(peer.handle, task[1], task[2]), got.value);
|
|
699
864
|
const state = taskView(got.value).state;
|
|
700
|
-
this.taskCache.set(key
|
|
865
|
+
this.taskCache.set(`${peer.handle}\n${key}`, { etag: got.etag, terminal: Boolean(state && TERMINAL_TASK_STATES.has(state)) });
|
|
701
866
|
}
|
|
702
867
|
/**
|
|
703
868
|
* Own `me/approvals/` (only the owner's view session can write there). A new decision on a request this agent
|
|
@@ -900,6 +1065,22 @@ export class Watcher {
|
|
|
900
1065
|
}
|
|
901
1066
|
catch (error) {
|
|
902
1067
|
await this.releaseClaim(pending.peer, pending.threadId, claimId);
|
|
1068
|
+
const now = this.now();
|
|
1069
|
+
const since = this.state.pending[key]?.deferredSince ?? now;
|
|
1070
|
+
if (now - since >= DEFERRAL_LIMIT_MS) {
|
|
1071
|
+
// Put off for 24 h without a verdict: dropped. The peer's human gets the one outage notice.
|
|
1072
|
+
await this.mutate(() => {
|
|
1073
|
+
delete this.state.pending[key];
|
|
1074
|
+
delete this.state.wakeRetries[key];
|
|
1075
|
+
delete this.state.approvalRetries[key];
|
|
1076
|
+
});
|
|
1077
|
+
this.deps.log('wake-dropped', { peer: pending.peer, threadId: pending.threadId, messageIds: woken, approvalIds: approvedIds, reason: 'authority-unverified-24h', ...errInfo(error) });
|
|
1078
|
+
return;
|
|
1079
|
+
}
|
|
1080
|
+
if (this.state.pending[key] && this.state.pending[key].deferredSince === undefined) {
|
|
1081
|
+
await this.mutate(() => { if (this.state.pending[key])
|
|
1082
|
+
this.state.pending[key].deferredSince ??= since; });
|
|
1083
|
+
}
|
|
903
1084
|
this.retryAt.set(key, at + WAKE_RETRY_MS);
|
|
904
1085
|
this.deps.log('wake-deferred', { peer: pending.peer, threadId: pending.threadId, reason: 'authority-unverified', ...errInfo(error) });
|
|
905
1086
|
return;
|
|
@@ -921,6 +1102,8 @@ export class Watcher {
|
|
|
921
1102
|
this.state.threadNotices[key] ??= threadNoticeState();
|
|
922
1103
|
if (woken.length)
|
|
923
1104
|
this.state.wakes[key] = [...this.recentWakes(key), this.now()];
|
|
1105
|
+
if (this.state.pending[key])
|
|
1106
|
+
delete this.state.pending[key].deferredSince;
|
|
924
1107
|
});
|
|
925
1108
|
}
|
|
926
1109
|
catch (error) {
|
|
@@ -1072,6 +1255,10 @@ export class Watcher {
|
|
|
1072
1255
|
}
|
|
1073
1256
|
}
|
|
1074
1257
|
noticeSlot(item) {
|
|
1258
|
+
if (item.notice.kind === 'access-unverified') {
|
|
1259
|
+
const record = this.state.unverified[item.notice.peer];
|
|
1260
|
+
return record?.eventId && !record.notified && record.eventId === item.notice.eventId ? record : undefined;
|
|
1261
|
+
}
|
|
1075
1262
|
const kind = needsKind(item.notice.kind);
|
|
1076
1263
|
if (kind) {
|
|
1077
1264
|
const slot = this.state.peerNotices[item.notice.peer]?.[kind];
|
|
@@ -1168,6 +1355,13 @@ export class Watcher {
|
|
|
1168
1355
|
} };
|
|
1169
1356
|
}
|
|
1170
1357
|
}
|
|
1358
|
+
for (const [peer, record] of Object.entries(this.state.unverified)) {
|
|
1359
|
+
if (record.eventId && !record.notified)
|
|
1360
|
+
yield { id: accessKey(peer), kind: 'notify',
|
|
1361
|
+
tries: record.tries, nextAttemptAt: record.nextAttemptAt, logged: record.logged, notice: {
|
|
1362
|
+
kind: 'access-unverified', peer, threadId: ZERO_ULID, eventId: record.eventId, detail: 'authority unverified for 24 h',
|
|
1363
|
+
} };
|
|
1364
|
+
}
|
|
1171
1365
|
}
|
|
1172
1366
|
/** Admit local work first, allowing eligible peers to pass deferred entries in a full queue. */
|
|
1173
1367
|
fillOutbox() {
|
|
@@ -1201,7 +1395,13 @@ export class Watcher {
|
|
|
1201
1395
|
const peer = item.kind === 'inform' ? item.peer : item.notice.peer;
|
|
1202
1396
|
const threadId = item.kind === 'inform' ? item.threadId : item.notice.threadId;
|
|
1203
1397
|
const record = this.state.threadNotices[`${peer}/${threadId}`];
|
|
1204
|
-
if (item.kind === 'notify' &&
|
|
1398
|
+
if (item.kind === 'notify' && item.notice.kind === 'access-unverified') {
|
|
1399
|
+
// One notice per outage, delivered or given up on: it is not queued again until a verdict ends the outage.
|
|
1400
|
+
const slot = this.noticeSlot(item);
|
|
1401
|
+
if (slot)
|
|
1402
|
+
slot.notified = true;
|
|
1403
|
+
}
|
|
1404
|
+
else if (item.kind === 'notify' && needsKind(item.notice.kind)) {
|
|
1205
1405
|
const slot = this.noticeSlot(item);
|
|
1206
1406
|
if (slot) {
|
|
1207
1407
|
const arrivals = slot.dispatchArrivals === '------' ? '000000' : slot.dispatchArrivals;
|
|
@@ -1335,11 +1535,22 @@ export class Watcher {
|
|
|
1335
1535
|
allowed = await this.authorized(item.peer);
|
|
1336
1536
|
}
|
|
1337
1537
|
catch (error) {
|
|
1538
|
+
const now = this.now();
|
|
1539
|
+
const stored = this.state.outbox.find(o => o.id === item.id);
|
|
1540
|
+
const since = stored?.kind === 'inform' ? stored.deferredSince ?? now : now;
|
|
1541
|
+
if (now - since >= DEFERRAL_LIMIT_MS) {
|
|
1542
|
+
// Put off for 24 h without a verdict: dropped. The peer's human gets the one outage notice.
|
|
1543
|
+
this.deps.log('inform-dropped', { id: item.id, peer: item.peer, threadId: item.threadId, reason: 'authority-unverified-24h', ...errInfo(error) });
|
|
1544
|
+
await this.mutate(() => this.finishOutbox(item));
|
|
1545
|
+
return;
|
|
1546
|
+
}
|
|
1338
1547
|
this.deps.log('inform-deferred', { id: item.id, peer: item.peer, threadId: item.threadId, reason: 'authority-unverified', ...errInfo(error) });
|
|
1339
1548
|
await this.mutate(() => {
|
|
1340
|
-
const
|
|
1341
|
-
if (
|
|
1342
|
-
|
|
1549
|
+
const current = this.state.outbox.find(o => o.id === item.id);
|
|
1550
|
+
if (current?.kind !== 'inform')
|
|
1551
|
+
return;
|
|
1552
|
+
current.nextAttemptAt = now + WAKE_RETRY_MS;
|
|
1553
|
+
current.deferredSince ??= since;
|
|
1343
1554
|
});
|
|
1344
1555
|
return;
|
|
1345
1556
|
}
|
|
@@ -1348,6 +1559,15 @@ export class Watcher {
|
|
|
1348
1559
|
await this.mutate(() => this.finishOutbox(item));
|
|
1349
1560
|
return;
|
|
1350
1561
|
}
|
|
1562
|
+
if (item.deferredSince !== undefined) {
|
|
1563
|
+
// A positive verdict ends the deferral: a post that then fails is retried on its own clock, never dropped
|
|
1564
|
+
// for the earlier outage.
|
|
1565
|
+
await this.mutate(() => {
|
|
1566
|
+
const current = this.state.outbox.find(o => o.id === item.id);
|
|
1567
|
+
if (current?.kind === 'inform')
|
|
1568
|
+
delete current.deferredSince;
|
|
1569
|
+
});
|
|
1570
|
+
}
|
|
1351
1571
|
await this.deps.postInform?.(item.peer, item.threadId, item.text, item.pauseId);
|
|
1352
1572
|
}
|
|
1353
1573
|
else {
|
|
@@ -1374,7 +1594,7 @@ export class Watcher {
|
|
|
1374
1594
|
if (episode) {
|
|
1375
1595
|
if ('repeats' in episode)
|
|
1376
1596
|
item.notice.repeats = stored.notice.repeats = episode.repeats;
|
|
1377
|
-
else {
|
|
1597
|
+
else if ('count' in episode) {
|
|
1378
1598
|
item.notice.threadId = stored.notice.threadId = episode.threadId;
|
|
1379
1599
|
item.notice.count = stored.notice.count = episode.count;
|
|
1380
1600
|
episode.arrivalFirstSeenAt = EPOCH;
|
|
@@ -1540,11 +1760,9 @@ export class Watcher {
|
|
|
1540
1760
|
await this.deps.beforePoll?.();
|
|
1541
1761
|
await this.checkAuthority();
|
|
1542
1762
|
let authorityAt = this.now();
|
|
1543
|
-
await this.migrateLegacyState();
|
|
1544
|
-
await this.loadState();
|
|
1763
|
+
await this.loadState(await this.migrateLegacyState());
|
|
1545
1764
|
log('start', { debounceMs: this.opts.debounceMs, pollMs: this.opts.pollMs, ticketSeconds: this.opts.ticketSeconds, once: Boolean(options.once) });
|
|
1546
|
-
await this.
|
|
1547
|
-
this.lastClaimPrune = this.now();
|
|
1765
|
+
await this.pruneClaimsIfDue();
|
|
1548
1766
|
const reconcile = () => {
|
|
1549
1767
|
const desired = new Map();
|
|
1550
1768
|
for (const peer of this.deps.loadPeers()) {
|
|
@@ -1618,10 +1836,8 @@ export class Watcher {
|
|
|
1618
1836
|
else {
|
|
1619
1837
|
await this.syncDirty();
|
|
1620
1838
|
}
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
await this.pruneClaims();
|
|
1624
|
-
}
|
|
1839
|
+
await this.pruneClaimsIfDue();
|
|
1840
|
+
this.saveLocal();
|
|
1625
1841
|
}
|
|
1626
1842
|
}
|
|
1627
1843
|
}
|
|
@@ -1636,6 +1852,7 @@ export class Watcher {
|
|
|
1636
1852
|
controller.abort();
|
|
1637
1853
|
external.removeEventListener('abort', onExternalAbort);
|
|
1638
1854
|
this.onFatal = undefined;
|
|
1855
|
+
this.saveLocal();
|
|
1639
1856
|
log('stop');
|
|
1640
1857
|
}
|
|
1641
1858
|
if (this.fatalError)
|