@staix/agent-hub 0.20.0 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +66 -3
- package/README.md +2 -2
- package/docs/agent-notes/README.md +11 -0
- package/docs/agent-notes/adapters.md +7 -3
- package/docs/agent-notes/benchmarks.md +27 -0
- package/docs/agent-notes/budget.md +2 -1
- package/docs/agent-notes/bus.md +3 -3
- package/docs/agent-notes/daemon.md +17 -8
- package/docs/agent-notes/local-worker.md +4 -0
- package/docs/agent-notes/models.md +2 -0
- package/docs/agent-notes/tasks.md +17 -9
- package/docs/agent-notes/tests.md +9 -2
- package/docs/bench.md +1 -1
- package/docs/data-policy.md +2 -0
- package/docs/dogfood.md +39 -4
- package/docs/events.md +2 -1
- package/docs/memory.md +9 -3
- package/docs/operations.md +610 -62
- package/docs/profiles.md +13 -1
- package/docs/quickstart.md +18 -9
- package/docs/research.md +7 -4
- package/docs/sdlc.md +54 -10
- package/docs/security.md +396 -34
- package/docs/smoke.md +133 -0
- package/docs/specs/2026-09-19-agent-hub-design.md +1101 -22
- package/docs/verification/2026-10-27-0.19.0.md +5 -5
- package/docs/verification/2026-10-28-claude-2.1.311-2.1.312-bash-permission.json +635 -0
- package/docs/verification/2026-10-28-claude-2.1.311-2.1.312-permission-transport.json +403 -0
- package/docs/verification/2026-10-28-claude-implementer-permission-triage.md +123 -0
- package/docs/verification/2026-10-28-jcode-0.99.1-usage.md +41 -0
- package/docs/verification/2026-10-28-realpath-name-audit.md +222 -0
- package/docs/verification/2026-10-28-routing-f0bb8ad0.md +1000 -0
- package/docs/verification/2026-10-29-0.21.0.md +61 -0
- package/docs/verification/2026-10-29-model-effort-capabilities.md +123 -0
- package/docs/verification/2026-10-29-modelrouting-b152ddb9.md +953 -0
- package/docs/verification/2026-10-30-0.22.0.md +49 -0
- package/docs/verification/2026-10-30-445-capability-probes.md +172 -0
- package/docs/verification/2026-10-30-bus-delivery-operational-settings.md +77 -0
- package/docs/verification/2026-10-30-jcode-ac4.md +24 -0
- package/docs/verification/2026-10-30-jcode-ac6.md +52 -0
- package/docs/verification/2026-10-30-recovery-wait-settings.md +141 -0
- package/docs/verification/claude-implementer-permission-triage.ts +160 -0
- package/docs/verification/claude-permission-transport-native.py +71 -14
- package/docs/verification/profiles/YYYY-MM-DD-research.md +1 -1
- package/docs/verified.json +15 -15
- package/package.json +2 -2
- package/plugins/agent-hub/.claude-plugin/plugin.json +1 -1
- package/plugins/agent-hub/server.js +1016 -280
- package/src/adapters/acp.ts +79 -3
- package/src/adapters/channel-peer.ts +4 -0
- package/src/adapters/claude-channel.ts +13 -6
- package/src/adapters/claude-implementer.ts +60 -7
- package/src/adapters/codex-appserver.ts +77 -4
- package/src/adapters/conformance/fakes/jcode-agent.ts +6 -2
- package/src/adapters/conformance/kit.ts +3 -9
- package/src/adapters/definitions/builtin-metadata.ts +3 -114
- package/src/adapters/definitions/native-config-defaults.ts +1 -1
- package/src/adapters/definitions/native-config-policy.ts +3 -2
- package/src/adapters/definitions/native-context.ts +12 -0
- package/src/adapters/external-metadata.ts +48 -0
- package/src/adapters/external-modules.ts +63 -3
- package/src/adapters/jcode.ts +6 -2
- package/src/adapters/local-worker.ts +40 -25
- package/src/adapters/native-proof-ports.ts +10 -0
- package/src/adapters/registry-types.ts +4 -0
- package/src/broker/protocol.ts +203 -0
- package/src/broker/server.ts +314 -0
- package/src/broker/store.ts +548 -0
- package/src/cli/bench.ts +3 -2
- package/src/cli/broker.ts +58 -0
- package/src/cli/conductor-recovery.ts +142 -0
- package/src/cli/configured-acp-entry.ts +26 -6
- package/src/cli/console-state.ts +59 -17
- package/src/cli/console.ts +17 -2
- package/src/cli/facts-hook.ts +3 -1
- package/src/cli/help.ts +29 -16
- package/src/cli/identity.ts +25 -10
- package/src/cli/init.ts +34 -7
- package/src/cli/intake-output.ts +33 -0
- package/src/cli/learn-output.ts +38 -0
- package/src/cli/main.ts +334 -73
- package/src/cli/output.ts +41 -1
- package/src/cli/recovery-package.ts +18 -1
- package/src/cli/recovery-waits.ts +26 -0
- package/src/cli/role-holders.ts +35 -0
- package/src/cli/room.ts +304 -0
- package/src/cli/say-input.ts +16 -0
- package/src/cli/schedule-output.ts +49 -0
- package/src/cli/sdlc-output.ts +44 -6
- package/src/cli/smoke-pty-bridge.ts +82 -0
- package/src/cli/smoke-pty.ts +153 -0
- package/src/cli/smoke-run.ts +139 -0
- package/src/cli/smoke-target.ts +130 -0
- package/src/cli/source-output.ts +29 -0
- package/src/cli/task-output.ts +3 -2
- package/src/cli/upgrade-runtime.ts +112 -28
- package/src/cli/upgrade.ts +68 -8
- package/src/hub/ask.ts +76 -12
- package/src/hub/board.ts +132 -22
- package/src/hub/bus.ts +416 -65
- package/src/hub/child-process.ts +2 -0
- package/src/hub/ci-watch.ts +42 -20
- package/src/hub/claude-failure-stage.ts +85 -0
- package/src/hub/claude-shell-wrapper.ts +11 -5
- package/src/hub/conductor.ts +17 -4
- package/src/hub/config-apply.ts +34 -6
- package/src/hub/config-reload.ts +7 -5
- package/src/hub/config-trust.ts +7 -1
- package/src/hub/context-window.ts +33 -3
- package/src/hub/control-client.ts +14 -2
- package/src/hub/corpus-config.ts +34 -0
- package/src/hub/corpus-evidence.ts +184 -0
- package/src/hub/corpus.ts +373 -0
- package/src/hub/cost-governance.ts +26 -0
- package/src/hub/daemon.ts +1209 -183
- package/src/hub/deliverable-files.ts +8 -2
- package/src/hub/delivery-journal.ts +200 -73
- package/src/hub/dogfood-profile.ts +32 -5
- package/src/hub/dogfood-read.ts +6 -2
- package/src/hub/envelope.ts +11 -0
- package/src/hub/eval-gate.ts +131 -0
- package/src/hub/events.ts +1 -1
- package/src/hub/external-instance.ts +69 -0
- package/src/hub/facts.ts +15 -5
- package/src/hub/folder-trigger.ts +223 -0
- package/src/hub/hub-tools.ts +2 -1
- package/src/hub/inference.ts +20 -5
- package/src/hub/instance-config.ts +69 -3
- package/src/hub/instance-execution.ts +6 -2
- package/src/hub/instances.ts +32 -4
- package/src/hub/intake.ts +125 -0
- package/src/hub/learning.ts +117 -0
- package/src/hub/lifecycle.ts +7 -3
- package/src/hub/limits.ts +12 -5
- package/src/hub/native-permission-request.ts +77 -0
- package/src/hub/operational-authority.ts +63 -5
- package/src/hub/operational-settings.ts +154 -0
- package/src/hub/operational-view.ts +1 -1
- package/src/hub/packs.ts +132 -0
- package/src/hub/path-policy.ts +7 -0
- package/src/hub/peer-doing.ts +130 -0
- package/src/hub/peers.ts +12 -0
- package/src/hub/queue-backpressure.ts +58 -0
- package/src/hub/read-tools.ts +2 -0
- package/src/hub/recovery-authority.ts +151 -0
- package/src/hub/recovery-policy.ts +220 -0
- package/src/hub/research.ts +24 -14
- package/src/hub/restart.ts +7 -0
- package/src/hub/routing.ts +54 -11
- package/src/hub/schedules.ts +266 -0
- package/src/hub/sdlc-merge.ts +38 -14
- package/src/hub/sdlc-metrics.ts +32 -10
- package/src/hub/sdlc-queue.ts +7 -1
- package/src/hub/sdlc-release.ts +1 -1
- package/src/hub/sdlc.ts +37 -7
- package/src/hub/security-baseline.ts +38 -0
- package/src/hub/settings.ts +27 -2
- package/src/hub/shipped-packs.ts +8 -0
- package/src/hub/snapshots.ts +72 -6
- package/src/hub/source-read.ts +160 -0
- package/src/hub/sources.ts +311 -0
- package/src/hub/tasks.ts +987 -110
- package/src/hub/workflow.ts +1 -1
- package/src/local/doc-read.ts +69 -0
- package/src/local/proxy.ts +4 -13
- package/src/local/tools.ts +37 -3
- package/src/local/web-fetch-config.ts +43 -0
- package/src/local/web-fetch.ts +183 -0
- package/src/memory/brief.ts +85 -13
- package/src/memory/capture.ts +1 -1
- package/src/rooms/application.ts +114 -0
- package/src/rooms/boundary.ts +103 -0
- package/src/rooms/commands.ts +294 -0
- package/src/rooms/crypto.ts +371 -0
- package/src/rooms/link.ts +584 -0
- package/src/rooms/policy.ts +82 -0
- package/src/rooms/remote-peer.ts +80 -0
- package/src/rooms/service.ts +426 -0
- package/src/rooms/status.ts +146 -0
- package/src/rooms/store.ts +1210 -0
- package/src/rooms/wire.ts +465 -0
- package/src/ui/index.html +91 -4
- package/templates/packs/example.pack.json +13 -0
package/src/hub/bus.ts
CHANGED
|
@@ -1,8 +1,11 @@
|
|
|
1
1
|
import { appendNote, DIGEST, frame, HUB, keepNotes, MAX_HOP, mergePrefaces, newEnvelope, noteLine, parseMarker, quoteNotes, replyAudience, uniqueMemoryLines, type Envelope, type EnvelopeOpts, type PeerId, type PeerState, type Priority } from "./envelope.ts";
|
|
2
2
|
import type { PeerAdapter, DeliveryReceipt } from "./peers.ts";
|
|
3
|
+
import type { DoingKind } from "./peer-doing.ts";
|
|
4
|
+
import type { QueuePressure } from "./queue-backpressure.ts";
|
|
3
5
|
import { DeliveryJournal, MAX_ENVELOPE_BODY, type JournalDelivery, type JournalDeliveryState } from "./delivery-journal.ts";
|
|
4
6
|
import { DELIVERY_RETRY_WARNING } from "./delivery-hints.ts";
|
|
5
7
|
import { MAX_DEFERRED_HUB, RecoveryDeferredHubError, validateDeferredHub } from "./deferred-hub.ts";
|
|
8
|
+
import { parseRemotePeerId } from "../rooms/application.ts";
|
|
6
9
|
import { PreEffectToolRefusal } from "./tool-refusal.ts";
|
|
7
10
|
|
|
8
11
|
export type { DeliveryReceipt } from "./peers.ts";
|
|
@@ -22,6 +25,10 @@ export type BusEvent =
|
|
|
22
25
|
| { t: "state"; peer: PeerId; state: PeerState };
|
|
23
26
|
|
|
24
27
|
export interface BusOptions {
|
|
28
|
+
/** Durable application outbox ids; the bus combines them with its own pending ids without recursion. */
|
|
29
|
+
remotePending?: (room: string) => readonly string[];
|
|
30
|
+
/** Host provenance at enqueue, never a body screen; unknown/throw marks the original private. */
|
|
31
|
+
remotePrivate?: (env: Readonly<Envelope>) => boolean | undefined;
|
|
25
32
|
/** Live project membership; absent preserves the existing audience and delivery behaviour. */
|
|
26
33
|
rostered?: (peer: PeerId) => boolean;
|
|
27
34
|
/** Transient owner lifecycle fence: retain new work without changing console/budget pauses. */
|
|
@@ -34,9 +41,16 @@ export interface BusOptions {
|
|
|
34
41
|
/** How long a status envelope may wait for company before it is delivered on its own. */
|
|
35
42
|
batchMs: number;
|
|
36
43
|
queueCap: number;
|
|
44
|
+
/** Accepted live operational settings; read only at the next matching delivery/note operation. */
|
|
45
|
+
maxAttempts?: () => number;
|
|
46
|
+
digestMax?: () => number;
|
|
47
|
+
pullBudgetBytes?: () => number;
|
|
48
|
+
notesKept?: () => number;
|
|
37
49
|
/** Optional: rewrite a delivery before it goes out (M6 digest condensation). Must return its input on any failure. */
|
|
38
50
|
condense?: (envs: Envelope[]) => Promise<Envelope[]>;
|
|
39
51
|
journal?: DeliveryJournal;
|
|
52
|
+
/** Host-owned opt-in; no delivery body or timestamp is a routing-policy authority. */
|
|
53
|
+
trackQueuePressure?: () => boolean;
|
|
40
54
|
/**
|
|
41
55
|
* Optional: may an agent send this envelope now (issue #38)? `parent` is the id of what it answers. A reason refuses
|
|
42
56
|
* it, and the sender hears it on its next delivery.
|
|
@@ -59,6 +73,8 @@ export interface BusOptions {
|
|
|
59
73
|
* dequeued, steered or journaled for it, and the queue still waits for `Bus.pull`; a throw is swallowed.
|
|
60
74
|
*/
|
|
61
75
|
onImportantQueued?: (peer: PeerId, counts: { queued: number; important: number }) => void;
|
|
76
|
+
/** Optional (#878): an attached adapter observed its session title or current plan item. Never published, journaled or tapped. */
|
|
77
|
+
doing?: (peer: PeerId, kind: DoingKind, text?: unknown) => void;
|
|
62
78
|
}
|
|
63
79
|
|
|
64
80
|
/** Serializable delivery state used by the controlled restart coordinator. Bodies stay in the private daemon file. */
|
|
@@ -70,8 +86,16 @@ export interface BusSnapshot {
|
|
|
70
86
|
attempts: Record<string, number>;
|
|
71
87
|
withdrawn: string[];
|
|
72
88
|
manualPaused?: string[];
|
|
89
|
+
/** Host enqueue provenance only. null means an age lost to a legacy/restored writer. */
|
|
90
|
+
queuedAt?: Record<string, Record<string, number | null>>;
|
|
73
91
|
/** Hub notices accepted during recovery, durable but outside the carried delivery queues. */
|
|
74
92
|
deferredHub?: Envelope[];
|
|
93
|
+
/** Local enqueue deadlines; absent old metadata is unknown, never a fresh TTL. */
|
|
94
|
+
remoteQueueExpires?: Record<string, Record<string, number>>;
|
|
95
|
+
/** Host room binding for retained queues, never reconstructed from a remote label. */
|
|
96
|
+
remoteQueueRooms?: Record<string, string>;
|
|
97
|
+
/** Original verified target identity; a reused room label cannot inherit queued text. */
|
|
98
|
+
remoteQueueMembers?: Record<string, string>;
|
|
75
99
|
journal?: ReturnType<DeliveryJournal["snapshot"]>;
|
|
76
100
|
}
|
|
77
101
|
|
|
@@ -99,14 +123,20 @@ function resolveTo(to: PeerId[], originals: Envelope[] | undefined): PeerId[] {
|
|
|
99
123
|
}
|
|
100
124
|
|
|
101
125
|
const SEEN_CAP = 2048;
|
|
102
|
-
const MAX_ATTEMPTS = 3;
|
|
126
|
+
export const MAX_ATTEMPTS = 3;
|
|
127
|
+
// ponytail: provenance is capped at 2048 ids/peer, 4096 peers, 100000 ids total and 256 KiB per snapshot.
|
|
128
|
+
// Omitted metadata means unknown age only; use a measured per-envelope store if larger queues need provenance.
|
|
129
|
+
const QUEUE_TIME_CAP = 2048, QUEUE_TIME_PEERS_CAP = 4096, QUEUE_TIME_TOTAL_CAP = 100_000, QUEUE_TIME_SNAPSHOT_BYTES = 256 * 1024;
|
|
130
|
+
const REMOTE_ROOM_CAP = 200, REMOTE_QUEUE_TTL_MS = 24 * 60 * 60 * 1000;
|
|
131
|
+
const remoteId = (id: string): boolean => id.includes("@");
|
|
132
|
+
const remoteFingerprint = (value: unknown): value is string => typeof value === "string" && /^[0-9a-f]{64}$/.test(value);
|
|
103
133
|
// ponytail: one digest carries at most ten originals; larger supervision windows split into bounded deliveries.
|
|
104
134
|
// Revisit this ceiling after measuring native supervision cost, rather than bypassing the delivery limit.
|
|
105
|
-
const DIGEST_MAX = 10;
|
|
135
|
+
export const DIGEST_MAX = 10;
|
|
106
136
|
// #713: a pull-only peer reads between steps, often minutes apart, so one pull drains its queue up to this many bytes of rendered text instead of ten items.
|
|
107
137
|
// Bounded, not unbounded: a queue larger than the budget takes more than one pull, and the reply says how many are still waiting. Push peers keep DIGEST_MAX.
|
|
108
|
-
const PULL_BUDGET_BYTES = 64 * 1024;
|
|
109
|
-
const NOTES_KEPT = 10;
|
|
138
|
+
export const PULL_BUDGET_BYTES = 64 * 1024;
|
|
139
|
+
export const NOTES_KEPT = 10;
|
|
110
140
|
|
|
111
141
|
/**
|
|
112
142
|
* N-peer fan-out. Never delivers back to `from`, caps hops, dedupes by id. Each peer has one queue,
|
|
@@ -118,7 +148,15 @@ export class Bus {
|
|
|
118
148
|
private readonly opts: BusOptions;
|
|
119
149
|
private readonly journal?: DeliveryJournal;
|
|
120
150
|
private readonly queues = new Map<PeerId, Envelope[]>();
|
|
151
|
+
private readonly queueTimes = new Map<PeerId, Map<string, number | null>>();
|
|
152
|
+
private queueTimeEntries = 0;
|
|
153
|
+
private queuePressureWasTracked = false;
|
|
121
154
|
private readonly prefaces = new Map<PeerId, Envelope>();
|
|
155
|
+
private readonly remoteQueueMembers = new Map<PeerId, string>();
|
|
156
|
+
private readonly remoteQueueRooms = new Map<PeerId, string>();
|
|
157
|
+
private readonly remoteQueueExpires = new Map<PeerId, Map<string, number>>();
|
|
158
|
+
private readonly remoteReservations = new Map<string, Set<string>>();
|
|
159
|
+
private readonly remoteTransport = new Map<string, { peer: string; originals: Envelope[] }>();
|
|
122
160
|
private readonly paused = new Set<PeerId>();
|
|
123
161
|
private readonly manualPaused = new Set<PeerId>();
|
|
124
162
|
private readonly timers = new Map<PeerId, ReturnType<typeof setTimeout>>();
|
|
@@ -168,6 +206,7 @@ export class Bus {
|
|
|
168
206
|
this.opts = { ...DEFAULT_BUS, ...opts };
|
|
169
207
|
this.onDeliver = opts.onDeliver;
|
|
170
208
|
this.journal = opts.journal;
|
|
209
|
+
this.trackingQueuePressure();
|
|
171
210
|
if (this.journal) {
|
|
172
211
|
const state = this.journal.snapshot();
|
|
173
212
|
const bus = state.bus as BusSnapshot;
|
|
@@ -191,9 +230,22 @@ export class Bus {
|
|
|
191
230
|
}
|
|
192
231
|
|
|
193
232
|
add(peer: PeerAdapter): void {
|
|
233
|
+
if (peer.remoteRoom !== undefined && (!parseRemotePeerId(peer.id) || typeof peer.remoteRoom !== "string" || !peer.remoteRoom || peer.remoteRoom.length > 256 || !remoteFingerprint(peer.remoteMember))) throw new Error("invalid remote peer binding");
|
|
234
|
+
const pendingRemote = peer.remoteRoom !== undefined && this.hasRemoteWork(peer.id);
|
|
235
|
+
if (peer.remoteRoom !== undefined && pendingRemote) {
|
|
236
|
+
const room = this.remoteQueueRooms.get(peer.id), member = this.remoteQueueMembers.get(peer.id);
|
|
237
|
+
if (room === undefined || member === undefined) {
|
|
238
|
+
this.expireRemoteQueues();
|
|
239
|
+
if (this.hasRemoteWork(peer.id)) throw new Error("remote queued identity binding unavailable");
|
|
240
|
+
} else if (room !== peer.remoteRoom) throw new Error("remote queued room binding changed");
|
|
241
|
+
else if (member !== peer.remoteMember) throw new Error("remote queued member binding changed");
|
|
242
|
+
}
|
|
243
|
+
if (peer.remoteRoom !== undefined && !pendingRemote) { this.remoteQueueRooms.delete(peer.id); this.remoteQueueMembers.delete(peer.id); }
|
|
194
244
|
this.peers.set(peer.id, peer);
|
|
195
245
|
if (!this.queues.has(peer.id)) this.queues.set(peer.id, []);
|
|
196
246
|
peer.onDelivery = (receipt) => this.deliveryReceipt(peer.id, receipt);
|
|
247
|
+
peer.onDoing = (kind, text) => { if (!this.closed && this.peers.get(peer.id) === peer) this.opts.doing?.(peer.id, kind, text); };
|
|
248
|
+
this.opts.doing?.(peer.id, "reset"); // a new adapter never shows what the one it replaces was doing
|
|
197
249
|
peer.onMessage = (text, opts) => {
|
|
198
250
|
if (!this.isRostered(peer.id)) return `off roster: ${peer.id}`;
|
|
199
251
|
const { priority, body } = parseMarker(text);
|
|
@@ -208,6 +260,12 @@ export class Bus {
|
|
|
208
260
|
const audienceOriginals = originals ?? (opts?.to?.includes(DIGEST) && condensed ? last?.originals : undefined);
|
|
209
261
|
const to = opts?.to?.length ? resolveTo(opts.to, audienceOriginals) : undefined;
|
|
210
262
|
let env = newEnvelope(peer.id, body, { ...opts, ...(to ? { to } : {}), priority: opts?.priority ?? capPriority(peer, priority, opts?.inReplyTo, originals) });
|
|
263
|
+
try { this.checkRemotePublication(env); }
|
|
264
|
+
catch (error) {
|
|
265
|
+
if (!(error instanceof PreEffectToolRefusal)) throw error;
|
|
266
|
+
this.note(peer.id, noteLine(HUB, "decision", `your message was not delivered: ${error.message}`));
|
|
267
|
+
return error.message;
|
|
268
|
+
}
|
|
211
269
|
const offRoster = this.offRosterRecipients(env);
|
|
212
270
|
if (offRoster.length && offRoster.length === new Set(env.to?.filter(id => id !== env.from)).size) {
|
|
213
271
|
const reason = `off roster: ${offRoster.join(", ")}`;
|
|
@@ -233,7 +291,13 @@ export class Bus {
|
|
|
233
291
|
this.note(peer.id, noteLine(HUB, "decision", `your message was not delivered: ${refused}`));
|
|
234
292
|
return refused;
|
|
235
293
|
}
|
|
236
|
-
|
|
294
|
+
let published: PublishResult;
|
|
295
|
+
try { published = this.publishResult(env); }
|
|
296
|
+
catch (error) {
|
|
297
|
+
if (!(error instanceof PreEffectToolRefusal)) throw error;
|
|
298
|
+
this.note(peer.id, noteLine(HUB, "decision", `your message was not delivered: ${error.message}`));
|
|
299
|
+
return error.message;
|
|
300
|
+
}
|
|
237
301
|
if (offRoster.length && published.dropped !== "hop") this.note(peer.id, noteLine(HUB, "decision", `your message is retained off roster for ${offRoster.join(", ")}; delivered to the other recipients`));
|
|
238
302
|
// The sender's result (issue #107): a turn answer has no caller, so it hears it on its next delivery. It counts as
|
|
239
303
|
// not sent only when nobody got it.
|
|
@@ -267,10 +331,11 @@ export class Bus {
|
|
|
267
331
|
|
|
268
332
|
private persist(): void {
|
|
269
333
|
if (!this.journal) return;
|
|
334
|
+
const tracking = this.trackingQueuePressure();
|
|
270
335
|
try {
|
|
271
336
|
this.journal.transaction(() => {
|
|
272
337
|
for (const outcome of this.pendingOutcomes) this.journal!.createDelivery(outcome);
|
|
273
|
-
this.journal!.persistBus(this.snapshotWithoutJournal(), [...this.manualPaused]);
|
|
338
|
+
this.journal!.persistBus(this.snapshotWithoutJournal(), [...this.manualPaused], tracking);
|
|
274
339
|
});
|
|
275
340
|
this.pendingOutcomes.length = 0;
|
|
276
341
|
this.storageError = undefined;
|
|
@@ -281,7 +346,7 @@ export class Bus {
|
|
|
281
346
|
private durableHandoff(peer: PeerId, deliveryId: string, originals: Envelope[], out: Envelope[], observe = true): void {
|
|
282
347
|
if (!this.journal) return;
|
|
283
348
|
const delivery = { id: deliveryId, peer, state: "dispatching" as const, createdAt: Date.now(), originals, out };
|
|
284
|
-
this.journal.checkpointDelivery(this.snapshotWithoutJournal(), delivery);
|
|
349
|
+
this.journal.checkpointDelivery(this.snapshotWithoutJournal(), delivery, this.trackingQueuePressure());
|
|
285
350
|
if (observe) this.memoryDelivery(peer, deliveryId, out, "rendered", originals);
|
|
286
351
|
}
|
|
287
352
|
|
|
@@ -375,16 +440,39 @@ export class Bus {
|
|
|
375
440
|
|
|
376
441
|
/** A bounded, JSON-safe representation of queued work and retry/dedupe state. */
|
|
377
442
|
snapshot(includeJournal = true): BusSnapshot {
|
|
443
|
+
const rooms = this.remoteRoomSnapshot(), members = this.remoteMemberSnapshot();
|
|
444
|
+
const queues = Object.fromEntries([...this.queues].map(([id, queue]) => [id, queue.map((e) => ({ ...e, ...(e.to ? { to: [...e.to] } : {}) }))]));
|
|
445
|
+
for (const peer of new Set([...Object.keys(rooms), ...Object.keys(members)])) queues[peer] ??= [];
|
|
378
446
|
const out: BusSnapshot = {
|
|
379
447
|
schemaVersion: 1,
|
|
380
|
-
queues
|
|
448
|
+
queues,
|
|
381
449
|
prefaces: Object.fromEntries([...this.prefaces].map(([id, e]) => [id, { ...e, ...(e.to ? { to: [...e.to] } : {}) }])),
|
|
382
450
|
seen: [...this.seen.values()].map((e) => ({ ...e, ...(e.to ? { to: [...e.to] } : {}) })),
|
|
383
451
|
attempts: Object.fromEntries(this.attempts),
|
|
384
452
|
withdrawn: [...this.withdrawn],
|
|
385
453
|
manualPaused: [...this.manualPaused],
|
|
386
454
|
...(this.deferredHub.size ? { deferredHub: structuredClone([...this.deferredHub.values()]) } : {}),
|
|
455
|
+
...(this.remoteQueueExpires.size ? { remoteQueueExpires: this.remoteExpirySnapshot() } : {}),
|
|
456
|
+
...(Object.keys(rooms).length ? { remoteQueueRooms: rooms } : {}),
|
|
457
|
+
...(Object.keys(members).length ? { remoteQueueMembers: members } : {}),
|
|
387
458
|
};
|
|
459
|
+
if (this.trackingQueuePressure()) {
|
|
460
|
+
// Metadata is advisory: omit capped or unknown entries rather than enlarge a delivery without bound.
|
|
461
|
+
const queuedAt = Object.create(null) as NonNullable<BusSnapshot["queuedAt"]>;
|
|
462
|
+
let bytes = 32;
|
|
463
|
+
for (const [peer, queue] of this.queues) {
|
|
464
|
+
const times = Object.create(null) as Record<string, number>;
|
|
465
|
+
const peerBytes = Buffer.byteLength(JSON.stringify(peer)) + 4;
|
|
466
|
+
for (const env of queue) {
|
|
467
|
+
const at = this.queueTimes.get(peer)?.get(env.id);
|
|
468
|
+
const entryBytes = Buffer.byteLength(JSON.stringify(env.id)) + 20;
|
|
469
|
+
if (typeof at !== "number" || bytes + peerBytes + entryBytes > QUEUE_TIME_SNAPSHOT_BYTES) continue;
|
|
470
|
+
times[env.id] = at; bytes += entryBytes;
|
|
471
|
+
}
|
|
472
|
+
if (Object.keys(times).length) { queuedAt[peer] = times; bytes += peerBytes; }
|
|
473
|
+
}
|
|
474
|
+
if (Object.keys(queuedAt).length) out.queuedAt = queuedAt;
|
|
475
|
+
}
|
|
388
476
|
if (includeJournal && this.journal) out.journal = this.journal.snapshot();
|
|
389
477
|
return out;
|
|
390
478
|
}
|
|
@@ -395,7 +483,7 @@ export class Bus {
|
|
|
395
483
|
if (_operationId && this.journal) {
|
|
396
484
|
const current = this.journal.snapshot();
|
|
397
485
|
if (current.revision === 0) {
|
|
398
|
-
this.journal.importSnapshot(snapshot.journal ?? { schemaVersion: 1, revision: 1, instanceId: this.journal.instanceId, bus: snapshot, deliveries: [], manualPaused: snapshot.manualPaused ?? [] }, _operationId);
|
|
486
|
+
this.journal.importSnapshot(snapshot.journal ?? { schemaVersion: 1, revision: 1, instanceId: this.journal.instanceId, bus: snapshot, deliveries: [], manualPaused: snapshot.manualPaused ?? [] }, _operationId, this.trackingQueuePressure());
|
|
399
487
|
}
|
|
400
488
|
const committedQueues = snapshot.queues;
|
|
401
489
|
snapshot = this.journal.snapshot().bus;
|
|
@@ -417,6 +505,24 @@ export class Bus {
|
|
|
417
505
|
for (const env of deferred) this.deferredHub.set(env.id, structuredClone(env));
|
|
418
506
|
this.queues.clear();
|
|
419
507
|
for (const [id, queue] of Object.entries(snapshot.queues ?? {})) this.queues.set(id, queue.map((e) => ({ ...e, ...(e.to ? { to: [...e.to] } : {}) })));
|
|
508
|
+
this.queueTimes.clear(); this.queueTimeEntries = 0;
|
|
509
|
+
if (this.trackingQueuePressure()) for (const [peer, queue] of this.queues) {
|
|
510
|
+
const provenance = snapshot.queuedAt;
|
|
511
|
+
const times = provenance && typeof provenance === "object" && !Array.isArray(provenance) && Object.hasOwn(provenance, peer) ? provenance[peer] : undefined;
|
|
512
|
+
for (const env of queue) {
|
|
513
|
+
const at = times && typeof times === "object" && !Array.isArray(times) && Object.hasOwn(times, env.id) ? times[env.id] : null;
|
|
514
|
+
this.recordQueueTime(peer, env.id, typeof at === "number" && Number.isSafeInteger(at) && at > 0 ? at : null);
|
|
515
|
+
}
|
|
516
|
+
}
|
|
517
|
+
this.remoteQueueMembers.clear();
|
|
518
|
+
for (const [peer, member] of Object.entries(snapshot.remoteQueueMembers ?? {})) if (parseRemotePeerId(peer) && remoteFingerprint(member)) this.remoteQueueMembers.set(peer, member);
|
|
519
|
+
this.remoteQueueRooms.clear();
|
|
520
|
+
for (const [peer, room] of Object.entries(snapshot.remoteQueueRooms ?? {})) if (parseRemotePeerId(peer) && typeof room === "string" && room.length > 0 && room.length <= 256) this.remoteQueueRooms.set(peer, room);
|
|
521
|
+
this.remoteQueueExpires.clear();
|
|
522
|
+
for (const [peer, values] of Object.entries(snapshot.remoteQueueExpires ?? {})) {
|
|
523
|
+
if (!remoteId(peer)) continue;
|
|
524
|
+
this.remoteQueueExpires.set(peer, new Map(Object.entries(values).filter(([, expiry]) => Number.isSafeInteger(expiry) && expiry >= 0)));
|
|
525
|
+
}
|
|
420
526
|
this.prefaces.clear();
|
|
421
527
|
for (const [id, e] of Object.entries(snapshot.prefaces ?? {})) this.prefaces.set(id, { ...e, body: uniqueMemoryLines(e.body), ...(e.to ? { to: [...e.to] } : {}) });
|
|
422
528
|
this.seen.clear();
|
|
@@ -460,6 +566,7 @@ export class Bus {
|
|
|
460
566
|
}
|
|
461
567
|
|
|
462
568
|
private retryOut(peer: string, originals: Envelope[], out: Envelope[]): Envelope[] {
|
|
569
|
+
if (remoteId(peer)) return out;
|
|
463
570
|
if (!out.length || !this.journal || !this.journal.retryWarning(peer, originals)) return out;
|
|
464
571
|
return out.map((env, index) => {
|
|
465
572
|
if (index !== 0) return env;
|
|
@@ -474,6 +581,7 @@ export class Bus {
|
|
|
474
581
|
|
|
475
582
|
resolveDelivery(id: string, revision: number, action: "completed" | "retry" | "discard", reason: string, actor: string = "user"): void {
|
|
476
583
|
if (!this.journal || this.storageError) throw new Error("delivery journal unavailable");
|
|
584
|
+
this.trackingQueuePressure();
|
|
477
585
|
const record = this.queueShow(id);
|
|
478
586
|
if (!record) throw new Error("unknown delivery");
|
|
479
587
|
const previous = this.journal.resolution(id);
|
|
@@ -481,6 +589,13 @@ export class Bus {
|
|
|
481
589
|
this.journal.resolve(id, revision, action, reason, revision, actor); // verifies identical request; no second enqueue
|
|
482
590
|
return;
|
|
483
591
|
}
|
|
592
|
+
if (action === "retry" && remoteId(record.peer)) {
|
|
593
|
+
const peer = this.peers.get(record.peer), room = this.remoteQueueRooms.get(record.peer), member = this.remoteQueueMembers.get(record.peer);
|
|
594
|
+
if (record.state !== "needs_review" || record.originals.length !== 1 || !room || !remoteFingerprint(member) || peer?.remoteRoom !== room || peer.remoteMember !== member) throw new Error("remote original identity unavailable or changed; discard it or send a new message explicitly");
|
|
595
|
+
const deadline = this.remoteQueueExpires.get(record.peer)?.get(record.originals[0]!.id);
|
|
596
|
+
if (!Number.isSafeInteger(deadline) || deadline! <= Date.now()) throw new PreEffectToolRefusal("remote original deadline unavailable or expired; discard it or send a new message explicitly");
|
|
597
|
+
this.checkRemotePublication(record.originals[0]!);
|
|
598
|
+
}
|
|
484
599
|
const recipientConsent = actor === `peer:${record.peer}` && action === "retry" && record.state === "needs_review";
|
|
485
600
|
// #713: a `queued` delivery was never handed to the peer, so nothing is in flight and a busy peer (a pull-only TUI is busy most of the time) does not block
|
|
486
601
|
// discarding it. `dispatching` and `accepted` stay refused, and every other resolution keeps the busy fence.
|
|
@@ -489,7 +604,7 @@ export class Bus {
|
|
|
489
604
|
if (this.peers.get(record.peer)?.state === "busy" && !recipientConsent && !queuedDiscard) throw new Error("peer busy: its delivery cannot be resolved while it works");
|
|
490
605
|
if (record.revision !== revision) throw new Error("stale delivery revision");
|
|
491
606
|
if (record.state === "queued" && action === "retry") throw new Error("queued delivery is already scheduled");
|
|
492
|
-
if (action === "retry" && (this.queues.get(record.peer)?.length ?? 0) + record.originals.length > this.opts.queueCap) throw new Error("recipient queue is full");
|
|
607
|
+
if (action === "retry" && !remoteId(record.peer) && (this.queues.get(record.peer)?.length ?? 0) + record.originals.length > this.opts.queueCap) throw new Error("recipient queue is full");
|
|
493
608
|
const queue = this.queues.get(record.peer) ?? [];
|
|
494
609
|
const hadQueue = this.queues.has(record.peer);
|
|
495
610
|
const before = [...queue];
|
|
@@ -549,8 +664,22 @@ export class Bus {
|
|
|
549
664
|
if (this.closed) return;
|
|
550
665
|
const owner = this.activeDeliveries.get(receipt.id);
|
|
551
666
|
if (!owner || owner !== peer || !this.journal) return;
|
|
667
|
+
this.trackingQueuePressure();
|
|
552
668
|
const id = receipt.id;
|
|
553
669
|
const before = this.journal.get(id);
|
|
670
|
+
if (this.peers.get(peer)?.remoteRoom !== undefined && receipt.state === "failed_safe") {
|
|
671
|
+
// A signed recipient NACK is a terminal known refusal, not a failed native
|
|
672
|
+
// turn eligible for poison retries. It records no recipient read evidence.
|
|
673
|
+
this.journal.transition(id, "failed", receipt.reason ?? "remote recipient refused safely");
|
|
674
|
+
this.activeDeliveries.delete(id); this.remoteTransport.delete(id);
|
|
675
|
+
for (const env of before?.originals ?? []) { this.attempts.delete(`${peer}:${env.id}`); this.remoteQueueExpires.get(peer)?.delete(env.id); }
|
|
676
|
+
const last = this.lastDelivery.get(peer);
|
|
677
|
+
if (before && last && last.out.length === before.out.length && last.out.every((env, index) => env.id === before.out[index]!.id)) this.lastDelivery.delete(peer);
|
|
678
|
+
this.refreshRecoveryHold(peer);
|
|
679
|
+
this.onQueues?.();
|
|
680
|
+
void this.drain(peer);
|
|
681
|
+
return;
|
|
682
|
+
}
|
|
554
683
|
const state: JournalDeliveryState = receipt.state === "accepted" ? "accepted" : receipt.state === "completed" ? "completed" : receipt.state === "failed_safe" ? "failed" : "needs_review";
|
|
555
684
|
if (state === "failed" || state === "needs_review") this.deliveryFailed(peer);
|
|
556
685
|
if (state === "completed") this.journal.completeDelivered(id, peer, "adapter", receipt.reason);
|
|
@@ -578,6 +707,7 @@ export class Bus {
|
|
|
578
707
|
|
|
579
708
|
completeReply(peer: PeerId, envelopeId: string): void {
|
|
580
709
|
if (!this.journal) return;
|
|
710
|
+
this.trackingQueuePressure();
|
|
581
711
|
for (const [id, owner] of this.activeDeliveries) {
|
|
582
712
|
if (owner !== peer) continue;
|
|
583
713
|
const row = this.journal.get(id);
|
|
@@ -654,6 +784,7 @@ export class Bus {
|
|
|
654
784
|
|
|
655
785
|
/** Context the hub wants the peer to see once: it rides in front of the next delivery instead of costing a turn of its own. */
|
|
656
786
|
preface(id: PeerId, text: string): void {
|
|
787
|
+
if (remoteId(id)) return;
|
|
657
788
|
const body = uniqueMemoryLines(quoteNotes(text));
|
|
658
789
|
const existing = this.prefaces.get(id);
|
|
659
790
|
if (existing) {
|
|
@@ -669,15 +800,17 @@ export class Bus {
|
|
|
669
800
|
|
|
670
801
|
/** Another peer's shared note: rides in the preface like recall. The newest NOTES_KEPT stay; older ones remain in amem. */
|
|
671
802
|
note(id: PeerId, line: string): void {
|
|
803
|
+
if (remoteId(id)) return;
|
|
672
804
|
const existing = this.prefaces.get(id);
|
|
673
|
-
this.prefaces.set(id, existing ? { ...existing, body: keepNotes(uniqueMemoryLines(appendNote(existing.body, line)), NOTES_KEPT) } : newEnvelope(HUB, uniqueMemoryLines(line), { to: [id], kind: "presence" }));
|
|
805
|
+
this.prefaces.set(id, existing ? { ...existing, body: keepNotes(uniqueMemoryLines(appendNote(existing.body, line)), this.opts.notesKept?.() ?? NOTES_KEPT) } : newEnvelope(HUB, uniqueMemoryLines(line), { to: [id], kind: "presence" }));
|
|
674
806
|
this.persist();
|
|
675
807
|
}
|
|
676
808
|
|
|
677
809
|
/** A preface that went out with a failed delivery comes back ahead of whatever was prefaced meanwhile, instead of replacing it. */
|
|
678
810
|
private restorePreface(id: PeerId, env: Envelope): void {
|
|
811
|
+
if (remoteId(id)) return;
|
|
679
812
|
const now = this.prefaces.get(id);
|
|
680
|
-
this.prefaces.set(id, !now ? { ...env, body: uniqueMemoryLines(env.body) } : now.id === env.id ? { ...now, body: uniqueMemoryLines(now.body) } : { ...env, body: keepNotes(mergePrefaces(env.body, now.body), NOTES_KEPT) });
|
|
813
|
+
this.prefaces.set(id, !now ? { ...env, body: uniqueMemoryLines(env.body) } : now.id === env.id ? { ...now, body: uniqueMemoryLines(now.body) } : { ...env, body: keepNotes(mergePrefaces(env.body, now.body), this.opts.notesKept?.() ?? NOTES_KEPT) });
|
|
681
814
|
}
|
|
682
815
|
|
|
683
816
|
/**
|
|
@@ -687,6 +820,7 @@ export class Bus {
|
|
|
687
820
|
*/
|
|
688
821
|
discardDeferredHub(reason: string): string[] {
|
|
689
822
|
if (this.peers.size || this.flushingHub || this.replayingHub || !this.journal || this.storageError) throw new Error("deferred reset requires a stopped, readable delivery journal");
|
|
823
|
+
this.trackingQueuePressure();
|
|
690
824
|
const pending = [...this.deferredHub];
|
|
691
825
|
const settled: string[] = [];
|
|
692
826
|
try {
|
|
@@ -713,6 +847,137 @@ export class Bus {
|
|
|
713
847
|
return [...(this.queues.get(peer) ?? []), ...[...this.deferredHub.values()].filter(e => e.to?.includes(peer))];
|
|
714
848
|
}
|
|
715
849
|
|
|
850
|
+
/** Own accepted queue/preparation/active ids, without consulting the host callback. */
|
|
851
|
+
remotePendingIds(room: string): string[] {
|
|
852
|
+
const ids = new Set<string>();
|
|
853
|
+
for (const [peer, queue] of this.queues) if ((this.remoteQueueRooms.get(peer) ?? this.peers.get(peer)?.remoteRoom) === room) for (const env of queue) ids.add(env.id);
|
|
854
|
+
for (const [peer, batch] of this.preparing) if ((this.remoteQueueRooms.get(peer) ?? this.peers.get(peer)?.remoteRoom) === room) for (const env of batch) ids.add(env.id);
|
|
855
|
+
for (const { peer, originals } of this.remoteTransport.values()) if ((this.remoteQueueRooms.get(peer) ?? this.peers.get(peer)?.remoteRoom) === room) for (const env of originals) ids.add(env.id);
|
|
856
|
+
for (const [id, peer] of this.activeDeliveries) if ((this.remoteQueueRooms.get(peer) ?? this.peers.get(peer)?.remoteRoom) === room) for (const env of this.journal?.get(id)?.originals ?? []) ids.add(env.id);
|
|
857
|
+
return [...ids];
|
|
858
|
+
}
|
|
859
|
+
/** Remote preflight precedes limiter admission; publication rechecks before effects. */
|
|
860
|
+
checkRemotePublication(env: Envelope): string[] {
|
|
861
|
+
if (env.to?.some(remoteId) && this.opts.remotePrivate) {
|
|
862
|
+
try { if (this.opts.remotePrivate(env) !== false) env.private = true; }
|
|
863
|
+
catch { env.private = true; }
|
|
864
|
+
}
|
|
865
|
+
const rooms = new Set<string>(), targets = [...new Set((env.to ?? []).filter(remoteId))];
|
|
866
|
+
if (remoteId(env.from) && !parseRemotePeerId(env.from)) throw new PreEffectToolRefusal("invalid remote sender");
|
|
867
|
+
for (const target of targets) {
|
|
868
|
+
const peer = this.peers.get(target);
|
|
869
|
+
if (!parseRemotePeerId(target) || !peer?.remoteRoom || !remoteFingerprint(peer.remoteMember)) throw new PreEffectToolRefusal("remote recipient unavailable");
|
|
870
|
+
const bound = this.remoteQueueRooms.get(target), member = this.remoteQueueMembers.get(target);
|
|
871
|
+
if (this.hasRemoteWork(target) && (!bound || !member || bound !== peer.remoteRoom || member !== peer.remoteMember)) throw new PreEffectToolRefusal("remote queued identity binding changed");
|
|
872
|
+
const ttl = peer.remoteQueueTtlMs ?? REMOTE_QUEUE_TTL_MS;
|
|
873
|
+
if (!Number.isSafeInteger(ttl) || ttl <= 0 || !Number.isSafeInteger(Date.now() + ttl)) throw new PreEffectToolRefusal("remote queue TTL unavailable");
|
|
874
|
+
rooms.add(peer.remoteRoom);
|
|
875
|
+
}
|
|
876
|
+
// ponytail: one original has one stable wire identity. A follow-up may derive
|
|
877
|
+
// stable per-target wire ids and account each delivery separately before fan-out.
|
|
878
|
+
if (targets.length > 1) throw new PreEffectToolRefusal(`remote targets ${targets.slice(0, 64).join(", ")}${targets.length > 64 ? ` and ${targets.length - 64} more` : ""}: send to each remote peer separately`);
|
|
879
|
+
for (const room of rooms) {
|
|
880
|
+
const pending = new Set([...this.remotePendingIds(room), ...(this.remoteReservations.get(room) ?? [])]);
|
|
881
|
+
let external: readonly string[];
|
|
882
|
+
try { external = this.opts.remotePending?.(room) ?? []; } catch { throw new PreEffectToolRefusal("remote queue unavailable"); }
|
|
883
|
+
if (!Array.isArray(external) || external.length > REMOTE_ROOM_CAP || external.some(id => typeof id !== "string" || id.length < 1 || id.length > 256)) throw new PreEffectToolRefusal("remote queue unavailable");
|
|
884
|
+
for (const id of external) pending.add(id);
|
|
885
|
+
pending.add(env.id);
|
|
886
|
+
if (pending.size > REMOTE_ROOM_CAP) throw new PreEffectToolRefusal("remote room queue is full");
|
|
887
|
+
}
|
|
888
|
+
return [...rooms];
|
|
889
|
+
}
|
|
890
|
+
private hasRemoteWork(peer: string, pending?: ReadonlySet<string>): boolean {
|
|
891
|
+
return (this.queues.get(peer)?.length ?? 0) > 0 || this.preparing.has(peer) || [...this.activeDeliveries.values()].includes(peer) || [...this.remoteTransport.values()].some(value => value.peer === peer) || (pending ?? new Set(this.journal?.pendingPeers() ?? [])).has(peer);
|
|
892
|
+
}
|
|
893
|
+
private remoteBindingSnapshot(values: ReadonlyMap<string, string>): Record<string, string> {
|
|
894
|
+
if (!values.size) return {};
|
|
895
|
+
const pending = new Set(this.journal?.pendingPeers() ?? []);
|
|
896
|
+
return Object.fromEntries([...values].filter(([peer]) => this.hasRemoteWork(peer, pending)));
|
|
897
|
+
}
|
|
898
|
+
private remoteMemberSnapshot(): Record<string, string> { return this.remoteBindingSnapshot(this.remoteQueueMembers); }
|
|
899
|
+
private remoteRoomSnapshot(): Record<string, string> { return this.remoteBindingSnapshot(this.remoteQueueRooms); }
|
|
900
|
+
private remoteExpirySnapshot(): Record<string, Record<string, number>> {
|
|
901
|
+
const snapshot: Record<string, Record<string, number>> = {};
|
|
902
|
+
for (const [peer, expiries] of this.remoteQueueExpires) {
|
|
903
|
+
const pending = new Set([...(this.queues.get(peer) ?? []), ...(this.preparing.get(peer) ?? [])].map(env => env.id));
|
|
904
|
+
for (const { peer: target, originals } of this.remoteTransport.values()) if (target === peer) for (const env of originals) pending.add(env.id);
|
|
905
|
+
for (const [id, target] of this.activeDeliveries) if (target === peer) for (const env of this.journal?.get(id)?.originals ?? []) pending.add(env.id);
|
|
906
|
+
// Uncertainty is still pending work. Keep its known TTL through checkpoints,
|
|
907
|
+
// using unresolved rows only, never completed delivery history.
|
|
908
|
+
for (const row of this.journal?.unresolved(peer) ?? []) for (const env of row.originals) pending.add(env.id);
|
|
909
|
+
for (const id of expiries.keys()) if (!pending.has(id)) expiries.delete(id);
|
|
910
|
+
if (expiries.size) snapshot[peer] = Object.fromEntries(expiries);
|
|
911
|
+
else this.remoteQueueExpires.delete(peer);
|
|
912
|
+
}
|
|
913
|
+
return snapshot;
|
|
914
|
+
}
|
|
915
|
+
/** Metadata-only timer hint; an unknown retained deadline/binding is due immediately. */
|
|
916
|
+
nextRemoteExpiry(): number | undefined {
|
|
917
|
+
let next: number | undefined;
|
|
918
|
+
for (const [peer, queue] of this.queues) {
|
|
919
|
+
if (!remoteId(peer) || !queue.length) continue;
|
|
920
|
+
if (!this.remoteQueueRooms.has(peer) || !this.remoteQueueMembers.has(peer)) return Date.now();
|
|
921
|
+
const expiries = this.remoteQueueExpires.get(peer);
|
|
922
|
+
for (const env of queue) {
|
|
923
|
+
const expiry = expiries?.get(env.id);
|
|
924
|
+
if (expiry === undefined) return Date.now();
|
|
925
|
+
next = next === undefined ? expiry : Math.min(next, expiry);
|
|
926
|
+
}
|
|
927
|
+
}
|
|
928
|
+
return next;
|
|
929
|
+
}
|
|
930
|
+
/** Expiry is a durable uncertain outcome, never a silent retry/drop or a fresh timestamp. */
|
|
931
|
+
expireRemoteQueues(now = Date.now()): number {
|
|
932
|
+
if (!Number.isSafeInteger(now) || now < 0) throw new Error("invalid remote queue clock");
|
|
933
|
+
if (!this.journal || this.closed) return 0; // Without a journal there is no durable sink for uncertain work.
|
|
934
|
+
const expired = [...this.queues].filter(([peer]) => remoteId(peer)).flatMap(([peer, queue]) => queue.filter(env => {
|
|
935
|
+
const expiry = this.remoteQueueExpires.get(peer)?.get(env.id);
|
|
936
|
+
return this.remoteQueueRooms.get(peer) === undefined || this.remoteQueueMembers.get(peer) === undefined || expiry === undefined || expiry <= now;
|
|
937
|
+
}).map(env => ({ peer, env })));
|
|
938
|
+
if (!expired.length) return 0;
|
|
939
|
+
const queues = new Map([...this.queues].map(([peer, queue]) => [peer, [...queue]]));
|
|
940
|
+
const expiries = new Map([...this.remoteQueueExpires].map(([peer, values]) => [peer, new Map(values)]));
|
|
941
|
+
const held = [...this.recoveryHeldPeers], outcomes = this.pendingOutcomes.length;
|
|
942
|
+
try {
|
|
943
|
+
for (const { peer, env } of expired) {
|
|
944
|
+
const queue = this.queues.get(peer)!;
|
|
945
|
+
const index = queue.findIndex(item => item.id === env.id);
|
|
946
|
+
if (index >= 0) queue.splice(index, 1);
|
|
947
|
+
const known = this.remoteQueueExpires.get(peer)?.has(env.id);
|
|
948
|
+
this.remoteQueueExpires.get(peer)?.delete(env.id);
|
|
949
|
+
this.pendingOutcomes.push({ id: crypto.randomUUID(), peer, state: "needs_review", createdAt: now, originals: [env], out: [env], reason: !known ? "remote queue expiry unavailable after restore" : !this.remoteQueueRooms.has(peer) ? "remote queue room binding unavailable after restore" : !this.remoteQueueMembers.has(peer) ? "remote queue member binding unavailable after restore" : "remote queue expired before recipient admission" });
|
|
950
|
+
this.recoveryHeldPeers.add(peer);
|
|
951
|
+
}
|
|
952
|
+
this.persist();
|
|
953
|
+
} catch (error) {
|
|
954
|
+
for (const [peer, before] of queues) this.queues.get(peer)!.splice(0, this.queues.get(peer)!.length, ...before);
|
|
955
|
+
this.remoteQueueExpires.clear(); for (const [peer, values] of expiries) this.remoteQueueExpires.set(peer, values);
|
|
956
|
+
this.recoveryHeldPeers.clear(); for (const peer of held) this.recoveryHeldPeers.add(peer);
|
|
957
|
+
this.pendingOutcomes.length = outcomes;
|
|
958
|
+
throw error;
|
|
959
|
+
}
|
|
960
|
+
this.onQueues?.();
|
|
961
|
+
return expired.length;
|
|
962
|
+
}
|
|
963
|
+
/** A room may acknowledge local admission only with a live, known-available journal. */
|
|
964
|
+
deliveryJournalAvailable(): boolean { return this.journal !== undefined && !this.storageError && !this.closed; }
|
|
965
|
+
/** Explicit person disposition metadata only, never inferred from transport completion. */
|
|
966
|
+
remoteDisposition(deliveryId: string): "completed" | "retry" | "discard" | undefined {
|
|
967
|
+
const resolution = this.journal?.resolution(deliveryId);
|
|
968
|
+
return resolution?.actor === "user" ? resolution.action : undefined;
|
|
969
|
+
}
|
|
970
|
+
/** A late signed queue receipt is evidence for the person; it never auto-settles uncertainty. */
|
|
971
|
+
noteLateRemoteReceipt(peer: string, deliveryId: string, outcome: "completed" | "failed_safe"): boolean {
|
|
972
|
+
if ((outcome !== "completed" && outcome !== "failed_safe") || !parseRemotePeerId(peer) || !this.peers.get(peer)?.remoteRoom || !this.journal || this.storageError || this.closed) return false;
|
|
973
|
+
const row = this.journal.get(deliveryId), adapter = this.peers.get(peer)!;
|
|
974
|
+
if (!row || row.peer !== peer || row.state !== "needs_review" || this.remoteQueueRooms.get(peer) !== adapter.remoteRoom || this.remoteQueueMembers.get(peer) !== adapter.remoteMember) return false;
|
|
975
|
+
const reason = outcome === "completed" ? "late signed completed receipt: remote queue admitted; remains needs_review" : "late signed failed_safe receipt; remains needs_review";
|
|
976
|
+
if (row.reason !== reason) this.journal.transition(row.id, "needs_review", reason, row.revision);
|
|
977
|
+
this.onQueues?.();
|
|
978
|
+
return true;
|
|
979
|
+
}
|
|
980
|
+
|
|
716
981
|
/** Returns the peers the envelope was accepted for, including durable deferred notices. */
|
|
717
982
|
publish(env: Envelope): PeerId[] {
|
|
718
983
|
return this.publishResult(env).targets;
|
|
@@ -722,6 +987,7 @@ export class Bus {
|
|
|
722
987
|
publishResult(env: Envelope): PublishResult {
|
|
723
988
|
if (this.storageError) throw new Error("delivery journal unavailable");
|
|
724
989
|
if (this.seen.has(env.id) || this.deferredHub.has(env.id)) return { targets: [] };
|
|
990
|
+
this.checkRemotePublication(env);
|
|
725
991
|
if (this.recoveryHeld && (this.deferHub || this.replayingHub) && env.from === HUB && env.priority !== "fyi" && env.hop <= MAX_HOP) {
|
|
726
992
|
if (this.deferredHub.size >= MAX_DEFERRED_HUB) throw new RecoveryDeferredHubError();
|
|
727
993
|
const targets = this.audience(env);
|
|
@@ -737,48 +1003,52 @@ export class Bus {
|
|
|
737
1003
|
private publishNow(env: Envelope): PublishResult {
|
|
738
1004
|
if (this.storageError) throw new Error("delivery journal unavailable");
|
|
739
1005
|
if (this.seen.has(env.id)) return { targets: [] };
|
|
740
|
-
this.
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
const
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
// Not queued while the steer is in flight, or an idle transition would deliver it a second time.
|
|
757
|
-
this.steering++;
|
|
758
|
-
const deliveryId = crypto.randomUUID();
|
|
759
|
-
this.activeDeliveries.set(deliveryId, id);
|
|
760
|
-
if (this.journal) this.durableHandoff(id, deliveryId, [env], [env]);
|
|
761
|
-
const steerResult = this.journal ? peer.steer([env], deliveryId) : peer.steer([env]);
|
|
762
|
-
steerResult.then(() => { this.delivered(id, [env]); }).catch(() => {
|
|
763
|
-
this.deliveryFailed(id);
|
|
764
|
-
if (this.journal) this.journal.transition(deliveryId, "needs_review", "adapter steering outcome is uncertain");
|
|
765
|
-
this.activeDeliveries.delete(deliveryId);
|
|
766
|
-
if (!this.journal) this.enqueue(id, env, true);
|
|
767
|
-
}).finally(() => { this.steering--; });
|
|
768
|
-
} else this.enqueue(id, env);
|
|
769
|
-
} } finally { this.suppressDrain = false; }
|
|
770
|
-
if (this.journal) {
|
|
771
|
-
this.persist();
|
|
772
|
-
if (!this.flushingHub) this.onQueues?.();
|
|
773
|
-
for (const id of targets) {
|
|
1006
|
+
const rooms = this.checkRemotePublication(env);
|
|
1007
|
+
for (const room of rooms) { const reserved = this.remoteReservations.get(room) ?? new Set<string>(); reserved.add(env.id); this.remoteReservations.set(room, reserved); }
|
|
1008
|
+
try {
|
|
1009
|
+
this.seen.set(env.id, env);
|
|
1010
|
+
if (this.seen.size > SEEN_CAP) this.seen.delete(this.seen.keys().next().value as string);
|
|
1011
|
+
|
|
1012
|
+
// fyi is for the record (console now, task board from M4), never worth an agent's turn.
|
|
1013
|
+
const dropped = env.hop > MAX_HOP ? "hop" : env.priority === "fyi" ? "fyi" : undefined;
|
|
1014
|
+
this.emit({ t: "envelope", env, ...(dropped ? { dropped } : {}) });
|
|
1015
|
+
if (dropped) { this.persist(); return { targets: [], dropped }; }
|
|
1016
|
+
|
|
1017
|
+
const hushed = this.hushed(env);
|
|
1018
|
+
if (hushed.length) this.emit({ t: "quiet", env, peers: hushed.map((h) => h.peer), reason: hushed[0]!.reason });
|
|
1019
|
+
const targets = this.audience(env).filter((id) => !hushed.some((h) => h.peer === id) || !this.isRostered(id));
|
|
1020
|
+
this.suppressDrain = !!this.journal;
|
|
1021
|
+
try { for (const id of targets) {
|
|
774
1022
|
const peer = this.peers.get(id);
|
|
775
|
-
if (!
|
|
776
|
-
|
|
1023
|
+
if (!peer) { this.enqueue(id, env); continue; }
|
|
1024
|
+
if (!this.journal && !this.held(id) && env.priority === "important" && !remoteId(id) && peer.steer && this.stateOf(id) === "busy") {
|
|
1025
|
+
// Not queued while the steer is in flight, or an idle transition would deliver it a second time.
|
|
1026
|
+
this.steering++;
|
|
1027
|
+
const deliveryId = crypto.randomUUID();
|
|
1028
|
+
this.activeDeliveries.set(deliveryId, id);
|
|
1029
|
+
if (this.journal) this.durableHandoff(id, deliveryId, [env], [env]);
|
|
1030
|
+
const steerResult = this.journal ? peer.steer([env], deliveryId) : peer.steer([env]);
|
|
1031
|
+
steerResult.then(() => { this.delivered(id, [env]); }).catch(() => {
|
|
1032
|
+
this.deliveryFailed(id);
|
|
1033
|
+
if (this.journal) this.journal.transition(deliveryId, "needs_review", "adapter steering outcome is uncertain");
|
|
1034
|
+
this.activeDeliveries.delete(deliveryId);
|
|
1035
|
+
if (!this.journal) this.enqueue(id, env, true);
|
|
1036
|
+
}).finally(() => { this.steering--; });
|
|
1037
|
+
} else this.enqueue(id, env);
|
|
1038
|
+
} } finally { this.suppressDrain = false; }
|
|
1039
|
+
if (this.journal) {
|
|
1040
|
+
this.persist();
|
|
1041
|
+
if (!this.flushingHub) this.onQueues?.();
|
|
1042
|
+
for (const id of targets) {
|
|
1043
|
+
const peer = this.peers.get(id);
|
|
1044
|
+
if (!this.held(id) && env.priority === "important" && !remoteId(id) && peer?.steer && this.stateOf(id) === "busy") void this.steerQueued(id, env);
|
|
1045
|
+
else void this.drain(id);
|
|
1046
|
+
}
|
|
777
1047
|
}
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
1048
|
+
const offRoster = targets.filter(id => !this.isRostered(id));
|
|
1049
|
+
if (offRoster.length) this.emit({ t: "off_roster", env, peers: offRoster });
|
|
1050
|
+
return { targets };
|
|
1051
|
+
} finally { for (const room of rooms) { const reserved = this.remoteReservations.get(room); reserved?.delete(env.id); if (!reserved?.size) this.remoteReservations.delete(room); } }
|
|
782
1052
|
}
|
|
783
1053
|
|
|
784
1054
|
isRostered(peer: PeerId): boolean { return this.opts.rostered?.(peer) ?? true; }
|
|
@@ -798,7 +1068,7 @@ export class Bus {
|
|
|
798
1068
|
/** Who an envelope is queued for: its `to`, or every peer but its sender. */
|
|
799
1069
|
audience(env: Envelope): PeerId[] {
|
|
800
1070
|
const known = new Set([...this.peers.keys(), ...this.queues.keys(), ...(this.journal?.list().map((d) => d.peer) ?? [])]);
|
|
801
|
-
return (env.to ?? [...this.peers.keys()]).filter((id) => id !== env.from && (known.has(id) || !this.isRostered(id)) && (env.to !== undefined || this.isRostered(id)));
|
|
1071
|
+
return (env.to ?? [...this.peers.keys()]).filter((id) => id !== env.from && (known.has(id) || !this.isRostered(id)) && (env.to !== undefined || this.isRostered(id)) && (env.to !== undefined || !remoteId(id)));
|
|
802
1072
|
}
|
|
803
1073
|
|
|
804
1074
|
/** The recipients of `env` that the silence policy holds it back from, each with why (issue #107). */
|
|
@@ -875,7 +1145,7 @@ export class Bus {
|
|
|
875
1145
|
}
|
|
876
1146
|
|
|
877
1147
|
private memoryDelivery(peer: PeerId, id: string, out: Envelope[], field: "rendered" | "delivered", originals: Envelope[] = out): void {
|
|
878
|
-
if ([...out, ...originals].some(env => env.private)) return;
|
|
1148
|
+
if (remoteId(peer) || [...out, ...originals].some(env => env.private || remoteId(env.from) || env.to?.some(remoteId))) return;
|
|
879
1149
|
try { this.onMemoryDelivery?.(peer, id, out, field); } catch { /* optional telemetry never alters delivery */ }
|
|
880
1150
|
}
|
|
881
1151
|
|
|
@@ -896,6 +1166,54 @@ export class Bus {
|
|
|
896
1166
|
return this.queues.get(id)?.length ?? 0;
|
|
897
1167
|
}
|
|
898
1168
|
|
|
1169
|
+
/** New automatic offers use only bounded pending metadata, never queue/history bodies. */
|
|
1170
|
+
assignmentQueue(peer: PeerId): QueuePressure {
|
|
1171
|
+
const queued = this.queued(peer), tracking = this.trackingQueuePressure();
|
|
1172
|
+
if (this.storageError) return { queued };
|
|
1173
|
+
if (queued === 0) return { queued: 0, oldestQueuedAt: null };
|
|
1174
|
+
if (!tracking) return { queued };
|
|
1175
|
+
// A durable summary cannot revive provenance that this host discarded while tracking was off.
|
|
1176
|
+
const times = this.queueTimes.get(peer);
|
|
1177
|
+
let oldest: number | undefined;
|
|
1178
|
+
for (const env of this.queues.get(peer) ?? []) {
|
|
1179
|
+
const at = times?.get(env.id);
|
|
1180
|
+
if (typeof at !== "number" || !Number.isSafeInteger(at) || at <= 0) return { queued };
|
|
1181
|
+
oldest = oldest === undefined ? at : Math.min(oldest, at);
|
|
1182
|
+
}
|
|
1183
|
+
if (this.journal) {
|
|
1184
|
+
try {
|
|
1185
|
+
const saved = this.journal.pendingQueueMetadata(peer);
|
|
1186
|
+
return saved?.queued === queued ? saved : { queued };
|
|
1187
|
+
} catch { return { queued }; }
|
|
1188
|
+
}
|
|
1189
|
+
return oldest === undefined ? { queued } : { queued, oldestQueuedAt: oldest };
|
|
1190
|
+
}
|
|
1191
|
+
|
|
1192
|
+
private trackingQueuePressure(): boolean {
|
|
1193
|
+
let tracking = false;
|
|
1194
|
+
try { tracking = this.opts.trackQueuePressure?.() === true; } catch { /* unknown policy is not enabled */ }
|
|
1195
|
+
if (!tracking && this.queuePressureWasTracked) { this.queueTimes.clear(); this.queueTimeEntries = 0; }
|
|
1196
|
+
this.queuePressureWasTracked = tracking;
|
|
1197
|
+
this.journal?.setQueueMetadataTracking(tracking);
|
|
1198
|
+
return tracking;
|
|
1199
|
+
}
|
|
1200
|
+
|
|
1201
|
+
private recordQueueTime(peer: PeerId, id: string, at: number | null): void {
|
|
1202
|
+
if (!this.trackingQueuePressure()) return;
|
|
1203
|
+
let times = this.queueTimes.get(peer);
|
|
1204
|
+
if (!times) {
|
|
1205
|
+
if (this.queueTimes.size >= QUEUE_TIME_PEERS_CAP) this.dropQueueTimes(this.queueTimes.keys().next().value!);
|
|
1206
|
+
this.queueTimes.set(peer, times = new Map());
|
|
1207
|
+
}
|
|
1208
|
+
if (!times.has(id)) { times.set(id, at); this.queueTimeEntries++; }
|
|
1209
|
+
while (times.size > QUEUE_TIME_CAP) { times.delete(times.keys().next().value!); this.queueTimeEntries--; }
|
|
1210
|
+
while (this.queueTimeEntries > QUEUE_TIME_TOTAL_CAP) this.dropQueueTimes(this.queueTimes.keys().next().value!);
|
|
1211
|
+
}
|
|
1212
|
+
private dropQueueTimes(peer: PeerId): void {
|
|
1213
|
+
this.queueTimeEntries -= this.queueTimes.get(peer)?.size ?? 0;
|
|
1214
|
+
this.queueTimes.delete(peer);
|
|
1215
|
+
}
|
|
1216
|
+
|
|
899
1217
|
/**
|
|
900
1218
|
* Queued envelopes that skip the batch window (issue #41: `queued 3` hid which ones the peer still waits out).
|
|
901
1219
|
* An `important` envelope that is in a queue is one that could not be steered, so it does not interrupt a
|
|
@@ -930,8 +1248,14 @@ export class Bus {
|
|
|
930
1248
|
const replace = !front && key ? queue.findIndex((old) => old.from === HUB && old.refs?.supervision && old.refs.supervisionKey === key && !this.attempts.has(`${id}:${old.id}`)) : -1;
|
|
931
1249
|
const preparing = !front && key ? this.preparing.get(id)?.find((old) => old.from === HUB && old.refs?.supervision && old.refs.supervisionKey === key && !this.attempts.has(`${id}:${old.id}`) && !this.withdrawn.has(old.id)) : undefined;
|
|
932
1250
|
let queuedEnv = preparing ? { ...env, ts: Math.min(preparing.ts, env.ts) } : env;
|
|
1251
|
+
const clock = this.trackingQueuePressure() ? Date.now() : 0;
|
|
1252
|
+
let enteredAt = this.queueTimes.get(id)?.has(env.id) ? this.queueTimes.get(id)!.get(env.id)!
|
|
1253
|
+
: front || this.flushingHub || !Number.isSafeInteger(clock) || clock <= 0 ? null : clock;
|
|
1254
|
+
if (this.trackingQueuePressure() && !this.queueTimes.get(id)?.has(env.id) && queue.some(old => old.id === env.id)) enteredAt = null;
|
|
1255
|
+
if (preparing) enteredAt = this.queueTimes.get(id)?.get(preparing.id) ?? null;
|
|
933
1256
|
if (replace >= 0) {
|
|
934
1257
|
const old = queue[replace]!;
|
|
1258
|
+
enteredAt = this.queueTimes.get(id)?.get(old.id) ?? null;
|
|
935
1259
|
// Keep the first milestone's deadline, so continuous updates cannot create another batching window.
|
|
936
1260
|
queuedEnv = { ...queuedEnv, ts: Math.min(old.ts, queuedEnv.ts) };
|
|
937
1261
|
queue[replace] = queuedEnv;
|
|
@@ -939,16 +1263,26 @@ export class Bus {
|
|
|
939
1263
|
this.discardQueued(id, old, "superseded supervision milestone");
|
|
940
1264
|
} else if (front) queue.unshift(queuedEnv);
|
|
941
1265
|
else queue.push(queuedEnv);
|
|
1266
|
+
this.recordQueueTime(id, queuedEnv.id, enteredAt);
|
|
942
1267
|
if (preparing && !this.withdrawn.has(preparing.id)) {
|
|
943
1268
|
this.markWithdrawn(preparing.id);
|
|
944
1269
|
this.discardQueued(id, preparing, "superseded supervision milestone during preparation");
|
|
945
1270
|
}
|
|
946
|
-
if (queue.length > this.opts.queueCap) {
|
|
1271
|
+
if (!remoteId(id) && queue.length > this.opts.queueCap) {
|
|
947
1272
|
const victim = queue.findIndex((e) => e.priority !== "important");
|
|
948
1273
|
const [lost] = queue.splice(victim === -1 ? 0 : victim, 1);
|
|
949
1274
|
if (this.journal) this.pendingOutcomes.push({ id: crypto.randomUUID(), peer: id, state: "failed", createdAt: Date.now(), originals: [lost!], out: [], reason: "queue capacity exceeded" });
|
|
950
1275
|
this.emit({ t: "overflow", env: lost!, peer: id });
|
|
951
1276
|
}
|
|
1277
|
+
if (remoteId(id) && !front) {
|
|
1278
|
+
const room = this.peers.get(id)?.remoteRoom;
|
|
1279
|
+
if (room !== undefined) this.remoteQueueRooms.set(id, room);
|
|
1280
|
+
const member = this.peers.get(id)?.remoteMember;
|
|
1281
|
+
if (member !== undefined) this.remoteQueueMembers.set(id, member);
|
|
1282
|
+
const expiries = this.remoteQueueExpires.get(id) ?? new Map<string, number>();
|
|
1283
|
+
if (!expiries.has(env.id)) expiries.set(env.id, Date.now() + (this.peers.get(id)?.remoteQueueTtlMs ?? REMOTE_QUEUE_TTL_MS));
|
|
1284
|
+
this.remoteQueueExpires.set(id, expiries);
|
|
1285
|
+
}
|
|
952
1286
|
if (queuedEnv.priority === "important" && this.peers.get(id)?.pullOnly) {
|
|
953
1287
|
// Advisory only (issue #398): the envelope stays queued for `pull`; the hook must never affect it.
|
|
954
1288
|
const counts = { queued: queue.length, important: queue.filter((e) => e.priority === "important").length };
|
|
@@ -1009,7 +1343,9 @@ export class Bus {
|
|
|
1009
1343
|
|
|
1010
1344
|
/** A push may go out now: nothing holds the queue, the peer is idle, and its session shows pushes (issue #205). */
|
|
1011
1345
|
private deliverable(id: PeerId): boolean {
|
|
1012
|
-
|
|
1346
|
+
const peer = this.peers.get(id);
|
|
1347
|
+
const remoteBinding = !remoteId(id) || !!peer?.remoteRoom && this.remoteQueueRooms.get(id) === peer.remoteRoom && this.remoteQueueMembers.get(id) === peer.remoteMember;
|
|
1348
|
+
return remoteBinding && !this.held(id) && this.stateOf(id) === "idle" && !peer?.pullOnly;
|
|
1013
1349
|
}
|
|
1014
1350
|
|
|
1015
1351
|
/**
|
|
@@ -1024,6 +1360,7 @@ export class Bus {
|
|
|
1024
1360
|
* steps if loss is ever observed.
|
|
1025
1361
|
*/
|
|
1026
1362
|
pull(id: PeerId): Envelope[] | undefined {
|
|
1363
|
+
if (remoteId(id)) return undefined; // A local inbox read is not signed remote queue admission.
|
|
1027
1364
|
if (this.storageError) throw new Error("delivery journal unavailable");
|
|
1028
1365
|
if (this.held(id)) return undefined;
|
|
1029
1366
|
const queue = this.queues.get(id) ?? [];
|
|
@@ -1061,22 +1398,23 @@ export class Bus {
|
|
|
1061
1398
|
|
|
1062
1399
|
private async drain(id: PeerId): Promise<void> {
|
|
1063
1400
|
if (this.storageError || this.draining.has(id) || !this.peers.has(id)) return;
|
|
1401
|
+
if (remoteId(id)) this.expireRemoteQueues();
|
|
1064
1402
|
this.draining.add(id);
|
|
1065
1403
|
try {
|
|
1066
1404
|
const peer = this.peers.get(id)!;
|
|
1067
1405
|
const queue = this.queues.get(id)!;
|
|
1068
1406
|
while (!this.storageError && queue.length && this.deliverable(id)) {
|
|
1069
1407
|
if (this.dropIrrelevant(id, queue) && !queue.length) break;
|
|
1070
|
-
const delay = this.wait(queue);
|
|
1408
|
+
const delay = remoteId(id) ? 0 : this.wait(queue);
|
|
1071
1409
|
if (delay > 0) { this.arm(id, delay); break; }
|
|
1072
|
-
const preface = this.prefaces.get(id);
|
|
1410
|
+
const preface = remoteId(id) ? undefined : this.prefaces.get(id);
|
|
1073
1411
|
// Durable queues retain the batch while asynchronous condensation runs. Another publish
|
|
1074
1412
|
// may checkpoint during that wait; it must not checkpoint the batch out of existence.
|
|
1075
1413
|
if (!this.journal) this.prefaces.delete(id);
|
|
1076
1414
|
const batch = this.take(id, queue, !this.journal);
|
|
1077
1415
|
this.preparing.set(id, batch);
|
|
1078
1416
|
const delivery = preface ? [preface, ...batch] : batch;
|
|
1079
|
-
const mayCondense = this.opts.condense && !delivery.some((e) => e.priority === "important");
|
|
1417
|
+
const mayCondense = this.opts.condense && !remoteId(id) && !delivery.some((e) => e.priority === "important" || remoteId(e.from));
|
|
1080
1418
|
this.condensing += mayCondense ? 1 : 0;
|
|
1081
1419
|
let out: Envelope[];
|
|
1082
1420
|
try {
|
|
@@ -1096,7 +1434,7 @@ export class Bus {
|
|
|
1096
1434
|
}
|
|
1097
1435
|
const deliveryId = crypto.randomUUID();
|
|
1098
1436
|
if (this.journal) {
|
|
1099
|
-
if (this.prefaces.get(id) !== preface || batch.some((e) => !queue.some((item) => item.id === e.id))) continue;
|
|
1437
|
+
if ((!remoteId(id) && this.prefaces.get(id) !== preface) || batch.some((e) => !queue.some((item) => item.id === e.id))) continue;
|
|
1100
1438
|
const queued = [...queue], seen = [...this.seen];
|
|
1101
1439
|
for (const env of batch) queue.splice(queue.findIndex((item) => item.id === env.id), 1);
|
|
1102
1440
|
this.prefaces.delete(id);
|
|
@@ -1112,8 +1450,14 @@ export class Bus {
|
|
|
1112
1450
|
this.activeDeliveries.set(deliveryId, id);
|
|
1113
1451
|
} else this.remember(out);
|
|
1114
1452
|
this.lastDelivery.set(id, { out, originals: delivery });
|
|
1453
|
+
if (remoteId(id)) this.remoteTransport.set(deliveryId, { peer: id, originals: delivery });
|
|
1115
1454
|
this.preparing.delete(id); // from here on the transport/journal owns settlement
|
|
1116
1455
|
try {
|
|
1456
|
+
if (remoteId(id)) {
|
|
1457
|
+
const env = delivery[0], expiry = env && this.remoteQueueExpires.get(id)?.get(env.id);
|
|
1458
|
+
if (!env || delivery.length !== 1 || expiry === undefined || expiry <= Date.now() || !peer.setRemoteDeadline) throw new Error("remote queue deadline unavailable");
|
|
1459
|
+
peer.setRemoteDeadline(env.id, expiry);
|
|
1460
|
+
}
|
|
1117
1461
|
try { this.onDeliver?.(id, delivery); } catch { /* observation must never affect delivery */ }
|
|
1118
1462
|
if (this.journal) await peer.deliver(out, deliveryId);
|
|
1119
1463
|
else { await peer.deliver(out); this.delivered(id, delivery); }
|
|
@@ -1121,7 +1465,7 @@ export class Bus {
|
|
|
1121
1465
|
if (this.journal) this.uncertain(id, deliveryId, "adapter delivery outcome is uncertain");
|
|
1122
1466
|
else this.failed(id, delivery);
|
|
1123
1467
|
break;
|
|
1124
|
-
}
|
|
1468
|
+
} finally { this.remoteTransport.delete(deliveryId); }
|
|
1125
1469
|
}
|
|
1126
1470
|
} catch { this.storageError = "delivery journal unavailable"; }
|
|
1127
1471
|
finally { this.preparing.delete(id); this.draining.delete(id); this.onQueues?.(); }
|
|
@@ -1166,13 +1510,14 @@ export class Bus {
|
|
|
1166
1510
|
* An envelope that already failed once goes alone, so a poison one cannot take a digest down with it.
|
|
1167
1511
|
*/
|
|
1168
1512
|
private take(id: PeerId, queue: Envelope[], remove = true): Envelope[] {
|
|
1513
|
+
const digestMax = this.opts.digestMax?.() ?? DIGEST_MAX;
|
|
1169
1514
|
const ordered = [...queue.filter((e) => e.priority === "important"), ...queue.filter((e) => e.priority !== "important")];
|
|
1170
1515
|
const batch: Envelope[] = [];
|
|
1171
1516
|
for (const env of ordered) {
|
|
1172
1517
|
const failedBefore = this.attempts.has(`${id}:${env.id}`);
|
|
1173
1518
|
if (failedBefore && batch.length) break;
|
|
1174
1519
|
batch.push(env);
|
|
1175
|
-
if (failedBefore || batch.length ===
|
|
1520
|
+
if (failedBefore || remoteId(id) || batch.length === digestMax) break;
|
|
1176
1521
|
}
|
|
1177
1522
|
if (remove) for (const env of batch) queue.splice(queue.indexOf(env), 1);
|
|
1178
1523
|
return batch;
|
|
@@ -1184,13 +1529,14 @@ export class Bus {
|
|
|
1184
1529
|
* envelope cannot stall the queue. An envelope that already failed once still goes alone, as in `take`.
|
|
1185
1530
|
*/
|
|
1186
1531
|
private takeBudget(id: PeerId, queue: Envelope[]): Envelope[] {
|
|
1532
|
+
const pullBudgetBytes = this.opts.pullBudgetBytes?.() ?? PULL_BUDGET_BYTES;
|
|
1187
1533
|
const batch: Envelope[] = [];
|
|
1188
1534
|
let bytes = 0;
|
|
1189
1535
|
for (const env of queue) {
|
|
1190
1536
|
const failedBefore = this.attempts.has(`${id}:${env.id}`);
|
|
1191
1537
|
if (failedBefore && batch.length) break;
|
|
1192
1538
|
const size = Buffer.byteLength(frame(env)) + 2;
|
|
1193
|
-
if (batch.length && bytes + size >
|
|
1539
|
+
if (batch.length && bytes + size > pullBudgetBytes) break;
|
|
1194
1540
|
batch.push(env); bytes += size;
|
|
1195
1541
|
if (failedBefore) break;
|
|
1196
1542
|
}
|
|
@@ -1216,11 +1562,12 @@ export class Bus {
|
|
|
1216
1562
|
}
|
|
1217
1563
|
|
|
1218
1564
|
private failed(id: PeerId, envs: Envelope[], reason = "delivery failed without an error detail", quota = false): void {
|
|
1565
|
+
const maxAttempts = this.opts.maxAttempts?.() ?? MAX_ATTEMPTS;
|
|
1219
1566
|
if (!this.journal) this.deliveryFailed(id);
|
|
1220
1567
|
// A quota refusal ran no work. The budget pause fences retries until reset; it is not a poison attempt.
|
|
1221
1568
|
const quotaPaused = quota && this.paused.has(id);
|
|
1222
1569
|
this.lastFailure.set(id, reason.replace(/\s+/g, " ").slice(0, 300));
|
|
1223
|
-
if (!quotaPaused && envs.some((env) => !(env.from === HUB && env.kind === "presence") && (this.attempts.get(`${id}:${env.id}`) ?? 0) + 1 >=
|
|
1570
|
+
if (!quotaPaused && envs.some((env) => !(env.from === HUB && env.kind === "presence") && (this.attempts.get(`${id}:${env.id}`) ?? 0) + 1 >= maxAttempts)) {
|
|
1224
1571
|
this.failureStreak.set(id, (this.failureStreak.get(id) ?? 0) + 1);
|
|
1225
1572
|
}
|
|
1226
1573
|
const keep = envs.filter((env) => {
|
|
@@ -1232,7 +1579,7 @@ export class Bus {
|
|
|
1232
1579
|
if (quotaPaused) return true;
|
|
1233
1580
|
const key = `${id}:${env.id}`;
|
|
1234
1581
|
const n = (this.attempts.get(key) ?? 0) + 1;
|
|
1235
|
-
if (n <
|
|
1582
|
+
if (n < maxAttempts) return this.attempts.set(key, n);
|
|
1236
1583
|
this.attempts.delete(key);
|
|
1237
1584
|
this.emit({ t: "undeliverable", env, peer: id, reason: this.lastFailure.get(id) });
|
|
1238
1585
|
return false;
|
|
@@ -1245,6 +1592,10 @@ export class Bus {
|
|
|
1245
1592
|
|
|
1246
1593
|
private emit(e: BusEvent): void {
|
|
1247
1594
|
if (this.flushingHub) { this.deferredEvents.push(e); return; }
|
|
1595
|
+
if ("env" in e && (remoteId(e.env.from) || e.env.to?.some(remoteId) || ("peer" in e && remoteId(e.peer)))) {
|
|
1596
|
+
const { refs: _, ...env } = e.env;
|
|
1597
|
+
e = { ...e, env: { ...env, body: "[remote room message withheld]", ...(env.to ? { to: env.to.filter(id => /^[a-z][a-z0-9-]{0,31}$/.test(id) || !!parseRemotePeerId(id)) } : {}) }, ...("reason" in e ? { reason: "remote room delivery" } : {}) };
|
|
1598
|
+
}
|
|
1248
1599
|
for (const tap of this.taps) tap(e);
|
|
1249
1600
|
}
|
|
1250
1601
|
}
|