@staix/agent-hub 0.30.0 → 0.32.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +52 -0
- package/README.md +13 -7
- package/docs/agent-notes/README.md +1 -0
- package/docs/agent-notes/adapters.md +5 -2
- package/docs/agent-notes/budget.md +1 -0
- package/docs/agent-notes/bus.md +11 -4
- package/docs/agent-notes/daemon.md +16 -8
- package/docs/agent-notes/local-worker.md +1 -0
- package/docs/agent-notes/tasks.md +7 -3
- package/docs/agent-notes/tests.md +6 -4
- package/docs/design/adapters.md +52 -0
- package/docs/design/agent-hub-architecture.md +46 -0
- package/docs/design/bus.md +178 -1
- package/docs/design/task-board.md +82 -0
- package/docs/dogfood.md +23 -4
- package/docs/events.md +3 -0
- package/docs/memory.md +17 -1
- package/docs/operations.md +355 -78
- package/docs/profiles.md +35 -0
- package/docs/quickstart.md +24 -5
- package/docs/security.md +178 -60
- package/docs/specs/2026-09-19-agent-hub-design.md +127 -14
- package/docs/specs/2026-09-19-multi-project-design.md +7 -2
- package/docs/verification/2026-11-04-1171-channel-client-evidence.md +31 -0
- package/docs/verification/2026-11-05-0.31.0.md +182 -0
- package/docs/verification/2026-11-05-claude-implementer-2.1.318.md +70 -0
- package/docs/verification/2026-11-05-hardcoded-values-classification-rules.md +311 -0
- package/docs/verification/2026-11-05-hardcoded-values-ratchet.md +55 -0
- package/docs/verification/2026-11-06-0.32.0.md +149 -0
- package/docs/verification/2026-11-06-rooms-presence-cost.md +152 -0
- package/docs/verification/claude-shell-wrapper-native.py +1 -1
- package/docs/verified.json +14 -14
- package/package.json +3 -4
- package/plugins/agent-hub/.claude-plugin/plugin.json +1 -1
- package/plugins/agent-hub/server.js +4393 -3609
- package/src/adapters/acp.ts +5 -4
- package/src/adapters/channel-peer.ts +72 -3
- package/src/adapters/claude-channel-evidence.ts +52 -0
- package/src/adapters/claude-channel.ts +61 -18
- package/src/adapters/claude-implementer.ts +29 -7
- package/src/adapters/claude-print-child.ts +12 -2
- package/src/adapters/codex-appserver.ts +13 -12
- package/src/adapters/compat.ts +3 -3
- package/src/adapters/definitions/builtin-metadata.ts +13 -13
- package/src/adapters/definitions/qwen.ts +1 -0
- package/src/adapters/local-worker.ts +61 -18
- package/src/adapters/pi.ts +48 -6
- package/src/adapters/registry-types.ts +12 -0
- package/src/adapters/registry.ts +2 -0
- package/src/adapters/reviewer-print.ts +50 -11
- package/src/broker/server.ts +44 -13
- package/src/broker/store.ts +38 -8
- package/src/cli/answer-args.ts +50 -0
- package/src/cli/attention.ts +17 -2
- package/src/cli/backlog-return-args.ts +37 -0
- package/src/cli/console-question.ts +110 -0
- package/src/cli/console-state.ts +78 -17
- package/src/cli/console.ts +54 -1
- package/src/cli/handoff-args.ts +15 -5
- package/src/cli/handoff-output.ts +15 -2
- package/src/cli/help.ts +8 -5
- package/src/cli/identity.ts +13 -3
- package/src/cli/main.ts +128 -16
- package/src/cli/memory.ts +107 -3
- package/src/cli/output.ts +29 -6
- package/src/cli/pet.ts +4 -4
- package/src/cli/results-args.ts +30 -0
- package/src/cli/room.ts +20 -10
- package/src/cli/runtime-version-warning.ts +50 -0
- package/src/cli/status-lines.ts +41 -1
- package/src/cli/tail-render.ts +1 -0
- package/src/cli/upgrade-interactive.ts +14 -2
- package/src/hub/action-class.ts +9 -1
- package/src/hub/board.ts +369 -18
- package/src/hub/bus.ts +511 -53
- package/src/hub/completion-detail.ts +50 -0
- package/src/hub/completions-store.ts +168 -0
- package/src/hub/completions.ts +120 -0
- package/src/hub/config-apply.ts +7 -5
- package/src/hub/config-reload.ts +10 -1
- package/src/hub/control-client.ts +40 -4
- package/src/hub/daemon.ts +1576 -139
- package/src/hub/delivery-failure.ts +14 -0
- package/src/hub/delivery-hints.ts +14 -4
- package/src/hub/delivery-journal.ts +751 -42
- package/src/hub/delivery-superseded.ts +4 -1
- package/src/hub/envelope.ts +3 -3
- package/src/hub/events.ts +6 -0
- package/src/hub/handoff.ts +145 -6
- package/src/hub/hub-tools.ts +56 -7
- package/src/hub/manager.ts +59 -3
- package/src/hub/memory-policy.ts +44 -1
- package/src/hub/memory-sync-policy.ts +109 -0
- package/src/hub/memory-sync-state.ts +145 -0
- package/src/hub/no-writes.ts +1 -1
- package/src/hub/operational-settings.ts +11 -2
- package/src/hub/operational-view.ts +12 -1
- package/src/hub/peers.ts +25 -2
- package/src/hub/planning.ts +6 -1
- package/src/hub/question-desk.ts +201 -0
- package/src/hub/question-detach.ts +6 -0
- package/src/hub/questions.ts +260 -0
- package/src/hub/research-evidence.ts +106 -0
- package/src/hub/restart.ts +14 -0
- package/src/hub/review-anchor.ts +181 -0
- package/src/hub/review-brief.ts +13 -0
- package/src/hub/review-comments.ts +115 -0
- package/src/hub/review-findings.ts +7 -2
- package/src/hub/review-material.ts +76 -0
- package/src/hub/routing.ts +3 -1
- package/src/hub/task-sweep.ts +2 -0
- package/src/hub/tasks.ts +577 -63
- package/src/memory/capture.ts +5 -2
- package/src/pet/config.ts +25 -13
- package/src/rooms/commands.ts +61 -16
- package/src/rooms/link.ts +338 -36
- package/src/rooms/liveness.ts +11 -0
- package/src/rooms/remote-peer.ts +186 -19
- package/src/rooms/service.ts +475 -35
- package/src/rooms/status.ts +8 -1
- package/src/rooms/store.ts +447 -65
- package/src/rooms/wire.ts +24 -6
- package/src/ui/index.html +142 -9
- package/src/ui/needs.ts +10 -4
- package/src/ui/plugin-questions.ts +56 -0
- package/src/ui/plugin-results.ts +81 -0
- package/src/ui/plugin.ts +207 -27
- package/src/pet/window.html +0 -114
- package/src/pet/window.jxa.js +0 -122
- package/src/pet/window.ts +0 -154
package/src/hub/bus.ts
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
|
+
import { isDeliveryFailureCause, type DeliveryFailureCause } from "./delivery-failure.ts";
|
|
2
|
+
import { isReviewObserverExcluded } from "./memory-policy.ts";
|
|
1
3
|
import { remoteEnvelope, appendNote, DIGEST, frame, HUB, USER, isInvocationKind, keepNotes, MAX_HOP, mergePrefaces, newEnvelope, noteLine, parseMarker, quoteNotes, replyAudience, uniqueMemoryLines, type Envelope, type EnvelopeOpts, type PeerId, type PeerState, type Priority } from "./envelope.ts";
|
|
2
4
|
import type { PeerAdapter, DeliveryReceipt } from "./peers.ts";
|
|
3
5
|
import type { DoingKind } from "./peer-doing.ts";
|
|
4
6
|
import type { QueuePressure } from "./queue-backpressure.ts";
|
|
5
|
-
import { DeliveryJournal, MAX_ENVELOPE_BODY, type JournalDelivery, type JournalDeliveryState, type JournalResolution } from "./delivery-journal.ts";
|
|
7
|
+
import { ACCEPTED_SETTLE_TIMEOUT_REASON, DeliveryJournal, MAX_ENVELOPE_BODY, type JournalDelivery, type JournalDeliveryState, type JournalResolution } from "./delivery-journal.ts";
|
|
6
8
|
import { DELIVERY_RETRY_WARNING } from "./delivery-hints.ts";
|
|
7
9
|
import { MAX_DEFERRED_HUB, RecoveryDeferredHubError, validateDeferredHub } from "./deferred-hub.ts";
|
|
8
10
|
import { parseRemotePeerId } from "../rooms/application.ts";
|
|
@@ -15,6 +17,12 @@ export interface PublishResult {
|
|
|
15
17
|
dropped?: "hop" | "fyi";
|
|
16
18
|
}
|
|
17
19
|
|
|
20
|
+
export interface AcceptedDeliveryExpiry { id: string; peer: PeerId; revision: number; notifyConductor: boolean }
|
|
21
|
+
|
|
22
|
+
export class ReviewInputHandoffError extends Error {
|
|
23
|
+
constructor() { super("review input provenance unavailable; the message remains queued. Reconnect the peer, check ahub status for journal health, then retry hub_inbox."); this.name = "ReviewInputHandoffError"; }
|
|
24
|
+
}
|
|
25
|
+
|
|
18
26
|
export type BusEvent =
|
|
19
27
|
| { t: "envelope"; env: Envelope; dropped?: "hop" | "fyi" }
|
|
20
28
|
| { t: "overflow"; env: Envelope; peer: PeerId }
|
|
@@ -22,9 +30,16 @@ export type BusEvent =
|
|
|
22
30
|
| { t: "stale"; env: Envelope; peer: PeerId; reason: string }
|
|
23
31
|
| { t: "quiet"; env: Envelope; peers: PeerId[]; reason: string }
|
|
24
32
|
| { t: "off_roster"; env: Envelope; peers: PeerId[] }
|
|
33
|
+
| { t: "remote_delivery_settled"; peer: PeerId; deliveryId: string; outcome: "completed" | "failed_safe" }
|
|
25
34
|
| { t: "state"; peer: PeerId; state: PeerState };
|
|
26
35
|
|
|
27
36
|
export interface BusOptions {
|
|
37
|
+
/** Durable host-only comment provenance; no peer-supplied marker or task authority. */
|
|
38
|
+
reviewObserverScope?: (peer: PeerId) => string | undefined;
|
|
39
|
+
/** Host task/input screening for observer-memory hooks; receives originals, never rendered output. */
|
|
40
|
+
reviewObserverInputExcluded?: (peer: PeerId, originals: Envelope[]) => boolean;
|
|
41
|
+
/** Required durable provenance before a review-scoped input crosses a native handoff boundary; throws to refuse handoff. */
|
|
42
|
+
beforeReviewInputHandoff?: (peer: PeerId, originals: Envelope[]) => void;
|
|
28
43
|
/** Durable application outbox ids; the bus combines them with its own pending ids without recursion. */
|
|
29
44
|
remotePending?: (room: string) => readonly string[];
|
|
30
45
|
/** Host provenance at enqueue, never a body screen; unknown/throw marks the original private. */
|
|
@@ -43,6 +58,8 @@ export interface BusOptions {
|
|
|
43
58
|
queueCap: number;
|
|
44
59
|
/** Accepted live operational settings; read only at the next matching delivery/note operation. */
|
|
45
60
|
maxAttempts?: () => number;
|
|
61
|
+
/** Maximum age of an accepted explicit-channel delivery before it needs review. */
|
|
62
|
+
acceptedSettleMs?: () => number;
|
|
46
63
|
digestMax?: () => number;
|
|
47
64
|
pullBudgetBytes?: () => number;
|
|
48
65
|
notesKept?: () => number;
|
|
@@ -143,6 +160,9 @@ export const NOTES_KEPT = 10;
|
|
|
143
160
|
* delivered as a whole (one envelope, or a digest) when the peer is idle and the queue is ready:
|
|
144
161
|
* it holds an `important` envelope, or `batchMax` envelopes, or its oldest one has waited `batchMs`.
|
|
145
162
|
*/
|
|
163
|
+
export const ROOM_PROOF_UNAVAILABLE_REASON = "room delivery proof unavailable; inspect ahub queue show";
|
|
164
|
+
export const ROOM_EPOCH_ROTATED_REASON = "room epoch rotated; inspect ahub queue show";
|
|
165
|
+
|
|
146
166
|
export class Bus {
|
|
147
167
|
readonly peers = new Map<PeerId, PeerAdapter>();
|
|
148
168
|
private readonly opts: BusOptions;
|
|
@@ -167,7 +187,13 @@ export class Bus {
|
|
|
167
187
|
private readonly taps = new Set<(e: BusEvent) => void>();
|
|
168
188
|
private readonly withdrawn = new Set<string>();
|
|
169
189
|
/** What each peer was last handed, next to what it stands for: an adapter that reports a failure later hands back the former. */
|
|
190
|
+
private readonly reviewParents = new Map<PeerId, Map<string, string>>();
|
|
191
|
+
private readonly reviewResponses = new Map<string, { peer: PeerId; source: string }>();
|
|
192
|
+
private reviewScopeRevision = -1;
|
|
193
|
+
private readonly reviewScopeCache = new Map<string, boolean>();
|
|
170
194
|
private readonly lastDelivery = new Map<PeerId, { out: Envelope[]; originals: Envelope[] }>();
|
|
195
|
+
/** Latest inbox handout; kept separately so it cannot replace a pushed invocation's answer parent. */
|
|
196
|
+
private readonly lastPulled = new Map<PeerId, { out: Envelope[]; originals: Envelope[] }>();
|
|
171
197
|
private readonly failureStreak = new Map<PeerId, number>();
|
|
172
198
|
private readonly lastFailure = new Map<PeerId, string>();
|
|
173
199
|
private readonly attempts = new Map<string, number>(); // `${peer}:${envelope id}` -> failed deliveries
|
|
@@ -203,7 +229,7 @@ export class Bus {
|
|
|
203
229
|
}
|
|
204
230
|
|
|
205
231
|
/** Hold the existing delivery rather than cancelling native work or replaying an uncertain effect. */
|
|
206
|
-
holdInvocation(id: string, reason: string, target?: PeerId): void {
|
|
232
|
+
holdInvocation(id: string, reason: string, target?: PeerId, failureCause?: DeliveryFailureCause): void {
|
|
207
233
|
const delivery = this.invocationDeliveries.get(id), row = delivery && this.journal?.get(delivery);
|
|
208
234
|
if (!row || !this.journal) {
|
|
209
235
|
this.storageError = "uncertain invocation delivery evidence unavailable; inspect the invocation ledger before recovery";
|
|
@@ -211,7 +237,7 @@ export class Bus {
|
|
|
211
237
|
this.onQueues?.();
|
|
212
238
|
return;
|
|
213
239
|
}
|
|
214
|
-
this.journal.transition(row.id, "needs_review", reason);
|
|
240
|
+
this.journal.transition(row.id, "needs_review", reason, undefined, failureCause ?? row.failureCause ?? "turn aborted");
|
|
215
241
|
this.activeDeliveries.delete(row.id);
|
|
216
242
|
this.recoveryHeldPeers.add(row.peer);
|
|
217
243
|
this.onQueues?.();
|
|
@@ -219,6 +245,10 @@ export class Bus {
|
|
|
219
245
|
|
|
220
246
|
/** Actual transport admission, using originals even when a digest was condensed. Never an enqueue metric. */
|
|
221
247
|
onDelivered?: (peer: PeerId, originals: Envelope[], deliveryId?: string) => void;
|
|
248
|
+
/** Metadata-only current-input observation after a successful inbox handout; never a transport ACK. */
|
|
249
|
+
onPulled?: (peer: PeerId, originals: Envelope[]) => void;
|
|
250
|
+
/** Metadata-only observation when an important message actually begins a steer; never a transport ACK. */
|
|
251
|
+
onSteering?: (peer: PeerId, originals: Envelope[]) => void;
|
|
222
252
|
/** Before deliver's synchronous busy transition, with originals rather than a condensed digest. */
|
|
223
253
|
onDeliver?: (peer: PeerId, originals: Envelope[]) => void;
|
|
224
254
|
/** Final rendered journal payload, and its first positive acknowledgement. Observation cannot alter settlement. */
|
|
@@ -243,13 +273,13 @@ export class Bus {
|
|
|
243
273
|
this.journal = opts.journal;
|
|
244
274
|
this.trackingQueuePressure();
|
|
245
275
|
if (this.journal) {
|
|
246
|
-
const state = this.journal.
|
|
276
|
+
const state = this.journal.transportSnapshot();
|
|
247
277
|
const bus = state.bus as BusSnapshot;
|
|
248
278
|
if (state.revision > 0 && bus && bus.schemaVersion === 1) this.loadSnapshot(bus);
|
|
249
279
|
for (const id of state.manualPaused) { this.manualPaused.add(id); this.paused.add(id); }
|
|
250
280
|
for (const row of this.journal.list()) {
|
|
251
281
|
for (const env of row.originals) if (env.kind === "invocation") this.invocationDeliveries.set(env.id, row.id);
|
|
252
|
-
if (row.state === "
|
|
282
|
+
if (row.state === "dispatching" || row.state === "accepted" || row.state === "needs_review" && (!remoteId(row.peer) || row.originals.some(isInvocationKind))) this.recoveryHeldPeers.add(row.peer);
|
|
253
283
|
}
|
|
254
284
|
}
|
|
255
285
|
}
|
|
@@ -277,29 +307,38 @@ export class Bus {
|
|
|
277
307
|
else if (member !== peer.remoteMember) throw new Error("remote queued member binding changed");
|
|
278
308
|
}
|
|
279
309
|
if (peer.remoteRoom !== undefined && !pendingRemote) { this.remoteQueueRooms.delete(peer.id); this.remoteQueueMembers.delete(peer.id); }
|
|
310
|
+
this.clearReviewProvenance(peer.id);
|
|
280
311
|
this.peers.set(peer.id, peer);
|
|
281
312
|
if (!this.queues.has(peer.id)) this.queues.set(peer.id, []);
|
|
282
|
-
peer.onDelivery = (receipt) => this.deliveryReceipt(peer.id, receipt);
|
|
283
|
-
peer.onDoing = (kind, text) => { if (!this.closed && this.peers.get(peer.id) === peer) this.opts.doing?.(peer.id, kind, text); };
|
|
313
|
+
peer.onDelivery = (receipt) => { if (this.peers.get(peer.id) === peer) this.deliveryReceipt(peer.id, receipt); };
|
|
314
|
+
peer.onDoing = (kind, text) => { if (kind === "reset") this.clearReviewProvenance(peer.id); if (!this.closed && this.peers.get(peer.id) === peer) this.opts.doing?.(peer.id, kind, text); };
|
|
284
315
|
this.opts.doing?.(peer.id, "reset"); // a new adapter never shows what the one it replaces was doing
|
|
285
316
|
peer.onMessage = (text, opts) => {
|
|
286
317
|
if (!this.isRostered(peer.id)) return `off roster: ${peer.id}`;
|
|
287
318
|
// #445 C2: an answer to an invocation is found by what the peer was handed, never by how the answer is addressed, so a headless agent whose final answer is
|
|
288
319
|
// normally shared with every peer cannot leak it. It is the invocation's answer when it names the request itself (an adapter may report its delivery complete before
|
|
289
|
-
// it answers, so the receipt cannot decide), or when it names nothing while that invocation's own delivery is still the peer's open turn.
|
|
290
|
-
//
|
|
291
|
-
const
|
|
292
|
-
const
|
|
293
|
-
|
|
320
|
+
// it answers, so the receipt cannot decide), or when it names nothing while that invocation's own pushed delivery is still the peer's open turn. A pull updates
|
|
321
|
+
// current-input provenance separately, so it never replaces this answer parent; after the invocation turn ends, a later unnamed message is ordinary.
|
|
322
|
+
const pushed = this.lastDelivery.get(peer.id), pulled = this.lastPulled.get(peer.id), parentId = opts?.inReplyTo?.id;
|
|
323
|
+
const namedPush = parentId !== undefined && pushed?.out.some(e => e.id === parentId);
|
|
324
|
+
const namedPull = parentId !== undefined && pulled?.out.some(e => e.id === parentId);
|
|
325
|
+
const pushedInvocation = pushed?.out.some(e => e.kind === "invocation") === true;
|
|
326
|
+
const pulledInvocation = pulled?.out.some(e => e.kind === "invocation") === true;
|
|
327
|
+
const answerParent = namedPush && pushedInvocation ? pushed
|
|
328
|
+
: namedPull && pulledInvocation ? pulled
|
|
329
|
+
: parentId === undefined && this.invocationTurnOpen(peer.id)
|
|
330
|
+
? pushedInvocation ? pushed : pulledInvocation ? pulled : undefined
|
|
331
|
+
: undefined;
|
|
332
|
+
if (this.onAnswer && answerParent && (namedPush || namedPull || parentId === undefined && this.invocationTurnOpen(peer.id))) {
|
|
294
333
|
let taken = false;
|
|
295
|
-
try { taken = this.onAnswer(peer.id, text,
|
|
334
|
+
try { taken = this.onAnswer(peer.id, text, answerParent.out) === true; } catch { taken = true; } // a throwing host never turns a callee's answer into a broadcast
|
|
296
335
|
if (taken) return;
|
|
297
336
|
}
|
|
298
337
|
const { priority, body } = parseMarker(text);
|
|
299
338
|
if (!body) return;
|
|
300
339
|
// What the peer was handed differs from what it stands for once a delivery was condensed; both the audience
|
|
301
340
|
// and the priority ceiling are about what it stands for.
|
|
302
|
-
const last =
|
|
341
|
+
const last = (parentId !== undefined && namedPush ? pushed : parentId !== undefined && namedPull ? pulled : pulled ?? pushed);
|
|
303
342
|
const answers = opts?.inReplyTo?.id !== undefined && last?.out.some((e) => e.id === opts.inReplyTo!.id);
|
|
304
343
|
const originals = answers ? last!.originals : undefined;
|
|
305
344
|
// `digest` in the audience stands for the condensed delivery even when a steer, not the digest, is the parent.
|
|
@@ -307,6 +346,7 @@ export class Bus {
|
|
|
307
346
|
const audienceOriginals = originals ?? (opts?.to?.includes(DIGEST) && condensed ? last?.originals : undefined);
|
|
308
347
|
const to = opts?.to?.length ? resolveTo(opts.to, audienceOriginals) : undefined;
|
|
309
348
|
let env = newEnvelope(peer.id, body, { ...opts, ...(to ? { to } : {}), priority: opts?.priority ?? capPriority(peer, priority, opts?.inReplyTo, originals) });
|
|
349
|
+
env = this.reviewResponse(env, opts?.inReplyTo?.id);
|
|
310
350
|
try { this.checkRemotePublication(env); }
|
|
311
351
|
catch (error) {
|
|
312
352
|
if (!(error instanceof PreEffectToolRefusal)) throw error;
|
|
@@ -360,19 +400,20 @@ export class Bus {
|
|
|
360
400
|
if (owner !== peer.id) continue;
|
|
361
401
|
const row = this.journal.get(deliveryId);
|
|
362
402
|
if (row && row.originals.some((e) => envs.some((incoming) => incoming.id === e.id))) {
|
|
363
|
-
this.deliveryReceipt(peer.id, { id: deliveryId, state: "needs_review", reason: "adapter reported delivery failure without safe-failure evidence" });
|
|
403
|
+
this.deliveryReceipt(peer.id, { id: deliveryId, state: "needs_review", reason: "adapter reported delivery failure without safe-failure evidence", failureCause: failure?.failureCause ?? "turn aborted" });
|
|
364
404
|
}
|
|
365
405
|
}
|
|
366
|
-
} else if (envs.some(env => env.kind === "invocation")) {
|
|
406
|
+
} else if (!remoteId(peer.id) && envs.some(env => env.kind === "invocation")) {
|
|
367
407
|
this.recoveryHeldPeers.add(peer.id);
|
|
368
408
|
for (const env of envs) if (env.kind === "invocation") this.onInvocationUncertain?.(env.id);
|
|
369
409
|
} else this.failed(peer.id, same ? last.originals : envs, undefined, failure?.quota === true);
|
|
370
410
|
};
|
|
371
411
|
peer.onState = () => {
|
|
412
|
+
if (peer.state === "offline") this.clearReviewProvenance(peer.id);
|
|
372
413
|
if (this.closed) return;
|
|
373
414
|
if (this.peers.get(peer.id) !== peer) return;
|
|
374
415
|
if (peer.state === "offline" && this.journal) {
|
|
375
|
-
for (const [id, owner] of [...this.activeDeliveries]) if (owner === peer.id) this.uncertain(peer.id, id, "peer disconnected before delivery settlement");
|
|
416
|
+
for (const [id, owner] of [...this.activeDeliveries]) if (owner === peer.id) this.uncertain(peer.id, id, "peer disconnected before delivery settlement", "peer offline");
|
|
376
417
|
}
|
|
377
418
|
this.emit({ t: "state", peer: peer.id, state: this.stateOf(peer.id) });
|
|
378
419
|
void this.drain(peer.id);
|
|
@@ -590,6 +631,13 @@ export class Bus {
|
|
|
590
631
|
return [...new Set([...this.peers.keys(), ...[...this.queues].filter(([, queue]) => queue.length).map(([id]) => id), ...pending])].sort();
|
|
591
632
|
}
|
|
592
633
|
|
|
634
|
+
/** Declare an already-rostered local destination whose adapter is not instantiated yet. */
|
|
635
|
+
ensureKnownRecipientQueue(peer: PeerId): void {
|
|
636
|
+
if (this.closed || this.recoveryHeld || this.storageError) throw new Error("delivery recipient queue unavailable");
|
|
637
|
+
if (typeof peer !== "string" || !/^[a-z][a-z0-9-]{0,31}$/.test(peer) || peer === HUB || peer === USER || peer === DIGEST || remoteId(peer) || !this.isRostered(peer)) throw new Error("invalid configured local recipient");
|
|
638
|
+
if (!this.queues.has(peer)) this.queues.set(peer, []);
|
|
639
|
+
}
|
|
640
|
+
|
|
593
641
|
queueList(peer?: string, unresolvedOnly = false): JournalDelivery[] {
|
|
594
642
|
if (this.journal) {
|
|
595
643
|
const rows = (unresolvedOnly ? this.journal.unresolved(peer) : this.journal.list(peer)).filter((row) => !row.reason?.startsWith("grouped into delivery "));
|
|
@@ -603,9 +651,68 @@ export class Bus {
|
|
|
603
651
|
return ids.flatMap((p) => (this.queues.get(p) ?? []).map((e) => ({ id: e.id, peer: p, state: "queued" as const, revision: 0, createdAt: e.ts, updatedAt: e.ts, originals: [e], out: [e] })));
|
|
604
652
|
}
|
|
605
653
|
|
|
654
|
+
/** Metadata only: status must not hydrate or sort retained delivery payloads. */
|
|
655
|
+
failedDeliveryCounts(): Record<string, number> | undefined {
|
|
656
|
+
if (this.closed || this.storageError || !this.journal) return undefined;
|
|
657
|
+
try { return this.journal.unresolvedFailedCounts(); }
|
|
658
|
+
catch { this.storageError = "delivery journal unavailable"; return undefined; }
|
|
659
|
+
}
|
|
660
|
+
|
|
606
661
|
queueShow(id: string): JournalDelivery | undefined { return this.journal?.get(id) ?? this.queueList().find((d) => d.id === id); }
|
|
607
662
|
|
|
608
|
-
heldDeliveryHints(peer: string
|
|
663
|
+
heldDeliveryHints(peer: string, includeAccepted = false) {
|
|
664
|
+
const adapter = this.peers.get(peer);
|
|
665
|
+
const canExposeAccepted = includeAccepted && adapter?.acceptedSettlement === "explicit";
|
|
666
|
+
return this.storageError ? undefined : this.journal?.heldDeliveryHints(peer, canExposeAccepted);
|
|
667
|
+
}
|
|
668
|
+
|
|
669
|
+
/** Whether timeout-originated holds still protect the recovery cause from supersession. */
|
|
670
|
+
hasAcceptedTimeoutHolds(): boolean {
|
|
671
|
+
if (this.closed || this.storageError || !this.journal) throw new Error("delivery timeout hold state unavailable");
|
|
672
|
+
return this.journal.hasAcceptedTimeoutHolds();
|
|
673
|
+
}
|
|
674
|
+
|
|
675
|
+
/** Expire only accepted channel notifications, without replaying or settling their effects. */
|
|
676
|
+
expireAcceptedDeliveries(now = Date.now()): AcceptedDeliveryExpiry[] {
|
|
677
|
+
if (this.closed || this.recoveryHeld || !this.journal || this.storageError) return [];
|
|
678
|
+
const settleMs = this.opts.acceptedSettleMs?.();
|
|
679
|
+
if (!Number.isSafeInteger(settleMs) || settleMs! < 1) return [];
|
|
680
|
+
if (!Number.isSafeInteger(now) || now < 0) return [];
|
|
681
|
+
if (now < settleMs!) return [];
|
|
682
|
+
const expired: AcceptedDeliveryExpiry[] = [];
|
|
683
|
+
const notify = () => { try { this.onQueues?.(); } catch { /* observers cannot consume a durable timeout notice */ } };
|
|
684
|
+
const eligiblePeers = [...this.peers.values()].filter(peer => peer.acceptedSettlement === "explicit").map(peer => peer.id);
|
|
685
|
+
try {
|
|
686
|
+
for (const row of this.journal.expiredAccepted(now - settleMs!, eligiblePeers)) {
|
|
687
|
+
// The synchronous read/transition pair is revision fenced; no await allows a confirmation to interleave.
|
|
688
|
+
const current = this.journal.get(row.id);
|
|
689
|
+
if (!current || current.peer !== row.peer || current.state !== "accepted" || current.revision !== row.revision) continue;
|
|
690
|
+
let updated: JournalDelivery;
|
|
691
|
+
try {
|
|
692
|
+
updated = this.journal.transition(row.id, "needs_review", ACCEPTED_SETTLE_TIMEOUT_REASON, row.revision, "timeout");
|
|
693
|
+
} catch (error) {
|
|
694
|
+
if (error instanceof Error && /stale delivery revision/.test(error.message)) continue;
|
|
695
|
+
this.storageError = "delivery journal unavailable";
|
|
696
|
+
notify();
|
|
697
|
+
return expired;
|
|
698
|
+
}
|
|
699
|
+
this.activeDeliveries.delete(row.id);
|
|
700
|
+
if (!remoteId(row.peer)) this.recoveryHeldPeers.add(row.peer);
|
|
701
|
+
expired.push({ id: row.id, peer: row.peer, revision: updated.revision, notifyConductor: row.notifyConductor });
|
|
702
|
+
this.deliveryFailed(current.peer);
|
|
703
|
+
for (const env of current.originals) if (isInvocationKind(env)) {
|
|
704
|
+
try { this.onInvocationReceipt?.(env.id, "needs_review"); } catch { /* an observer cannot suppress timeout metadata */ }
|
|
705
|
+
if (env.kind === "invocation") try { this.onInvocationUncertain?.(env.id); } catch { /* preserve uncertain invocation handling for other ids */ }
|
|
706
|
+
}
|
|
707
|
+
}
|
|
708
|
+
} catch {
|
|
709
|
+
this.storageError = "delivery journal unavailable";
|
|
710
|
+
notify();
|
|
711
|
+
return expired;
|
|
712
|
+
}
|
|
713
|
+
if (expired.length) notify();
|
|
714
|
+
return expired;
|
|
715
|
+
}
|
|
609
716
|
|
|
610
717
|
/** Consent is for this peer's held row only, never authority to complete/discard or release a pause. */
|
|
611
718
|
retryDelivery(peer: string, id: string): void {
|
|
@@ -628,8 +735,19 @@ export class Bus {
|
|
|
628
735
|
const row = this.queueShow(id), actor = `peer:${peer}`, cleanReason = reason.trim();
|
|
629
736
|
const notYours = "no needs_review delivery with this id is held for you; call hub_inbox to list your held deliveries and their revisions";
|
|
630
737
|
if (!row || row.peer !== peer) return refuse(notYours);
|
|
738
|
+
if (this.activeDeliveries.has(id) && this.activeDeliveries.get(id) !== peer) return refuse(notYours);
|
|
631
739
|
// An uncertain invocation may still be running: a peer claiming it completed would make the caller wait for a result that may never come. Only the person settles it.
|
|
632
740
|
if (row.originals.some(env => isInvocationKind(env))) return refuse(`delivery ${id} carries an invocation whose outcome is uncertain; only the person settles it (ahub queue show ${id}, then ahub queue resolve ${id})`);
|
|
741
|
+
if (row.state === "accepted") {
|
|
742
|
+
if (this.peers.get(peer)?.acceptedSettlement !== "explicit") return refuse("only an explicit push channel session can settle its accepted delivery; ask the person to resolve it");
|
|
743
|
+
if (row.revision !== revision) return refuse("held delivery revision changed; read hub_inbox again for the current revision");
|
|
744
|
+
try { this.resolveDelivery(id, revision, action, cleanReason, actor); }
|
|
745
|
+
catch (error) {
|
|
746
|
+
if (error instanceof Error && /stale delivery revision/.test(error.message)) return refuse("held delivery revision changed; read hub_inbox again for the current revision");
|
|
747
|
+
return refuse("delivery storage refused this disposition; inspect ahub status and ask the person to resolve it");
|
|
748
|
+
}
|
|
749
|
+
return;
|
|
750
|
+
}
|
|
633
751
|
const previous = this.journal.resolution(id);
|
|
634
752
|
if (previous) {
|
|
635
753
|
if (previous.actor !== actor || previous.action !== action || previous.reason !== cleanReason) return refuse(`delivery ${id} was already settled with a different action or reason; the person can inspect it with ahub queue show ${id}`);
|
|
@@ -684,10 +802,11 @@ export class Bus {
|
|
|
684
802
|
if (!Number.isSafeInteger(deadline) || deadline! <= Date.now()) throw new PreEffectToolRefusal("remote original deadline unavailable or expired; discard it or send a new message explicitly");
|
|
685
803
|
this.checkRemotePublication(record.originals[0]!);
|
|
686
804
|
}
|
|
687
|
-
const
|
|
805
|
+
const explicitAcceptedDisposition = actor === `peer:${record.peer}` && record.state === "accepted" && this.peers.get(record.peer)?.acceptedSettlement === "explicit" && (action === "completed" || action === "discard");
|
|
806
|
+
const recipientConsent = actor === `peer:${record.peer}` && (record.state === "needs_review" && (action === "retry" || action === "completed" || action === "discard") || explicitAcceptedDisposition);
|
|
688
807
|
// #713: a `queued` delivery was never handed to the peer, so nothing is in flight and a busy peer (a pull-only TUI is busy most of the time) does not block
|
|
689
|
-
// discarding it. `dispatching`
|
|
690
|
-
if (
|
|
808
|
+
// discarding it. `dispatching` stays refused; `accepted` permits only its explicit channel owner's completed/discard disposition.
|
|
809
|
+
if (record.state === "dispatching" || record.state === "accepted" && !explicitAcceptedDisposition) throw new Error("active delivery cannot be resolved");
|
|
691
810
|
const queuedDiscard = record.state === "queued" && action === "discard";
|
|
692
811
|
// #1096: a `failed` row was refused before it ran and discarding it touches no queue, so a busy peer has nothing in flight to protect either.
|
|
693
812
|
const failedDiscard = record.state === "failed" && action === "discard";
|
|
@@ -703,7 +822,7 @@ export class Bus {
|
|
|
703
822
|
this.journal.transaction(() => {
|
|
704
823
|
if (!this.journal!.get(id)) this.journal!.createDelivery({ ...record });
|
|
705
824
|
const current = this.journal!.get(id)!;
|
|
706
|
-
this.journal!.resolve(id, current.revision, action, reason, revision, actor);
|
|
825
|
+
this.journal!.resolve(id, current.revision, action, reason, revision, actor, explicitAcceptedDisposition);
|
|
707
826
|
if (record.state === "queued") {
|
|
708
827
|
const ids = new Set(record.originals.map((env) => env.id));
|
|
709
828
|
queue.splice(0, queue.length, ...queue.filter((env) => !ids.has(env.id)));
|
|
@@ -724,6 +843,7 @@ export class Bus {
|
|
|
724
843
|
else this.prefaces.delete(record.peer);
|
|
725
844
|
throw error;
|
|
726
845
|
}
|
|
846
|
+
if (explicitAcceptedDisposition) this.activeDeliveries.delete(id);
|
|
727
847
|
this.refreshRecoveryHold(record.peer);
|
|
728
848
|
this.onQueues?.();
|
|
729
849
|
void this.drain(record.peer);
|
|
@@ -734,7 +854,7 @@ export class Bus {
|
|
|
734
854
|
const queued = this.queues.get(peer) ?? [];
|
|
735
855
|
const review = rows.filter((r) => r.state === "needs_review");
|
|
736
856
|
const needsReview = review.length;
|
|
737
|
-
const blocked = rows.find((r) => r.state === "needs_review" || (["dispatching", "accepted"].includes(r.state) && !this.activeDeliveries.has(r.id)));
|
|
857
|
+
const blocked = rows.find((r) => (r.state === "needs_review" && (!remoteId(peer) || r.originals.some(isInvocationKind))) || (["dispatching", "accepted"].includes(r.state) && !this.activeDeliveries.has(r.id)));
|
|
738
858
|
const liveAccepted = rows.filter((r) => r.state === "accepted" && this.activeDeliveries.get(r.id) === peer).map((r) => r.id);
|
|
739
859
|
const hold = { ...(blocked ? { heldBy: blocked.id } : {}), ...(liveAccepted.length ? { liveAccepted } : {}) };
|
|
740
860
|
const oldest = queued.reduce<number | undefined>((a, r) => a === undefined ? r.ts : Math.min(a, r.ts), undefined);
|
|
@@ -744,6 +864,7 @@ export class Bus {
|
|
|
744
864
|
manualPausedPeers(): string[] { return [...this.manualPaused].sort(); }
|
|
745
865
|
setManualPaused(ids: string[]): void { this.manualPaused.clear(); for (const id of ids) this.manualPaused.add(id); this.persist(); }
|
|
746
866
|
closeJournal(): void {
|
|
867
|
+
this.reviewParents.clear(); this.reviewResponses.clear();
|
|
747
868
|
this.closed = true;
|
|
748
869
|
this.recoveryHeld = true;
|
|
749
870
|
for (const timer of this.timers.values()) clearTimeout(timer);
|
|
@@ -760,22 +881,33 @@ export class Bus {
|
|
|
760
881
|
if (this.peers.get(peer)?.remoteRoom !== undefined && receipt.state === "failed_safe") {
|
|
761
882
|
// A signed recipient NACK is a terminal known refusal, not a failed native
|
|
762
883
|
// turn eligible for poison retries. It records no recipient read evidence.
|
|
763
|
-
this.journal.transition(id, "failed", receipt.reason ?? "remote recipient refused safely");
|
|
764
|
-
this.activeDeliveries.delete(id);
|
|
884
|
+
this.journal.transition(id, "failed", receipt.reason ?? "remote recipient refused safely", undefined, "adapter refused");
|
|
885
|
+
this.activeDeliveries.delete(id);
|
|
886
|
+
if (this.remoteTransport.get(id)?.peer === peer) this.remoteTransport.delete(id);
|
|
765
887
|
for (const env of before?.originals ?? []) { this.attempts.delete(`${peer}:${env.id}`); this.remoteQueueExpires.get(peer)?.delete(env.id); }
|
|
766
888
|
const last = this.lastDelivery.get(peer);
|
|
767
889
|
if (before && last && last.out.length === before.out.length && last.out.every((env, index) => env.id === before.out[index]!.id)) this.lastDelivery.delete(peer);
|
|
768
890
|
this.refreshRecoveryHold(peer);
|
|
769
|
-
for (const env of before?.originals ?? []) if (isInvocationKind(env))
|
|
891
|
+
for (const env of before?.originals ?? []) if (isInvocationKind(env)) {
|
|
892
|
+
try { this.onInvocationReceipt?.(env.id, "failed_safe"); } catch { /* a ledger observer cannot undo a durable signed refusal */ }
|
|
893
|
+
}
|
|
770
894
|
this.onQueues?.();
|
|
771
895
|
void this.drain(peer);
|
|
772
896
|
return;
|
|
773
897
|
}
|
|
774
898
|
const state: JournalDeliveryState = receipt.state === "accepted" ? "accepted" : receipt.state === "completed" ? "completed" : receipt.state === "failed_safe" ? "failed" : "needs_review";
|
|
775
|
-
if (state === "failed" || state === "needs_review") this.deliveryFailed(peer);
|
|
776
|
-
if (state === "completed") this.journal.completeDelivered(id, peer, "adapter", receipt.reason);
|
|
777
|
-
else
|
|
778
|
-
|
|
899
|
+
if (!remoteId(peer) && (state === "failed" || state === "needs_review")) this.deliveryFailed(peer);
|
|
900
|
+
if (state === "completed" && !remoteId(peer)) this.journal.completeDelivered(id, peer, "adapter", receipt.reason);
|
|
901
|
+
else {
|
|
902
|
+
let cause: DeliveryFailureCause | undefined;
|
|
903
|
+
if (state === "failed" || state === "needs_review") {
|
|
904
|
+
cause = isDeliveryFailureCause(receipt.failureCause) ? receipt.failureCause : undefined;
|
|
905
|
+
if (!cause && state === "failed") cause = this.stateOf(peer) === "offline" ? "peer offline" : "adapter refused";
|
|
906
|
+
if (!cause) cause = "turn aborted";
|
|
907
|
+
}
|
|
908
|
+
this.journal.transition(id, state, receipt.reason, undefined, cause);
|
|
909
|
+
}
|
|
910
|
+
if (!remoteId(peer) && before?.state === "dispatching" && (state === "accepted" || state === "completed")) {
|
|
779
911
|
this.memoryDelivery(peer, id, before.out, "delivered", before.originals);
|
|
780
912
|
this.delivered(peer, before.originals, id);
|
|
781
913
|
}
|
|
@@ -783,20 +915,27 @@ export class Bus {
|
|
|
783
915
|
if (before) this.failed(peer, before.originals, before.originals.some((e) => e.private) ? "private delivery failed" : receipt.reason, receipt.quota === true && before.state === "dispatching");
|
|
784
916
|
}
|
|
785
917
|
if (state === "completed") { this.failureStreak.delete(peer); this.lastFailure.delete(peer); }
|
|
786
|
-
if (state === "completed" || state === "failed" || state === "needs_review")
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
918
|
+
if (state === "completed" || state === "failed" || state === "needs_review") {
|
|
919
|
+
this.activeDeliveries.delete(id);
|
|
920
|
+
if (remoteId(peer) && this.remoteTransport.get(id)?.peer === peer) this.remoteTransport.delete(id);
|
|
921
|
+
}
|
|
922
|
+
for (const env of before?.originals ?? []) if (isInvocationKind(env) && (receipt.state === "completed" || receipt.state === "needs_review")) {
|
|
923
|
+
try { this.onInvocationReceipt?.(env.id, receipt.state); } catch { /* a ledger observer cannot undo a durable transport receipt */ }
|
|
924
|
+
}
|
|
925
|
+
if (state === "needs_review" && (!remoteId(peer) || before?.originals.some(isInvocationKind))) {
|
|
790
926
|
this.recoveryHeldPeers.add(peer);
|
|
791
|
-
for (const env of before?.originals ?? []) if (env.kind === "invocation")
|
|
927
|
+
for (const env of before?.originals ?? []) if (env.kind === "invocation") {
|
|
928
|
+
try { this.onInvocationUncertain?.(env.id); } catch { /* uncertainty hold remains durable */ }
|
|
929
|
+
}
|
|
792
930
|
}
|
|
931
|
+
if (state === "completed" || state === "failed" || state === "needs_review" && remoteId(peer)) { this.refreshRecoveryHold(peer); void this.drain(peer); }
|
|
793
932
|
this.onQueues?.();
|
|
794
933
|
}
|
|
795
934
|
|
|
796
935
|
private refreshRecoveryHold(peer: PeerId): void {
|
|
797
936
|
if (!this.journal) return;
|
|
798
937
|
// Live accepted notifications are awaiting explicit correlation, not crash recovery.
|
|
799
|
-
const blocked = this.journal.list(peer).some((r) => r.state === "needs_review" || (["dispatching", "accepted"].includes(r.state) && this.activeDeliveries.get(r.id) !== peer));
|
|
938
|
+
const blocked = this.journal.list(peer).some((r) => (r.state === "needs_review" && (!remoteId(peer) || r.originals.some(isInvocationKind))) || (["dispatching", "accepted"].includes(r.state) && this.activeDeliveries.get(r.id) !== peer));
|
|
800
939
|
if (blocked) this.recoveryHeldPeers.add(peer); else this.recoveryHeldPeers.delete(peer);
|
|
801
940
|
}
|
|
802
941
|
|
|
@@ -865,14 +1004,13 @@ export class Bus {
|
|
|
865
1004
|
}
|
|
866
1005
|
|
|
867
1006
|
pause(id: PeerId): void {
|
|
1007
|
+
// Transient holds are not in BusSnapshot; their owners persist durable policy.
|
|
868
1008
|
this.paused.add(id);
|
|
869
|
-
this.persist();
|
|
870
1009
|
this.emit({ t: "state", peer: id, state: this.stateOf(id) });
|
|
871
1010
|
}
|
|
872
1011
|
|
|
873
1012
|
resume(id: PeerId): void {
|
|
874
1013
|
if (!this.paused.delete(id)) return;
|
|
875
|
-
this.persist();
|
|
876
1014
|
this.emit({ t: "state", peer: id, state: this.stateOf(id) });
|
|
877
1015
|
void this.drain(id);
|
|
878
1016
|
}
|
|
@@ -1040,9 +1178,9 @@ export class Bus {
|
|
|
1040
1178
|
const index = queue.findIndex(item => item.id === env.id);
|
|
1041
1179
|
if (index >= 0) queue.splice(index, 1);
|
|
1042
1180
|
const known = this.remoteQueueExpires.get(peer)?.has(env.id);
|
|
1181
|
+
const failureCause: DeliveryFailureCause = known && this.remoteQueueRooms.has(peer) && this.remoteQueueMembers.has(peer) ? "timeout" : "push error";
|
|
1043
1182
|
this.remoteQueueExpires.get(peer)?.delete(env.id);
|
|
1044
|
-
this.pendingOutcomes.push({ id: crypto.randomUUID(), peer, state: "needs_review", createdAt: now, originals: [env], out: [env], reason: !known ? "remote queue expiry unavailable after restore" : !this.remoteQueueRooms.has(peer) ? "remote queue room binding unavailable after restore" : !this.remoteQueueMembers.has(peer) ? "remote queue member binding unavailable after restore" : "remote queue expired before recipient admission" });
|
|
1045
|
-
this.recoveryHeldPeers.add(peer);
|
|
1183
|
+
this.pendingOutcomes.push({ id: crypto.randomUUID(), peer, state: "needs_review", createdAt: now, originals: [env], out: [env], failureCause, reason: !known ? "remote queue expiry unavailable after restore" : !this.remoteQueueRooms.has(peer) ? "remote queue room binding unavailable after restore" : !this.remoteQueueMembers.has(peer) ? "remote queue member binding unavailable after restore" : "remote queue expired before recipient admission" });
|
|
1046
1184
|
}
|
|
1047
1185
|
this.persist();
|
|
1048
1186
|
} catch (error) {
|
|
@@ -1073,13 +1211,252 @@ export class Bus {
|
|
|
1073
1211
|
return true;
|
|
1074
1212
|
}
|
|
1075
1213
|
|
|
1214
|
+
/** Settle an active or held remote row only when the current room adapter owns this receipt path. */
|
|
1215
|
+
settleRemoteReceipt(source: PeerAdapter, deliveryId: string, outcome: "completed" | "failed_safe", reason: string, currentGuard: () => boolean): boolean {
|
|
1216
|
+
const peer = source.id;
|
|
1217
|
+
if ((outcome !== "completed" && outcome !== "failed_safe") || !parseRemotePeerId(peer) || this.peers.get(peer) !== source ||
|
|
1218
|
+
!source.remoteRoom || !remoteFingerprint(source.remoteMember) || !this.journal || this.storageError || this.closed) return false;
|
|
1219
|
+
if (this.remoteQueueRooms.get(peer) !== source.remoteRoom || this.remoteQueueMembers.get(peer) !== source.remoteMember) return false;
|
|
1220
|
+
const row = this.journal.get(deliveryId);
|
|
1221
|
+
if (!row || row.peer !== peer || !["needs_review", "dispatching", "accepted"].includes(row.state) || row.originals.length !== 1 || row.originals[0]?.to?.includes(peer) !== true || this.journal.resolution(deliveryId)) return false;
|
|
1222
|
+
if (row.state === "needs_review" && row.originals.some(isInvocationKind)) return false;
|
|
1223
|
+
const activeOwner = this.activeDeliveries.get(deliveryId);
|
|
1224
|
+
if (activeOwner !== undefined && activeOwner !== peer) return false;
|
|
1225
|
+
try { if (currentGuard() !== true) return false; } catch { return false; }
|
|
1226
|
+
try {
|
|
1227
|
+
if (outcome === "completed") this.journal.completeRemoteReceipt(row.id, peer, row.revision, reason);
|
|
1228
|
+
else this.journal.failRemoteReceipt(row.id, peer, row.revision, reason);
|
|
1229
|
+
} catch (error) {
|
|
1230
|
+
if (error instanceof Error && /stale delivery revision|already has a person disposition|not unresolved/.test(error.message)) return false;
|
|
1231
|
+
this.storageError = "delivery journal unavailable";
|
|
1232
|
+
this.onQueues?.();
|
|
1233
|
+
return false;
|
|
1234
|
+
}
|
|
1235
|
+
this.activeDeliveries.delete(row.id);
|
|
1236
|
+
this.remoteTransport.delete(row.id);
|
|
1237
|
+
for (const env of row.originals) {
|
|
1238
|
+
this.attempts.delete(`${peer}:${env.id}`);
|
|
1239
|
+
this.remoteQueueExpires.get(peer)?.delete(env.id);
|
|
1240
|
+
}
|
|
1241
|
+
if (row.state !== "needs_review") for (const env of row.originals) if (isInvocationKind(env)) {
|
|
1242
|
+
try { this.onInvocationReceipt?.(env.id, outcome); } catch { /* the durable room receipt remains settled */ }
|
|
1243
|
+
}
|
|
1244
|
+
this.refreshRecoveryHold(peer);
|
|
1245
|
+
this.emit({ t: "remote_delivery_settled", peer, deliveryId: row.id, outcome });
|
|
1246
|
+
this.onQueues?.();
|
|
1247
|
+
void this.drain(peer);
|
|
1248
|
+
return true;
|
|
1249
|
+
}
|
|
1250
|
+
|
|
1251
|
+
/** A typed room refusal before durable queue admission is safe to retry without failure accounting. */
|
|
1252
|
+
requeueRemoteNotSent(source: PeerAdapter, deliveryId: string): boolean {
|
|
1253
|
+
const peer = source.id;
|
|
1254
|
+
if (!parseRemotePeerId(peer) || this.peers.get(peer) !== source || !source.remoteRoom ||
|
|
1255
|
+
!remoteFingerprint(source.remoteMember) || !this.journal || this.storageError || this.closed ||
|
|
1256
|
+
this.remoteQueueRooms.get(peer) !== source.remoteRoom || this.remoteQueueMembers.get(peer) !== source.remoteMember ||
|
|
1257
|
+
this.activeDeliveries.get(deliveryId) !== peer) return false;
|
|
1258
|
+
const row = this.journal.get(deliveryId);
|
|
1259
|
+
if (!row || row.peer !== peer || row.state !== "dispatching" || row.originals.length !== 1) return false;
|
|
1260
|
+
const env = row.originals[0]!;
|
|
1261
|
+
if (env.to?.includes(peer) !== true) return false;
|
|
1262
|
+
const expiry = this.remoteQueueExpires.get(peer)?.get(env.id);
|
|
1263
|
+
if (expiry === undefined || expiry <= Date.now()) {
|
|
1264
|
+
this.deliveryReceipt(peer, { id: deliveryId, state: "needs_review", failureCause: expiry === undefined ? "push error" : "timeout", reason: expiry === undefined ? "remote queue expiry unavailable after restore" : "remote queue expired before recipient admission" });
|
|
1265
|
+
return true;
|
|
1266
|
+
}
|
|
1267
|
+
const queue = this.queues.get(peer) ?? [], beforeQueue = [...queue];
|
|
1268
|
+
if (queue.some(item => item.id === env.id)) return false;
|
|
1269
|
+
try {
|
|
1270
|
+
this.journal.transaction(() => {
|
|
1271
|
+
this.journal!.transition(deliveryId, "queued", "room transport confirmed not sent", row.revision);
|
|
1272
|
+
queue.unshift(env);
|
|
1273
|
+
this.queues.set(peer, queue);
|
|
1274
|
+
this.persist();
|
|
1275
|
+
});
|
|
1276
|
+
} catch {
|
|
1277
|
+
queue.splice(0, queue.length, ...beforeQueue);
|
|
1278
|
+
if (!beforeQueue.length && !this.queues.has(peer)) this.queues.delete(peer);
|
|
1279
|
+
this.storageError = "delivery journal unavailable";
|
|
1280
|
+
this.onQueues?.();
|
|
1281
|
+
return false;
|
|
1282
|
+
}
|
|
1283
|
+
this.activeDeliveries.delete(deliveryId);
|
|
1284
|
+
this.remoteTransport.delete(deliveryId);
|
|
1285
|
+
this.refreshRecoveryHold(peer);
|
|
1286
|
+
this.onQueues?.();
|
|
1287
|
+
return true;
|
|
1288
|
+
}
|
|
1289
|
+
|
|
1290
|
+
/** Host-only recovery port: expose one matching room original for proof revalidation. */
|
|
1291
|
+
remoteDeliveryEnvelope(source: PeerAdapter, deliveryId: string): Envelope | undefined {
|
|
1292
|
+
const peer = source.id;
|
|
1293
|
+
if (this.peers.get(peer) !== source || !parseRemotePeerId(peer) || !source.remoteRoom || !remoteFingerprint(source.remoteMember) ||
|
|
1294
|
+
this.remoteQueueRooms.get(peer) !== source.remoteRoom || this.remoteQueueMembers.get(peer) !== source.remoteMember ||
|
|
1295
|
+
!this.journal || this.storageError || this.closed) return undefined;
|
|
1296
|
+
const row = this.journal.get(deliveryId), original = row?.originals[0];
|
|
1297
|
+
if (!row || row.peer !== peer || !["needs_review", "dispatching", "accepted"].includes(row.state) || this.journal.resolution(deliveryId) || row.originals.length !== 1 || original?.to?.includes(peer) !== true) return undefined;
|
|
1298
|
+
return structuredClone(original);
|
|
1299
|
+
}
|
|
1300
|
+
|
|
1301
|
+
/** Current host admission for sending remote work, separate from immutable room trust proof. */
|
|
1302
|
+
remoteSendAllowed(source: PeerAdapter): boolean {
|
|
1303
|
+
const peer = source.id;
|
|
1304
|
+
if (!parseRemotePeerId(peer) || this.peers.get(peer) !== source || !source.remoteRoom || !remoteFingerprint(source.remoteMember) ||
|
|
1305
|
+
!this.journal || this.storageError || this.closed || this.recoveryHeld ||
|
|
1306
|
+
this.remoteQueueRooms.get(peer) !== source.remoteRoom || this.remoteQueueMembers.get(peer) !== source.remoteMember) return false;
|
|
1307
|
+
try {
|
|
1308
|
+
return this.isRostered(peer) && !this.paused.has(peer) && !this.manualPaused.has(peer) && this.opts.deliveryHeld?.(peer) !== true;
|
|
1309
|
+
} catch { return false; }
|
|
1310
|
+
}
|
|
1311
|
+
|
|
1312
|
+
private holdRemoteDeliveryMetadata(peer: PeerId, row: JournalDelivery, reason: string): boolean {
|
|
1313
|
+
try { this.journal!.transition(row.id, "needs_review", reason, row.revision, row.failureCause ?? "push error"); }
|
|
1314
|
+
catch (error) {
|
|
1315
|
+
if (error instanceof Error && /stale delivery revision|unknown delivery|already has a person disposition/.test(error.message)) return false;
|
|
1316
|
+
this.storageError = "delivery journal unavailable";
|
|
1317
|
+
this.onQueues?.();
|
|
1318
|
+
return false;
|
|
1319
|
+
}
|
|
1320
|
+
this.activeDeliveries.delete(row.id);
|
|
1321
|
+
if (this.remoteTransport.get(row.id)?.peer === peer) this.remoteTransport.delete(row.id);
|
|
1322
|
+
this.refreshRecoveryHold(peer);
|
|
1323
|
+
this.onQueues?.();
|
|
1324
|
+
void this.drain(peer);
|
|
1325
|
+
return true;
|
|
1326
|
+
}
|
|
1327
|
+
|
|
1328
|
+
/** Hold a remote row for human inspection when its durable intent or delivery proof is unavailable. */
|
|
1329
|
+
holdRemoteMissingIntent(source: PeerAdapter, deliveryId: string, currentGuard: () => boolean, origin: "restored" | "known-not-sent" | "proof-unavailable" = "restored"): boolean {
|
|
1330
|
+
if (origin !== "restored" && origin !== "known-not-sent" && origin !== "proof-unavailable") return false;
|
|
1331
|
+
const peer = source.id;
|
|
1332
|
+
if (!parseRemotePeerId(peer) || this.peers.get(peer) !== source || !source.remoteRoom || !remoteFingerprint(source.remoteMember) ||
|
|
1333
|
+
!this.journal || this.storageError || this.closed ||
|
|
1334
|
+
this.remoteQueueRooms.get(peer) !== source.remoteRoom || this.remoteQueueMembers.get(peer) !== source.remoteMember) return false;
|
|
1335
|
+
const row = this.journal.get(deliveryId);
|
|
1336
|
+
if (!row || row.peer !== peer || !["dispatching", "accepted", ...(origin === "proof-unavailable" ? ["needs_review"] : [])].includes(row.state) || row.originals.length !== 1 ||
|
|
1337
|
+
row.originals[0]?.to?.includes(peer) !== true || this.journal.resolution(deliveryId)) return false;
|
|
1338
|
+
if (origin === "proof-unavailable" && row.originals.some(isInvocationKind)) return false;
|
|
1339
|
+
if (origin === "known-not-sent") {
|
|
1340
|
+
if (this.activeDeliveries.get(deliveryId) !== peer) return false;
|
|
1341
|
+
} else if (origin === "restored") {
|
|
1342
|
+
if (this.activeDeliveries.has(deliveryId) || this.remoteTransport.has(deliveryId)) return false;
|
|
1343
|
+
} else {
|
|
1344
|
+
const activeOwner = this.activeDeliveries.get(deliveryId), transportOwner = this.remoteTransport.get(deliveryId)?.peer;
|
|
1345
|
+
if (activeOwner !== undefined && activeOwner !== peer || transportOwner !== undefined && transportOwner !== peer) return false;
|
|
1346
|
+
}
|
|
1347
|
+
try { if (currentGuard() !== true) return false; } catch { return false; }
|
|
1348
|
+
if (origin === "proof-unavailable" && row.state === "needs_review") return true;
|
|
1349
|
+
const reason = origin === "restored"
|
|
1350
|
+
? `room durable intent unavailable after restart; inspect ahub queue show ${deliveryId}`
|
|
1351
|
+
: origin === "known-not-sent"
|
|
1352
|
+
? `room not-sent checkpoint unavailable; inspect ahub queue show ${deliveryId}`
|
|
1353
|
+
: `${ROOM_PROOF_UNAVAILABLE_REASON} ${deliveryId}`;
|
|
1354
|
+
return this.holdRemoteDeliveryMetadata(peer, row, reason);
|
|
1355
|
+
}
|
|
1356
|
+
|
|
1357
|
+
/** Host-only quarantine when a signed room outcome belongs to an invalidated recipient tuple. */
|
|
1358
|
+
holdInvalidatedRemoteOutcome(peerId: string, deliveryId: string, expectedRoom: string, expectedMember: string, currentGuard: () => boolean): boolean {
|
|
1359
|
+
// Exact equality below inherits the room bound enforced at add/loadSnapshot.
|
|
1360
|
+
if (!parseRemotePeerId(peerId) || typeof expectedRoom !== "string" || !expectedRoom ||
|
|
1361
|
+
!remoteFingerprint(expectedMember) || !this.journal || this.storageError || this.closed ||
|
|
1362
|
+
this.remoteQueueRooms.get(peerId) !== expectedRoom || this.remoteQueueMembers.get(peerId) !== expectedMember) return false;
|
|
1363
|
+
const row = this.journal.get(deliveryId);
|
|
1364
|
+
if (!row || row.peer !== peerId || !["dispatching", "accepted"].includes(row.state) || row.originals.length !== 1 ||
|
|
1365
|
+
row.originals[0]?.to?.includes(peerId) !== true || this.journal.resolution(deliveryId) ||
|
|
1366
|
+
this.activeDeliveries.has(deliveryId) || this.remoteTransport.has(deliveryId)) return false;
|
|
1367
|
+
try { if (currentGuard() !== true) return false; } catch { return false; }
|
|
1368
|
+
return this.holdRemoteDeliveryMetadata(peerId, row, `room recipient proof no longer current after restart; inspect ahub queue show ${deliveryId}`);
|
|
1369
|
+
}
|
|
1370
|
+
|
|
1371
|
+
/** Person-authorized epoch change: hold metadata only, without cancellation, replay or completion. */
|
|
1372
|
+
holdRemoteEpochRotated(peerId: string, deliveryId: string, expectedRoom: string, expectedMember: string, currentGuard: () => boolean): boolean {
|
|
1373
|
+
if (!parseRemotePeerId(peerId) || typeof expectedRoom !== "string" || !expectedRoom || !remoteFingerprint(expectedMember)
|
|
1374
|
+
|| !this.journal || this.storageError || this.closed
|
|
1375
|
+
|| this.remoteQueueRooms.get(peerId) !== expectedRoom || this.remoteQueueMembers.get(peerId) !== expectedMember) return false;
|
|
1376
|
+
const row = this.journal.get(deliveryId), reason = `${ROOM_EPOCH_ROTATED_REASON} ${deliveryId}`;
|
|
1377
|
+
if (!row || row.peer !== peerId || row.originals.length !== 1 || row.originals[0]?.to?.includes(peerId) !== true
|
|
1378
|
+
|| this.journal.resolution(deliveryId)) return false;
|
|
1379
|
+
const active = this.activeDeliveries.get(deliveryId), transport = this.remoteTransport.get(deliveryId)?.peer;
|
|
1380
|
+
if (active !== undefined && active !== peerId || transport !== undefined && transport !== peerId) return false;
|
|
1381
|
+
if (!["dispatching", "accepted", "needs_review"].includes(row.state)) return false;
|
|
1382
|
+
if (row.state === "needs_review" && row.reason !== reason && row.reason !== `${ROOM_PROOF_UNAVAILABLE_REASON} ${deliveryId}`) return false;
|
|
1383
|
+
try { if (currentGuard() !== true) return false; } catch { return false; }
|
|
1384
|
+
if (row.state === "needs_review" && row.reason === reason) return true;
|
|
1385
|
+
return this.holdRemoteDeliveryMetadata(peerId, row, reason);
|
|
1386
|
+
}
|
|
1387
|
+
|
|
1388
|
+
/** Resume the same delivery after the host revalidates its durable outbox proof. */
|
|
1389
|
+
resumeRemotePending(source: PeerAdapter, deliveryId: string, state: "dispatching" | "accepted" | "needs_review", currentGuard: () => boolean): boolean {
|
|
1390
|
+
const peer = source.id;
|
|
1391
|
+
const journal = this.journal;
|
|
1392
|
+
if (!journal || !this.remoteSendAllowed(source)) return false;
|
|
1393
|
+
const row = journal.get(deliveryId), env = row?.originals[0], expiry = env && this.remoteQueueExpires.get(peer)?.get(env.id);
|
|
1394
|
+
if (!row || row.peer !== peer || !["dispatching", "accepted", "needs_review"].includes(row.state) || row.originals.length !== 1 || env?.to?.includes(peer) !== true || journal.resolution(deliveryId) || !env || expiry === undefined || expiry <= Date.now()) return false;
|
|
1395
|
+
if (row.state === "needs_review" && row.originals.some(isInvocationKind)) return false;
|
|
1396
|
+
if (state === "needs_review") return false;
|
|
1397
|
+
if ([...(this.queues.get(peer) ?? [])].some(item => item.id === env.id)) return false;
|
|
1398
|
+
if ([...this.activeDeliveries].some(([id, owner]) => owner === peer && id !== deliveryId)) return false;
|
|
1399
|
+
if ([...this.remoteTransport].some(([id, active]) => active.peer === peer && id !== deliveryId)) return false;
|
|
1400
|
+
try { if (currentGuard() !== true) return false; } catch { return false; }
|
|
1401
|
+
try {
|
|
1402
|
+
// Store and journal commit independently. An already-active journal row is
|
|
1403
|
+
// authoritative; a different Store state must not strand or downgrade it.
|
|
1404
|
+
if (row.state === "needs_review") journal.transition(deliveryId, state, "room outbox proof confirmed pending", row.revision);
|
|
1405
|
+
}
|
|
1406
|
+
catch {
|
|
1407
|
+
this.storageError = "delivery journal unavailable";
|
|
1408
|
+
this.onQueues?.();
|
|
1409
|
+
return false;
|
|
1410
|
+
}
|
|
1411
|
+
this.activeDeliveries.set(deliveryId, peer);
|
|
1412
|
+
this.remoteTransport.set(deliveryId, { peer, originals: row.originals });
|
|
1413
|
+
this.refreshRecoveryHold(peer);
|
|
1414
|
+
this.onQueues?.();
|
|
1415
|
+
void this.drain(peer);
|
|
1416
|
+
return true;
|
|
1417
|
+
}
|
|
1418
|
+
|
|
1076
1419
|
/** Returns the peers the envelope was accepted for, including durable deferred notices. */
|
|
1077
1420
|
publish(env: Envelope): PeerId[] {
|
|
1078
1421
|
return this.publishResult(env).targets;
|
|
1079
1422
|
}
|
|
1080
1423
|
|
|
1424
|
+
private clearReviewProvenance(peer: PeerId): void {
|
|
1425
|
+
this.reviewParents.delete(peer);
|
|
1426
|
+
for (const [id, source] of this.reviewResponses) if (source.peer === peer) this.reviewResponses.delete(id);
|
|
1427
|
+
}
|
|
1428
|
+
|
|
1429
|
+
/** Host-derived observer provenance. Caller-supplied response/package markers are stripped.
|
|
1430
|
+
* ponytail: active turns and 128 delivered parent IDs per peer are attributable; arbitrary later
|
|
1431
|
+
* native-session paraphrases need explicit session provenance, never content matching or PII authority.
|
|
1432
|
+
*/
|
|
1433
|
+
reviewResponse(env: Envelope, parentId?: string): Envelope {
|
|
1434
|
+
if (env.from === HUB) return env;
|
|
1435
|
+
const pulled = this.lastPulled.get(env.from)?.originals ?? [], pushed = this.lastDelivery.get(env.from)?.originals ?? [];
|
|
1436
|
+
const active = pulled.length ? pulled : pushed;
|
|
1437
|
+
const parent = parentId
|
|
1438
|
+
? pulled.find(item => item.id === parentId && isReviewObserverExcluded(item)) ?? pushed.find(item => item.id === parentId && isReviewObserverExcluded(item))
|
|
1439
|
+
: active.find(isReviewObserverExcluded);
|
|
1440
|
+
const known = this.reviewResponses.get(env.id);
|
|
1441
|
+
let durableScope: string | undefined;
|
|
1442
|
+
try { durableScope = this.opts.reviewObserverScope?.(env.from); }
|
|
1443
|
+
catch { durableScope = "review-scope-unavailable"; } // Unknown provenance cannot clear an observer exclusion.
|
|
1444
|
+
const source = parent?.refs?.reviewPackage ?? parent?.refs?.reviewResponse
|
|
1445
|
+
?? (parentId ? this.reviewParents.get(env.from)?.get(parentId) : undefined)
|
|
1446
|
+
?? (known?.peer === env.from ? known.source : undefined) ?? durableScope;
|
|
1447
|
+
const { reviewPackage: _package, reviewResponse: _response, ...refs } = env.refs ?? {};
|
|
1448
|
+
if (source && (env.kind === "chat" || env.kind === "status")) {
|
|
1449
|
+
this.reviewResponses.set(env.id, { peer: env.from, source });
|
|
1450
|
+
if (this.reviewResponses.size > SEEN_CAP) this.reviewResponses.delete(this.reviewResponses.keys().next().value!);
|
|
1451
|
+
return { ...env, refs: { ...refs, reviewResponse: source } };
|
|
1452
|
+
}
|
|
1453
|
+
return _package !== undefined || _response !== undefined ? { ...env, refs } : env;
|
|
1454
|
+
}
|
|
1455
|
+
|
|
1081
1456
|
/** Publication outcome distinguishes a policy drop from an empty audience. */
|
|
1082
1457
|
publishResult(env: Envelope): PublishResult {
|
|
1458
|
+
if (this.closed) return { targets: [] };
|
|
1459
|
+
env = this.reviewResponse(env);
|
|
1083
1460
|
if (this.storageError) throw new Error("delivery journal unavailable");
|
|
1084
1461
|
if (this.seen.has(env.id) || this.deferredHub.has(env.id)) return { targets: [] };
|
|
1085
1462
|
this.checkRemotePublication(env);
|
|
@@ -1096,6 +1473,7 @@ export class Bus {
|
|
|
1096
1473
|
}
|
|
1097
1474
|
|
|
1098
1475
|
private publishNow(env: Envelope): PublishResult {
|
|
1476
|
+
if (this.closed) return { targets: [] };
|
|
1099
1477
|
if (this.storageError) throw new Error("delivery journal unavailable");
|
|
1100
1478
|
if (this.seen.has(env.id)) return { targets: [] };
|
|
1101
1479
|
const rooms = this.checkRemotePublication(env);
|
|
@@ -1118,14 +1496,17 @@ export class Bus {
|
|
|
1118
1496
|
if (!peer) { this.enqueue(id, env); continue; }
|
|
1119
1497
|
if (!this.journal && !this.held(id) && env.priority === "important" && !remoteId(id) && peer.steer && this.stateOf(id) === "busy" && !this.invocationTurnOpen(id)) {
|
|
1120
1498
|
// Not queued while the steer is in flight, or an idle transition would deliver it a second time.
|
|
1499
|
+
try { this.beforeReviewInputHandoff(id, [env]); }
|
|
1500
|
+
catch { this.enqueue(id, env); continue; }
|
|
1121
1501
|
this.steering++;
|
|
1122
1502
|
const deliveryId = crypto.randomUUID();
|
|
1123
1503
|
this.activeDeliveries.set(deliveryId, id);
|
|
1124
1504
|
if (this.journal) this.durableHandoff(id, deliveryId, [env], [env]);
|
|
1505
|
+
this.noteSteering(id, [env]);
|
|
1125
1506
|
const steerResult = this.journal ? peer.steer([env], deliveryId) : peer.steer([env]);
|
|
1126
1507
|
steerResult.then(() => { this.delivered(id, [env]); }).catch(() => {
|
|
1127
1508
|
this.deliveryFailed(id);
|
|
1128
|
-
if (this.journal) this.journal.transition(deliveryId, "needs_review", "adapter steering outcome is uncertain");
|
|
1509
|
+
if (this.journal) this.journal.transition(deliveryId, "needs_review", "adapter steering outcome is uncertain", undefined, "push error");
|
|
1129
1510
|
this.activeDeliveries.delete(deliveryId);
|
|
1130
1511
|
if (!this.journal) this.enqueue(id, env, true);
|
|
1131
1512
|
}).finally(() => { this.steering--; });
|
|
@@ -1179,6 +1560,45 @@ export class Bus {
|
|
|
1179
1560
|
get(id: string): Envelope | undefined {
|
|
1180
1561
|
return this.seen.get(id) ?? this.deferredHub.get(id);
|
|
1181
1562
|
}
|
|
1563
|
+
/** Only newly saved versioned review packages may interpret absent as permission for explicit publication. */
|
|
1564
|
+
reviewEnvelopeStatus(id: string): "published" | "absent" | "unknown" | "failed" | "discarded" {
|
|
1565
|
+
if (this.closed || this.storageError || !this.journal) return "unknown";
|
|
1566
|
+
try {
|
|
1567
|
+
const state = this.journal.reviewEnvelopeState(id);
|
|
1568
|
+
if (state === "failed" || state === "discarded") return state;
|
|
1569
|
+
return state ? "published" : "absent";
|
|
1570
|
+
}
|
|
1571
|
+
catch { return "unknown"; }
|
|
1572
|
+
}
|
|
1573
|
+
hasUnsettledReviewScope(peer: PeerId, task?: number): boolean {
|
|
1574
|
+
if (this.closed || this.storageError) return true;
|
|
1575
|
+
if (!this.journal) return false; // Ephemeral test buses retain the existing delivered-parent scope.
|
|
1576
|
+
try {
|
|
1577
|
+
const revision = this.journal.revision();
|
|
1578
|
+
if (revision !== this.reviewScopeRevision) { this.reviewScopeCache.clear(); this.reviewScopeRevision = revision; }
|
|
1579
|
+
const key = JSON.stringify([peer, task ?? null]), cached = this.reviewScopeCache.get(key);
|
|
1580
|
+
if (cached !== undefined) return cached;
|
|
1581
|
+
const held = this.journal.hasUnsettledReviewScope(peer, task);
|
|
1582
|
+
this.reviewScopeCache.set(key, held);
|
|
1583
|
+
if (this.reviewScopeCache.size > 512) this.reviewScopeCache.delete(this.reviewScopeCache.keys().next().value!);
|
|
1584
|
+
return held;
|
|
1585
|
+
} catch { return true; }
|
|
1586
|
+
}
|
|
1587
|
+
/** Explicit recovery reuses the immutable package envelope; a durable publication is never replayed. */
|
|
1588
|
+
publishReviewEnvelope(env: Envelope): PeerId[] {
|
|
1589
|
+
if (env.from !== HUB || env.kind !== "review" || !env.refs?.reviewPackage || !env.to?.length) throw new Error("review package envelope is invalid; refresh the saved package before retrying");
|
|
1590
|
+
const status = this.reviewEnvelopeStatus(env.id);
|
|
1591
|
+
if (status === "published") {
|
|
1592
|
+
this.journal!.reviewEnvelopePublished(env.id, env);
|
|
1593
|
+
return [...env.to];
|
|
1594
|
+
}
|
|
1595
|
+
if (status === "failed" || status === "discarded")
|
|
1596
|
+
throw new Error(`saved review package delivery is ${status}; inspect ahub queue list --peer ${env.to[0]} and ahub queue show <delivery-id> before retrying`);
|
|
1597
|
+
if (status !== "absent") throw new Error("review package publication evidence is unavailable; inspect ahub queue list before retrying");
|
|
1598
|
+
// A pre-checkpoint observer may have thrown after remembering the id but before any durable enqueue or native effect.
|
|
1599
|
+
this.seen.delete(env.id);
|
|
1600
|
+
return this.publish(env);
|
|
1601
|
+
}
|
|
1182
1602
|
|
|
1183
1603
|
/**
|
|
1184
1604
|
* #445 C2: whether the peer's current turn is one that holds an invocation request. With a journal that is a delivery of the peer's own that has not reached a
|
|
@@ -1187,7 +1607,7 @@ export class Bus {
|
|
|
1187
1607
|
* everything its turn holds, and a steered one would become what the answer names instead of the request.
|
|
1188
1608
|
*/
|
|
1189
1609
|
private invocationTurnOpen(peer: PeerId): boolean {
|
|
1190
|
-
if (!this.journal) return this.stateOf(peer) === "busy" && (this.lastDelivery.get(peer)?.out ?? []).some((e) => e.kind === "invocation");
|
|
1610
|
+
if (!this.journal) return this.stateOf(peer) === "busy" && [...(this.lastDelivery.get(peer)?.out ?? []), ...(this.lastPulled.get(peer)?.out ?? [])].some((e) => e.kind === "invocation");
|
|
1191
1611
|
for (const [deliveryId, owner] of this.activeDeliveries) {
|
|
1192
1612
|
if (owner === peer && this.journal.get(deliveryId)?.out.some((e) => e.kind === "invocation")) return true;
|
|
1193
1613
|
}
|
|
@@ -1196,9 +1616,12 @@ export class Bus {
|
|
|
1196
1616
|
|
|
1197
1617
|
/** The envelopes most recently handed to a peer, used to distinguish a received request from an unread queue. */
|
|
1198
1618
|
lastDeliveredTo(peer: PeerId): Envelope[] {
|
|
1199
|
-
return this.lastDelivery.get(peer)?.out ?? [];
|
|
1619
|
+
return this.lastPulled.get(peer)?.out ?? this.lastDelivery.get(peer)?.out ?? [];
|
|
1200
1620
|
}
|
|
1201
1621
|
|
|
1622
|
+
/** A confirmed new native session invalidates only this session's current-input cache; durable named-parent rows remain. */
|
|
1623
|
+
resetLastDelivery(peer: PeerId): void { this.lastDelivery.delete(peer); this.lastPulled.delete(peer); }
|
|
1624
|
+
|
|
1202
1625
|
/**
|
|
1203
1626
|
* Take back an envelope that has not been delivered yet (a request that has expired). True when it was removed from a
|
|
1204
1627
|
* queue now; an envelope whose steer is still in flight is not in any queue, so its id is remembered and it is
|
|
@@ -1254,11 +1677,21 @@ export class Bus {
|
|
|
1254
1677
|
}
|
|
1255
1678
|
|
|
1256
1679
|
private memoryDelivery(peer: PeerId, id: string, out: Envelope[], field: "rendered" | "delivered", originals: Envelope[] = out): void {
|
|
1257
|
-
if (remoteId(peer) || [...out, ...originals].some(env => env.private || remoteId(env.from) || env.to?.some(remoteId))) return;
|
|
1680
|
+
if (remoteId(peer) || this.opts.reviewObserverInputExcluded?.(peer, originals) === true || [...out, ...originals].some(env => isReviewObserverExcluded(env) || env.private || remoteId(env.from) || env.to?.some(remoteId))) return;
|
|
1258
1681
|
try { this.onMemoryDelivery?.(peer, id, out, field); } catch { /* optional telemetry never alters delivery */ }
|
|
1259
1682
|
}
|
|
1260
1683
|
|
|
1684
|
+
private rememberReviewParents(peer: PeerId, originals: Envelope[], requireAttached = true): void {
|
|
1685
|
+
for (const env of originals) if ((!requireAttached || !this.closed && this.peers.has(peer) && this.stateOf(peer) !== "offline") && isReviewObserverExcluded(env)) {
|
|
1686
|
+
const parents = this.reviewParents.get(peer) ?? new Map<string, string>();
|
|
1687
|
+
parents.set(env.id, (env.refs?.reviewPackage ?? env.refs?.reviewResponse)!);
|
|
1688
|
+
if (parents.size > 128) parents.delete(parents.keys().next().value!);
|
|
1689
|
+
this.reviewParents.set(peer, parents);
|
|
1690
|
+
}
|
|
1691
|
+
}
|
|
1692
|
+
|
|
1261
1693
|
private delivered(peer: PeerId, originals: Envelope[], deliveryId?: string): void {
|
|
1694
|
+
this.rememberReviewParents(peer, originals);
|
|
1262
1695
|
try { this.onDelivered?.(peer, originals, deliveryId); } catch { /* observation must never affect delivery settlement */ }
|
|
1263
1696
|
}
|
|
1264
1697
|
|
|
@@ -1266,6 +1699,15 @@ export class Bus {
|
|
|
1266
1699
|
try { this.onDeliveryFailed?.(peer); } catch { /* observation must never affect delivery settlement */ }
|
|
1267
1700
|
}
|
|
1268
1701
|
|
|
1702
|
+
private noteSteering(peer: PeerId, originals: Envelope[]): void {
|
|
1703
|
+
try { this.onSteering?.(peer, originals); } catch { /* metadata observation cannot alter the steer attempt */ }
|
|
1704
|
+
}
|
|
1705
|
+
|
|
1706
|
+
private beforeReviewInputHandoff(peer: PeerId, originals: Envelope[]): void {
|
|
1707
|
+
try { this.opts.beforeReviewInputHandoff?.(peer, originals); }
|
|
1708
|
+
catch { throw new ReviewInputHandoffError(); }
|
|
1709
|
+
}
|
|
1710
|
+
|
|
1269
1711
|
private markWithdrawn(id: string): void {
|
|
1270
1712
|
this.withdrawn.add(id);
|
|
1271
1713
|
if (this.withdrawn.size > 256) this.withdrawn.delete(this.withdrawn.values().next().value as string);
|
|
@@ -1380,7 +1822,7 @@ export class Bus {
|
|
|
1380
1822
|
if (!remoteId(id) && queue.length > this.opts.queueCap) {
|
|
1381
1823
|
const victim = queue.findIndex((e) => e.priority !== "important");
|
|
1382
1824
|
const [lost] = queue.splice(victim === -1 ? 0 : victim, 1);
|
|
1383
|
-
if (this.journal) this.pendingOutcomes.push({ id: crypto.randomUUID(), peer: id, state: "failed", createdAt: Date.now(), originals: [lost!], out: [], reason: "queue capacity exceeded" });
|
|
1825
|
+
if (this.journal) this.pendingOutcomes.push({ id: crypto.randomUUID(), peer: id, state: "failed", createdAt: Date.now(), originals: [lost!], out: [], reason: "queue capacity exceeded", failureCause: "adapter refused" });
|
|
1384
1826
|
this.emit({ t: "overflow", env: lost!, peer: id });
|
|
1385
1827
|
}
|
|
1386
1828
|
if (remoteId(id) && !front) {
|
|
@@ -1410,9 +1852,9 @@ export class Bus {
|
|
|
1410
1852
|
return Math.max(0, queue[0]!.ts + this.opts.batchMs - Date.now());
|
|
1411
1853
|
}
|
|
1412
1854
|
|
|
1413
|
-
private uncertain(peer: PeerId, deliveryId: string, reason: string): void {
|
|
1855
|
+
private uncertain(peer: PeerId, deliveryId: string, reason: string, failureCause: DeliveryFailureCause = "push error"): void {
|
|
1414
1856
|
const row = this.journal?.get(deliveryId);
|
|
1415
|
-
if (row && ["dispatching", "accepted"].includes(row.state)) this.deliveryReceipt(peer, { id: deliveryId, state: "needs_review", reason });
|
|
1857
|
+
if (row && ["dispatching", "accepted"].includes(row.state)) this.deliveryReceipt(peer, { id: deliveryId, state: "needs_review", reason, failureCause });
|
|
1416
1858
|
}
|
|
1417
1859
|
|
|
1418
1860
|
private async steerQueued(id: PeerId, env: Envelope): Promise<void> {
|
|
@@ -1428,16 +1870,18 @@ export class Bus {
|
|
|
1428
1870
|
let removed = false, handedOff = false;
|
|
1429
1871
|
try {
|
|
1430
1872
|
const out = this.retryOut(id, [env], [env]);
|
|
1873
|
+
this.beforeReviewInputHandoff(id, [env]);
|
|
1431
1874
|
queue.splice(index, 1);
|
|
1432
1875
|
removed = true;
|
|
1433
1876
|
this.durableHandoff(id, deliveryId, [env], out);
|
|
1434
1877
|
handedOff = true;
|
|
1435
1878
|
this.activeDeliveries.set(deliveryId, id);
|
|
1879
|
+
this.noteSteering(id, [env]);
|
|
1436
1880
|
try { await peer.steer(out, deliveryId); }
|
|
1437
1881
|
catch { this.uncertain(id, deliveryId, "steering outcome is uncertain"); }
|
|
1438
|
-
} catch {
|
|
1882
|
+
} catch (error) {
|
|
1439
1883
|
if (removed && !handedOff) queue.splice(index, 0, env);
|
|
1440
|
-
this.storageError = "delivery journal unavailable";
|
|
1884
|
+
if (!(error instanceof ReviewInputHandoffError)) this.storageError = "delivery journal unavailable";
|
|
1441
1885
|
} finally { this.steering--; this.onQueues?.(); }
|
|
1442
1886
|
}
|
|
1443
1887
|
|
|
@@ -1488,6 +1932,7 @@ export class Bus {
|
|
|
1488
1932
|
const deliveryId = crypto.randomUUID();
|
|
1489
1933
|
try {
|
|
1490
1934
|
out = this.retryOut(id, batch, batch);
|
|
1935
|
+
this.beforeReviewInputHandoff(id, batch);
|
|
1491
1936
|
this.prefaces.delete(id);
|
|
1492
1937
|
// Before the checkpoint, as in a drain, so the persisted snapshot carries it; a failed policy read rolls back too.
|
|
1493
1938
|
this.remember(out);
|
|
@@ -1498,15 +1943,18 @@ export class Bus {
|
|
|
1498
1943
|
} catch (error) {
|
|
1499
1944
|
queue.splice(0, queue.length, ...queued);
|
|
1500
1945
|
if (preface) this.prefaces.set(id, preface);
|
|
1501
|
-
this.storageError = "delivery journal unavailable";
|
|
1946
|
+
if (!(error instanceof ReviewInputHandoffError)) this.storageError = "delivery journal unavailable";
|
|
1502
1947
|
throw error;
|
|
1503
1948
|
}
|
|
1504
1949
|
if (this.journal) {
|
|
1505
1950
|
this.memoryDelivery(id, deliveryId, out, "rendered", batch);
|
|
1506
1951
|
this.memoryDelivery(id, deliveryId, out, "delivered", batch);
|
|
1507
1952
|
}
|
|
1953
|
+
this.lastPulled.set(id, { out, originals: batch });
|
|
1954
|
+
this.rememberReviewParents(id, batch, false);
|
|
1955
|
+
if (batch.length) { try { this.onPulled?.(id, batch); } catch { /* metadata observation cannot change a committed inbox result */ } }
|
|
1508
1956
|
this.failureStreak.delete(id); this.lastFailure.delete(id); // read, as a completed push receipt clears them
|
|
1509
|
-
this.onQueues?.();
|
|
1957
|
+
try { this.onQueues?.(); } catch { /* a committed inbox result must still reach its caller */ }
|
|
1510
1958
|
return out;
|
|
1511
1959
|
}
|
|
1512
1960
|
|
|
@@ -1558,6 +2006,7 @@ export class Bus {
|
|
|
1558
2006
|
const deliveryId = crypto.randomUUID();
|
|
1559
2007
|
if (this.journal) {
|
|
1560
2008
|
if ((!remoteId(id) && this.prefaces.get(id) !== preface) || batch.some((e) => !queue.some((item) => item.id === e.id))) continue;
|
|
2009
|
+
this.beforeReviewInputHandoff(id, delivery);
|
|
1561
2010
|
const queued = [...queue], seen = [...this.seen];
|
|
1562
2011
|
for (const env of batch) queue.splice(queue.findIndex((item) => item.id === env.id), 1);
|
|
1563
2012
|
this.prefaces.delete(id);
|
|
@@ -1571,7 +2020,16 @@ export class Bus {
|
|
|
1571
2020
|
throw error;
|
|
1572
2021
|
}
|
|
1573
2022
|
this.activeDeliveries.set(deliveryId, id);
|
|
1574
|
-
} else
|
|
2023
|
+
} else {
|
|
2024
|
+
try { this.beforeReviewInputHandoff(id, delivery); }
|
|
2025
|
+
catch (error) {
|
|
2026
|
+
if (preface) this.restorePreface(id, preface);
|
|
2027
|
+
queue.unshift(...batch.filter(e => !this.withdrawn.has(e.id)));
|
|
2028
|
+
throw error;
|
|
2029
|
+
}
|
|
2030
|
+
this.remember(out);
|
|
2031
|
+
}
|
|
2032
|
+
this.lastPulled.delete(id);
|
|
1575
2033
|
this.lastDelivery.set(id, { out, originals: delivery });
|
|
1576
2034
|
for (const env of delivery) if (env.kind === "invocation") this.invocationDeliveries.set(env.id, deliveryId);
|
|
1577
2035
|
if (remoteId(id)) this.remoteTransport.set(deliveryId, { peer: id, originals: delivery });
|
|
@@ -1594,7 +2052,7 @@ export class Bus {
|
|
|
1594
2052
|
break;
|
|
1595
2053
|
} finally { this.remoteTransport.delete(deliveryId); }
|
|
1596
2054
|
}
|
|
1597
|
-
} catch { this.storageError = "delivery journal unavailable"; }
|
|
2055
|
+
} catch (error) { if (!(error instanceof ReviewInputHandoffError)) this.storageError = "delivery journal unavailable"; }
|
|
1598
2056
|
finally { this.preparing.delete(id); this.draining.delete(id); this.onQueues?.(); }
|
|
1599
2057
|
}
|
|
1600
2058
|
|