@bastani/atomic 0.9.39-alpha.2 → 0.9.40-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +42 -0
- package/dist/builtin/intercom/CHANGELOG.md +20 -0
- package/dist/builtin/intercom/broker/broker.ts +523 -72
- package/dist/builtin/intercom/broker/client.ts +136 -5
- package/dist/builtin/intercom/broker/database-wake-up-bus.ts +274 -0
- package/dist/builtin/intercom/broker/group-membership.ts +2 -2
- package/dist/builtin/intercom/broker/multinode-router.ts +1500 -0
- package/dist/builtin/intercom/broker/multinode-state-store.ts +569 -0
- package/dist/builtin/intercom/broker/open-broker-state.ts +160 -0
- package/dist/builtin/intercom/broker/paths.ts +6 -0
- package/dist/builtin/intercom/broker/pending-send-registry.ts +14 -0
- package/dist/builtin/intercom/broker/recipient-receipts.ts +93 -0
- package/dist/builtin/intercom/broker/restoring-recipients.ts +263 -0
- package/dist/builtin/intercom/broker/send-handler.ts +241 -25
- package/dist/builtin/intercom/broker/session-id-reclaim.ts +41 -0
- package/dist/builtin/intercom/broker/socket-writes.ts +47 -0
- package/dist/builtin/intercom/broker/spawn.ts +10 -2
- package/dist/builtin/intercom/broker/sql-held-send-store.ts +96 -0
- package/dist/builtin/intercom/broker/sql-known-session-store.ts +129 -0
- package/dist/builtin/intercom/broker/sql-lease-store.ts +194 -0
- package/dist/builtin/intercom/broker/sql-outbox-store.ts +249 -0
- package/dist/builtin/intercom/broker/sql-pending-ask-store.ts +105 -0
- package/dist/builtin/intercom/broker/sql-restoring-reservation-store.ts +295 -0
- package/dist/builtin/intercom/broker/sql-sent-message-authority.ts +126 -0
- package/dist/builtin/intercom/broker/supervisor-channel.ts +10 -0
- package/dist/builtin/intercom/broker/wake-up-bus.ts +60 -0
- package/dist/builtin/intercom/index.bundle.mjs +793 -91
- package/dist/builtin/intercom/package.json +1 -1
- package/dist/builtin/intercom/retry-policy.ts +4 -0
- package/dist/builtin/intercom/session-target.ts +24 -0
- package/dist/builtin/intercom/types.ts +57 -3
- package/dist/builtin/mcp/package.json +1 -1
- package/dist/builtin/subagents/CHANGELOG.md +37 -0
- package/dist/builtin/subagents/README.md +18 -12
- package/dist/builtin/subagents/package.json +1 -1
- package/dist/builtin/subagents/skills/agent-browser/SKILL.md +1 -1
- package/dist/builtin/subagents/skills/cua-driver/BROWSER.md +55 -8
- package/dist/builtin/subagents/skills/cua-driver/EMBEDDING.md +7 -2
- package/dist/builtin/subagents/skills/cua-driver/LINUX.md +7 -1
- package/dist/builtin/subagents/skills/cua-driver/MACOS.md +151 -27
- package/dist/builtin/subagents/skills/cua-driver/RUNTIME.md +1 -1
- package/dist/builtin/subagents/skills/cua-driver/SKILL.md +33 -17
- package/dist/builtin/subagents/skills/cua-driver/WINDOWS.md +5 -0
- package/dist/builtin/subagents/skills/cua-driver/WORKFLOW.md +46 -8
- package/dist/builtin/subagents/skills/subagent/SKILL.md +18 -20
- package/dist/builtin/subagents/src/extension/index.bundle.mjs +8032 -6034
- package/dist/builtin/web-access/package.json +1 -1
- package/dist/builtin/workflows/CHANGELOG.md +32 -0
- package/dist/builtin/workflows/README.md +1 -1
- package/dist/builtin/workflows/package.json +1 -1
- package/dist/builtin/workflows/skills/impeccable/SKILL.md +8 -8
- package/dist/builtin/workflows/skills/impeccable/reference/adapt.md +3 -3
- package/dist/builtin/workflows/skills/impeccable/reference/adapt.native.md +1 -1
- package/dist/builtin/workflows/skills/impeccable/reference/audit.md +4 -2
- package/dist/builtin/workflows/skills/impeccable/reference/audit.native.md +2 -2
- package/dist/builtin/workflows/skills/impeccable/reference/craft-floor.md +1 -1
- package/dist/builtin/workflows/skills/impeccable/reference/critique.md +22 -9
- package/dist/builtin/workflows/skills/impeccable/reference/degraded/finish-reviewer.md +2 -2
- package/dist/builtin/workflows/skills/impeccable/reference/distill.md +3 -3
- package/dist/builtin/workflows/skills/impeccable/reference/extract.md +3 -3
- package/dist/builtin/workflows/skills/impeccable/reference/harden.md +2 -4
- package/dist/builtin/workflows/skills/impeccable/reference/hooks.md +4 -4
- package/dist/builtin/workflows/skills/impeccable/reference/live.md +1 -1
- package/dist/builtin/workflows/skills/impeccable/reference/new-work.md +6 -6
- package/dist/builtin/workflows/skills/impeccable/reference/onboard.md +3 -3
- package/dist/builtin/workflows/skills/impeccable/reference/optimize.md +3 -3
- package/dist/builtin/workflows/skills/impeccable/reference/overdrive.md +3 -3
- package/dist/builtin/workflows/skills/impeccable/reference/quieter.md +3 -3
- package/dist/builtin/workflows/skills/impeccable/reference/typeset.md +10 -0
- package/dist/builtin/workflows/skills/impeccable/reference/visualize.md +1 -1
- package/dist/builtin/workflows/skills/impeccable/scripts/VERSION +1 -1
- package/dist/builtin/workflows/skills/impeccable/scripts/impeccable +6 -0
- package/dist/builtin/workflows/skills/impeccable/scripts/impeccable.cmd +47 -22
- package/dist/builtin/workflows/skills/impeccable/scripts/live-browser-dom.js +34 -1
- package/dist/builtin/workflows/skills/impeccable/scripts/live-browser-session.js +25 -1
- package/dist/builtin/workflows/skills/impeccable/scripts/live-browser.js +33 -8
- package/dist/builtin/workflows/src/durable/journal/resolution.d.ts +7 -7
- package/dist/builtin/workflows/src/extension/index.bundle.mjs +22855 -22308
- package/dist/builtin/workflows/src/index.js +106 -35
- package/dist/cli/args.d.ts +2 -0
- package/dist/cli/args.d.ts.map +1 -1
- package/dist/cli/args.js +16 -0
- package/dist/cli/args.js.map +1 -1
- package/dist/core/agent-session-extension-bindings.d.ts +2 -0
- package/dist/core/agent-session-extension-bindings.d.ts.map +1 -1
- package/dist/core/agent-session-extension-bindings.js +6 -6
- package/dist/core/agent-session-extension-bindings.js.map +1 -1
- package/dist/core/agent-session-methods.d.ts +6 -3
- package/dist/core/agent-session-methods.d.ts.map +1 -1
- package/dist/core/agent-session-methods.js.map +1 -1
- package/dist/core/agent-session-persistent-custom-messages.d.ts +7 -2
- package/dist/core/agent-session-persistent-custom-messages.d.ts.map +1 -1
- package/dist/core/agent-session-persistent-custom-messages.js +29 -12
- package/dist/core/agent-session-persistent-custom-messages.js.map +1 -1
- package/dist/core/agent-session-recovery.d.ts +14 -0
- package/dist/core/agent-session-recovery.d.ts.map +1 -0
- package/dist/core/agent-session-recovery.js +83 -0
- package/dist/core/agent-session-recovery.js.map +1 -0
- package/dist/core/agent-session-tasks.d.ts.map +1 -1
- package/dist/core/agent-session-tasks.js +4 -6
- package/dist/core/agent-session-tasks.js.map +1 -1
- package/dist/core/agent-session-types.d.ts +13 -0
- package/dist/core/agent-session-types.d.ts.map +1 -1
- package/dist/core/agent-session-types.js.map +1 -1
- package/dist/core/agent-session.d.ts +2 -1
- package/dist/core/agent-session.d.ts.map +1 -1
- package/dist/core/agent-session.js +4 -1
- package/dist/core/agent-session.js.map +1 -1
- package/dist/core/extensions/api-types.d.ts +6 -2
- package/dist/core/extensions/api-types.d.ts.map +1 -1
- package/dist/core/extensions/api-types.js.map +1 -1
- package/dist/core/extensions/event-results.d.ts +21 -0
- package/dist/core/extensions/event-results.d.ts.map +1 -1
- package/dist/core/extensions/event-results.js.map +1 -1
- package/dist/core/extensions/index.d.ts +1 -1
- package/dist/core/extensions/index.d.ts.map +1 -1
- package/dist/core/extensions/index.js.map +1 -1
- package/dist/core/extensions/loader-api.d.ts.map +1 -1
- package/dist/core/extensions/loader-api.js +4 -0
- package/dist/core/extensions/loader-api.js.map +1 -1
- package/dist/core/extensions/runner-events.d.ts +9 -2
- package/dist/core/extensions/runner-events.d.ts.map +1 -1
- package/dist/core/extensions/runner-events.js +29 -0
- package/dist/core/extensions/runner-events.js.map +1 -1
- package/dist/core/extensions/runner.d.ts +5 -3
- package/dist/core/extensions/runner.d.ts.map +1 -1
- package/dist/core/extensions/runner.js +7 -2
- package/dist/core/extensions/runner.js.map +1 -1
- package/dist/core/extensions/runtime-types.d.ts +1 -0
- package/dist/core/extensions/runtime-types.d.ts.map +1 -1
- package/dist/core/extensions/runtime-types.js.map +1 -1
- package/dist/core/extensions/session-events.d.ts +30 -2
- package/dist/core/extensions/session-events.d.ts.map +1 -1
- package/dist/core/extensions/session-events.js.map +1 -1
- package/dist/core/sdk-types.d.ts +3 -2
- package/dist/core/sdk-types.d.ts.map +1 -1
- package/dist/core/sdk-types.js.map +1 -1
- package/dist/core/sdk.d.ts.map +1 -1
- package/dist/core/sdk.js +3 -0
- package/dist/core/sdk.js.map +1 -1
- package/dist/core/session-storage.d.ts.map +1 -1
- package/dist/core/session-storage.js +15 -9
- package/dist/core/session-storage.js.map +1 -1
- package/dist/core/storage/process-selection.d.ts +39 -0
- package/dist/core/storage/process-selection.d.ts.map +1 -0
- package/dist/core/storage/process-selection.js +154 -0
- package/dist/core/storage/process-selection.js.map +1 -0
- package/dist/core/storage/provider.d.ts +6 -1
- package/dist/core/storage/provider.d.ts.map +1 -1
- package/dist/core/storage/provider.js +3 -2
- package/dist/core/storage/provider.js.map +1 -1
- package/dist/core/storage/selection.d.ts +1 -1
- package/dist/core/storage/selection.d.ts.map +1 -1
- package/dist/core/storage/selection.js.map +1 -1
- package/dist/core/storage/session/jsonl-session-store.d.ts +2 -0
- package/dist/core/storage/session/jsonl-session-store.d.ts.map +1 -1
- package/dist/core/storage/session/jsonl-session-store.js +38 -2
- package/dist/core/storage/session/jsonl-session-store.js.map +1 -1
- package/dist/core/tasks/completion.d.ts +11 -2
- package/dist/core/tasks/completion.d.ts.map +1 -1
- package/dist/core/tasks/completion.js +40 -16
- package/dist/core/tasks/completion.js.map +1 -1
- package/dist/core/tasks/pause.d.ts +4 -1
- package/dist/core/tasks/pause.d.ts.map +1 -1
- package/dist/core/tasks/pause.js +8 -5
- package/dist/core/tasks/pause.js.map +1 -1
- package/dist/index-extensions.d.ts +1 -1
- package/dist/index-extensions.d.ts.map +1 -1
- package/dist/index-extensions.js.map +1 -1
- package/dist/index.d.ts +2 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/main.d.ts.map +1 -1
- package/dist/main.js +7 -1
- package/dist/main.js.map +1 -1
- package/dist/modes/interactive/components/chat-session-host-rendering.d.ts.map +1 -1
- package/dist/modes/interactive/components/chat-session-host-rendering.js +11 -1
- package/dist/modes/interactive/components/chat-session-host-rendering.js.map +1 -1
- package/dist/modes/interactive/components/chat-session-host-state.d.ts +2 -0
- package/dist/modes/interactive/components/chat-session-host-state.d.ts.map +1 -1
- package/dist/modes/interactive/components/chat-session-host-state.js +2 -0
- package/dist/modes/interactive/components/chat-session-host-state.js.map +1 -1
- package/dist/modes/interactive/interactive-deferred-startup.d.ts.map +1 -1
- package/dist/modes/interactive/interactive-deferred-startup.js +4 -0
- package/dist/modes/interactive/interactive-deferred-startup.js.map +1 -1
- package/dist/modes/interactive/interactive-mode-surface.d.ts +2 -1
- package/dist/modes/interactive/interactive-mode-surface.d.ts.map +1 -1
- package/dist/modes/interactive/interactive-mode-surface.js.map +1 -1
- package/dist/modes/interactive/interactive-render-chat.js +3 -0
- package/dist/modes/interactive/interactive-render-chat.js.map +1 -1
- package/dist/modes/interactive/interactive-session-routing.js +2 -0
- package/dist/modes/interactive/interactive-session-routing.js.map +1 -1
- package/dist/modes/interactive/interactive-session-runtime.d.ts +2 -1
- package/dist/modes/interactive/interactive-session-runtime.d.ts.map +1 -1
- package/dist/modes/interactive/interactive-session-runtime.js +29 -0
- package/dist/modes/interactive/interactive-session-runtime.js.map +1 -1
- package/dist/modes/interactive/interactive-startup.d.ts.map +1 -1
- package/dist/modes/interactive/interactive-startup.js +4 -1
- package/dist/modes/interactive/interactive-startup.js.map +1 -1
- package/dist/modes/interactive-engine/engine-args.d.ts.map +1 -1
- package/dist/modes/interactive-engine/engine-args.js +4 -0
- package/dist/modes/interactive-engine/engine-args.js.map +1 -1
- package/dist/modes/interactive-engine/engine-render-service.d.ts.map +1 -1
- package/dist/modes/interactive-engine/engine-render-service.js +4 -4
- package/dist/modes/interactive-engine/engine-render-service.js.map +1 -1
- package/dist/modes/interactive-engine/isolated-runtime.d.ts.map +1 -1
- package/dist/modes/interactive-engine/isolated-runtime.js +4 -0
- package/dist/modes/interactive-engine/isolated-runtime.js.map +1 -1
- package/dist/modes/interactive-engine/remote-renderer.js +1 -1
- package/dist/modes/interactive-engine/remote-renderer.js.map +1 -1
- package/dist/modes/rpc/rpc-client-api.d.ts +4 -1
- package/dist/modes/rpc/rpc-client-api.d.ts.map +1 -1
- package/dist/modes/rpc/rpc-client-api.js +4 -0
- package/dist/modes/rpc/rpc-client-api.js.map +1 -1
- package/dist/modes/rpc/rpc-command-handler.d.ts.map +1 -1
- package/dist/modes/rpc/rpc-command-handler.js +3 -0
- package/dist/modes/rpc/rpc-command-handler.js.map +1 -1
- package/dist/modes/rpc/rpc-command-timeouts.d.ts.map +1 -1
- package/dist/modes/rpc/rpc-command-timeouts.js +1 -0
- package/dist/modes/rpc/rpc-command-timeouts.js.map +1 -1
- package/dist/modes/rpc/rpc-input-scheduler.d.ts.map +1 -1
- package/dist/modes/rpc/rpc-input-scheduler.js +6 -1
- package/dist/modes/rpc/rpc-input-scheduler.js.map +1 -1
- package/dist/modes/rpc/rpc-types.d.ts +12 -1
- package/dist/modes/rpc/rpc-types.d.ts.map +1 -1
- package/dist/modes/rpc/rpc-types.js.map +1 -1
- package/dist/utils/shell.d.ts.map +1 -1
- package/dist/utils/shell.js +44 -11
- package/dist/utils/shell.js.map +1 -1
- package/dist/utils/tools-manager.js +1 -1
- package/dist/utils/tools-manager.js.map +1 -1
- package/docs/background-tasks.md +5 -3
- package/docs/computer-use.md +2 -2
- package/docs/environment-variables.md +4 -1
- package/docs/extensions/events.md +29 -2
- package/docs/getting-started/first-session.md +1 -1
- package/docs/guides/intercom.md +1 -1
- package/docs/intercom/operations.md +103 -1
- package/docs/intercom/reference.md +10 -3
- package/docs/intercom.md +8 -6
- package/docs/keybindings.md +1 -1
- package/docs/models.md +1 -1
- package/docs/providers.md +4 -2
- package/docs/reference/cli.md +5 -0
- package/docs/rpc/protocol.md +19 -0
- package/docs/rpc.md +3 -0
- package/docs/sdk/web-server.md +1 -1
- package/docs/sdk.md +5 -2
- package/docs/sessions.md +17 -1
- package/docs/settings.md +5 -3
- package/docs/skills.md +1 -1
- package/docs/storage.md +54 -3
- package/docs/subagents.md +49 -9
- package/docs/usage.md +1 -1
- package/docs/windows.md +14 -4
- package/docs/workflows/api-reference.md +1 -1
- package/docs/workflows/authoring.md +1 -1
- package/docs/workflows/operations.md +10 -6
- package/npm-shrinkwrap.json +37 -37
- package/package.json +4 -4
|
@@ -0,0 +1,1500 @@
|
|
|
1
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
2
|
+
import { readFileSync } from "node:fs";
|
|
3
|
+
import { hostname } from "node:os";
|
|
4
|
+
import { join } from "node:path";
|
|
5
|
+
import { normalizeGroup } from "../group.js";
|
|
6
|
+
import { isAgentRecipient, NON_AGENT_RECIPIENT_REFUSAL, TERMINAL_CHILD_ASK_REFUSAL } from "../recipient-purpose.js";
|
|
7
|
+
import { DELIVERED_MESSAGE_TTL_MS, RECIPIENT_RECEIPT_TIMEOUT_MS } from "../retry-policy.js";
|
|
8
|
+
import {
|
|
9
|
+
resolveSessionTarget,
|
|
10
|
+
resolveSessionTargetByHostSessionId,
|
|
11
|
+
type SessionTargetResolution,
|
|
12
|
+
sessionTargetFailureReason,
|
|
13
|
+
} from "../session-target.js";
|
|
14
|
+
import type { SessionInfo } from "../types.js";
|
|
15
|
+
import type { DeliveredMessageCache } from "./delivered-message-cache.js";
|
|
16
|
+
import { sameGroup } from "./group-isolation.js";
|
|
17
|
+
import { sessionGroups } from "./group-membership.js";
|
|
18
|
+
import {
|
|
19
|
+
BROKER_LEASE_RENEW_INTERVAL_MS,
|
|
20
|
+
BROKER_LEASE_STALE_AFTER_MS,
|
|
21
|
+
type HeldSendRequest,
|
|
22
|
+
type IntercomBrokerStateStore,
|
|
23
|
+
type IntercomDeliveryFrame,
|
|
24
|
+
KNOWN_SESSION_RETENTION_MS,
|
|
25
|
+
type KnownSession,
|
|
26
|
+
type KnownSessionState,
|
|
27
|
+
type Lease,
|
|
28
|
+
type LeaseState,
|
|
29
|
+
OUTBOX_BACKUP_POLL_INTERVAL_MS,
|
|
30
|
+
OUTBOX_CLAIM_STALE_AFTER_MS,
|
|
31
|
+
type OutboxClaim,
|
|
32
|
+
type OutboxRow,
|
|
33
|
+
type RecipientPresenceSubject,
|
|
34
|
+
type RestoringReservationLease,
|
|
35
|
+
reservationKeyFor,
|
|
36
|
+
} from "./multinode-state-store.js";
|
|
37
|
+
import { BROKER_STORAGE_ENV } from "./paths.js";
|
|
38
|
+
import type { PendingQuestionIndex } from "./pending-question-index.js";
|
|
39
|
+
import { UNCERTAIN_RECEIPT_REASON } from "./recipient-receipts.js";
|
|
40
|
+
import { RESTORING_RECIPIENT_HOLD_MS, restoringRecipientRetryReason } from "./restoring-recipients.js";
|
|
41
|
+
import {
|
|
42
|
+
type BrokerConnectedSession,
|
|
43
|
+
deliveryTargetIdentity,
|
|
44
|
+
type RemoteSendRequest,
|
|
45
|
+
type SendClientMessage,
|
|
46
|
+
senderGroupIdentity,
|
|
47
|
+
} from "./send-handler.js";
|
|
48
|
+
import type { BrokerOutboundFrame } from "./socket-writes.js";
|
|
49
|
+
import type { SupervisorChannelCache } from "./supervisor-channel.js";
|
|
50
|
+
import type { WakeUpBus, WakeUpEvent, WakeUpSubscription } from "./wake-up-bus.js";
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Multi-node S5 (specs/2026-10-19-intercom-multi-node.md §4–§8, §14): routes sends to sessions hosted by other
|
|
54
|
+
* brokers through the shared state, and delivers the shared outbox to this broker's own sessions. Local delivery
|
|
55
|
+
* never comes here. Brokers coordinate only through the shared store and the wake-up bus; hints are only hints, so
|
|
56
|
+
* catch-up scans run on every resync, presence acquisition and backup poll.
|
|
57
|
+
*/
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* How often the origin re-reads the outcome of sends it is waiting on and of sends it holds for a restoring
|
|
61
|
+
* recipient. Shorter than the 20 s hold, so a lost `outcome` hint for a held send is noticed before it times out.
|
|
62
|
+
*/
|
|
63
|
+
export const MULTI_NODE_OUTCOME_POLL_INTERVAL_MS = 5_000;
|
|
64
|
+
/** The `queued` frame's target for a send waiting in the shared outbox for an offline session (decision 1). */
|
|
65
|
+
export const OFFLINE_SESSION_TARGET = "offline session";
|
|
66
|
+
/** A send for another node while the shared database cannot be reached (spec §8, "Database unavailable"). */
|
|
67
|
+
export const SHARED_STORAGE_UNAVAILABLE_REASON =
|
|
68
|
+
"Session not found on this node, and Intercom could not reach the shared storage database to look for it on other nodes. Try again once the database is available.";
|
|
69
|
+
const MAX_HELD_SENDS = 256;
|
|
70
|
+
/** A session that moved here may find its presence lease still held by the broker it just left. */
|
|
71
|
+
const PRESENCE_HANDOVER_ATTEMPTS = 4;
|
|
72
|
+
const PRESENCE_HANDOVER_RETRY_MS = 50;
|
|
73
|
+
/**
|
|
74
|
+
* The longest a registration waits for its presence lease. A hung shared database must not hang registration (the
|
|
75
|
+
* client gives up after 10 s); the lease is still written when the database answers.
|
|
76
|
+
*/
|
|
77
|
+
export const PRESENCE_REGISTRATION_BOUND_MS = 5_000;
|
|
78
|
+
/**
|
|
79
|
+
* The longest a send to a local recipient waits for the shared outbox to say whether it is a retry of a cross-node
|
|
80
|
+
* send. Local delivery does not depend on the shared database (spec §8): past this bound the send is delivered
|
|
81
|
+
* locally, with the local accepted-delivery cache as its duplicate protection.
|
|
82
|
+
*/
|
|
83
|
+
export const LOCAL_RETRY_LOOKUP_BOUND_MS = 300;
|
|
84
|
+
/** The longest `list` waits for other nodes' sessions; the client's list request gives up after 5 s. */
|
|
85
|
+
const REMOTE_SESSIONS_BOUND_MS = 2_000;
|
|
86
|
+
const MAX_SUPERSEDED_HOPS = 8;
|
|
87
|
+
/**
|
|
88
|
+
* How often renewal refreshes a live session's known-session record. Far below `KNOWN_SESSION_RETENTION_MS`, so a
|
|
89
|
+
* session that stays connected for longer than the retention is still known (queued for) after its broker crashes.
|
|
90
|
+
*/
|
|
91
|
+
export const KNOWN_SESSION_REFRESH_INTERVAL_MS = 60 * 60 * 1000;
|
|
92
|
+
const TARGET_GROUP_REFUSAL = "Target session is in a different intercom group";
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* The raw shared `storage` selection, read without the host package so the broker stays self-contained: the
|
|
96
|
+
* selection the launching process handed this broker in `ATOMIC_STORAGE` (its `--storage`, `--storage-file` or own
|
|
97
|
+
* `ATOMIC_STORAGE`), otherwise the top-level `storage` value of the agent directory's settings. Relative paths in
|
|
98
|
+
* either resolve against the agent directory's parent, as for sessions and the journal. `undefined` leaves
|
|
99
|
+
* multi-node routing off: no selection, a per-process selection that is not a JSON object (the launching process
|
|
100
|
+
* validated it; settings are never consulted instead), and the selections the storage layer reads as in-memory
|
|
101
|
+
* (`memory: true` without a path, or `path: ":memory:"`), which no other broker can share. `path: ""` is not one of
|
|
102
|
+
* them: it selects the default storage file.
|
|
103
|
+
*/
|
|
104
|
+
export function readSharedStorageSetting(
|
|
105
|
+
agentDir: string,
|
|
106
|
+
env: NodeJS.ProcessEnv = process.env,
|
|
107
|
+
): Record<string, unknown> | undefined {
|
|
108
|
+
const processSelection = env[BROKER_STORAGE_ENV];
|
|
109
|
+
let parsed: unknown;
|
|
110
|
+
try {
|
|
111
|
+
parsed =
|
|
112
|
+
processSelection === undefined || processSelection.length === 0
|
|
113
|
+
? (JSON.parse(readFileSync(join(agentDir, "settings.json"), "utf8")) as Record<string, unknown>).storage
|
|
114
|
+
: JSON.parse(processSelection);
|
|
115
|
+
} catch {
|
|
116
|
+
return undefined;
|
|
117
|
+
}
|
|
118
|
+
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) return undefined;
|
|
119
|
+
const selection = parsed as Record<string, unknown>;
|
|
120
|
+
if (selection.adapter === undefined) {
|
|
121
|
+
if (selection.memory === true && !selection.path) return undefined;
|
|
122
|
+
if (selection.path === ":memory:") return undefined;
|
|
123
|
+
}
|
|
124
|
+
return selection;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/** What the broker uses of `open-broker-state.ts`, which it loads lazily (replaceable in tests). */
|
|
128
|
+
export interface BrokerStateOpener<Source> {
|
|
129
|
+
resolveBrokerStateSource(
|
|
130
|
+
setting: Record<string, unknown>,
|
|
131
|
+
agentDir: string,
|
|
132
|
+
): Promise<{ readonly status: "off"; readonly reason: string } | { readonly status: "shared"; readonly source: Source }>;
|
|
133
|
+
openIntercomBrokerState(source: Source): Promise<{ readonly state: IntercomBrokerStateStore; readonly bus: WakeUpBus }>;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
export type MultiNodeStartup<Source> =
|
|
137
|
+
| { readonly status: "off"; readonly reason: string; readonly error?: Error }
|
|
138
|
+
| { readonly status: "shared"; readonly opener: BrokerStateOpener<Source>; readonly source: Source };
|
|
139
|
+
|
|
140
|
+
/**
|
|
141
|
+
* Decide whether a top-level `storage` value gives this broker a shared database. Multi-node routing is off, and
|
|
142
|
+
* Intercom behaves exactly as a single-node broker, unless it does: also when the host storage module cannot be
|
|
143
|
+
* loaded (a standalone binary) or the selection cannot be resolved to a database. Only a selection that resolved to a
|
|
144
|
+
* database whose connection then fails makes cross-node sends fail explicitly.
|
|
145
|
+
*/
|
|
146
|
+
export async function resolveMultiNodeStartup<Source>(
|
|
147
|
+
setting: Record<string, unknown>,
|
|
148
|
+
agentDir: string,
|
|
149
|
+
load: () => Promise<BrokerStateOpener<Source>>,
|
|
150
|
+
): Promise<MultiNodeStartup<Source>> {
|
|
151
|
+
try {
|
|
152
|
+
const opener = await load();
|
|
153
|
+
const resolved = await opener.resolveBrokerStateSource(setting, agentDir);
|
|
154
|
+
return resolved.status === "off"
|
|
155
|
+
? { status: "off", reason: resolved.reason }
|
|
156
|
+
: { status: "shared", opener, source: resolved.source };
|
|
157
|
+
} catch (cause) {
|
|
158
|
+
const error = cause instanceof Error ? cause : new Error(String(cause));
|
|
159
|
+
return { status: "off", reason: error.message, error };
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/** The unkeyed SHA-256 of the canonical send, the digest of every cross-node row (decision 2). */
|
|
164
|
+
export function crossNodeDigest(signature: string): string {
|
|
165
|
+
return createHash("sha256").update(signature, "utf8").digest("hex");
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
const normalizeTarget = (target: string): string => target.trim().toLowerCase();
|
|
169
|
+
|
|
170
|
+
export interface MultiNodeHost {
|
|
171
|
+
readonly sessions: ReadonlyMap<string, BrokerConnectedSession>;
|
|
172
|
+
/** Write a frame unless the socket is closed; false when nothing was written. */
|
|
173
|
+
write(socket: BrokerConnectedSession["socket"], frame: BrokerOutboundFrame): boolean;
|
|
174
|
+
/** Write a frame and report the socket write callback's outcome. */
|
|
175
|
+
writeConfirmed(
|
|
176
|
+
socket: BrokerConnectedSession["socket"],
|
|
177
|
+
frame: BrokerOutboundFrame,
|
|
178
|
+
onSettled: (written: boolean) => void,
|
|
179
|
+
): void;
|
|
180
|
+
readonly supervisorCache: SupervisorChannelCache;
|
|
181
|
+
/** This broker's own ask routes: an ask whose asker and recipient are both here is answered on the local path. */
|
|
182
|
+
readonly pendingQuestions: PendingQuestionIndex;
|
|
183
|
+
/** This broker's local accepted deliveries: a retry whose recipient moved to another broker is answered from them. */
|
|
184
|
+
readonly deliveredMessages: Pick<DeliveredMessageCache, "lookupForTarget" | "lookupQuestionTarget">;
|
|
185
|
+
/** Route a held send again as its sender, exactly like a new send (local delivery or the outbox). */
|
|
186
|
+
dispatch(socket: BrokerConnectedSession["socket"], clientMessage: SendClientMessage, senderId: string): void;
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
export interface MultiNodeRouterOptions {
|
|
190
|
+
readonly state: IntercomBrokerStateStore;
|
|
191
|
+
readonly bus: WakeUpBus;
|
|
192
|
+
readonly host: MultiNodeHost;
|
|
193
|
+
readonly brokerId?: string;
|
|
194
|
+
readonly nodeId?: string;
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
interface Inflight {
|
|
198
|
+
readonly claim: OutboxClaim;
|
|
199
|
+
readonly row: OutboxRow;
|
|
200
|
+
readonly session: BrokerConnectedSession;
|
|
201
|
+
readonly identity: string;
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
interface Waiter {
|
|
205
|
+
readonly messageId: string;
|
|
206
|
+
readonly digest: string;
|
|
207
|
+
readonly socket: BrokerConnectedSession["socket"];
|
|
208
|
+
readonly attemptId?: string;
|
|
209
|
+
readonly senderSessionId: string;
|
|
210
|
+
readonly expectsReply: boolean;
|
|
211
|
+
readonly resolveReplyTarget: boolean;
|
|
212
|
+
readonly targetSessionId: string;
|
|
213
|
+
readonly targetIdentity: string;
|
|
214
|
+
/** Told `queued` already: `delivered` follows only while the sender is connected, and it never expires. */
|
|
215
|
+
readonly queued: boolean;
|
|
216
|
+
timer: ReturnType<typeof setTimeout> | undefined;
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
interface TrackedHold {
|
|
220
|
+
readonly messageId: string;
|
|
221
|
+
readonly reservationKey: string;
|
|
222
|
+
readonly childName: string;
|
|
223
|
+
readonly socket: BrokerConnectedSession["socket"];
|
|
224
|
+
readonly senderId: string;
|
|
225
|
+
readonly attemptId?: string;
|
|
226
|
+
readonly clientMessage: SendClientMessage;
|
|
227
|
+
readonly timer: ReturnType<typeof setTimeout>;
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
interface PresenceEntry {
|
|
231
|
+
lease?: Lease;
|
|
232
|
+
/** `Date.now()` when this broker last wrote the session's known-session record. */
|
|
233
|
+
knownAt?: number;
|
|
234
|
+
/** The session's asks were failed once it turned terminal. */
|
|
235
|
+
terminal?: boolean;
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
interface MirroredReservation {
|
|
239
|
+
lease: RestoringReservationLease;
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
interface RemoteTarget {
|
|
243
|
+
readonly identity: string;
|
|
244
|
+
readonly sessionId: string;
|
|
245
|
+
readonly groups: ReadonlySet<string>;
|
|
246
|
+
/**
|
|
247
|
+
* The recipient attributes that gate delivery, from its live presence or its known-session record: every route
|
|
248
|
+
* to a session, live or offline, refuses non-agent recipients and asks to terminal ones alike.
|
|
249
|
+
*/
|
|
250
|
+
readonly gates: Pick<SessionInfo, "recipientPurpose" | "replyCapability">;
|
|
251
|
+
/** The broker hosting it now; absent for an offline session. */
|
|
252
|
+
readonly hostBrokerId?: string;
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
function returnAddressOf(identity: string): string | undefined {
|
|
256
|
+
try {
|
|
257
|
+
const parsed = JSON.parse(identity) as unknown;
|
|
258
|
+
return Array.isArray(parsed) && parsed[0] === "return" && typeof parsed[1] === "string" ? parsed[1] : undefined;
|
|
259
|
+
} catch {
|
|
260
|
+
return undefined;
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
function sharesGroup(groups: ReadonlySet<string>, sender: SessionInfo): boolean {
|
|
265
|
+
const senderGroups = sessionGroups(sender);
|
|
266
|
+
return [...groups].some((group) => senderGroups.has(group));
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
export class MultiNodeRouter {
|
|
270
|
+
readonly brokerId: string;
|
|
271
|
+
private readonly state: IntercomBrokerStateStore;
|
|
272
|
+
private readonly bus: WakeUpBus;
|
|
273
|
+
private readonly host: MultiNodeHost;
|
|
274
|
+
private readonly nodeId: string;
|
|
275
|
+
private membership: Lease | undefined;
|
|
276
|
+
private subscription: WakeUpSubscription | undefined;
|
|
277
|
+
private readonly presence = new Map<BrokerConnectedSession, PresenceEntry>();
|
|
278
|
+
private readonly inflight = new Map<string, Inflight>();
|
|
279
|
+
private readonly waiters = new Map<string, Waiter>();
|
|
280
|
+
private readonly holds = new Map<string, TrackedHold>();
|
|
281
|
+
private readonly mirrored = new Map<string, MirroredReservation>();
|
|
282
|
+
private readonly heldIdentities = new Set<string>();
|
|
283
|
+
private readonly scanning = new Map<string, { rescan: boolean }>();
|
|
284
|
+
private readonly serials = new Map<string, Promise<void>>();
|
|
285
|
+
private readonly settling = new Set<Promise<void>>();
|
|
286
|
+
private readonly timers: ReturnType<typeof setInterval>[] = [];
|
|
287
|
+
private checkingOutcomes: { again: boolean } | undefined;
|
|
288
|
+
private closed = false;
|
|
289
|
+
private lastError: string | undefined;
|
|
290
|
+
|
|
291
|
+
private constructor(options: MultiNodeRouterOptions) {
|
|
292
|
+
this.state = options.state;
|
|
293
|
+
this.bus = options.bus;
|
|
294
|
+
this.host = options.host;
|
|
295
|
+
this.brokerId = options.brokerId ?? randomUUID();
|
|
296
|
+
this.nodeId = options.nodeId ?? hostname();
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
/** Take the membership lease, subscribe to wake-ups and adopt the sessions already connected. */
|
|
300
|
+
static async start(options: MultiNodeRouterOptions): Promise<MultiNodeRouter> {
|
|
301
|
+
const router = new MultiNodeRouter(options);
|
|
302
|
+
const acquired = await router.state.leases.acquire(
|
|
303
|
+
{ kind: "broker", brokerId: router.brokerId, nodeId: router.nodeId, startedAt: Date.now() },
|
|
304
|
+
router.brokerId,
|
|
305
|
+
BROKER_LEASE_STALE_AFTER_MS,
|
|
306
|
+
);
|
|
307
|
+
if (acquired.status === "acquired") router.membership = acquired.lease;
|
|
308
|
+
const subscribed = await router.bus.subscribe(router.brokerId, (event) => router.onWakeUp(event));
|
|
309
|
+
if (subscribed.status === "subscribed") router.subscription = subscribed.subscription;
|
|
310
|
+
router.every(BROKER_LEASE_RENEW_INTERVAL_MS, () => router.renew());
|
|
311
|
+
router.every(OUTBOX_BACKUP_POLL_INTERVAL_MS, () => router.backupPoll());
|
|
312
|
+
router.every(MULTI_NODE_OUTCOME_POLL_INTERVAL_MS, () => router.outcomePoll());
|
|
313
|
+
for (const session of options.host.sessions.values()) void router.sessionRegistered(session);
|
|
314
|
+
return router;
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
// ---------------------------------------------------------------------------------------------------------
|
|
318
|
+
// Session lifecycle
|
|
319
|
+
|
|
320
|
+
/**
|
|
321
|
+
* A session registered here: record it as known and take its presence lease, retrying briefly while the broker
|
|
322
|
+
* it left still holds it. Resolves once other brokers can resolve it, or after `PRESENCE_REGISTRATION_BOUND_MS`
|
|
323
|
+
* while the database does not answer; what waits for it is delivered after.
|
|
324
|
+
*/
|
|
325
|
+
sessionRegistered(
|
|
326
|
+
session: BrokerConnectedSession,
|
|
327
|
+
consumed?: { readonly capability: string; readonly childName?: string; readonly groups: ReadonlySet<string> },
|
|
328
|
+
): Promise<void> {
|
|
329
|
+
if (this.closed || this.presence.has(session)) return Promise.resolve();
|
|
330
|
+
const entry: PresenceEntry = {};
|
|
331
|
+
this.presence.set(session, entry);
|
|
332
|
+
const identity = deliveryTargetIdentity(session);
|
|
333
|
+
return new Promise<void>((resolve) => {
|
|
334
|
+
const bound = setTimeout(resolve, PRESENCE_REGISTRATION_BOUND_MS);
|
|
335
|
+
bound.unref?.();
|
|
336
|
+
this.serial(identity, async () => {
|
|
337
|
+
try {
|
|
338
|
+
await this.recordKnown(session, entry);
|
|
339
|
+
for (let attempt = 0; ; attempt++) {
|
|
340
|
+
await this.acquirePresence(session, entry);
|
|
341
|
+
if (entry.lease !== undefined || attempt >= PRESENCE_HANDOVER_ATTEMPTS || this.presence.get(session) !== entry)
|
|
342
|
+
break;
|
|
343
|
+
await new Promise<void>((wait) => setTimeout(wait, PRESENCE_HANDOVER_RETRY_MS * 2 ** attempt).unref());
|
|
344
|
+
}
|
|
345
|
+
if (consumed !== undefined) await this.consumeReservation(consumed);
|
|
346
|
+
} finally {
|
|
347
|
+
clearTimeout(bound);
|
|
348
|
+
resolve();
|
|
349
|
+
}
|
|
350
|
+
this.scanIdentity(identity);
|
|
351
|
+
});
|
|
352
|
+
});
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
/** Presence or memberships changed: the shared subject follows them. */
|
|
356
|
+
sessionUpdated(session: BrokerConnectedSession): void {
|
|
357
|
+
const entry = this.presence.get(session);
|
|
358
|
+
if (this.closed || entry === undefined) return;
|
|
359
|
+
this.serial(deliveryTargetIdentity(session), async () => {
|
|
360
|
+
await this.recordKnown(session, entry);
|
|
361
|
+
if (session.info.replyCapability === "terminal" && entry.terminal !== true) {
|
|
362
|
+
entry.terminal = true;
|
|
363
|
+
await this.failTerminalAsks(session);
|
|
364
|
+
}
|
|
365
|
+
if (entry.lease === undefined) return;
|
|
366
|
+
const renewed = await this.state.leases.renew(entry.lease, this.presenceSubject(session));
|
|
367
|
+
if (renewed.status !== "renewed") entry.lease = undefined;
|
|
368
|
+
});
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
/**
|
|
372
|
+
* The session can no longer answer: the asks other nodes delivered to it get the terminal error a local asker gets
|
|
373
|
+
* (`failTerminalQuestions`), as a reply correlated to each ask. A remote asker's error travels the outbox, so its
|
|
374
|
+
* route stays open until that reply is delivered and then closes like any answered ask.
|
|
375
|
+
*/
|
|
376
|
+
private async failTerminalAsks(target: BrokerConnectedSession): Promise<void> {
|
|
377
|
+
const taken = await this.state.pendingAsks.takeForTarget(target.info.id);
|
|
378
|
+
if (taken.status !== "taken" || taken.asks.length === 0) return;
|
|
379
|
+
const live = (await this.state.leases.listLive("presence", BROKER_LEASE_STALE_AFTER_MS)).filter(
|
|
380
|
+
(lease) => lease.subject.kind === "presence",
|
|
381
|
+
);
|
|
382
|
+
for (const ask of taken.asks) {
|
|
383
|
+
const frame: IntercomDeliveryFrame = {
|
|
384
|
+
type: "message",
|
|
385
|
+
from: target.info,
|
|
386
|
+
message: {
|
|
387
|
+
id: `terminal:${ask.messageId}`,
|
|
388
|
+
timestamp: Date.now(),
|
|
389
|
+
replyTo: ask.messageId,
|
|
390
|
+
replyError: TERMINAL_CHILD_ASK_REFUSAL,
|
|
391
|
+
content: { text: TERMINAL_CHILD_ASK_REFUSAL },
|
|
392
|
+
},
|
|
393
|
+
};
|
|
394
|
+
const local = this.host.sessions.get(ask.senderSessionId);
|
|
395
|
+
if (local !== undefined) {
|
|
396
|
+
this.host.write(local.socket, frame);
|
|
397
|
+
continue;
|
|
398
|
+
}
|
|
399
|
+
const hosted = live.find((lease) => (lease.subject as RecipientPresenceSubject).session.id === ask.senderSessionId);
|
|
400
|
+
let identity = hosted === undefined ? undefined : (hosted.subject as RecipientPresenceSubject).targetIdentity;
|
|
401
|
+
if (identity === undefined) {
|
|
402
|
+
const known = await this.state.knownSessions.lookup(ask.senderSessionId, KNOWN_SESSION_RETENTION_MS);
|
|
403
|
+
identity =
|
|
404
|
+
known.status === "found"
|
|
405
|
+
? known.sessions.find((session) => session.intercomSessionId === ask.senderSessionId)?.targetIdentity
|
|
406
|
+
: undefined;
|
|
407
|
+
}
|
|
408
|
+
if (identity === undefined) continue;
|
|
409
|
+
await this.state.pendingAsks.record(ask);
|
|
410
|
+
const enqueued = await this.state.outbox.enqueue({
|
|
411
|
+
targetIdentity: identity,
|
|
412
|
+
originBrokerId: this.brokerId,
|
|
413
|
+
digest: crossNodeDigest(JSON.stringify(frame)),
|
|
414
|
+
frame,
|
|
415
|
+
});
|
|
416
|
+
if (enqueued.status === "inserted" && hosted !== undefined) {
|
|
417
|
+
await this.bus.publish({ brokerId: hosted.holderId, reason: "outbox", version: enqueued.row.sequence });
|
|
418
|
+
}
|
|
419
|
+
}
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
/** A session left (cleanly or not): it stays known, so sends to it are queued until it returns anywhere. */
|
|
423
|
+
sessionLeft(session: BrokerConnectedSession): void {
|
|
424
|
+
const entry = this.presence.get(session);
|
|
425
|
+
if (this.closed || entry === undefined) return;
|
|
426
|
+
this.presence.delete(session);
|
|
427
|
+
const identity = deliveryTargetIdentity(session);
|
|
428
|
+
const claims = [...this.inflight.values()].filter((inflight) => inflight.session === session);
|
|
429
|
+
for (const inflight of claims) this.inflight.delete(inflight.row.messageId);
|
|
430
|
+
for (const waiter of [...this.waiters.values()]) {
|
|
431
|
+
if (waiter.socket === session.socket) this.dropWaiter(waiter);
|
|
432
|
+
}
|
|
433
|
+
const holds = [...this.holds.values()].filter((hold) => hold.senderId === session.info.id);
|
|
434
|
+
for (const hold of holds) {
|
|
435
|
+
clearTimeout(hold.timer);
|
|
436
|
+
this.holds.delete(hold.messageId);
|
|
437
|
+
}
|
|
438
|
+
const sessionId = session.info.id;
|
|
439
|
+
this.serial(identity, async () => {
|
|
440
|
+
for (const inflight of claims) await this.state.outbox.releaseClaim(inflight.claim);
|
|
441
|
+
if (entry.lease !== undefined) await this.state.leases.release(entry.lease);
|
|
442
|
+
await this.recordKnown(session, entry);
|
|
443
|
+
await this.state.pendingAsks.pruneSender(sessionId);
|
|
444
|
+
await this.state.pendingAsks.takeForTarget(sessionId);
|
|
445
|
+
for (const hold of holds) await this.state.heldSends.drop(hold.messageId);
|
|
446
|
+
for (const [key, mirror] of [...this.mirrored]) {
|
|
447
|
+
if (mirror.lease.ownerSessionId !== sessionId) continue;
|
|
448
|
+
this.mirrored.delete(key);
|
|
449
|
+
await this.state.reservations.release(mirror.lease);
|
|
450
|
+
await this.wakeHeldOrigins(key);
|
|
451
|
+
}
|
|
452
|
+
});
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
/** #3761 `received` from a local recipient; true when it confirms a row this broker claimed for it. */
|
|
456
|
+
received(recipient: BrokerConnectedSession, messageId: string): boolean {
|
|
457
|
+
const inflight = this.inflight.get(messageId);
|
|
458
|
+
if (inflight === undefined || inflight.identity !== deliveryTargetIdentity(recipient)) return false;
|
|
459
|
+
this.settleDelivered(inflight, recipient);
|
|
460
|
+
return true;
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
/**
|
|
464
|
+
* Sessions other brokers host now (live presence leases), keyed by session id, for `list`: a session connected
|
|
465
|
+
* through another node is a live peer exactly like a local one. Group visibility is the caller's, as for local
|
|
466
|
+
* sessions. A database that fails or does not answer within `REMOTE_SESSIONS_BOUND_MS` yields none.
|
|
467
|
+
*/
|
|
468
|
+
remoteSessions(): Promise<Map<string, { readonly info: SessionInfo }>> {
|
|
469
|
+
const remote = new Map<string, { readonly info: SessionInfo }>();
|
|
470
|
+
if (this.closed) return Promise.resolve(remote);
|
|
471
|
+
return new Promise((resolve) => {
|
|
472
|
+
const bound = setTimeout(() => resolve(new Map()), REMOTE_SESSIONS_BOUND_MS);
|
|
473
|
+
bound.unref?.();
|
|
474
|
+
void this.state.leases.listLive("presence", BROKER_LEASE_STALE_AFTER_MS).then(
|
|
475
|
+
(live) => {
|
|
476
|
+
clearTimeout(bound);
|
|
477
|
+
for (const lease of live) {
|
|
478
|
+
if (lease.subject.kind !== "presence" || lease.holderId === this.brokerId) continue;
|
|
479
|
+
const info = lease.subject.session;
|
|
480
|
+
if (!this.host.sessions.has(info.id) && !remote.has(info.id)) remote.set(info.id, { info });
|
|
481
|
+
}
|
|
482
|
+
resolve(remote);
|
|
483
|
+
},
|
|
484
|
+
(error: Error) => {
|
|
485
|
+
clearTimeout(bound);
|
|
486
|
+
this.report(error);
|
|
487
|
+
resolve(new Map());
|
|
488
|
+
},
|
|
489
|
+
);
|
|
490
|
+
});
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
// ---------------------------------------------------------------------------------------------------------
|
|
494
|
+
// Restoring reservations (§5.7): mirror this broker's reservations so senders on any host hold their sends.
|
|
495
|
+
|
|
496
|
+
restoringReserved(
|
|
497
|
+
capability: string,
|
|
498
|
+
ownerSessionId: string,
|
|
499
|
+
childName: string,
|
|
500
|
+
group: string,
|
|
501
|
+
aliases: readonly string[],
|
|
502
|
+
dormant: boolean,
|
|
503
|
+
): void {
|
|
504
|
+
if (this.closed) return;
|
|
505
|
+
const reservationKey = reservationKeyFor(capability);
|
|
506
|
+
const owner = this.host.sessions.get(ownerSessionId);
|
|
507
|
+
this.serial(`reservation:${reservationKey}`, async () => {
|
|
508
|
+
const reserved = await this.state.reservations.reserve({
|
|
509
|
+
reservationKey,
|
|
510
|
+
ownerSessionId,
|
|
511
|
+
ownerBrokerId: this.brokerId,
|
|
512
|
+
childName,
|
|
513
|
+
childTarget: normalizeTarget(childName),
|
|
514
|
+
aliases: aliases.map(normalizeTarget).filter((alias) => alias.length > 0),
|
|
515
|
+
group,
|
|
516
|
+
dormant,
|
|
517
|
+
});
|
|
518
|
+
if (reserved.status !== "reserved") return;
|
|
519
|
+
// The owner left (or its id was taken over) while the reserve was being written: its departure scanned the
|
|
520
|
+
// mirrors before this one existed, so end the reservation here instead of renewing it for a gone owner.
|
|
521
|
+
if (this.host.sessions.get(ownerSessionId) !== owner) {
|
|
522
|
+
await this.state.reservations.release(reserved.lease);
|
|
523
|
+
await this.wakeHeldOrigins(reservationKey);
|
|
524
|
+
return;
|
|
525
|
+
}
|
|
526
|
+
this.mirrored.set(reservationKey, { lease: reserved.lease });
|
|
527
|
+
});
|
|
528
|
+
}
|
|
529
|
+
|
|
530
|
+
/** The owner released the reservation; `gone` is the formatted gone refusal kept as a marker. */
|
|
531
|
+
restoringReleased(capability: string, ownerSessionId: string, gone?: string, refusal?: string): void {
|
|
532
|
+
if (this.closed) return;
|
|
533
|
+
const reservationKey = reservationKeyFor(capability);
|
|
534
|
+
// Behind the reserve on the same key: a release while the reserve is still being written ends what it wrote.
|
|
535
|
+
this.serial(`reservation:${reservationKey}`, async () => {
|
|
536
|
+
const mirror = this.mirrored.get(reservationKey);
|
|
537
|
+
if (mirror === undefined || mirror.lease.ownerSessionId !== ownerSessionId) return;
|
|
538
|
+
if (gone === undefined) this.mirrored.delete(reservationKey);
|
|
539
|
+
await this.state.reservations.release(
|
|
540
|
+
mirror.lease,
|
|
541
|
+
gone !== undefined ? { gone } : refusal !== undefined ? { refusal } : undefined,
|
|
542
|
+
);
|
|
543
|
+
await this.wakeHeldOrigins(reservationKey);
|
|
544
|
+
});
|
|
545
|
+
}
|
|
546
|
+
|
|
547
|
+
private async consumeReservation(consumed: {
|
|
548
|
+
readonly capability: string;
|
|
549
|
+
readonly childName?: string;
|
|
550
|
+
readonly groups: ReadonlySet<string>;
|
|
551
|
+
}): Promise<void> {
|
|
552
|
+
const reservationKey = reservationKeyFor(consumed.capability);
|
|
553
|
+
const result = await this.state.reservations.consume(
|
|
554
|
+
reservationKey,
|
|
555
|
+
consumed.childName === undefined
|
|
556
|
+
? undefined
|
|
557
|
+
: { childTarget: normalizeTarget(consumed.childName), groups: [...consumed.groups] },
|
|
558
|
+
);
|
|
559
|
+
if (result.status !== "consumed") return;
|
|
560
|
+
for (const key of result.reservationKeys) {
|
|
561
|
+
this.mirrored.delete(key);
|
|
562
|
+
await this.wakeHeldOrigins(key);
|
|
563
|
+
}
|
|
564
|
+
}
|
|
565
|
+
|
|
566
|
+
/** Tell every origin holding a send for this reservation to read it; drop rows no live broker owns. */
|
|
567
|
+
private async wakeHeldOrigins(reservationKey: string): Promise<void> {
|
|
568
|
+
const listed = await this.state.heldSends.list(reservationKey);
|
|
569
|
+
if (listed.status !== "listed" || listed.sends.length === 0) return;
|
|
570
|
+
const live = new Set(
|
|
571
|
+
(await this.state.leases.listLive("broker", BROKER_LEASE_STALE_AFTER_MS)).map((lease) => lease.holderId),
|
|
572
|
+
);
|
|
573
|
+
const origins = new Set<string>();
|
|
574
|
+
for (const send of listed.sends) {
|
|
575
|
+
if (live.has(send.originBrokerId) || send.originBrokerId === this.brokerId) origins.add(send.originBrokerId);
|
|
576
|
+
else await this.state.heldSends.drop(send.request.message.id);
|
|
577
|
+
}
|
|
578
|
+
for (const origin of origins) {
|
|
579
|
+
if (origin === this.brokerId) void this.checkHolds();
|
|
580
|
+
else await this.bus.publish({ brokerId: origin, reason: "outcome", version: 0 });
|
|
581
|
+
}
|
|
582
|
+
}
|
|
583
|
+
|
|
584
|
+
// ---------------------------------------------------------------------------------------------------------
|
|
585
|
+
// Sending to another node
|
|
586
|
+
|
|
587
|
+
/** Answer a send no local session resolved: another node's session, a shared hold, an offline session, or none. */
|
|
588
|
+
routeSend(request: RemoteSendRequest): void {
|
|
589
|
+
void this.routeSendSafely(request);
|
|
590
|
+
}
|
|
591
|
+
|
|
592
|
+
private async routeSendSafely(request: RemoteSendRequest): Promise<void> {
|
|
593
|
+
try {
|
|
594
|
+
if (request.localTarget !== undefined) await this.answerLocalRetry(request, request.localTarget);
|
|
595
|
+
else await this.route(request);
|
|
596
|
+
} catch (error) {
|
|
597
|
+
this.report(error);
|
|
598
|
+
// Local delivery does not depend on the shared database (spec §8).
|
|
599
|
+
if (request.localTarget !== undefined) return request.notFound();
|
|
600
|
+
this.fail(request, SHARED_STORAGE_UNAVAILABLE_REASON, request.requirePendingReply ? "session_not_found" : undefined);
|
|
601
|
+
}
|
|
602
|
+
}
|
|
603
|
+
|
|
604
|
+
private async route(request: RemoteSendRequest): Promise<void> {
|
|
605
|
+
if (request.message.replyTo !== undefined) await Promise.all([...this.settling]);
|
|
606
|
+
const from = request.from;
|
|
607
|
+
const live = (await this.state.leases.listLive("presence", BROKER_LEASE_STALE_AFTER_MS)).filter(
|
|
608
|
+
(lease) =>
|
|
609
|
+
lease.subject.kind === "presence" &&
|
|
610
|
+
lease.holderId !== this.brokerId &&
|
|
611
|
+
!this.host.sessions.has(lease.subject.session.id),
|
|
612
|
+
);
|
|
613
|
+
const presenceOf = (lease: LeaseState) => lease.subject as RecipientPresenceSubject;
|
|
614
|
+
const exact = live.find((lease) => presenceOf(lease).session.id === request.target);
|
|
615
|
+
let resolved: LeaseState | undefined = exact;
|
|
616
|
+
if (resolved === undefined) {
|
|
617
|
+
const replyAuthorized = await this.replyAuthorizedTargets(
|
|
618
|
+
request,
|
|
619
|
+
live.map((lease) => presenceOf(lease).session.id),
|
|
620
|
+
);
|
|
621
|
+
const visible = live.filter((lease) => {
|
|
622
|
+
const info = presenceOf(lease).session;
|
|
623
|
+
return (
|
|
624
|
+
sameGroup(info, from.info) ||
|
|
625
|
+
(request.supervisorSend && info.id === from.supervisorId) ||
|
|
626
|
+
replyAuthorized.has(info.id)
|
|
627
|
+
);
|
|
628
|
+
});
|
|
629
|
+
const nonAgentNamed = visible.some(
|
|
630
|
+
(lease) =>
|
|
631
|
+
!isAgentRecipient(presenceOf(lease).session) &&
|
|
632
|
+
presenceOf(lease).session.name?.toLowerCase() === request.target.toLowerCase(),
|
|
633
|
+
);
|
|
634
|
+
const agents = visible.filter((lease) => isAgentRecipient(presenceOf(lease).session));
|
|
635
|
+
const infos = agents.map((lease) => presenceOf(lease).session);
|
|
636
|
+
let resolution: SessionTargetResolution = resolveSessionTarget(infos, request.target);
|
|
637
|
+
if (resolution.kind === "not_found") {
|
|
638
|
+
resolution = resolveSessionTargetByHostSessionId(infos, request.target, (info) => {
|
|
639
|
+
const lease = agents.find((candidate) => presenceOf(candidate).session === info);
|
|
640
|
+
return lease === undefined ? undefined : returnAddressOf(presenceOf(lease).targetIdentity);
|
|
641
|
+
});
|
|
642
|
+
}
|
|
643
|
+
if (resolution.kind === "resolved") {
|
|
644
|
+
const session = resolution.session;
|
|
645
|
+
resolved = agents.find((lease) => presenceOf(lease).session === session);
|
|
646
|
+
} else if (resolution.kind !== "not_found") {
|
|
647
|
+
this.fail(request, sessionTargetFailureReason(request.clientMessage.to, resolution));
|
|
648
|
+
return;
|
|
649
|
+
} else if (nonAgentNamed) {
|
|
650
|
+
this.fail(request, NON_AGENT_RECIPIENT_REFUSAL);
|
|
651
|
+
return;
|
|
652
|
+
}
|
|
653
|
+
}
|
|
654
|
+
if (resolved !== undefined) {
|
|
655
|
+
const subject = presenceOf(resolved);
|
|
656
|
+
await this.deliverThroughOutbox(request, {
|
|
657
|
+
identity: subject.targetIdentity,
|
|
658
|
+
sessionId: subject.session.id,
|
|
659
|
+
groups: sessionGroups(subject.session),
|
|
660
|
+
gates: subject.session,
|
|
661
|
+
hostBrokerId: resolved.holderId,
|
|
662
|
+
});
|
|
663
|
+
return;
|
|
664
|
+
}
|
|
665
|
+
if (!request.supervisorSend && !request.requirePendingReply && request.expectedRecipientId === undefined) {
|
|
666
|
+
if (await this.holdForRemoteReservation(request)) return;
|
|
667
|
+
}
|
|
668
|
+
const known = await this.resolveKnownSession(request);
|
|
669
|
+
if (known === "answered") return;
|
|
670
|
+
if (known !== undefined) {
|
|
671
|
+
await this.deliverThroughOutbox(request, {
|
|
672
|
+
identity: known.targetIdentity,
|
|
673
|
+
sessionId: known.intercomSessionId,
|
|
674
|
+
groups: new Set(known.groups),
|
|
675
|
+
gates: known,
|
|
676
|
+
});
|
|
677
|
+
return;
|
|
678
|
+
}
|
|
679
|
+
request.notFound();
|
|
680
|
+
}
|
|
681
|
+
|
|
682
|
+
/**
|
|
683
|
+
* A send that resolved to a session on this broker: a same-id retry of a send that already went through the shared
|
|
684
|
+
* outbox (its recipient has since moved here) is answered from that row, never delivered again; any other send
|
|
685
|
+
* takes the local path. A lookup the shared database does not answer within `LOCAL_RETRY_LOOKUP_BOUND_MS` takes the
|
|
686
|
+
* local path too, and its late answer is ignored: the send has been answered once already.
|
|
687
|
+
*/
|
|
688
|
+
private async answerLocalRetry(request: RemoteSendRequest, target: BrokerConnectedSession): Promise<void> {
|
|
689
|
+
const lookup = this.state.outbox.read(request.message.id);
|
|
690
|
+
let bound: ReturnType<typeof setTimeout> | undefined;
|
|
691
|
+
const expired = new Promise<"expired">((resolve) => {
|
|
692
|
+
bound = setTimeout(() => resolve("expired"), LOCAL_RETRY_LOOKUP_BOUND_MS);
|
|
693
|
+
bound.unref?.();
|
|
694
|
+
});
|
|
695
|
+
const existing = await Promise.race([lookup, expired]).finally(() => clearTimeout(bound));
|
|
696
|
+
if (existing === "expired") {
|
|
697
|
+
lookup.catch((error: Error) => this.report(error));
|
|
698
|
+
request.notFound();
|
|
699
|
+
return;
|
|
700
|
+
}
|
|
701
|
+
if (existing.status !== "found") {
|
|
702
|
+
request.notFound();
|
|
703
|
+
return;
|
|
704
|
+
}
|
|
705
|
+
await this.answerExistingRow(
|
|
706
|
+
request,
|
|
707
|
+
{
|
|
708
|
+
identity: deliveryTargetIdentity(target),
|
|
709
|
+
sessionId: target.info.id,
|
|
710
|
+
groups: sessionGroups(target.info),
|
|
711
|
+
gates: target.info,
|
|
712
|
+
hostBrokerId: this.brokerId,
|
|
713
|
+
},
|
|
714
|
+
crossNodeDigest(request.signature),
|
|
715
|
+
existing.row,
|
|
716
|
+
);
|
|
717
|
+
}
|
|
718
|
+
|
|
719
|
+
/**
|
|
720
|
+
* The sessions among `sessionIds` this send's `replyTo` answers: a recorded vertical crossing or an open ask route,
|
|
721
|
+
* exactly as a local reply's candidates (`handleBrokerSend`). Such a session resolves by name across groups; the
|
|
722
|
+
* outbox write still checks the same authorization.
|
|
723
|
+
*/
|
|
724
|
+
private async replyAuthorizedTargets(request: RemoteSendRequest, sessionIds: readonly string[]): Promise<Set<string>> {
|
|
725
|
+
const authorized = new Set<string>();
|
|
726
|
+
const replyTo = request.message.replyTo;
|
|
727
|
+
if (replyTo === undefined) return authorized;
|
|
728
|
+
const replySenderSessionId = request.from.info.id;
|
|
729
|
+
for (const sessionId of new Set(sessionIds)) {
|
|
730
|
+
if (this.host.supervisorCache.matchReply(replyTo, replySenderSessionId, sessionId)) authorized.add(sessionId);
|
|
731
|
+
else if (
|
|
732
|
+
(await this.state.pendingAsks.matchesReply({ replySenderSessionId, replyTargetSessionId: sessionId, replyTo }))
|
|
733
|
+
.status === "match"
|
|
734
|
+
) {
|
|
735
|
+
authorized.add(sessionId);
|
|
736
|
+
}
|
|
737
|
+
}
|
|
738
|
+
return authorized;
|
|
739
|
+
}
|
|
740
|
+
|
|
741
|
+
/**
|
|
742
|
+
* An offline session, resolved exactly as a live remote one is: an exact id first (its recipient checks follow in
|
|
743
|
+
* `deliverThroughOutbox`), otherwise a visible agent by name or Atomic session id; a visible non-agent of that
|
|
744
|
+
* name alone is refused as a live one is.
|
|
745
|
+
*/
|
|
746
|
+
private async resolveKnownSession(request: RemoteSendRequest): Promise<KnownSessionState | "answered" | undefined> {
|
|
747
|
+
const found = await this.state.knownSessions.lookup(request.target, KNOWN_SESSION_RETENTION_MS);
|
|
748
|
+
if (found.status !== "found") return undefined;
|
|
749
|
+
const localIdentities = new Set([...this.host.sessions.values()].map(deliveryTargetIdentity));
|
|
750
|
+
const sessions = found.sessions.filter((session) => !localIdentities.has(session.targetIdentity));
|
|
751
|
+
const byId = sessions.find((session) => session.intercomSessionId === request.target);
|
|
752
|
+
if (byId !== undefined) return byId;
|
|
753
|
+
const replyAuthorized = await this.replyAuthorizedTargets(
|
|
754
|
+
request,
|
|
755
|
+
sessions.map((session) => session.intercomSessionId),
|
|
756
|
+
);
|
|
757
|
+
const visible = sessions.filter(
|
|
758
|
+
(session) =>
|
|
759
|
+
sharesGroup(new Set(session.groups), request.from.info) ||
|
|
760
|
+
(request.supervisorSend && session.intercomSessionId === request.from.supervisorId) ||
|
|
761
|
+
replyAuthorized.has(session.intercomSessionId),
|
|
762
|
+
);
|
|
763
|
+
const target = request.target.toLowerCase();
|
|
764
|
+
const agents = visible.filter(isAgentRecipient);
|
|
765
|
+
const candidates = [
|
|
766
|
+
["ambiguous_name", agents.filter((session) => session.name?.toLowerCase() === target)],
|
|
767
|
+
["ambiguous_session_id", agents.filter((session) => session.atomicSessionId === request.target)],
|
|
768
|
+
] as const;
|
|
769
|
+
for (const [ambiguity, matches] of candidates) {
|
|
770
|
+
if (matches.length === 1) return matches[0];
|
|
771
|
+
if (matches.length > 1) {
|
|
772
|
+
const infos = matches.map((session) => ({ id: session.intercomSessionId, name: session.name }));
|
|
773
|
+
this.fail(request, sessionTargetFailureReason(request.clientMessage.to, { kind: ambiguity, matches: infos }));
|
|
774
|
+
return "answered";
|
|
775
|
+
}
|
|
776
|
+
}
|
|
777
|
+
if (visible.some((session) => !isAgentRecipient(session) && session.name?.toLowerCase() === target)) {
|
|
778
|
+
this.fail(request, NON_AGENT_RECIPIENT_REFUSAL);
|
|
779
|
+
return "answered";
|
|
780
|
+
}
|
|
781
|
+
return undefined;
|
|
782
|
+
}
|
|
783
|
+
|
|
784
|
+
/** Run the origin's routing decisions on the shared state, then enqueue the exact local frame. */
|
|
785
|
+
private async deliverThroughOutbox(request: RemoteSendRequest, target: RemoteTarget): Promise<void> {
|
|
786
|
+
const { message, from } = request;
|
|
787
|
+
if (!isAgentRecipient(target.gates)) {
|
|
788
|
+
this.fail(request, NON_AGENT_RECIPIENT_REFUSAL);
|
|
789
|
+
return;
|
|
790
|
+
}
|
|
791
|
+
if (request.expectedRecipientId !== undefined && request.expectedRecipientId !== target.sessionId) {
|
|
792
|
+
this.fail(request, "Reply target does not match the expected recipient");
|
|
793
|
+
return;
|
|
794
|
+
}
|
|
795
|
+
const digest = crossNodeDigest(request.signature);
|
|
796
|
+
const existing = await this.state.outbox.read(message.id);
|
|
797
|
+
if (existing.status === "found") {
|
|
798
|
+
await this.answerExistingRow(request, target, digest, existing.row);
|
|
799
|
+
return;
|
|
800
|
+
}
|
|
801
|
+
if (this.answerLocalDelivery(request, target)) return;
|
|
802
|
+
const correlatedReply =
|
|
803
|
+
message.replyTo !== undefined &&
|
|
804
|
+
(
|
|
805
|
+
await this.state.pendingAsks.matchesReply({
|
|
806
|
+
replySenderSessionId: from.info.id,
|
|
807
|
+
replyTargetSessionId: target.sessionId,
|
|
808
|
+
replyTo: message.replyTo,
|
|
809
|
+
})
|
|
810
|
+
).status === "match";
|
|
811
|
+
if (request.requirePendingReply && !correlatedReply) {
|
|
812
|
+
this.fail(request, "Pending question route does not authorize this public reply");
|
|
813
|
+
return;
|
|
814
|
+
}
|
|
815
|
+
const verticalReply =
|
|
816
|
+
message.replyTo !== undefined &&
|
|
817
|
+
this.host.supervisorCache.matchReply(message.replyTo, from.info.id, target.sessionId);
|
|
818
|
+
if (!request.supervisorSend && !verticalReply && !correlatedReply && !sharesGroup(target.groups, from.info)) {
|
|
819
|
+
this.fail(request, TARGET_GROUP_REFUSAL);
|
|
820
|
+
return;
|
|
821
|
+
}
|
|
822
|
+
if (message.expectsReply === true && target.gates.replyCapability === "terminal") {
|
|
823
|
+
this.fail(request, TERMINAL_CHILD_ASK_REFUSAL);
|
|
824
|
+
return;
|
|
825
|
+
}
|
|
826
|
+
const reserved = await this.state.sentMessages.reserve({
|
|
827
|
+
messageId: message.id,
|
|
828
|
+
digest,
|
|
829
|
+
targetIdentity: target.identity,
|
|
830
|
+
...(message.expectsReply === true
|
|
831
|
+
? { questionRoute: { targetSessionId: target.sessionId, senderGroupIdentity: senderGroupIdentity(from) } }
|
|
832
|
+
: {}),
|
|
833
|
+
});
|
|
834
|
+
if (reserved.status === "conflict") {
|
|
835
|
+
this.failConflict(request);
|
|
836
|
+
return;
|
|
837
|
+
}
|
|
838
|
+
const frame: IntercomDeliveryFrame = request.supervisorSend
|
|
839
|
+
? { type: "message", from: from.info, message, channel: "supervisor" }
|
|
840
|
+
: { type: "message", from: from.info, message };
|
|
841
|
+
const enqueued = await this.state.outbox.enqueue({
|
|
842
|
+
targetIdentity: target.identity,
|
|
843
|
+
originBrokerId: this.brokerId,
|
|
844
|
+
digest,
|
|
845
|
+
frame,
|
|
846
|
+
});
|
|
847
|
+
if (enqueued.status === "conflict") {
|
|
848
|
+
this.failConflict(request);
|
|
849
|
+
return;
|
|
850
|
+
}
|
|
851
|
+
if (enqueued.status === "invalid") {
|
|
852
|
+
this.fail(request, "Session not found");
|
|
853
|
+
return;
|
|
854
|
+
}
|
|
855
|
+
await this.answerExistingRow(request, target, digest, enqueued.row);
|
|
856
|
+
}
|
|
857
|
+
|
|
858
|
+
/**
|
|
859
|
+
* A same-id retry of a send this broker delivered locally, whose recipient has since moved to another broker, is
|
|
860
|
+
* answered from that local delivery (the same check the local path makes), never sent through the outbox again.
|
|
861
|
+
* True when the send was answered.
|
|
862
|
+
*/
|
|
863
|
+
private answerLocalDelivery(request: RemoteSendRequest, target: RemoteTarget): boolean {
|
|
864
|
+
const { message } = request;
|
|
865
|
+
const delivered = this.host.deliveredMessages.lookupForTarget(message.id, request.signature, target.identity);
|
|
866
|
+
if (delivered === "miss") return false;
|
|
867
|
+
if (delivered === "conflict") {
|
|
868
|
+
this.failConflict(request);
|
|
869
|
+
return true;
|
|
870
|
+
}
|
|
871
|
+
if (delivered === "invalid") {
|
|
872
|
+
this.fail(request, "Intercom accepted-delivery authority is invalid; refusing possible duplicate delivery");
|
|
873
|
+
return true;
|
|
874
|
+
}
|
|
875
|
+
if (delivered === "uncertain") {
|
|
876
|
+
this.fail(
|
|
877
|
+
request,
|
|
878
|
+
"Intercom cannot prove whether this reserved operation reached its recipient; refusing redelivery",
|
|
879
|
+
);
|
|
880
|
+
return true;
|
|
881
|
+
}
|
|
882
|
+
if (message.expectsReply === true) {
|
|
883
|
+
const questionTarget = this.host.deliveredMessages.lookupQuestionTarget(
|
|
884
|
+
message.id,
|
|
885
|
+
request.signature,
|
|
886
|
+
senderGroupIdentity(request.from),
|
|
887
|
+
);
|
|
888
|
+
if (questionTarget === undefined) {
|
|
889
|
+
this.fail(request, "Accepted Intercom question route could not be securely rebound");
|
|
890
|
+
return true;
|
|
891
|
+
}
|
|
892
|
+
if (request.resolveReplyTarget && request.attemptId !== undefined) {
|
|
893
|
+
this.host.write(request.socket, {
|
|
894
|
+
type: "question_target",
|
|
895
|
+
messageId: message.id,
|
|
896
|
+
attemptId: request.attemptId,
|
|
897
|
+
sessionId: target.sessionId,
|
|
898
|
+
});
|
|
899
|
+
}
|
|
900
|
+
}
|
|
901
|
+
this.host.write(request.socket, { type: "delivered", messageId: message.id, attemptId: request.attemptId });
|
|
902
|
+
return true;
|
|
903
|
+
}
|
|
904
|
+
|
|
905
|
+
/** Report a row's current state to its sender, and keep waiting on it until it settles. */
|
|
906
|
+
private async answerExistingRow(
|
|
907
|
+
request: RemoteSendRequest,
|
|
908
|
+
target: RemoteTarget,
|
|
909
|
+
digest: string,
|
|
910
|
+
row: OutboxRow,
|
|
911
|
+
): Promise<void> {
|
|
912
|
+
const { message } = request;
|
|
913
|
+
if (row.digest !== digest || row.targetIdentity !== target.identity) {
|
|
914
|
+
this.failConflict(request);
|
|
915
|
+
return;
|
|
916
|
+
}
|
|
917
|
+
if (row.state === "failed") {
|
|
918
|
+
this.fail(request, row.failureReason ?? "Session not found");
|
|
919
|
+
return;
|
|
920
|
+
}
|
|
921
|
+
const asksTarget = message.expectsReply === true && request.resolveReplyTarget && request.attemptId !== undefined;
|
|
922
|
+
if (row.state === "delivered") {
|
|
923
|
+
await this.state.sentMessages.accept({ messageId: message.id, digest });
|
|
924
|
+
if (asksTarget) await this.writeQuestionTarget(request.socket, message.id, request.attemptId, target);
|
|
925
|
+
this.host.write(request.socket, { type: "delivered", messageId: message.id, attemptId: request.attemptId });
|
|
926
|
+
return;
|
|
927
|
+
}
|
|
928
|
+
const offline = target.hostBrokerId === undefined;
|
|
929
|
+
const previous = this.waiters.get(message.id);
|
|
930
|
+
if (previous !== undefined) this.dropWaiter(previous);
|
|
931
|
+
const waiter: Waiter = {
|
|
932
|
+
messageId: message.id,
|
|
933
|
+
digest,
|
|
934
|
+
socket: request.socket,
|
|
935
|
+
...(request.attemptId === undefined ? {} : { attemptId: request.attemptId }),
|
|
936
|
+
senderSessionId: request.from.info.id,
|
|
937
|
+
expectsReply: message.expectsReply === true,
|
|
938
|
+
resolveReplyTarget: request.resolveReplyTarget,
|
|
939
|
+
targetSessionId: target.sessionId,
|
|
940
|
+
targetIdentity: target.identity,
|
|
941
|
+
queued: offline,
|
|
942
|
+
timer: undefined,
|
|
943
|
+
};
|
|
944
|
+
waiter.timer = setTimeout(
|
|
945
|
+
() => {
|
|
946
|
+
if (this.waiters.get(message.id) !== waiter) return;
|
|
947
|
+
this.dropWaiter(waiter);
|
|
948
|
+
if (!offline) {
|
|
949
|
+
this.host.write(request.socket, {
|
|
950
|
+
type: "delivery_failed",
|
|
951
|
+
messageId: message.id,
|
|
952
|
+
attemptId: request.attemptId,
|
|
953
|
+
reason: UNCERTAIN_RECEIPT_REASON,
|
|
954
|
+
});
|
|
955
|
+
}
|
|
956
|
+
},
|
|
957
|
+
offline ? DELIVERED_MESSAGE_TTL_MS : RECIPIENT_RECEIPT_TIMEOUT_MS,
|
|
958
|
+
);
|
|
959
|
+
waiter.timer.unref?.();
|
|
960
|
+
this.waiters.set(message.id, waiter);
|
|
961
|
+
if (offline) {
|
|
962
|
+
const listed = await this.state.outbox.listUndelivered(target.identity);
|
|
963
|
+
const index = listed.status === "listed" ? listed.rows.findIndex((entry) => entry.messageId === message.id) : -1;
|
|
964
|
+
this.host.write(request.socket, {
|
|
965
|
+
type: "queued",
|
|
966
|
+
messageId: message.id,
|
|
967
|
+
...(request.attemptId === undefined ? {} : { attemptId: request.attemptId }),
|
|
968
|
+
target: OFFLINE_SESSION_TARGET,
|
|
969
|
+
position: index + 1 || 1,
|
|
970
|
+
});
|
|
971
|
+
return;
|
|
972
|
+
}
|
|
973
|
+
if (asksTarget) await this.writeQuestionTarget(request.socket, message.id, request.attemptId, target);
|
|
974
|
+
await this.bus.publish({ brokerId: target.hostBrokerId!, reason: "outbox", version: row.sequence });
|
|
975
|
+
}
|
|
976
|
+
|
|
977
|
+
private async writeQuestionTarget(
|
|
978
|
+
socket: BrokerConnectedSession["socket"],
|
|
979
|
+
messageId: string,
|
|
980
|
+
attemptId: string | undefined,
|
|
981
|
+
target: Pick<RemoteTarget, "sessionId">,
|
|
982
|
+
): Promise<void> {
|
|
983
|
+
if (attemptId === undefined) return;
|
|
984
|
+
this.host.write(socket, { type: "question_target", messageId, attemptId, sessionId: target.sessionId });
|
|
985
|
+
}
|
|
986
|
+
|
|
987
|
+
/** §5.7: a reservation shared by another broker holds this send here, at its origin. */
|
|
988
|
+
private async holdForRemoteReservation(request: RemoteSendRequest): Promise<boolean> {
|
|
989
|
+
const { from, message } = request;
|
|
990
|
+
const lookup = await this.state.reservations.lookup(
|
|
991
|
+
normalizeTarget(request.target),
|
|
992
|
+
{ sessionId: from.info.id, groups: [...sessionGroups(from.info)] },
|
|
993
|
+
BROKER_LEASE_STALE_AFTER_MS,
|
|
994
|
+
);
|
|
995
|
+
if (lookup.status === "none" || lookup.status === "invalid") return false;
|
|
996
|
+
const reservation = lookup.reservation;
|
|
997
|
+
if (lookup.status === "gone") {
|
|
998
|
+
this.fail(request, reservation.gone ?? restoringRecipientRetryReason(reservation.childName));
|
|
999
|
+
return true;
|
|
1000
|
+
}
|
|
1001
|
+
const listed = await this.state.heldSends.list(reservation.reservationKey);
|
|
1002
|
+
const count = listed.status === "listed" ? listed.sends.length : 0;
|
|
1003
|
+
if (!request.awaitRestoringRecipient || count >= MAX_HELD_SENDS) {
|
|
1004
|
+
this.fail(request, restoringRecipientRetryReason(reservation.childName));
|
|
1005
|
+
return true;
|
|
1006
|
+
}
|
|
1007
|
+
const held = {
|
|
1008
|
+
reservationKey: reservation.reservationKey,
|
|
1009
|
+
originBrokerId: this.brokerId,
|
|
1010
|
+
senderSessionId: from.info.id,
|
|
1011
|
+
request: request.clientMessage as HeldSendRequest,
|
|
1012
|
+
};
|
|
1013
|
+
let result = await this.state.heldSends.hold(held);
|
|
1014
|
+
if (result.status === "duplicate" && (await this.state.heldSends.drop(message.id)).status === "dropped") {
|
|
1015
|
+
result = await this.state.heldSends.hold(held);
|
|
1016
|
+
}
|
|
1017
|
+
if (result.status !== "held") return true;
|
|
1018
|
+
const previous = this.holds.get(message.id);
|
|
1019
|
+
if (previous !== undefined) clearTimeout(previous.timer);
|
|
1020
|
+
const timer = setTimeout(() => void this.expireHold(message.id), RESTORING_RECIPIENT_HOLD_MS);
|
|
1021
|
+
timer.unref?.();
|
|
1022
|
+
this.holds.set(message.id, {
|
|
1023
|
+
messageId: message.id,
|
|
1024
|
+
reservationKey: reservation.reservationKey,
|
|
1025
|
+
childName: reservation.childName,
|
|
1026
|
+
socket: request.socket,
|
|
1027
|
+
senderId: from.info.id,
|
|
1028
|
+
...(request.attemptId === undefined ? {} : { attemptId: request.attemptId }),
|
|
1029
|
+
clientMessage: request.clientMessage,
|
|
1030
|
+
timer,
|
|
1031
|
+
});
|
|
1032
|
+
this.host.write(request.socket, {
|
|
1033
|
+
type: "send_held",
|
|
1034
|
+
messageId: message.id,
|
|
1035
|
+
...(request.attemptId === undefined ? {} : { attemptId: request.attemptId }),
|
|
1036
|
+
timeoutMs: RESTORING_RECIPIENT_HOLD_MS,
|
|
1037
|
+
});
|
|
1038
|
+
return true;
|
|
1039
|
+
}
|
|
1040
|
+
|
|
1041
|
+
private async expireHold(messageId: string): Promise<void> {
|
|
1042
|
+
const hold = this.holds.get(messageId);
|
|
1043
|
+
if (hold === undefined) return;
|
|
1044
|
+
this.holds.delete(messageId);
|
|
1045
|
+
try {
|
|
1046
|
+
if ((await this.state.heldSends.drop(messageId)).status !== "dropped") return;
|
|
1047
|
+
} catch (error) {
|
|
1048
|
+
this.report(error);
|
|
1049
|
+
}
|
|
1050
|
+
this.writeHoldFailure(hold, restoringRecipientRetryReason(hold.childName));
|
|
1051
|
+
}
|
|
1052
|
+
|
|
1053
|
+
private writeHoldFailure(hold: TrackedHold, reason: string): void {
|
|
1054
|
+
this.host.write(hold.socket, {
|
|
1055
|
+
type: "delivery_failed",
|
|
1056
|
+
messageId: hold.messageId,
|
|
1057
|
+
...(hold.attemptId === undefined ? {} : { attemptId: hold.attemptId }),
|
|
1058
|
+
reason,
|
|
1059
|
+
});
|
|
1060
|
+
}
|
|
1061
|
+
|
|
1062
|
+
/** Read the reservation of every send held here and dispatch or refuse the ones whose hold ended (§5.7). */
|
|
1063
|
+
private async checkHolds(): Promise<void> {
|
|
1064
|
+
for (const hold of [...this.holds.values()]) {
|
|
1065
|
+
let key = hold.reservationKey;
|
|
1066
|
+
let decision: { readonly kind: "keep" } | { readonly kind: "refuse"; readonly reason: string } | { readonly kind: "dispatch" } = {
|
|
1067
|
+
kind: "keep",
|
|
1068
|
+
};
|
|
1069
|
+
for (let hop = 0; hop < MAX_SUPERSEDED_HOPS; hop++) {
|
|
1070
|
+
const read = await this.state.reservations.read(key);
|
|
1071
|
+
if (read.status === "active") {
|
|
1072
|
+
decision = read.reservation.gone === undefined ? { kind: "keep" } : { kind: "refuse", reason: read.reservation.gone };
|
|
1073
|
+
} else if (read.status === "ended" && read.end.outcome === "superseded") {
|
|
1074
|
+
key = read.end.by;
|
|
1075
|
+
continue;
|
|
1076
|
+
} else if (read.status === "ended" && read.end.outcome === "released" && read.end.refusal !== undefined) {
|
|
1077
|
+
decision = { kind: "refuse", reason: read.end.refusal };
|
|
1078
|
+
} else if (read.status !== "invalid") {
|
|
1079
|
+
decision = { kind: "dispatch" };
|
|
1080
|
+
}
|
|
1081
|
+
break;
|
|
1082
|
+
}
|
|
1083
|
+
if (decision.kind === "keep" || this.holds.get(hold.messageId) !== hold) continue;
|
|
1084
|
+
this.holds.delete(hold.messageId);
|
|
1085
|
+
clearTimeout(hold.timer);
|
|
1086
|
+
if ((await this.state.heldSends.drop(hold.messageId)).status !== "dropped") continue;
|
|
1087
|
+
if (decision.kind === "refuse") {
|
|
1088
|
+
this.writeHoldFailure(hold, decision.reason);
|
|
1089
|
+
continue;
|
|
1090
|
+
}
|
|
1091
|
+
const sender = this.host.sessions.get(hold.senderId);
|
|
1092
|
+
if (sender?.socket === hold.socket) this.host.dispatch(hold.socket, hold.clientMessage, hold.senderId);
|
|
1093
|
+
}
|
|
1094
|
+
}
|
|
1095
|
+
|
|
1096
|
+
// ---------------------------------------------------------------------------------------------------------
|
|
1097
|
+
// Delivering the shared outbox to this broker's sessions
|
|
1098
|
+
|
|
1099
|
+
private scanIdentity(identity: string): void {
|
|
1100
|
+
if (this.closed) return;
|
|
1101
|
+
const running = this.scanning.get(identity);
|
|
1102
|
+
if (running !== undefined) {
|
|
1103
|
+
running.rescan = true;
|
|
1104
|
+
return;
|
|
1105
|
+
}
|
|
1106
|
+
const state = { rescan: false };
|
|
1107
|
+
this.scanning.set(identity, state);
|
|
1108
|
+
void (async () => {
|
|
1109
|
+
try {
|
|
1110
|
+
do {
|
|
1111
|
+
state.rescan = false;
|
|
1112
|
+
await this.claimFor(identity);
|
|
1113
|
+
} while (state.rescan && !this.closed);
|
|
1114
|
+
} catch (error) {
|
|
1115
|
+
this.report(error);
|
|
1116
|
+
} finally {
|
|
1117
|
+
this.scanning.delete(identity);
|
|
1118
|
+
}
|
|
1119
|
+
})();
|
|
1120
|
+
}
|
|
1121
|
+
|
|
1122
|
+
private scanAll(): void {
|
|
1123
|
+
for (const identity of new Set([...this.host.sessions.values()].map(deliveryTargetIdentity))) {
|
|
1124
|
+
this.scanIdentity(identity);
|
|
1125
|
+
}
|
|
1126
|
+
}
|
|
1127
|
+
|
|
1128
|
+
private localSessionFor(identity: string): BrokerConnectedSession | undefined {
|
|
1129
|
+
for (const session of this.host.sessions.values()) {
|
|
1130
|
+
if (deliveryTargetIdentity(session) === identity) return session;
|
|
1131
|
+
}
|
|
1132
|
+
return undefined;
|
|
1133
|
+
}
|
|
1134
|
+
|
|
1135
|
+
private async claimFor(identity: string): Promise<void> {
|
|
1136
|
+
if (this.localSessionFor(identity) === undefined) return;
|
|
1137
|
+
const claimed = await this.state.outbox.claim({
|
|
1138
|
+
claimantId: this.brokerId,
|
|
1139
|
+
targetIdentity: identity,
|
|
1140
|
+
staleAfterMs: OUTBOX_CLAIM_STALE_AFTER_MS,
|
|
1141
|
+
});
|
|
1142
|
+
if (claimed.status === "held") {
|
|
1143
|
+
this.heldIdentities.add(identity);
|
|
1144
|
+
return;
|
|
1145
|
+
}
|
|
1146
|
+
this.heldIdentities.delete(identity);
|
|
1147
|
+
if (claimed.status !== "claimed") return;
|
|
1148
|
+
for (const { row, claim } of claimed.rows) {
|
|
1149
|
+
const session = this.localSessionFor(identity);
|
|
1150
|
+
if (session === undefined) {
|
|
1151
|
+
await this.state.outbox.releaseClaim(claim);
|
|
1152
|
+
continue;
|
|
1153
|
+
}
|
|
1154
|
+
const refusal = await this.transitRefusal(row.frame, session);
|
|
1155
|
+
if (refusal !== undefined) {
|
|
1156
|
+
const failed = await this.state.outbox.markFailed(claim, refusal);
|
|
1157
|
+
if (failed.status === "failed") await this.wakeOrigin(row);
|
|
1158
|
+
continue;
|
|
1159
|
+
}
|
|
1160
|
+
const inflight: Inflight = { claim, row, session, identity };
|
|
1161
|
+
this.inflight.set(row.messageId, inflight);
|
|
1162
|
+
this.host.writeConfirmed(session.socket, row.frame, (written) => {
|
|
1163
|
+
if (this.inflight.get(row.messageId) !== inflight) return;
|
|
1164
|
+
if (!written) {
|
|
1165
|
+
this.inflight.delete(row.messageId);
|
|
1166
|
+
void this.state.outbox.releaseClaim(claim).catch((error: Error) => this.report(error));
|
|
1167
|
+
return;
|
|
1168
|
+
}
|
|
1169
|
+
// A recipient that never confirms saves is delivered once the frame is written, as locally.
|
|
1170
|
+
if (session.acknowledgesReceipt !== true) this.settleDelivered(inflight, session);
|
|
1171
|
+
});
|
|
1172
|
+
}
|
|
1173
|
+
}
|
|
1174
|
+
|
|
1175
|
+
/**
|
|
1176
|
+
* What can change in transit, and what a row written without it never checked: the recipient's purpose (only
|
|
1177
|
+
* agents receive messages), groups and reply capability (§8, "Split ownership"). A `replyTo` alone authorizes
|
|
1178
|
+
* nothing: only a reply on an open ask route, or a vertical reply between a child and its supervisor, may cross
|
|
1179
|
+
* groups, exactly as the origin decided.
|
|
1180
|
+
*/
|
|
1181
|
+
private async transitRefusal(
|
|
1182
|
+
frame: IntercomDeliveryFrame,
|
|
1183
|
+
recipient: BrokerConnectedSession,
|
|
1184
|
+
): Promise<string | undefined> {
|
|
1185
|
+
if (!isAgentRecipient(recipient.info)) return NON_AGENT_RECIPIENT_REFUSAL;
|
|
1186
|
+
if (frame.message.expectsReply === true && recipient.info.replyCapability === "terminal") {
|
|
1187
|
+
return TERMINAL_CHILD_ASK_REFUSAL;
|
|
1188
|
+
}
|
|
1189
|
+
if (frame.channel === "supervisor" || sameGroup(recipient.info, frame.from)) return undefined;
|
|
1190
|
+
const replyTo = frame.message.replyTo;
|
|
1191
|
+
if (replyTo === undefined) return TARGET_GROUP_REFUSAL;
|
|
1192
|
+
if (recipient.supervisorId === frame.from.id) return undefined;
|
|
1193
|
+
if (this.host.supervisorCache.matchReply(replyTo, frame.from.id, recipient.info.id)) return undefined;
|
|
1194
|
+
const route = await this.state.pendingAsks.matchesReply({
|
|
1195
|
+
replySenderSessionId: frame.from.id,
|
|
1196
|
+
replyTargetSessionId: recipient.info.id,
|
|
1197
|
+
replyTo,
|
|
1198
|
+
});
|
|
1199
|
+
return route.status === "match" ? undefined : TARGET_GROUP_REFUSAL;
|
|
1200
|
+
}
|
|
1201
|
+
|
|
1202
|
+
private settleDelivered(inflight: Inflight, recipient: BrokerConnectedSession): void {
|
|
1203
|
+
this.inflight.delete(inflight.row.messageId);
|
|
1204
|
+
const { frame } = inflight.row;
|
|
1205
|
+
const settling = (async () => {
|
|
1206
|
+
// Route first (spec §5.4): a settled row is never claimed again, so a route written after `markDelivered`
|
|
1207
|
+
// would be lost if this broker stopped between the writes. Redelivery repeats the route write safely.
|
|
1208
|
+
if (frame.message.expectsReply === true) {
|
|
1209
|
+
const asker = this.host.sessions.get(frame.from.id);
|
|
1210
|
+
// The asker is here too: the reply takes the local path, which answers only local routes.
|
|
1211
|
+
if (asker !== undefined) this.host.pendingQuestions.record(frame.from.id, recipient.info.id, frame.message.id);
|
|
1212
|
+
else {
|
|
1213
|
+
await this.state.pendingAsks.record({
|
|
1214
|
+
senderSessionId: frame.from.id,
|
|
1215
|
+
targetSessionId: recipient.info.id,
|
|
1216
|
+
messageId: frame.message.id,
|
|
1217
|
+
});
|
|
1218
|
+
}
|
|
1219
|
+
}
|
|
1220
|
+
const delivered = await this.state.outbox.markDelivered(inflight.claim);
|
|
1221
|
+
if (delivered.status !== "delivered") return;
|
|
1222
|
+
if (frame.message.replyTo !== undefined) {
|
|
1223
|
+
await this.state.pendingAsks.clearReply({
|
|
1224
|
+
replySenderSessionId: frame.from.id,
|
|
1225
|
+
replyTargetSessionId: recipient.info.id,
|
|
1226
|
+
replyTo: frame.message.replyTo,
|
|
1227
|
+
});
|
|
1228
|
+
}
|
|
1229
|
+
if (frame.channel === "supervisor") {
|
|
1230
|
+
this.host.supervisorCache.record(frame.message.id, frame.from.id, recipient.info.id);
|
|
1231
|
+
}
|
|
1232
|
+
await this.wakeOrigin(inflight.row);
|
|
1233
|
+
})().catch((error: Error) => {
|
|
1234
|
+
this.report(error);
|
|
1235
|
+
// Not settled: hand the row back so it is delivered again (the recipient deduplicates) and settled then.
|
|
1236
|
+
void this.state.outbox
|
|
1237
|
+
.releaseClaim(inflight.claim)
|
|
1238
|
+
.then(() => this.scanIdentity(inflight.identity))
|
|
1239
|
+
.catch((released: Error) => this.report(released));
|
|
1240
|
+
});
|
|
1241
|
+
this.settling.add(settling);
|
|
1242
|
+
void settling.finally(() => this.settling.delete(settling));
|
|
1243
|
+
}
|
|
1244
|
+
|
|
1245
|
+
private async wakeOrigin(row: OutboxRow): Promise<void> {
|
|
1246
|
+
if (row.originBrokerId === this.brokerId) void this.checkOutcomes();
|
|
1247
|
+
else await this.bus.publish({ brokerId: row.originBrokerId, reason: "outcome", version: row.sequence });
|
|
1248
|
+
}
|
|
1249
|
+
|
|
1250
|
+
// ---------------------------------------------------------------------------------------------------------
|
|
1251
|
+
// Outcomes for the senders this broker serves
|
|
1252
|
+
|
|
1253
|
+
private async checkOutcomes(): Promise<void> {
|
|
1254
|
+
if (this.checkingOutcomes !== undefined) {
|
|
1255
|
+
this.checkingOutcomes.again = true;
|
|
1256
|
+
return;
|
|
1257
|
+
}
|
|
1258
|
+
const state = { again: false };
|
|
1259
|
+
this.checkingOutcomes = state;
|
|
1260
|
+
try {
|
|
1261
|
+
do {
|
|
1262
|
+
state.again = false;
|
|
1263
|
+
for (const waiter of [...this.waiters.values()]) {
|
|
1264
|
+
const read = await this.state.outbox.read(waiter.messageId);
|
|
1265
|
+
if (this.waiters.get(waiter.messageId) !== waiter || read.status !== "found") continue;
|
|
1266
|
+
const { row } = read;
|
|
1267
|
+
if (row.state === "delivered") await this.finishDelivered(waiter);
|
|
1268
|
+
else if (row.state === "failed") {
|
|
1269
|
+
this.dropWaiter(waiter);
|
|
1270
|
+
this.host.write(waiter.socket, {
|
|
1271
|
+
type: "delivery_failed",
|
|
1272
|
+
messageId: waiter.messageId,
|
|
1273
|
+
attemptId: waiter.attemptId,
|
|
1274
|
+
reason: row.failureReason ?? "Session not found",
|
|
1275
|
+
});
|
|
1276
|
+
}
|
|
1277
|
+
}
|
|
1278
|
+
} while (state.again && !this.closed);
|
|
1279
|
+
} catch (error) {
|
|
1280
|
+
this.report(error);
|
|
1281
|
+
} finally {
|
|
1282
|
+
this.checkingOutcomes = undefined;
|
|
1283
|
+
}
|
|
1284
|
+
}
|
|
1285
|
+
|
|
1286
|
+
private async finishDelivered(waiter: Waiter): Promise<void> {
|
|
1287
|
+
this.dropWaiter(waiter);
|
|
1288
|
+
await this.state.sentMessages.accept({ messageId: waiter.messageId, digest: waiter.digest });
|
|
1289
|
+
if (waiter.expectsReply && waiter.resolveReplyTarget && waiter.attemptId !== undefined) {
|
|
1290
|
+
// #3777: a recipient that reconnected before confirming answers under its new session id.
|
|
1291
|
+
const current = await this.state.leases.read({ kind: "presence", targetIdentity: waiter.targetIdentity });
|
|
1292
|
+
if (
|
|
1293
|
+
current.status === "found" &&
|
|
1294
|
+
current.lease.subject.kind === "presence" &&
|
|
1295
|
+
current.lease.subject.session.id !== waiter.targetSessionId
|
|
1296
|
+
) {
|
|
1297
|
+
await this.writeQuestionTarget(waiter.socket, waiter.messageId, waiter.attemptId, {
|
|
1298
|
+
sessionId: current.lease.subject.session.id,
|
|
1299
|
+
});
|
|
1300
|
+
}
|
|
1301
|
+
}
|
|
1302
|
+
this.host.write(waiter.socket, { type: "delivered", messageId: waiter.messageId, attemptId: waiter.attemptId });
|
|
1303
|
+
}
|
|
1304
|
+
|
|
1305
|
+
private dropWaiter(waiter: Waiter): void {
|
|
1306
|
+
if (this.waiters.get(waiter.messageId) === waiter) this.waiters.delete(waiter.messageId);
|
|
1307
|
+
if (waiter.timer !== undefined) clearTimeout(waiter.timer);
|
|
1308
|
+
}
|
|
1309
|
+
|
|
1310
|
+
// ---------------------------------------------------------------------------------------------------------
|
|
1311
|
+
// Hints, polls and renewals
|
|
1312
|
+
|
|
1313
|
+
private onWakeUp(event: WakeUpEvent): void {
|
|
1314
|
+
if (this.closed) return;
|
|
1315
|
+
if (event.kind === "resync") {
|
|
1316
|
+
this.scanAll();
|
|
1317
|
+
void this.checkOutcomes();
|
|
1318
|
+
void this.guard(this.checkHolds());
|
|
1319
|
+
return;
|
|
1320
|
+
}
|
|
1321
|
+
if (event.hint.reason === "outbox") this.scanAll();
|
|
1322
|
+
else {
|
|
1323
|
+
void this.checkOutcomes();
|
|
1324
|
+
void this.guard(this.checkHolds());
|
|
1325
|
+
}
|
|
1326
|
+
}
|
|
1327
|
+
|
|
1328
|
+
private async renew(): Promise<void> {
|
|
1329
|
+
if (this.membership !== undefined) {
|
|
1330
|
+
const renewed = await this.state.leases.renew(this.membership);
|
|
1331
|
+
if (renewed.status !== "renewed") this.membership = undefined;
|
|
1332
|
+
}
|
|
1333
|
+
if (this.membership === undefined) {
|
|
1334
|
+
const acquired = await this.state.leases.acquire(
|
|
1335
|
+
{ kind: "broker", brokerId: this.brokerId, nodeId: this.nodeId, startedAt: Date.now() },
|
|
1336
|
+
this.brokerId,
|
|
1337
|
+
BROKER_LEASE_STALE_AFTER_MS,
|
|
1338
|
+
);
|
|
1339
|
+
if (acquired.status === "acquired") this.membership = acquired.lease;
|
|
1340
|
+
}
|
|
1341
|
+
for (const [session, entry] of this.presence) {
|
|
1342
|
+
if (entry.lease !== undefined) {
|
|
1343
|
+
const renewed = await this.state.leases.renew(entry.lease, this.presenceSubject(session));
|
|
1344
|
+
if (renewed.status !== "renewed") entry.lease = undefined;
|
|
1345
|
+
else if (entry.knownAt === undefined || Date.now() - entry.knownAt >= KNOWN_SESSION_REFRESH_INTERVAL_MS) {
|
|
1346
|
+
// A live session stays known: retention counts from its last renewal, not from its registration.
|
|
1347
|
+
this.serial(deliveryTargetIdentity(session), () => this.recordKnown(session, entry));
|
|
1348
|
+
}
|
|
1349
|
+
} else {
|
|
1350
|
+
this.serial(deliveryTargetIdentity(session), async () => {
|
|
1351
|
+
if (this.presence.get(session) !== entry) return;
|
|
1352
|
+
await this.acquirePresence(session, entry);
|
|
1353
|
+
if (entry.lease !== undefined) this.scanIdentity(deliveryTargetIdentity(session));
|
|
1354
|
+
});
|
|
1355
|
+
}
|
|
1356
|
+
}
|
|
1357
|
+
for (const inflight of [...this.inflight.values()]) {
|
|
1358
|
+
const renewed = await this.state.outbox.renewClaim(inflight.claim);
|
|
1359
|
+
if (renewed.status !== "renewed" && this.inflight.get(inflight.row.messageId) === inflight) {
|
|
1360
|
+
this.inflight.delete(inflight.row.messageId);
|
|
1361
|
+
}
|
|
1362
|
+
}
|
|
1363
|
+
for (const [key, mirror] of this.mirrored) {
|
|
1364
|
+
const renewed = await this.state.reservations.renew(mirror.lease);
|
|
1365
|
+
if (renewed.status !== "renewed" && this.mirrored.get(key) === mirror) this.mirrored.delete(key);
|
|
1366
|
+
}
|
|
1367
|
+
for (const identity of [...this.heldIdentities]) this.scanIdentity(identity);
|
|
1368
|
+
}
|
|
1369
|
+
|
|
1370
|
+
private async backupPoll(): Promise<void> {
|
|
1371
|
+
this.scanAll();
|
|
1372
|
+
await this.checkOutcomes();
|
|
1373
|
+
await this.checkHolds();
|
|
1374
|
+
await this.state.outbox.prune(DELIVERED_MESSAGE_TTL_MS);
|
|
1375
|
+
await this.state.sentMessages.prune(DELIVERED_MESSAGE_TTL_MS);
|
|
1376
|
+
await this.state.reservations.prune(DELIVERED_MESSAGE_TTL_MS);
|
|
1377
|
+
await this.state.knownSessions.prune(KNOWN_SESSION_RETENTION_MS);
|
|
1378
|
+
}
|
|
1379
|
+
|
|
1380
|
+
private async outcomePoll(): Promise<void> {
|
|
1381
|
+
if ([...this.waiters.values()].some((waiter) => !waiter.queued)) await this.checkOutcomes();
|
|
1382
|
+
if (this.holds.size > 0) await this.checkHolds();
|
|
1383
|
+
}
|
|
1384
|
+
|
|
1385
|
+
private async recordKnown(session: BrokerConnectedSession, entry: PresenceEntry): Promise<void> {
|
|
1386
|
+
await this.state.knownSessions.record(this.knownOf(session));
|
|
1387
|
+
entry.knownAt = Date.now();
|
|
1388
|
+
}
|
|
1389
|
+
|
|
1390
|
+
private async acquirePresence(session: BrokerConnectedSession, entry: PresenceEntry): Promise<void> {
|
|
1391
|
+
const acquired = await this.state.leases.acquire(
|
|
1392
|
+
this.presenceSubject(session),
|
|
1393
|
+
this.brokerId,
|
|
1394
|
+
BROKER_LEASE_STALE_AFTER_MS,
|
|
1395
|
+
);
|
|
1396
|
+
if (acquired.status === "acquired") entry.lease = acquired.lease;
|
|
1397
|
+
}
|
|
1398
|
+
|
|
1399
|
+
private presenceSubject(session: BrokerConnectedSession): RecipientPresenceSubject {
|
|
1400
|
+
return { kind: "presence", targetIdentity: deliveryTargetIdentity(session), session: session.info };
|
|
1401
|
+
}
|
|
1402
|
+
|
|
1403
|
+
private knownOf(session: BrokerConnectedSession): KnownSession {
|
|
1404
|
+
return {
|
|
1405
|
+
targetIdentity: deliveryTargetIdentity(session),
|
|
1406
|
+
intercomSessionId: session.info.id,
|
|
1407
|
+
...(session.info.name === undefined ? {} : { name: session.info.name }),
|
|
1408
|
+
...(session.registrationReturnAddress === undefined ? {} : { atomicSessionId: session.registrationReturnAddress }),
|
|
1409
|
+
group: normalizeGroup(session.registrationGroup),
|
|
1410
|
+
groups: [...sessionGroups(session.info)],
|
|
1411
|
+
...(session.info.recipientPurpose === undefined ? {} : { recipientPurpose: session.info.recipientPurpose }),
|
|
1412
|
+
...(session.info.replyCapability === undefined ? {} : { replyCapability: session.info.replyCapability }),
|
|
1413
|
+
};
|
|
1414
|
+
}
|
|
1415
|
+
|
|
1416
|
+
private fail(request: RemoteSendRequest, reason: string, reasonCode?: "session_not_found"): void {
|
|
1417
|
+
this.host.write(request.socket, {
|
|
1418
|
+
type: "delivery_failed",
|
|
1419
|
+
messageId: request.message.id,
|
|
1420
|
+
attemptId: request.attemptId,
|
|
1421
|
+
reason,
|
|
1422
|
+
...(reasonCode === undefined ? {} : { reasonCode }),
|
|
1423
|
+
});
|
|
1424
|
+
}
|
|
1425
|
+
|
|
1426
|
+
private failConflict(request: RemoteSendRequest): void {
|
|
1427
|
+
this.host.write(request.socket, {
|
|
1428
|
+
type: "delivery_failed",
|
|
1429
|
+
messageId: request.message.id,
|
|
1430
|
+
attemptId: request.attemptId,
|
|
1431
|
+
reason: `Intercom message ID '${request.message.id}' was already delivered with a different target or payload`,
|
|
1432
|
+
reasonCode: "message_id_conflict",
|
|
1433
|
+
});
|
|
1434
|
+
}
|
|
1435
|
+
|
|
1436
|
+
/** Run `operation` after every earlier operation for the same key, so a leave never overtakes its register. */
|
|
1437
|
+
private serial(key: string, operation: () => Promise<void>): void {
|
|
1438
|
+
const previous = this.serials.get(key) ?? Promise.resolve();
|
|
1439
|
+
const next = previous.then(operation).catch((error: Error) => this.report(error));
|
|
1440
|
+
this.serials.set(key, next);
|
|
1441
|
+
void next.finally(() => {
|
|
1442
|
+
if (this.serials.get(key) === next) this.serials.delete(key);
|
|
1443
|
+
});
|
|
1444
|
+
}
|
|
1445
|
+
|
|
1446
|
+
private every(intervalMs: number, tick: () => Promise<void>): void {
|
|
1447
|
+
let running = false;
|
|
1448
|
+
const timer = setInterval(() => {
|
|
1449
|
+
if (running || this.closed) return;
|
|
1450
|
+
running = true;
|
|
1451
|
+
void tick()
|
|
1452
|
+
.catch((error: Error) => this.report(error))
|
|
1453
|
+
.finally(() => {
|
|
1454
|
+
running = false;
|
|
1455
|
+
});
|
|
1456
|
+
}, intervalMs);
|
|
1457
|
+
timer.unref?.();
|
|
1458
|
+
this.timers.push(timer);
|
|
1459
|
+
}
|
|
1460
|
+
|
|
1461
|
+
private async guard(operation: Promise<void>): Promise<void> {
|
|
1462
|
+
try {
|
|
1463
|
+
await operation;
|
|
1464
|
+
} catch (error) {
|
|
1465
|
+
this.report(error);
|
|
1466
|
+
}
|
|
1467
|
+
}
|
|
1468
|
+
|
|
1469
|
+
private report(error: unknown): void {
|
|
1470
|
+
const text = error instanceof Error ? error.message : String(error);
|
|
1471
|
+
if (text === this.lastError) return;
|
|
1472
|
+
this.lastError = text;
|
|
1473
|
+
console.error(`Intercom multi-node routing: ${text}`);
|
|
1474
|
+
}
|
|
1475
|
+
|
|
1476
|
+
/** Give this broker's presence and membership leases up, then close the bus and the store. */
|
|
1477
|
+
async close(): Promise<void> {
|
|
1478
|
+
if (this.closed) return;
|
|
1479
|
+
this.closed = true;
|
|
1480
|
+
for (const timer of this.timers) clearInterval(timer);
|
|
1481
|
+
for (const waiter of [...this.waiters.values()]) this.dropWaiter(waiter);
|
|
1482
|
+
for (const hold of this.holds.values()) clearTimeout(hold.timer);
|
|
1483
|
+
this.holds.clear();
|
|
1484
|
+
await Promise.all([...this.serials.values()]);
|
|
1485
|
+
try {
|
|
1486
|
+
for (const inflight of this.inflight.values()) await this.state.outbox.releaseClaim(inflight.claim);
|
|
1487
|
+
for (const entry of this.presence.values()) {
|
|
1488
|
+
if (entry.lease !== undefined) await this.state.leases.release(entry.lease);
|
|
1489
|
+
}
|
|
1490
|
+
if (this.membership !== undefined) await this.state.leases.release(this.membership);
|
|
1491
|
+
} catch (error) {
|
|
1492
|
+
this.report(error);
|
|
1493
|
+
}
|
|
1494
|
+
this.inflight.clear();
|
|
1495
|
+
this.presence.clear();
|
|
1496
|
+
await this.subscription?.close();
|
|
1497
|
+
await this.bus.close();
|
|
1498
|
+
await this.state.close();
|
|
1499
|
+
}
|
|
1500
|
+
}
|