@cotal-ai/core 0.167.0 → 0.169.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-file.d.ts.map +1 -1
- package/dist/agent-file.js +12 -9
- package/dist/agent-file.js.map +1 -1
- package/dist/connector-setup.d.ts +2 -0
- package/dist/connector-setup.d.ts.map +1 -1
- package/dist/connector.d.ts +8 -11
- package/dist/connector.d.ts.map +1 -1
- package/dist/control-exchange.d.ts +114 -0
- package/dist/control-exchange.d.ts.map +1 -0
- package/dist/control-exchange.js +416 -0
- package/dist/control-exchange.js.map +1 -0
- package/dist/endpoint-call.d.ts +6 -3
- package/dist/endpoint-call.d.ts.map +1 -1
- package/dist/endpoint-call.js +30 -14
- package/dist/endpoint-call.js.map +1 -1
- package/dist/endpoint-delivery-plane.d.ts +337 -0
- package/dist/endpoint-delivery-plane.d.ts.map +1 -0
- package/dist/endpoint-delivery-plane.js +1291 -0
- package/dist/endpoint-delivery-plane.js.map +1 -0
- package/dist/endpoint-envelope.d.ts +2 -1
- package/dist/endpoint-envelope.d.ts.map +1 -1
- package/dist/endpoint-envelope.js +9 -3
- package/dist/endpoint-envelope.js.map +1 -1
- package/dist/endpoint-history.d.ts +370 -0
- package/dist/endpoint-history.d.ts.map +1 -0
- package/dist/endpoint-history.js +969 -0
- package/dist/endpoint-history.js.map +1 -0
- package/dist/endpoint-inbound.d.ts +10 -0
- package/dist/endpoint-inbound.d.ts.map +1 -0
- package/dist/endpoint-inbound.js +28 -0
- package/dist/endpoint-inbound.js.map +1 -0
- package/dist/endpoint-provenance.d.ts +11 -0
- package/dist/endpoint-provenance.d.ts.map +1 -0
- package/dist/endpoint-provenance.js +20 -0
- package/dist/endpoint-provenance.js.map +1 -0
- package/dist/endpoint-serve.d.ts +11 -1
- package/dist/endpoint-serve.d.ts.map +1 -1
- package/dist/endpoint-serve.js +43 -14
- package/dist/endpoint-serve.js.map +1 -1
- package/dist/endpoint-subjects.d.ts +3 -0
- package/dist/endpoint-subjects.d.ts.map +1 -1
- package/dist/endpoint-subjects.js +3 -0
- package/dist/endpoint-subjects.js.map +1 -1
- package/dist/endpoint.d.ts +80 -252
- package/dist/endpoint.d.ts.map +1 -1
- package/dist/endpoint.js +248 -2027
- package/dist/endpoint.js.map +1 -1
- package/dist/index.d.ts +2 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -1
- package/dist/index.js.map +1 -1
- package/dist/launch-material.d.ts +9 -3
- package/dist/launch-material.d.ts.map +1 -1
- package/dist/launch-material.js +22 -2
- package/dist/launch-material.js.map +1 -1
- package/dist/provision-jwt.d.ts +518 -0
- package/dist/provision-jwt.d.ts.map +1 -0
- package/dist/provision-jwt.js +641 -0
- package/dist/provision-jwt.js.map +1 -0
- package/dist/provision-permissions.d.ts +72 -0
- package/dist/provision-permissions.d.ts.map +1 -0
- package/dist/provision-permissions.js +2015 -0
- package/dist/provision-permissions.js.map +1 -0
- package/dist/provision-resources.d.ts +66 -0
- package/dist/provision-resources.d.ts.map +1 -0
- package/dist/provision-resources.js +61 -0
- package/dist/provision-resources.js.map +1 -0
- package/dist/provision-server-config.d.ts +115 -0
- package/dist/provision-server-config.d.ts.map +1 -0
- package/dist/provision-server-config.js +169 -0
- package/dist/provision-server-config.js.map +1 -0
- package/dist/provision.d.ts +19 -707
- package/dist/provision.d.ts.map +1 -1
- package/dist/provision.js +4 -2860
- package/dist/provision.js.map +1 -1
- package/dist/remote-manager-authority.d.ts +1 -1
- package/dist/run-journal.d.ts +2 -0
- package/dist/run-journal.d.ts.map +1 -1
- package/dist/run-journal.js +1 -1
- package/dist/run-journal.js.map +1 -1
- package/package.json +5 -1
|
@@ -0,0 +1,1291 @@
|
|
|
1
|
+
// The plane-3 delivery region of CotalEndpoint, as an internal component. Moved verbatim from
|
|
2
|
+
// endpoint.ts; the endpoint keeps the public methods and forwards to it. Not re-exported by the index.
|
|
3
|
+
import { aclRetired, readAcl } from "./acls.js";
|
|
4
|
+
import { ANYCAST_CANCEL_OP, ANYCAST_STATUS_OP, ANYCAST_TAKES_OP } from "./anycast-requests.js";
|
|
5
|
+
import { ARTIFACT_PART_KIND, isArtifactPart } from "./artifact.js";
|
|
6
|
+
import { DELIVERY_LIMIT_HEADER, HISTORY_NO_COUNT } from "./backup-config.js";
|
|
7
|
+
import { isRestoredConsumer } from "./backup.js";
|
|
8
|
+
import { ackWaitOf, authenticatedChannelMessage, durableDelivery, isRecord } from "./endpoint-inbound.js";
|
|
9
|
+
import { deliveryLimitOf, fitHistoryPage, isFromSubjectSender } from "./endpoint.js";
|
|
10
|
+
import { EVICT_PRINCIPALS_MAX } from "./evict.js";
|
|
11
|
+
import { issuedPermitsSubject } from "./issued-authority.js";
|
|
12
|
+
import { isConsumerAlreadyExists, isJetStreamMissing } from "./jetstream-missing.js";
|
|
13
|
+
import { deleteMember, durableEligible, listLifecycleMemberChannels, listMembers, readMember } from "./members.js";
|
|
14
|
+
import { parseSecretStoreIdentity } from "./secret-store.js";
|
|
15
|
+
import { fanoutDurableConfig, inboxReaderConfig } from "./streams.js";
|
|
16
|
+
import { CONTROL_DELIVERY, CONTROL_DELIVERY_ADMIN, FANOUT_DURABLE, INBOX_READER_DURABLE, assertLifecycleToken, assertValidChannel, channelInAllow, chatStream, dlvDurable, dlvStream, dlvSubject, familiesCovering, inboxStream, isChannelFamily, isConcreteChannel, membershipInAllow, parseDinboxPrincipal, parsePrincipalKey, parseSubject } from "./subjects.js";
|
|
17
|
+
import { JetStreamApiCodes } from "@nats-io/jetstream";
|
|
18
|
+
import { headers } from "@nats-io/transport-node";
|
|
19
|
+
const PLANE3_FRAME_HEADER = "Cotal-Delivery-Frame";
|
|
20
|
+
/** The `ctl.delivery` ops a sender calls about one of its own anycast requests (SPEC §8.1). */
|
|
21
|
+
const ANYCAST_CONTROL_OPS = new Set([ANYCAST_STATUS_OP, ANYCAST_CANCEL_OP, ANYCAST_TAKES_OP]);
|
|
22
|
+
/** How long the trusted writer reuses its read of the DLV count before reading it again. */
|
|
23
|
+
const DELIVERY_LIMIT_READ_MS = 5_000;
|
|
24
|
+
/** Plane-3 trusted-reader redelivery ceiling: a dinbox entry that keeps failing re-auth-defer
|
|
25
|
+
* (unknown owner) or DELIVER transfer is `term()`d + surfaced after this many redeliveries, so one
|
|
26
|
+
* stuck/poison entry can't head-of-line the single shared reader forever. */
|
|
27
|
+
const READER_MAX_REDELIVERIES = 10;
|
|
28
|
+
/** Page size for the mediated history read when the caller names none — matches `channelHistory`'s
|
|
29
|
+
* own default, since the mediated read exists to be a drop-in for it. */
|
|
30
|
+
const READ_HISTORY_DEFAULT_LIMIT = 100;
|
|
31
|
+
/** Hard server-side ceiling on one mediated page. The caller PROPOSES a limit and the mediator
|
|
32
|
+
* decides: the reader is pooled and privileged, so an unbounded caller-chosen limit would let one
|
|
33
|
+
* request pull a channel's whole retained set through it. Clamped, not refused — a UI asking for
|
|
34
|
+
* more than a page should get a page, and `complete: false` already tells it more remains. */
|
|
35
|
+
const READ_HISTORY_MAX_LIMIT = 200;
|
|
36
|
+
function isPlane3DeliveryFrame(value) {
|
|
37
|
+
if (!isRecord(value))
|
|
38
|
+
return false;
|
|
39
|
+
const keys = Object.keys(value);
|
|
40
|
+
if (keys.length !== 3 || !keys.includes("version") || !keys.includes("channel") || !keys.includes("msg"))
|
|
41
|
+
return false;
|
|
42
|
+
if (value.version !== 1 || typeof value.channel !== "string" || !isConcreteChannel(value.channel))
|
|
43
|
+
return false;
|
|
44
|
+
return isCotalMessage(value.msg) && value.msg.channel === value.channel;
|
|
45
|
+
}
|
|
46
|
+
function isCotalMessage(value) {
|
|
47
|
+
if (!isRecord(value))
|
|
48
|
+
return false;
|
|
49
|
+
const routes = ["channel", "to", "toService"].filter((key) => key in value);
|
|
50
|
+
if (routes.length !== 1 || typeof value[routes[0]] !== "string")
|
|
51
|
+
return false;
|
|
52
|
+
if (typeof value.id !== "string" || typeof value.ts !== "number" || !Number.isFinite(value.ts) ||
|
|
53
|
+
typeof value.space !== "string" || !isEndpointRef(value.from) || !Array.isArray(value.parts) ||
|
|
54
|
+
!value.parts.every(isMessagePart))
|
|
55
|
+
return false;
|
|
56
|
+
if (value.mentions !== undefined && (!Array.isArray(value.mentions) || !value.mentions.every((name) => typeof name === "string")))
|
|
57
|
+
return false;
|
|
58
|
+
if (value.replyTo !== undefined && typeof value.replyTo !== "string")
|
|
59
|
+
return false;
|
|
60
|
+
if (value.contextId !== undefined && typeof value.contextId !== "string")
|
|
61
|
+
return false;
|
|
62
|
+
return true;
|
|
63
|
+
}
|
|
64
|
+
function isEndpointRef(value) {
|
|
65
|
+
return isRecord(value) && typeof value.id === "string" && typeof value.name === "string" &&
|
|
66
|
+
(value.role === undefined || typeof value.role === "string");
|
|
67
|
+
}
|
|
68
|
+
/** A conformant message part: the three CORE kinds, or a reverse-DNS extension kind (SPEC §5).
|
|
69
|
+
*
|
|
70
|
+
* Reached from {@link isCotalMessage}, which gates the Plane-3 delivery frame — so a core kind
|
|
71
|
+
* missing an arm here is not a schema nicety: the durable backstop DROPS every message carrying
|
|
72
|
+
* it, silently, and the drop surfaces nowhere near the feature that added the part. `artifact`
|
|
73
|
+
* was exactly that case before it was added (a bare kind has no dot, so it fell through to the
|
|
74
|
+
* extension regex and failed). */
|
|
75
|
+
function isMessagePart(value) {
|
|
76
|
+
if (!isRecord(value) || typeof value.kind !== "string")
|
|
77
|
+
return false;
|
|
78
|
+
if (value.kind === "text")
|
|
79
|
+
return typeof value.text === "string";
|
|
80
|
+
if (value.kind === "data")
|
|
81
|
+
return Object.prototype.hasOwnProperty.call(value, "data");
|
|
82
|
+
if (value.kind === ARTIFACT_PART_KIND)
|
|
83
|
+
return isArtifactPart(value);
|
|
84
|
+
return /^[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)+$/.test(value.kind);
|
|
85
|
+
}
|
|
86
|
+
/** The plane-3 delivery daemon's side of one endpoint: the fan-out writer, the trusted reader, the
|
|
87
|
+
* `ctl.delivery` and `ctl.delivery-admin` responders and the quiesce fence around them, plus the agent's
|
|
88
|
+
* own DELIVER durable pump. Internal: `CotalEndpoint` keeps the public methods and forwards to it. */
|
|
89
|
+
export class EndpointDeliveryPlane {
|
|
90
|
+
deps;
|
|
91
|
+
/** The live `ctl.delivery` serve subscription (delivery daemon) — re-created on every (re)connect by
|
|
92
|
+
* {@link armDeliveryControl}; tracked so the stale one is dropped on reconnect. */
|
|
93
|
+
deliveryServeSub;
|
|
94
|
+
deliveryAdminServeSub;
|
|
95
|
+
/** The lifecycles whose membership inventory this daemon has been asked for, as principal to uids
|
|
96
|
+
* (#4965). A durable join or leave bound to one of them is refused, so no row of a retiring lifecycle
|
|
97
|
+
* is written after its inventory read starts. An entry goes when a later inventory for the same
|
|
98
|
+
* principal finds its ACL row retired; an alias that never retires again keeps its one entry until
|
|
99
|
+
* the daemon restarts. This is the second layer of the fence: it covers a manager that does not
|
|
100
|
+
* narrow the ACL row before the inventory, on this daemon process only. The first is the ACL row
|
|
101
|
+
* each join re-reads around its write (`durableJoinFor`'s `aclFence`), which holds across daemons. */
|
|
102
|
+
retiringLifecycles = new Map();
|
|
103
|
+
/** The durable joins and leaves running now, by `<principal>/<lifecycleUid>`: the writes an
|
|
104
|
+
* inventory of that lifecycle waits for before it reads. */
|
|
105
|
+
membershipWrites = new Map();
|
|
106
|
+
/** True once {@link quiescePlane3} has stopped serving this shard pending an ownership answer.
|
|
107
|
+
* Guards {@link armPlane3} so a RECONNECT cannot silently resume serving mid-question. */
|
|
108
|
+
plane3Quiesced = false;
|
|
109
|
+
/** When set, this endpoint hosts the Plane-3 fan-out writer + trusted reader (the server-side delivery
|
|
110
|
+
* daemon). `aclFor` maps an owner id to its current read ACL (`allowSubscribe`) for the reader's
|
|
111
|
+
* re-authorization — read FRESH per entry from the durable ACL registry KV, hence async. */
|
|
112
|
+
plane3;
|
|
113
|
+
deliveryLimitRead;
|
|
114
|
+
aclForAlias;
|
|
115
|
+
channelHistory;
|
|
116
|
+
deliveryClassFresh;
|
|
117
|
+
durableJoinFor;
|
|
118
|
+
durableLeaveFor;
|
|
119
|
+
ownerMemberships;
|
|
120
|
+
ownsDeliveryLease;
|
|
121
|
+
readDeliveryLease;
|
|
122
|
+
readDeliveryLeaseEntry;
|
|
123
|
+
reloadCreds;
|
|
124
|
+
serveControl;
|
|
125
|
+
aclRegistry;
|
|
126
|
+
membersRegistry;
|
|
127
|
+
armAnycastRequests;
|
|
128
|
+
deliveryAnycast;
|
|
129
|
+
dropAnycastRequests;
|
|
130
|
+
manager;
|
|
131
|
+
notLiveMsg;
|
|
132
|
+
plane3MsgId;
|
|
133
|
+
publishDinbox;
|
|
134
|
+
resolveOwnerByName;
|
|
135
|
+
scheduleResidentSwap;
|
|
136
|
+
sendMentionNotice;
|
|
137
|
+
emit;
|
|
138
|
+
constructor(deps) {
|
|
139
|
+
this.deps = deps;
|
|
140
|
+
this.aclForAlias = deps.aclForAlias;
|
|
141
|
+
this.channelHistory = deps.channelHistory;
|
|
142
|
+
this.deliveryClassFresh = deps.deliveryClassFresh;
|
|
143
|
+
this.durableJoinFor = deps.durableJoinFor;
|
|
144
|
+
this.durableLeaveFor = deps.durableLeaveFor;
|
|
145
|
+
this.ownerMemberships = deps.ownerMemberships;
|
|
146
|
+
this.ownsDeliveryLease = deps.ownsDeliveryLease;
|
|
147
|
+
this.readDeliveryLease = deps.readDeliveryLease;
|
|
148
|
+
this.readDeliveryLeaseEntry = deps.readDeliveryLeaseEntry;
|
|
149
|
+
this.reloadCreds = deps.reloadCreds;
|
|
150
|
+
this.serveControl = deps.serveControl;
|
|
151
|
+
this.aclRegistry = deps.aclRegistry;
|
|
152
|
+
this.membersRegistry = deps.membersRegistry;
|
|
153
|
+
this.armAnycastRequests = deps.armAnycastRequests;
|
|
154
|
+
this.deliveryAnycast = deps.deliveryAnycast;
|
|
155
|
+
this.dropAnycastRequests = deps.dropAnycastRequests;
|
|
156
|
+
this.manager = deps.manager;
|
|
157
|
+
this.notLiveMsg = deps.notLiveMsg;
|
|
158
|
+
this.plane3MsgId = deps.plane3MsgId;
|
|
159
|
+
this.publishDinbox = deps.publishDinbox;
|
|
160
|
+
this.resolveOwnerByName = deps.resolveOwnerByName;
|
|
161
|
+
this.scheduleResidentSwap = deps.scheduleResidentSwap;
|
|
162
|
+
this.sendMentionNotice = deps.sendMentionNotice;
|
|
163
|
+
this.emit = deps.emit;
|
|
164
|
+
}
|
|
165
|
+
get ackWaitMs() { return this.deps.ackWaitMs(); }
|
|
166
|
+
get actor() { return this.deps.actor(); }
|
|
167
|
+
get owner() { return this.deps.owner(); }
|
|
168
|
+
get allowSubscribe() { return this.deps.allowSubscribe(); }
|
|
169
|
+
get anycastUnavailable() { return this.deps.anycastUnavailable(); }
|
|
170
|
+
get card() { return this.deps.card(); }
|
|
171
|
+
get js() { return this.deps.js(); }
|
|
172
|
+
get jsm() { return this.deps.jsm(); }
|
|
173
|
+
get nc() { return this.deps.nc(); }
|
|
174
|
+
get ownLifecycleUid() { return this.deps.ownLifecycleUid(); }
|
|
175
|
+
get residentSwap() { return this.deps.residentSwap(); }
|
|
176
|
+
get space() { return this.deps.space(); }
|
|
177
|
+
get stopped() { return this.deps.stopped(); }
|
|
178
|
+
get streamMsgs() { return this.deps.streamMsgs(); }
|
|
179
|
+
get subs() { return this.deps.subs(); }
|
|
180
|
+
get userMode() { return this.deps.userMode(); }
|
|
181
|
+
get watchDeletePermissions() { return this.deps.watchDeletePermissions(); }
|
|
182
|
+
get maxPayload() { return this.deps.maxPayload(); }
|
|
183
|
+
/** Start the Plane-3 fan-out writer + trusted reader on THIS (privileged, server-side delivery-daemon)
|
|
184
|
+
* endpoint, AND serve the `ctl.delivery` control service (runtime durable join/leave/list). `aclFor`
|
|
185
|
+
* maps an owner id to its current read ACL for the reader's re-authorization — read FRESH per entry
|
|
186
|
+
* from the durable ACL registry (async). Call once after connect; idempotent durable creation lets it
|
|
187
|
+
* resume on a daemon restart. Both the JS loops AND the `ctl.delivery` subscription are (re)bound by
|
|
188
|
+
* {@link armPlane3} on EVERY (re)connect — a reconnect drains the old connection, so re-binding both
|
|
189
|
+
* is required, not optional (the responder would otherwise be lost on a broker blip). */
|
|
190
|
+
async startPlane3(aclFor, opts = {}) {
|
|
191
|
+
if (!this.js)
|
|
192
|
+
throw new Error(this.notLiveMsg());
|
|
193
|
+
this.plane3 = { aclFor, reloadMembershipCreds: opts.reloadMembershipCreds, evictPrincipal: opts.evictPrincipal, evictPrincipals: opts.evictPrincipals, planeConnLiveness: opts.planeConnLiveness, principalLiveness: opts.principalLiveness, reloadStoreIdentity: opts.reloadStoreIdentity, onDeliveryCredsAdopted: opts.onDeliveryCredsAdopted, startupComplete: opts.startupComplete };
|
|
194
|
+
await this.armPlane3();
|
|
195
|
+
}
|
|
196
|
+
/** Whether an ALREADY-DISPATCHED unit of Plane-3 work may still take effect.
|
|
197
|
+
*
|
|
198
|
+
* Unsubscribing stops NEW work; it cannot recall work already in flight. A handler that entered
|
|
199
|
+
* before {@link quiescePlane3} and awaited broker I/O inside it resumes AFTER the freeze, and a
|
|
200
|
+
* reviewer traced the ordering that makes that a split rather than a latency blip: the loser
|
|
201
|
+
* accepts a unit and awaits, the loser is descheduled, the successor acquires the shard and flips
|
|
202
|
+
* its lease READY, then the loser resumes and answers or acks for a shard it no longer holds.
|
|
203
|
+
* Subscription counts and parked-pull readings cannot see it, because the effect is the reply and
|
|
204
|
+
* the ack rather than the binding.
|
|
205
|
+
*
|
|
206
|
+
* So each work path re-asks HERE, at the point of effect, after its awaits and before it acts.
|
|
207
|
+
* The lease keeps one ROW, not one SERVER; this is the half that keeps one server. An unacked
|
|
208
|
+
* message is not lost by refusing: a stopped consumer redelivers it to whoever holds the shard
|
|
209
|
+
* next, which is the same property quiescing already relies on. */
|
|
210
|
+
plane3MayAct() {
|
|
211
|
+
return !this.plane3Quiesced;
|
|
212
|
+
}
|
|
213
|
+
/** Serve one runtime durable-membership control request (the server-side delivery daemon). The caller
|
|
214
|
+
* id is the authenticated subject sender ({@link serveControl} fail-closes on a mismatch). Validation
|
|
215
|
+
* is against the durable ACL registry — the SAME KV the reader re-auths against (single source of
|
|
216
|
+
* truth, no in-memory ledger to drift). */
|
|
217
|
+
async handleDeliveryControl(req) {
|
|
218
|
+
// FENCE: entered before a quiesce, resuming after it. Answering now would put a second server on
|
|
219
|
+
// this shard's control rail while the winner is already READY.
|
|
220
|
+
if (!this.plane3MayAct())
|
|
221
|
+
return { ok: false, error: "delivery: this daemon is not serving this shard (it is re-checking ownership); retry" };
|
|
222
|
+
const caller = req.from.id;
|
|
223
|
+
const args = req.args ?? {};
|
|
224
|
+
if (req.op === "durableJoin")
|
|
225
|
+
return this.deliveryJoin(caller, args);
|
|
226
|
+
if (req.op === "durableLeave")
|
|
227
|
+
return this.deliveryLeave(caller, args);
|
|
228
|
+
if (req.op === "readHistory")
|
|
229
|
+
return this.deliveryReadHistory(caller, args);
|
|
230
|
+
if (req.op === ANYCAST_TAKES_OP)
|
|
231
|
+
return this.anycastUnavailable ? { ok: false, error: `delivery: ${this.anycastUnavailable}` } : { ok: true, data: { takes: true } };
|
|
232
|
+
if (req.op === ANYCAST_STATUS_OP || req.op === ANYCAST_CANCEL_OP)
|
|
233
|
+
return this.deliveryAnycast(req.op, caller, args);
|
|
234
|
+
if (req.op === "listMemberships") {
|
|
235
|
+
if (typeof args.lifecycleUid !== "string")
|
|
236
|
+
return { ok: false, error: "listMemberships: the caller's lifecycleUid is required (membership rows are lifecycle-keyed, SPEC 13.1)" };
|
|
237
|
+
let uid;
|
|
238
|
+
try {
|
|
239
|
+
uid = assertLifecycleToken(args.lifecycleUid);
|
|
240
|
+
}
|
|
241
|
+
catch (e) {
|
|
242
|
+
return { ok: false, error: e.message };
|
|
243
|
+
}
|
|
244
|
+
let memberships;
|
|
245
|
+
if (args.channel !== undefined) {
|
|
246
|
+
const channel = this.checkDurableChannelArg(args, "listMemberships", { family: true });
|
|
247
|
+
if (typeof channel !== "string")
|
|
248
|
+
return channel;
|
|
249
|
+
memberships = (await listMembers(await this.membersRegistry(), { owner: caller }))
|
|
250
|
+
.filter((r) => r.channel === channel && r.leaveCursor === undefined)
|
|
251
|
+
.map((r) => ({ channel: r.channel, generation: r.generation, activated: r.activated === true, lifecycleUid: r.lifecycleUid }));
|
|
252
|
+
}
|
|
253
|
+
else {
|
|
254
|
+
memberships = await this.ownerMemberships(caller, uid);
|
|
255
|
+
}
|
|
256
|
+
// FENCE AFTER THE AWAIT, which is the one that matters. The entry check above cannot catch the
|
|
257
|
+
// case this exists for: a request admitted WHILE SERVING, then descheduled inside its broker
|
|
258
|
+
// read, resuming after a successor has taken the shard and reached READY. Re-ask at the point
|
|
259
|
+
// of effect, so the answer is never served by a daemon that has stood down.
|
|
260
|
+
if (!this.plane3MayAct())
|
|
261
|
+
return { ok: false, error: "delivery: this daemon stopped serving this shard while the request was in flight; retry" };
|
|
262
|
+
return { ok: true, data: { memberships } };
|
|
263
|
+
}
|
|
264
|
+
return { ok: false, error: `op "${req.op}" not supported on the delivery control service` };
|
|
265
|
+
}
|
|
266
|
+
/** Validate the channel ARG shape only: non-blank, valid, concrete (NO ACL check, that is op-specific).
|
|
267
|
+
* `family` also admits a channel family such as `review.>` (a membership join or leave, #5404).
|
|
268
|
+
* Returns the channel on success or a ControlReply error to short-circuit. */
|
|
269
|
+
checkDurableChannelArg(args, op, opts = {}) {
|
|
270
|
+
const channel = typeof args.channel === "string" ? args.channel.trim() : "";
|
|
271
|
+
if (!channel)
|
|
272
|
+
return { ok: false, error: `${op}: channel must be a non-blank string` };
|
|
273
|
+
try {
|
|
274
|
+
assertValidChannel(channel);
|
|
275
|
+
}
|
|
276
|
+
catch (e) {
|
|
277
|
+
return { ok: false, error: e.message };
|
|
278
|
+
}
|
|
279
|
+
if (opts.family && isChannelFamily(channel))
|
|
280
|
+
return channel;
|
|
281
|
+
if (!isConcreteChannel(channel))
|
|
282
|
+
return {
|
|
283
|
+
ok: false,
|
|
284
|
+
error: opts.family
|
|
285
|
+
? `${op}: "${channel}" stays live only: a durable membership takes a concrete channel or a family such as review.>, never the root > or a pattern with *`
|
|
286
|
+
: `${op}: "${channel}" must be a concrete channel (durable membership is per-concrete-channel, not wildcard)`,
|
|
287
|
+
};
|
|
288
|
+
return channel;
|
|
289
|
+
}
|
|
290
|
+
/** JOIN requires the channel be within the caller's CURRENT read ACL (you can't durable-subscribe a
|
|
291
|
+
* channel you may not read). A family must lie inside one ACL entry, so every channel under it is
|
|
292
|
+
* readable. */
|
|
293
|
+
async deliveryJoin(caller, args) {
|
|
294
|
+
const channel = this.checkDurableChannelArg(args, "durableJoin", { family: true });
|
|
295
|
+
if (typeof channel !== "string")
|
|
296
|
+
return channel; // a ControlReply error
|
|
297
|
+
// The caller arrives as an ALIAS (the control subject carries owner+actor, never a uid): resolve
|
|
298
|
+
// its single live lifecycle-keyed ACL row SERVER-SIDE — the trusted registry, never a caller
|
|
299
|
+
// assertion, decides which lifecycle joins. Two live rows (a reservation breach / unfinished
|
|
300
|
+
// teardown) refuse loudly rather than binding a membership to a guessed lifecycle.
|
|
301
|
+
let acl;
|
|
302
|
+
try {
|
|
303
|
+
acl = await this.aclForAlias(caller);
|
|
304
|
+
}
|
|
305
|
+
catch (e) {
|
|
306
|
+
return { ok: false, error: e.message };
|
|
307
|
+
}
|
|
308
|
+
if (acl === undefined)
|
|
309
|
+
return { ok: false, error: `durableJoin: no read ACL on record for ${caller} (not provisioned for durable delivery)` };
|
|
310
|
+
if (!membershipInAllow(acl.allowSubscribe, channel))
|
|
311
|
+
return { ok: false, error: `channel "${channel}" is not within your read ACL [${acl.allowSubscribe.join(", ")}]` };
|
|
312
|
+
const uid = acl.lifecycleUid;
|
|
313
|
+
// The lookup above may be stale by the time the row is written (a retirement can fence and purge
|
|
314
|
+
// this lifecycle meanwhile), so the write re-checks the exact ACL row itself (`aclFence`).
|
|
315
|
+
try {
|
|
316
|
+
return { ok: true, data: await this.membershipWrite(caller, uid, () => this.durableJoinFor(caller, channel, uid, { aclFence: true })) };
|
|
317
|
+
}
|
|
318
|
+
catch (e) {
|
|
319
|
+
return { ok: false, error: e.message };
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
/** Whether `lifecycleUid`'s exact ACL row still allows reads, so an inventory of it came from a
|
|
323
|
+
* caller that did not fence. A row narrowed to no reads, or retired or absent, is fenced. A read
|
|
324
|
+
* that fails answers true: the caller keeps its in-memory mark, which only ever refuses. */
|
|
325
|
+
async lifecycleAclUnfenced(principal, lifecycleUid) {
|
|
326
|
+
try {
|
|
327
|
+
const row = await readAcl(await this.aclRegistry(), principal, lifecycleUid);
|
|
328
|
+
return row !== undefined && row.record.allowSubscribe.length > 0;
|
|
329
|
+
}
|
|
330
|
+
catch {
|
|
331
|
+
return true;
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
/** Run one durable join or leave that writes `principal`'s rows of `lifecycleUid`, refused once that
|
|
335
|
+
* lifecycle's inventory has been asked for ({@link retiringLifecycles}). The check and the
|
|
336
|
+
* registration have no await between them, so an inventory either finds this write running and
|
|
337
|
+
* waits for it, or this write finds the inventory's mark and is refused. */
|
|
338
|
+
async membershipWrite(principal, lifecycleUid, write) {
|
|
339
|
+
if (this.retiringLifecycles.get(principal)?.has(lifecycleUid))
|
|
340
|
+
throw new Error(`lifecycle ${lifecycleUid} of ${principal} is retiring: its membership inventory was requested, so its durable memberships no longer change`);
|
|
341
|
+
const key = `${principal}/${lifecycleUid}`;
|
|
342
|
+
const running = write();
|
|
343
|
+
let writes = this.membershipWrites.get(key);
|
|
344
|
+
if (writes === undefined)
|
|
345
|
+
this.membershipWrites.set(key, (writes = new Set()));
|
|
346
|
+
writes.add(running);
|
|
347
|
+
try {
|
|
348
|
+
return await running;
|
|
349
|
+
}
|
|
350
|
+
finally {
|
|
351
|
+
writes.delete(running);
|
|
352
|
+
if (writes.size === 0)
|
|
353
|
+
this.membershipWrites.delete(key);
|
|
354
|
+
}
|
|
355
|
+
}
|
|
356
|
+
/** LEAVE must NOT require current-ACL coverage. Leave fires precisely when the ACL was narrowed/revoked
|
|
357
|
+
* (a refused live sub → {@link closeRefusedMembership}); gating the tombstone on the current ACL would
|
|
358
|
+
* loop forever and leave the SPEC §7 boundary open (the membership could resume if the ACL is later
|
|
359
|
+
* restored). The guards are: authenticated caller (serveControl), concrete channel, a finite generation
|
|
360
|
+
* (the join epoch — without it a stale/replayed leave could tombstone a newer rejoin), and an EXISTING
|
|
361
|
+
* own membership; `durableLeaveFor` → `tombstoneMember` then enforces the generation match. */
|
|
362
|
+
async deliveryLeave(caller, args) {
|
|
363
|
+
const channel = this.checkDurableChannelArg(args, "durableLeave", { family: true });
|
|
364
|
+
if (typeof channel !== "string")
|
|
365
|
+
return channel; // a ControlReply error
|
|
366
|
+
if (typeof args.generation !== "number" || !Number.isFinite(args.generation))
|
|
367
|
+
return { ok: false, error: "durableLeave: a finite generation is required (fail-closed stale-leave guard)" };
|
|
368
|
+
// The LEAVE carries the leaver's own lifecycleUid: leave must work precisely when the ACL row was
|
|
369
|
+
// narrowed or already purged (see the method doc), so the alias→row resolution join uses is not
|
|
370
|
+
// available here. The uid is caller-asserted but harmless to lie about: the member key it selects
|
|
371
|
+
// is confined to the AUTHENTICATED caller's own alias (`<channel>/<caller>.<uid>`), so the worst a
|
|
372
|
+
// false uid reaches is the caller's own retired incarnation's row — legitimate cleanup — and the
|
|
373
|
+
// generation guard still applies.
|
|
374
|
+
if (typeof args.lifecycleUid !== "string")
|
|
375
|
+
return { ok: false, error: "durableLeave: the leaving incarnation's lifecycleUid is required (membership rows are lifecycle-keyed, SPEC 13.1)" };
|
|
376
|
+
let uid;
|
|
377
|
+
try {
|
|
378
|
+
uid = assertLifecycleToken(args.lifecycleUid);
|
|
379
|
+
}
|
|
380
|
+
catch (e) {
|
|
381
|
+
return { ok: false, error: e.message };
|
|
382
|
+
}
|
|
383
|
+
const existing = await readMember(await this.membersRegistry(), channel, caller, uid);
|
|
384
|
+
if (!existing)
|
|
385
|
+
return { ok: true, data: { channel, alreadyLeft: true } }; // nothing to tombstone — idempotent
|
|
386
|
+
const generation = args.generation;
|
|
387
|
+
try {
|
|
388
|
+
await this.membershipWrite(caller, uid, () => this.durableLeaveFor(caller, channel, uid, generation));
|
|
389
|
+
}
|
|
390
|
+
catch (e) {
|
|
391
|
+
return { ok: false, error: e.message };
|
|
392
|
+
}
|
|
393
|
+
return { ok: true, data: { channel } };
|
|
394
|
+
}
|
|
395
|
+
/** Serve one MEDIATED HISTORY READ (`readHistory`, the client side is {@link readHistory}). The daemon
|
|
396
|
+
* holds the consumer so the caller does not have to: this is a privilege reduction, not a new
|
|
397
|
+
* capability, and it is the shape SPEC's "Mediated reads (normative)" rule asks for.
|
|
398
|
+
*
|
|
399
|
+
* THE ONE INVARIANT THIS METHOD EXISTS FOR: authorization is read FRESH on every call and never
|
|
400
|
+
* cached across calls. A consumer pins its authorization when it is created, so a revoked caller
|
|
401
|
+
* keeps being served by a consumer it already holds; re-reading here is what makes a revocation
|
|
402
|
+
* stop the very NEXT read. Cache this and the verb loses its only advantage over the raw consumer
|
|
403
|
+
* path.
|
|
404
|
+
*
|
|
405
|
+
* AUTHORITY (SPEC §9.6): "current read ACL" is the effective broker-accepted credential. The
|
|
406
|
+
* durable registry is a live mirror of that credential, not a second grant source. History
|
|
407
|
+
* authorizes against `allowSubscribe ∩ issuedAllowSubscribe` — the live row (so revocation via
|
|
408
|
+
* plain commitAcl still stops the next read) intersected with the mint-time ceiling (so a
|
|
409
|
+
* registry widen without a remint cannot grant what the JWT does not). Raising the ceiling is
|
|
410
|
+
* {@link reissueAcl}, which is what provision does when it bakes the list into the JWT.
|
|
411
|
+
*
|
|
412
|
+
* The caller arrives as an alias (the control subject carries owner+actor, never a uid) and is the
|
|
413
|
+
* AUTHENTICATED subject sender — `serveControl` has already fail-closed on any payload that names a
|
|
414
|
+
* different principal, so `caller` cannot be self-asserted. */
|
|
415
|
+
async deliveryReadHistory(caller, args) {
|
|
416
|
+
const channel = this.checkDurableChannelArg(args, "readHistory");
|
|
417
|
+
if (typeof channel !== "string")
|
|
418
|
+
return channel; // a ControlReply error
|
|
419
|
+
// The caller PROPOSES a limit; the mediator decides. Reject a nonsense limit loudly rather than
|
|
420
|
+
// silently substituting a default — a caller asking for -1 has a bug and should hear about it.
|
|
421
|
+
const asked = args.limit === undefined ? READ_HISTORY_DEFAULT_LIMIT : args.limit;
|
|
422
|
+
if (typeof asked !== "number" || !Number.isInteger(asked) || asked < 1)
|
|
423
|
+
return { ok: false, error: `readHistory: limit must be a positive integer (got ${JSON.stringify(args.limit)})` };
|
|
424
|
+
const limit = Math.min(asked, READ_HISTORY_MAX_LIMIT);
|
|
425
|
+
// FRESH per call — see the method doc. Same registry, same alias resolution, and the same loud
|
|
426
|
+
// refusal on two live rows that `durableJoin` uses: a stale lifecycle must never authorize its
|
|
427
|
+
// successor's read.
|
|
428
|
+
//
|
|
429
|
+
// SPEC §9.6: "current read ACL" is the effective broker-accepted credential. The registry row is
|
|
430
|
+
// a live mirror and can be rewritten by plain commitAcl without a remint. Authorizing history
|
|
431
|
+
// from the row ALONE let a widen grant reads channelHistory still broker-denies — a new
|
|
432
|
+
// capability wearing a privilege-reduction label (panel BLOCKING at 914fd7b0). History therefore
|
|
433
|
+
// requires the channel in BOTH the live row AND the mint-time ceiling (`issuedAllowSubscribe`).
|
|
434
|
+
// Revocation still works: narrowing allowSubscribe stops the next read. Raising the ceiling
|
|
435
|
+
// requires reissueAcl (provision/remint), which is the same act that bakes the list into the JWT.
|
|
436
|
+
let acl;
|
|
437
|
+
try {
|
|
438
|
+
acl = await this.aclForAlias(caller);
|
|
439
|
+
}
|
|
440
|
+
catch (e) {
|
|
441
|
+
return { ok: false, error: e.message };
|
|
442
|
+
}
|
|
443
|
+
if (acl === undefined)
|
|
444
|
+
return { ok: false, error: `readHistory: no read ACL on record for ${caller} - not permitted` };
|
|
445
|
+
const inLive = channelInAllow(acl.allowSubscribe, channel);
|
|
446
|
+
const inIssued = channelInAllow(acl.issuedAllowSubscribe, channel);
|
|
447
|
+
if (!inLive || !inIssued) {
|
|
448
|
+
// Name the live list in the error — that is what operators edit and what revocation clears.
|
|
449
|
+
// A ceiling miss (registry widened past the credential) is the same refusal shape as a live
|
|
450
|
+
// miss so a caller cannot probe which half failed.
|
|
451
|
+
return {
|
|
452
|
+
ok: false,
|
|
453
|
+
error: `readHistory: channel "${channel}" is not within your read ACL [${acl.allowSubscribe.join(", ")}] - refused`,
|
|
454
|
+
};
|
|
455
|
+
}
|
|
456
|
+
// One message MORE than the page is the completeness probe: if the backlog hands back limit+1,
|
|
457
|
+
// something older exists behind the page and `complete` is false. Cheaper and more honest than a
|
|
458
|
+
// separate count, which could race the page it describes.
|
|
459
|
+
let page;
|
|
460
|
+
try {
|
|
461
|
+
page = await this.channelHistory(channel, { limit: limit + 1 });
|
|
462
|
+
}
|
|
463
|
+
catch (e) {
|
|
464
|
+
// Surface the read failure. `streamHistory` deliberately raises rather than returning empty on a
|
|
465
|
+
// denial or a cut-short window, and that distinction must survive the rail: an error here would
|
|
466
|
+
// otherwise reach the caller as an empty page and render as "no history".
|
|
467
|
+
return { ok: false, error: `readHistory: ${e.message}` };
|
|
468
|
+
}
|
|
469
|
+
const reachedStart = page.length <= limit;
|
|
470
|
+
const wanted = reachedStart ? page : page.slice(-limit);
|
|
471
|
+
// A page that cannot be SENT is not a page. The reply rides one NATS message, so `limit` alone is
|
|
472
|
+
// the wrong bound: 200 large messages serialize past `max_payload`, and `serveControl` can then only
|
|
473
|
+
// answer with a refusal in place of the page.
|
|
474
|
+
//
|
|
475
|
+
// So bound by BYTES too, keeping the NEWEST that fit — which needs no new vocabulary, because
|
|
476
|
+
// `complete: false` already means "older history remains behind this page". Trimming here is the
|
|
477
|
+
// documented truncation signal doing its job, not a silent degradation.
|
|
478
|
+
const fitted = fitHistoryPage(wanted, this.maxPayload);
|
|
479
|
+
// `wanted.length > 0` is load-bearing, not defensive: an EMPTY channel also fits nothing, and
|
|
480
|
+
// without this guard a channel nobody has posted to was refused with "the newest message exceeds
|
|
481
|
+
// the payload budget" — a confident, entirely wrong explanation for a legitimately empty result.
|
|
482
|
+
// Genuine emptiness is `{ items: [], complete: true }`; only a message too large to ever send is
|
|
483
|
+
// the error.
|
|
484
|
+
if (wanted.length > 0 && fitted.length === 0)
|
|
485
|
+
return { ok: false, error: `readHistory: the newest message on "${channel}" alone exceeds the broker payload budget (${this.maxPayload} bytes) - refused loudly rather than served as an empty page` };
|
|
486
|
+
return { ok: true, data: { items: fitted, complete: reachedStart && fitted.length === wanted.length } };
|
|
487
|
+
}
|
|
488
|
+
/** Stop serving Plane-3 WITHOUT tearing down the connection, so a daemon that has just learned its
|
|
489
|
+
* lease may no longer be its own can stop acting on the shard while it finds out for certain.
|
|
490
|
+
*
|
|
491
|
+
* A COMPARE-AND-SWAP KEEPS ONE LEASE ROW; IT DOES NOT KEEP ONE SERVER. That distinction is the
|
|
492
|
+
* reason this exists, and it was a review finding. When a renew fails, the daemon re-reads the
|
|
493
|
+
* key and may then re-acquire it, and across that read-then-create it was still consuming the
|
|
494
|
+
* fan-out durable, still running the inbox reader, and still answering ctl.delivery. If a
|
|
495
|
+
* replacement acquired the shard in that window, both processes served the same durables until
|
|
496
|
+
* the loser's create was refused and its teardown finished. The old code did not have this
|
|
497
|
+
* window, because it began shutting down on the first renew failure; treating that failure as a
|
|
498
|
+
* question instead of a verdict is right, but asking the question while still serving is not.
|
|
499
|
+
*
|
|
500
|
+
* So the daemon goes quiet FIRST and re-arms only once it has proof: `held` on a re-read, or a
|
|
501
|
+
* won atomic create. `unknown` stays quiet, the whole point is that not being able to ask is not
|
|
502
|
+
* permission to keep acting. Quiescing costs delivery latency for a few seconds; the alternative
|
|
503
|
+
* costs a SPLIT durable, which is a correctness failure rather than an availability one.
|
|
504
|
+
*
|
|
505
|
+
* Deliberately not `stop()`: the connection, the lease KV handles and the control rails must stay
|
|
506
|
+
* up, because the daemon still has to ask the broker who owns the shard. */
|
|
507
|
+
async quiescePlane3() {
|
|
508
|
+
this.plane3Quiesced = true;
|
|
509
|
+
if (this.deliveryServeSub) {
|
|
510
|
+
try {
|
|
511
|
+
this.deliveryServeSub.unsubscribe();
|
|
512
|
+
}
|
|
513
|
+
catch { /* already dead */ }
|
|
514
|
+
this.deliveryServeSub = undefined;
|
|
515
|
+
}
|
|
516
|
+
if (this.deliveryAdminServeSub) {
|
|
517
|
+
try {
|
|
518
|
+
this.deliveryAdminServeSub.unsubscribe();
|
|
519
|
+
}
|
|
520
|
+
catch { /* already dead */ }
|
|
521
|
+
this.deliveryAdminServeSub = undefined;
|
|
522
|
+
}
|
|
523
|
+
this.dropAnycastRequests();
|
|
524
|
+
// Stopping the consumers ends the `for await` loops that drive fan-out and the reader. In-flight
|
|
525
|
+
// messages are NOT acked by a stopped consumer, so they redeliver to whoever holds the shard
|
|
526
|
+
// next: quiescing loses no message, it only stops this process from claiming them.
|
|
527
|
+
for (const msgs of this.streamMsgs.splice(0)) {
|
|
528
|
+
try {
|
|
529
|
+
msgs.stop();
|
|
530
|
+
}
|
|
531
|
+
catch { /* already draining */ }
|
|
532
|
+
}
|
|
533
|
+
}
|
|
534
|
+
/** Whether Plane-3 is currently quiesced: unbound, and refusing to re-bind until ownership is proven.
|
|
535
|
+
* Read-only. The flag is the daemon's own answer to "am I serving this shard", so a cell that asserts
|
|
536
|
+
* a recovery worked can check the endpoint's state rather than inferring it from a log line. */
|
|
537
|
+
plane3IsQuiesced() {
|
|
538
|
+
return this.plane3Quiesced;
|
|
539
|
+
}
|
|
540
|
+
/** Resume serving Plane-3 after {@link quiescePlane3}, once ownership has been re-established.
|
|
541
|
+
* Idempotent, and a no-op when the daemon was never quiesced.
|
|
542
|
+
*
|
|
543
|
+
* The flag is cleared ONLY after every binding is up, and restored if any of them throws. Clearing
|
|
544
|
+
* it first looks equivalent and is not: `armPlane3` binds in four stages (`manager`, the two control
|
|
545
|
+
* responders, the fan-out consumer, the reader), so a failure at any one leaves the endpoint recorded
|
|
546
|
+
* as un-quiesced while some of those are missing. From there every later `rearmPlane3` returns at the
|
|
547
|
+
* `!plane3Quiesced` guard WITHOUT attempting to bind, and the caller goes on to flip the lease READY.
|
|
548
|
+
* That is a readiness lie surviving a transient broker error, the daemon claims a responder it does
|
|
549
|
+
* not have, which is the #1318 outage wearing the readiness flag instead of the exit path. */
|
|
550
|
+
async rearmPlane3() {
|
|
551
|
+
if (!this.plane3Quiesced)
|
|
552
|
+
return;
|
|
553
|
+
// `armPlane3` refuses while quiesced (a reconnect must not resume mid-question), so the flag is
|
|
554
|
+
// lowered for the attempt and RAISED AGAIN on failure. A partial bind leaves the endpoint quiesced,
|
|
555
|
+
// which is the honest state: not serving, and a later retry will attempt the whole sequence again.
|
|
556
|
+
this.plane3Quiesced = false;
|
|
557
|
+
try {
|
|
558
|
+
await this.armPlane3();
|
|
559
|
+
}
|
|
560
|
+
catch (e) {
|
|
561
|
+
this.plane3Quiesced = true;
|
|
562
|
+
// Drop whatever DID bind. Leaving half-bound responders up would answer control requests for a
|
|
563
|
+
// shard this process has just declared itself not to be serving.
|
|
564
|
+
this.quiescePlane3();
|
|
565
|
+
throw e;
|
|
566
|
+
}
|
|
567
|
+
}
|
|
568
|
+
/** (Re)bind the Plane-3 fan-out writer + trusted reader. Idempotent — the durables resume from their
|
|
569
|
+
* cursor. Called by {@link startPlane3} once AND by {@link connectAndBind} on every (re)connect, so
|
|
570
|
+
* the delivery daemon's reconnect RE-ARMS the backstop + the ctl.delivery responder. Without this, a broker blip would silently kill
|
|
571
|
+
* the loops while `durableJoinFor` kept reporting `durable:true` (the impl-review's BLOCKER-1). No-op
|
|
572
|
+
* unless this endpoint hosts Plane-3 (`this.plane3` set). */
|
|
573
|
+
async armPlane3() {
|
|
574
|
+
if (!this.plane3 || !this.js)
|
|
575
|
+
return;
|
|
576
|
+
// A quiesced endpoint must not be re-armed by a RECONNECT: the reconnect path calls this too,
|
|
577
|
+
// and silently resuming there would restore exactly the double-serving this guards against
|
|
578
|
+
// while the ownership question is still open.
|
|
579
|
+
if (this.plane3Quiesced)
|
|
580
|
+
return;
|
|
581
|
+
await this.manager(); // the manager runs consume:false, so this.jsm is lazy — ensure it
|
|
582
|
+
this.armDeliveryControl();
|
|
583
|
+
await this.armAnycastRequests();
|
|
584
|
+
await this.runFanout();
|
|
585
|
+
await this.runReader();
|
|
586
|
+
}
|
|
587
|
+
/** (Re)register the `ctl.delivery` control responder on the CURRENT connection. A reconnect drains the
|
|
588
|
+
* old connection (the old sub is dead and `clearConnectionScoped` leaves caller-owned subs alone), so
|
|
589
|
+
* this MUST run on every arm — otherwise durable join/leave/list silently lose their responder after a
|
|
590
|
+
* broker blip. The stale sub is dropped (unsubscribed + removed from `this.subs`) before re-creating.
|
|
591
|
+
* `boundReply` is essential here: the daemon holds a wildcard reply-publish grant, so the serve path
|
|
592
|
+
* must reject any reply target outside the authenticated sender's own subtree (confused-deputy fix). */
|
|
593
|
+
armDeliveryControl() {
|
|
594
|
+
if (this.deliveryServeSub) {
|
|
595
|
+
try {
|
|
596
|
+
this.deliveryServeSub.unsubscribe();
|
|
597
|
+
}
|
|
598
|
+
catch { /* dead with the old connection */ }
|
|
599
|
+
const i = this.subs.indexOf(this.deliveryServeSub);
|
|
600
|
+
if (i >= 0)
|
|
601
|
+
this.subs.splice(i, 1);
|
|
602
|
+
}
|
|
603
|
+
// A request's status and cancel touch only the TASK queue and the anycast records, so they do
|
|
604
|
+
// not wait behind a channel join on the in-order chain.
|
|
605
|
+
this.deliveryServeSub = this.serveControl(CONTROL_DELIVERY, (req) => this.handleDeliveryControl(req), {
|
|
606
|
+
boundReply: true,
|
|
607
|
+
concurrent: (req) => ANYCAST_CONTROL_OPS.has(req.op),
|
|
608
|
+
});
|
|
609
|
+
if (this.deliveryAdminServeSub) {
|
|
610
|
+
try {
|
|
611
|
+
this.deliveryAdminServeSub.unsubscribe();
|
|
612
|
+
}
|
|
613
|
+
catch { /* dead with the old connection */ }
|
|
614
|
+
const i = this.subs.indexOf(this.deliveryAdminServeSub);
|
|
615
|
+
if (i >= 0)
|
|
616
|
+
this.subs.splice(i, 1);
|
|
617
|
+
}
|
|
618
|
+
// The membership inventory is read-only and shares nothing with the other admin commands, so it
|
|
619
|
+
// does not queue behind them. In a mass stop each retirement's inventory followed every other
|
|
620
|
+
// seat's `evictPrincipal` on this rail and missed its 5 s deadline (#4965).
|
|
621
|
+
this.deliveryAdminServeSub = this.serveControl(CONTROL_DELIVERY_ADMIN, (req) => this.handleDeliveryAdmin(req), {
|
|
622
|
+
boundReply: true,
|
|
623
|
+
concurrent: (req) => req.op === "lifecycleMemberships",
|
|
624
|
+
});
|
|
625
|
+
}
|
|
626
|
+
/** Serve one PRIVILEGED delivery-admin request (the D5 rail-split). The cred layer is the caller
|
|
627
|
+
* boundary — only the supervisor profile can publish here — and `serveControl`'s sender check +
|
|
628
|
+
* bounded reply still apply on top. `reloadCreds` is the class-2 renewal ADOPTION step: re-read
|
|
629
|
+
* the renewal-owner-re-signed creds file, pin, swap the live connection, reconnect the membership
|
|
630
|
+
* feed's rw connection, and reply with proof (identities + the adopted JWT windows) — or a
|
|
631
|
+
* structured failure (e.g. the file was never re-signed), never a silent partial. */
|
|
632
|
+
async handleDeliveryAdmin(req) {
|
|
633
|
+
// A reload's resident swap reconnects the connection every reply here rides. A caller's next
|
|
634
|
+
// request, such as a manager's eviction right after its reload, is answered once that
|
|
635
|
+
// connection is back, not while it is down.
|
|
636
|
+
await this.residentSwap;
|
|
637
|
+
// FENCE: same reason as the runtime rail above.
|
|
638
|
+
if (!this.plane3MayAct())
|
|
639
|
+
return { ok: false, error: "delivery: this daemon is not serving this shard (it is re-checking ownership); retry" };
|
|
640
|
+
if (req.op === "reloadCreds") {
|
|
641
|
+
// NOT BEFORE START-UP IS DONE (#2304). Both halves below would run: the delivery half commits
|
|
642
|
+
// and schedules `nc.reconnect()`, which lands underneath start-up awaits still in flight on
|
|
643
|
+
// this connection (the lease watch's consumer create times out and the daemon exits). Refuse
|
|
644
|
+
// before either half acts, so nothing is adopted and the renewal owner records why.
|
|
645
|
+
if (this.plane3?.startupComplete?.() === false)
|
|
646
|
+
return { ok: false, error: "delivery: this daemon has not finished starting; nothing adopted - the next renewal pass or the daemon's 75% re-read adopts the re-signed creds" };
|
|
647
|
+
// The renewal owner's EXPECTED-generation tokens (SHA-256 of each JWT it re-signed), per
|
|
648
|
+
// component. A missing entry means "no expectation" (the passive backstop still adopts).
|
|
649
|
+
const expected = (req.args?.expected ?? {});
|
|
650
|
+
// Prove BOTH components INDEPENDENTLY (non-short-circuit): one failing must neither hide the
|
|
651
|
+
// other's outcome nor be masked by it. Top-level `ok` is false if EITHER proof failed, so the
|
|
652
|
+
// manager can never record a green adoption while a component was refused.
|
|
653
|
+
// Each component's outcome claims only what is PROVEN: `brokerAccepted` (the preflight) carries
|
|
654
|
+
// the accepted generation's window; `residentSwap` records that the wire swap is best-effort and
|
|
655
|
+
// self-healing, NOT witnessed. Neither claims verified resident reauth (that would need a
|
|
656
|
+
// generation-tied reconnect witness on both connections).
|
|
657
|
+
const [delivery, membership] = await Promise.all([
|
|
658
|
+
this.reloadCreds(expected.delivery).then((brokerAccepted) => ({ ok: true, brokerAccepted, residentSwap: "scheduled" }), (e) => ({ ok: false, error: e.message })),
|
|
659
|
+
this.plane3?.reloadMembershipCreds
|
|
660
|
+
? this.plane3.reloadMembershipCreds(expected.membership).then((result) => ({ ok: true, ...result }), (e) => ({ ok: false, error: e.message }))
|
|
661
|
+
: Promise.resolve({ ok: true, skipped: "no-membership-hook" }),
|
|
662
|
+
]);
|
|
663
|
+
const failures = [
|
|
664
|
+
delivery.ok ? undefined : `delivery: ${delivery.error}`,
|
|
665
|
+
membership.ok ? undefined : `membership: ${membership.error}`,
|
|
666
|
+
].filter((m) => m !== undefined);
|
|
667
|
+
// Arm the resident wire swap ONLY now — after BOTH proofs settled, right before the reply is
|
|
668
|
+
// returned+responded — so a slow membership proof can never let the delivery reconnect strand
|
|
669
|
+
// this reply. Only when delivery actually adopted a new candidate (currentCreds was updated).
|
|
670
|
+
if (delivery.ok) {
|
|
671
|
+
this.plane3?.onDeliveryCredsAdopted?.();
|
|
672
|
+
this.scheduleResidentSwap();
|
|
673
|
+
}
|
|
674
|
+
return failures.length
|
|
675
|
+
? { ok: false, error: failures.join("; "), data: { delivery, membership } }
|
|
676
|
+
: { ok: true, data: { delivery, membership } };
|
|
677
|
+
}
|
|
678
|
+
if (req.op === "evictPrincipal") {
|
|
679
|
+
// The LIVE-EVICTION executor: force-drop a denied principal's connections.
|
|
680
|
+
// Composition-root hook because the $SYS observer/evictor creds live outside this endpoint's
|
|
681
|
+
// trust boundary; absent hook = a daemon build without the executor, refused loudly.
|
|
682
|
+
if (!this.plane3?.evictPrincipal)
|
|
683
|
+
return { ok: false, error: "evictPrincipal: no eviction executor wired on this daemon" };
|
|
684
|
+
const principal = typeof req.args?.principal === "string" ? req.args.principal.trim() : "";
|
|
685
|
+
if (!principal)
|
|
686
|
+
return { ok: false, error: "evictPrincipal: a principal (owner.actor dot-form) is required" };
|
|
687
|
+
try {
|
|
688
|
+
return { ok: true, data: await this.plane3.evictPrincipal(principal) };
|
|
689
|
+
}
|
|
690
|
+
catch (e) {
|
|
691
|
+
return { ok: false, error: e.message };
|
|
692
|
+
}
|
|
693
|
+
}
|
|
694
|
+
if (req.op === "evictPrincipals") {
|
|
695
|
+
// The same executor over a SET: one shared sweep instead of one request and one scan per
|
|
696
|
+
// principal. The set is bounded so one request stays inside one sweep's work.
|
|
697
|
+
if (!this.plane3?.evictPrincipals)
|
|
698
|
+
return { ok: false, error: "evictPrincipals: no batch eviction executor wired on this daemon" };
|
|
699
|
+
const raw = req.args?.principals;
|
|
700
|
+
const principals = Array.isArray(raw) ? raw.map((p) => (typeof p === "string" ? p.trim() : "")) : [];
|
|
701
|
+
if (principals.length === 0 || principals.some((p) => !p))
|
|
702
|
+
return { ok: false, error: "evictPrincipals: principals must be a non-empty array of owner.actor dot-form principals" };
|
|
703
|
+
if (principals.length > EVICT_PRINCIPALS_MAX || new Set(principals).size !== principals.length)
|
|
704
|
+
return { ok: false, error: `evictPrincipals: principals must be distinct and at most ${EVICT_PRINCIPALS_MAX}` };
|
|
705
|
+
try {
|
|
706
|
+
return { ok: true, data: await this.plane3.evictPrincipals(principals) };
|
|
707
|
+
}
|
|
708
|
+
catch (e) {
|
|
709
|
+
return { ok: false, error: e.message };
|
|
710
|
+
}
|
|
711
|
+
}
|
|
712
|
+
if (req.op === "planeConnLiveness") {
|
|
713
|
+
// The plane-claim liveness oracle (#29 HIGH 3): a CLOSED read-only verb — two claimed
|
|
714
|
+
// scanner tuples in, two bound verdicts + sweep completeness out. The executor hook owns
|
|
715
|
+
// the closed query validation (it holds the $SYS observer cred outside this trust boundary);
|
|
716
|
+
// absent hook = a daemon build without the oracle, refused loudly (the auth plane treats
|
|
717
|
+
// that refusal as UNKNOWN and never reclaims over it).
|
|
718
|
+
if (!this.plane3?.planeConnLiveness)
|
|
719
|
+
return { ok: false, error: "planeConnLiveness: no plane-liveness oracle wired on this daemon" };
|
|
720
|
+
try {
|
|
721
|
+
return { ok: true, data: await this.plane3.planeConnLiveness(req.args?.query) };
|
|
722
|
+
}
|
|
723
|
+
catch (e) {
|
|
724
|
+
return { ok: false, error: e.message };
|
|
725
|
+
}
|
|
726
|
+
}
|
|
727
|
+
if (req.op === "lifecycleMemberships") {
|
|
728
|
+
// The terminal teardown's membership INVENTORY: a complete, read-only, lifecycle-exact listing
|
|
729
|
+
// of one principal's durable membership rows (tombstones included) from the trusted daemon that
|
|
730
|
+
// owns the members bucket. The caller deletes only the exact keys it gets back, through its
|
|
731
|
+
// target-pinned deprovisioner grant; this verb deletes nothing and returns no other lifecycle.
|
|
732
|
+
const principal = typeof req.args?.principal === "string" ? req.args.principal.trim() : "";
|
|
733
|
+
const uid = typeof req.args?.lifecycleUid === "string" ? req.args.lifecycleUid : "";
|
|
734
|
+
if (!parsePrincipalKey(principal))
|
|
735
|
+
return { ok: false, error: "lifecycleMemberships: a principal (owner.actor dot-form) is required" };
|
|
736
|
+
try {
|
|
737
|
+
assertLifecycleToken(uid);
|
|
738
|
+
}
|
|
739
|
+
catch (e) {
|
|
740
|
+
return { ok: false, error: `lifecycleMemberships: ${e.message}` };
|
|
741
|
+
}
|
|
742
|
+
// A retirement fences the lifecycle's ACL row before it asks (fenceLifecycleAcl), and every join
|
|
743
|
+
// re-reads that row around the row it creates, so a row written after this read begins removes
|
|
744
|
+
// itself, on whichever daemon wrote it. For a manager that does not fence, this daemon's own
|
|
745
|
+
// memberships of the lifecycle stop changing here (#4965): a join or leave bound to it is refused
|
|
746
|
+
// from now on, and the read starts after the ones already running have finished.
|
|
747
|
+
let marked = this.retiringLifecycles.get(principal);
|
|
748
|
+
if (marked === undefined)
|
|
749
|
+
this.retiringLifecycles.set(principal, (marked = new Set()));
|
|
750
|
+
// An earlier lifecycle of this principal whose ACL row is retired can bind no join or leave any
|
|
751
|
+
// more, so its mark goes. A failed read keeps the mark, which only ever refuses.
|
|
752
|
+
for (const earlier of [...marked])
|
|
753
|
+
if (earlier !== uid && await aclRetired(await this.aclRegistry(), principal, earlier).catch(() => false))
|
|
754
|
+
marked.delete(earlier);
|
|
755
|
+
// A mark is kept only for an inventory whose caller did not fence (#4996): a lifecycle whose ACL
|
|
756
|
+
// row is narrowed to no reads, or already retired, needs none, since every join re-reads that
|
|
757
|
+
// row around its write. A read that fails proves nothing, so the mark is kept, which only refuses.
|
|
758
|
+
if (await this.lifecycleAclUnfenced(principal, uid))
|
|
759
|
+
marked.add(uid);
|
|
760
|
+
else if (marked.size === 0)
|
|
761
|
+
this.retiringLifecycles.delete(principal);
|
|
762
|
+
await Promise.allSettled(this.membershipWrites.get(`${principal}/${uid}`) ?? []);
|
|
763
|
+
try {
|
|
764
|
+
const channels = await listLifecycleMemberChannels(await this.membersRegistry(), principal, uid);
|
|
765
|
+
return { ok: true, data: { complete: true, channels } };
|
|
766
|
+
}
|
|
767
|
+
catch (e) {
|
|
768
|
+
return { ok: false, error: `lifecycleMemberships: the inventory read did not complete (${e.message})` };
|
|
769
|
+
}
|
|
770
|
+
}
|
|
771
|
+
if (req.op === "principalLiveness") {
|
|
772
|
+
// The freeze-holder liveness probe (#391): the READ-ONLY half of `evictPrincipal`. A repair
|
|
773
|
+
// that must REFUSE while the holder is alive cannot use eviction as its own precheck — that
|
|
774
|
+
// kills the holder before anything can refuse on its behalf. Same executor-hook shape as the
|
|
775
|
+
// verbs above (the $SYS observer cred lives outside this trust boundary); absent hook =
|
|
776
|
+
// refused loudly, and the caller maps a refusal to UNKNOWN and never repairs over it.
|
|
777
|
+
if (!this.plane3?.principalLiveness)
|
|
778
|
+
return { ok: false, error: "principalLiveness: no liveness oracle wired on this daemon" };
|
|
779
|
+
const principal = typeof req.args?.principal === "string" ? req.args.principal.trim() : "";
|
|
780
|
+
if (!principal)
|
|
781
|
+
return { ok: false, error: "principalLiveness: a principal (owner.actor dot-form) is required" };
|
|
782
|
+
try {
|
|
783
|
+
return { ok: true, data: await this.plane3.principalLiveness(principal) };
|
|
784
|
+
}
|
|
785
|
+
catch (e) {
|
|
786
|
+
return { ok: false, error: e.message };
|
|
787
|
+
}
|
|
788
|
+
}
|
|
789
|
+
if (req.op === "reloadStoreIdentity") {
|
|
790
|
+
// Proof that this daemon reloads standing creds from ONE named store. The manager
|
|
791
|
+
// compares it to its remint store before every renewal pass. Absent hook
|
|
792
|
+
// is a daemon that cannot name its store, which is itself a divergent composition.
|
|
793
|
+
if (!this.plane3?.reloadStoreIdentity)
|
|
794
|
+
return { ok: false, error: "reloadStoreIdentity: this daemon did not name the SecretStore it reloads from" };
|
|
795
|
+
try {
|
|
796
|
+
const identity = this.plane3.reloadStoreIdentity();
|
|
797
|
+
// WHO IS ANSWERING is part of the answer. This rail is queue-grouped, so the broker hands
|
|
798
|
+
// the request to any bound responder, and every process with a `delivery` credential for
|
|
799
|
+
// the space can bind it, while only the DELIVERY LEASE HOLDER actually reloads the standing
|
|
800
|
+
// credentials. A bare store identity therefore lets a second responder's store stand in for
|
|
801
|
+
// the reloading process's, and the manager's remint decision is then made about a process
|
|
802
|
+
// that reloads nothing. So the reply carries this responder's wire identity and whether it
|
|
803
|
+
// holds the lease, and the caller requires the binding (#1694).
|
|
804
|
+
//
|
|
805
|
+
// The lease read is the LIVE row, not a cached claim: a holder that lost the shard must not
|
|
806
|
+
// keep asserting it. A read that fails is reported as `false` rather than thrown, because
|
|
807
|
+
// "I cannot prove I hold the shard" is what a non-holder's answer says, and it sends the
|
|
808
|
+
// caller down the same fail-closed path rather than inventing an authority.
|
|
809
|
+
let holdsDeliveryLease = false;
|
|
810
|
+
try {
|
|
811
|
+
const own = await this.readDeliveryLeaseEntry(0);
|
|
812
|
+
holdsDeliveryLease = own !== undefined && this.ownsDeliveryLease(own.info);
|
|
813
|
+
}
|
|
814
|
+
catch {
|
|
815
|
+
holdsDeliveryLease = false;
|
|
816
|
+
}
|
|
817
|
+
// Round-trip through the closed parser so a hook cannot smuggle extra fields onto the rail.
|
|
818
|
+
return { ok: true, data: { identity: parseSecretStoreIdentity(identity), responder: this.card.id, holdsDeliveryLease } };
|
|
819
|
+
}
|
|
820
|
+
catch (e) {
|
|
821
|
+
return { ok: false, error: e.message };
|
|
822
|
+
}
|
|
823
|
+
}
|
|
824
|
+
return { ok: false, error: `op "${req.op}" not supported on the delivery admin service` };
|
|
825
|
+
}
|
|
826
|
+
/** Create one of the daemon's Plane-3 durables. A named create is idempotent for the same config,
|
|
827
|
+
* and a denial is never taken for existence. The one conflict accepted is this durable as a full
|
|
828
|
+
* restore recreated it, starting after its recorded ACK floor ({@link isRestoredConsumer}); every
|
|
829
|
+
* other failure propagates. */
|
|
830
|
+
async addPlane3Durable(stream, config) {
|
|
831
|
+
try {
|
|
832
|
+
await this.jsm.consumers.add(stream, config);
|
|
833
|
+
}
|
|
834
|
+
catch (e) {
|
|
835
|
+
if (!isConsumerAlreadyExists(e) || !isRestoredConsumer(this.space, stream, await this.jsm.consumers.info(stream, config.durable_name), config))
|
|
836
|
+
throw e;
|
|
837
|
+
}
|
|
838
|
+
}
|
|
839
|
+
/** Fan-out loop: bind the privileged `fanout` durable on CHAT and route each message (routing only —
|
|
840
|
+
* the trusted reader is the auth gate). */
|
|
841
|
+
async runFanout() {
|
|
842
|
+
if (!this.js || !this.jsm)
|
|
843
|
+
return;
|
|
844
|
+
await this.addPlane3Durable(chatStream(this.space), fanoutDurableConfig(this.space, { ackWaitMs: this.ackWaitMs }));
|
|
845
|
+
const consumer = await this.js.consumers.get(chatStream(this.space), FANOUT_DURABLE);
|
|
846
|
+
const msgs = await consumer.consume();
|
|
847
|
+
this.streamMsgs.push(msgs);
|
|
848
|
+
void (async () => {
|
|
849
|
+
for await (const m of msgs) {
|
|
850
|
+
try {
|
|
851
|
+
await this.fanOutMessage(m);
|
|
852
|
+
}
|
|
853
|
+
catch (e) {
|
|
854
|
+
if (!this.stopped)
|
|
855
|
+
this.emit("error", e);
|
|
856
|
+
try {
|
|
857
|
+
m.nak();
|
|
858
|
+
}
|
|
859
|
+
catch { /* draining */ }
|
|
860
|
+
}
|
|
861
|
+
}
|
|
862
|
+
})().catch((e) => { if (!this.stopped)
|
|
863
|
+
this.emit("error", e); });
|
|
864
|
+
}
|
|
865
|
+
/** Route ONE chat message to eligible owners' mixed inboxes. `durable` channel → its `durable-active`
|
|
866
|
+
* members within interval; `live` channel → `@mention` targets authorized to read it (ACL only).
|
|
867
|
+
* On either class, a mention of an agent whose read ACL excludes the channel is reported to the
|
|
868
|
+
* sender with a mention notice. Members KV is scanned FRESH per message (no cache — red-team
|
|
869
|
+
* BLOCKER-1 catch-up correctness). */
|
|
870
|
+
async fanOutMessage(m) {
|
|
871
|
+
// FENCE, and NOT followed by an ack: this unit was dispatched before the quiesce and is resuming
|
|
872
|
+
// after it, so writing the fan-out now would put entries in member inboxes on behalf of a shard
|
|
873
|
+
// this daemon no longer serves. Returning WITHOUT acking is the whole point - a stopped consumer
|
|
874
|
+
// redelivers to whoever holds the shard next, so refusing here loses no message, while acking it
|
|
875
|
+
// would consume the successor's work.
|
|
876
|
+
if (!this.plane3MayAct())
|
|
877
|
+
return;
|
|
878
|
+
const parsed = parseSubject(m.subject);
|
|
879
|
+
if (!parsed || parsed.kind !== "chat") {
|
|
880
|
+
m.ack();
|
|
881
|
+
return;
|
|
882
|
+
}
|
|
883
|
+
const channel = parsed.rest;
|
|
884
|
+
let msg;
|
|
885
|
+
try {
|
|
886
|
+
msg = m.json();
|
|
887
|
+
}
|
|
888
|
+
catch {
|
|
889
|
+
m.ack();
|
|
890
|
+
return;
|
|
891
|
+
}
|
|
892
|
+
if (!isFromSubjectSender(msg, parsed)) {
|
|
893
|
+
m.ack();
|
|
894
|
+
return;
|
|
895
|
+
} // authenticity (SPEC §5)
|
|
896
|
+
const seq = m.seq;
|
|
897
|
+
const normalizedMsg = authenticatedChannelMessage(msg, channel);
|
|
898
|
+
// SECOND FENCE, AT THE COMMIT POINT. The entry check above cannot cover this unit's own awaits:
|
|
899
|
+
// the class read and the member scan below both hit the broker, and a daemon descheduled inside
|
|
900
|
+
// them can resume after a successor owns the shard. Re-ask immediately before the first WRITE,
|
|
901
|
+
// and again return without acking so the entry redelivers rather than being consumed here.
|
|
902
|
+
if (!this.plane3MayAct())
|
|
903
|
+
return;
|
|
904
|
+
const durable = (await this.deliveryClassFresh(channel)) === "durable";
|
|
905
|
+
// Each mention is resolved once. A live channel delivers to the agents among them that may read
|
|
906
|
+
// it, and on either class a mention of an agent that may not read the channel is reported to the
|
|
907
|
+
// sender below, so a post never claims a reader it cannot have.
|
|
908
|
+
const mentioned = [];
|
|
909
|
+
for (const name of msg.mentions ?? []) {
|
|
910
|
+
const owner = this.resolveOwnerByName(name);
|
|
911
|
+
if (!owner || owner === msg.from.id)
|
|
912
|
+
continue;
|
|
913
|
+
// A mention arrives as an ALIAS (the roster names no lifecycle): resolve its single live ACL
|
|
914
|
+
// row for BOTH the read-authorization and the lifecycle to address. Ambiguity (two live rows)
|
|
915
|
+
// refuses THIS copy loudly rather than guessing a lifecycle.
|
|
916
|
+
let row;
|
|
917
|
+
try {
|
|
918
|
+
row = await this.aclForAlias(owner);
|
|
919
|
+
}
|
|
920
|
+
catch (e) {
|
|
921
|
+
this.emit("error", e);
|
|
922
|
+
continue;
|
|
923
|
+
}
|
|
924
|
+
if (row)
|
|
925
|
+
mentioned.push({ name, owner, row });
|
|
926
|
+
}
|
|
927
|
+
if (durable) {
|
|
928
|
+
// The channel's own rows and every family row above it (`review.>`). A lifecycle holding two of
|
|
929
|
+
// them gets two publishes and keeps one copy: the dedupe id is per message and lifecycle (#5404).
|
|
930
|
+
for (const rec of await listMembers(await this.membersRegistry(), { covering: channel })) {
|
|
931
|
+
if (rec.owner === msg.from.id)
|
|
932
|
+
continue; // never backstop the sender's own post
|
|
933
|
+
if (!durableEligible(rec, seq))
|
|
934
|
+
continue; // routing fast-filter (reader re-checks)
|
|
935
|
+
// Address the member row's RECORDED lifecycle: a retired row (tombstone pending) routes to the
|
|
936
|
+
// retired lifecycle's inbox, never the alias's new occupant (SPEC 13.1 cross-plane scoping).
|
|
937
|
+
// Store the AUTHENTICATED-channel copy (main's normalization): the durable frame validates
|
|
938
|
+
// msg.channel === frame.channel, and payload to/toService are stripped.
|
|
939
|
+
// Per-copy, because the scan above and every publish below is a broker round trip: a fan-out
|
|
940
|
+
// to fifty members can straddle a quiesce in its middle. Stopping part-way is safe where
|
|
941
|
+
// stopping late is not - the entry is never acked here, so the successor redelivers it and
|
|
942
|
+
// re-sends the whole set; publishDinbox is idempotent per (owner, lifecycle, msgID).
|
|
943
|
+
if (!this.plane3MayAct())
|
|
944
|
+
return;
|
|
945
|
+
await this.publishDinbox(rec.owner, rec.lifecycleUid, {
|
|
946
|
+
msg: normalizedMsg, channel, seq, reason: "durable-channel", generation: rec.generation,
|
|
947
|
+
...(rec.channel !== channel ? { row: rec.channel } : {}),
|
|
948
|
+
});
|
|
949
|
+
}
|
|
950
|
+
}
|
|
951
|
+
else {
|
|
952
|
+
for (const { owner, row } of mentioned) {
|
|
953
|
+
if (!channelInAllow(row.allowSubscribe, channel))
|
|
954
|
+
continue; // @mention can't bypass the read ACL
|
|
955
|
+
if (!this.plane3MayAct())
|
|
956
|
+
return; // same rule on the live-mention path
|
|
957
|
+
await this.publishDinbox(owner, row.lifecycleUid, { msg: normalizedMsg, channel, seq, reason: "live-mention", generation: 0 });
|
|
958
|
+
}
|
|
959
|
+
}
|
|
960
|
+
const unreadable = mentioned.filter(({ row }) => !channelInAllow(row.allowSubscribe, channel)).map(({ name }) => name);
|
|
961
|
+
if (unreadable.length) {
|
|
962
|
+
if (!this.plane3MayAct())
|
|
963
|
+
return;
|
|
964
|
+
await this.sendMentionNotice(msg, channel, seq, unreadable);
|
|
965
|
+
}
|
|
966
|
+
// FINAL FENCE, BEFORE THE ACK ITSELF, which is the real commit: acking tells the stream this
|
|
967
|
+
// shard's work is done. A daemon that stopped serving mid-fan-out must not make that claim, or
|
|
968
|
+
// the successor never sees the entry and the message is lost outright rather than duplicated.
|
|
969
|
+
if (!this.plane3MayAct())
|
|
970
|
+
return;
|
|
971
|
+
m.ack();
|
|
972
|
+
}
|
|
973
|
+
/** The DLV count the trusted writer stamps on a copy (SPEC §8): the stream's `max_msgs_per_subject`,
|
|
974
|
+
* read at most {@link DELIVERY_LIMIT_READ_MS} before the write, as a whole number or `none`. */
|
|
975
|
+
async deliveryLimitStamp() {
|
|
976
|
+
const at = Date.now();
|
|
977
|
+
if (this.deliveryLimitRead && at - this.deliveryLimitRead.at < DELIVERY_LIMIT_READ_MS)
|
|
978
|
+
return this.deliveryLimitRead.stamp;
|
|
979
|
+
const count = (await this.jsm.streams.info(dlvStream(this.space))).config.max_msgs_per_subject;
|
|
980
|
+
const stamp = count === HISTORY_NO_COUNT || count < 1 ? "none" : String(count);
|
|
981
|
+
this.deliveryLimitRead = { at, stamp };
|
|
982
|
+
return stamp;
|
|
983
|
+
}
|
|
984
|
+
/** Trusted-reader loop: bind the single privileged `reader` durable over `dinbox.>` and re-authorize
|
|
985
|
+
* + transfer each entry. */
|
|
986
|
+
async runReader() {
|
|
987
|
+
if (!this.js || !this.jsm)
|
|
988
|
+
return;
|
|
989
|
+
await this.addPlane3Durable(inboxStream(this.space), inboxReaderConfig(this.space, { ackWaitMs: this.ackWaitMs }));
|
|
990
|
+
const consumer = await this.js.consumers.get(inboxStream(this.space), INBOX_READER_DURABLE);
|
|
991
|
+
const msgs = await consumer.consume();
|
|
992
|
+
this.streamMsgs.push(msgs);
|
|
993
|
+
void (async () => {
|
|
994
|
+
for await (const m of msgs) {
|
|
995
|
+
try {
|
|
996
|
+
await this.readerHandle(m);
|
|
997
|
+
}
|
|
998
|
+
catch (e) {
|
|
999
|
+
if (!this.stopped)
|
|
1000
|
+
this.emit("error", e);
|
|
1001
|
+
try {
|
|
1002
|
+
m.nak();
|
|
1003
|
+
}
|
|
1004
|
+
catch { /* draining */ }
|
|
1005
|
+
}
|
|
1006
|
+
}
|
|
1007
|
+
})().catch((e) => { if (!this.stopped)
|
|
1008
|
+
this.emit("error", e); });
|
|
1009
|
+
}
|
|
1010
|
+
/** Whether one of the lifecycle's rows covering concrete `channel`, the channel's own or a family
|
|
1011
|
+
* above it, is eligible for the post at `seq`. One exact read per row, the channel's own first. */
|
|
1012
|
+
async lifecycleCoversPost(owner, lifecycleUid, channel, seq) {
|
|
1013
|
+
const kv = await this.membersRegistry();
|
|
1014
|
+
for (const row of [channel, ...familiesCovering(channel).reverse()]) {
|
|
1015
|
+
const rec = await readMember(kv, row, owner, lifecycleUid);
|
|
1016
|
+
if (rec && durableEligible(rec.record, seq))
|
|
1017
|
+
return true;
|
|
1018
|
+
}
|
|
1019
|
+
return false;
|
|
1020
|
+
}
|
|
1021
|
+
/** Remove a retired lifecycle's member row for `channel`, when one exists. Best effort: the row
|
|
1022
|
+
* authorizes nothing (its ACL row is retired), and the next entry fan-out writes against it
|
|
1023
|
+
* brings the reader back here. */
|
|
1024
|
+
async removeRetiredMemberRow(channel, owner, lifecycleUid) {
|
|
1025
|
+
try {
|
|
1026
|
+
const kv = await this.membersRegistry();
|
|
1027
|
+
if (await readMember(kv, channel, owner, lifecycleUid) === undefined)
|
|
1028
|
+
return;
|
|
1029
|
+
await deleteMember(kv, channel, owner, lifecycleUid);
|
|
1030
|
+
}
|
|
1031
|
+
catch (e) {
|
|
1032
|
+
this.emit("warning", new Error(`plane-3 reader: could not remove retired lifecycle ${owner}.${lifecycleUid}'s "${channel}" member row: ${e.message}`));
|
|
1033
|
+
}
|
|
1034
|
+
}
|
|
1035
|
+
/** Re-authorize ONE mixed-inbox entry and transfer it to the owner's DELIVER store. Deny (drop) on a
|
|
1036
|
+
* revoked/narrowed ACL or out-of-interval seq; on transfer success, ack the mixed entry (durability
|
|
1037
|
+
* has moved to DLV — an §8 equivalent per-member at-least-once mechanism). The agent acks DLV. */
|
|
1038
|
+
async readerHandle(m) {
|
|
1039
|
+
// FENCE, same rule as the fan-out writer: no effect and no ack once this daemon has stopped
|
|
1040
|
+
// serving the shard. The entry redelivers to the holder rather than being consumed here.
|
|
1041
|
+
if (!this.plane3MayAct())
|
|
1042
|
+
return;
|
|
1043
|
+
const pr = parseDinboxPrincipal(m.subject);
|
|
1044
|
+
if (!pr) {
|
|
1045
|
+
m.ack();
|
|
1046
|
+
return;
|
|
1047
|
+
} // unparseable subject (incl. a pre-cut 5-segment form) — not a real entry
|
|
1048
|
+
const owner = `${pr.owner}.${pr.actor}`; // the member principal dot-form (acl/member keys, msgID)
|
|
1049
|
+
let entry;
|
|
1050
|
+
try {
|
|
1051
|
+
entry = m.json();
|
|
1052
|
+
}
|
|
1053
|
+
catch {
|
|
1054
|
+
m.ack();
|
|
1055
|
+
return;
|
|
1056
|
+
} // undecodable — drop
|
|
1057
|
+
if (!isRecord(entry)) {
|
|
1058
|
+
m.ack();
|
|
1059
|
+
return;
|
|
1060
|
+
} // non-object envelope — permanently invalid
|
|
1061
|
+
const redeliveries = m.info?.deliveryCount ?? 1; // JsMsg delivery attempts (1 on first delivery)
|
|
1062
|
+
// Lifecycle-exact ACL re-auth (SPEC 13.1): the entry was addressed to pr.lifecycleUid's inbox, so
|
|
1063
|
+
// the row read is that lifecycle's exact key — a retired lifecycle's purged row reads as unknown
|
|
1064
|
+
// and its residual entries are removed below, never authorized against the successor's row.
|
|
1065
|
+
const acl = await this.plane3?.aclFor(owner, pr.lifecycleUid);
|
|
1066
|
+
if (acl === undefined) {
|
|
1067
|
+
// RETIRED lifecycle: its exact ACL key carries retirement's tombstone, so no attempt can ever
|
|
1068
|
+
// deliver this entry. Drop it AND remove it from the store. Deferring it to the ceiling and
|
|
1069
|
+
// keeping it would make every reader that starts from a fresh cursor pay the same sweep again.
|
|
1070
|
+
// An ACL row that is merely absent proves nothing and still defers below.
|
|
1071
|
+
if (await aclRetired(await this.aclRegistry(), owner, pr.lifecycleUid)) {
|
|
1072
|
+
if (!this.plane3MayAct())
|
|
1073
|
+
return;
|
|
1074
|
+
// Remove BEFORE the ack. The ack ends the reader's work on this entry, so acking first and
|
|
1075
|
+
// then failing the delete would keep the entry with nothing left to remove it. A failed delete
|
|
1076
|
+
// stays pending and retries, bounded by the same ceiling; an entry the stream no longer holds
|
|
1077
|
+
// is already removed (the broker answers a delete of a missing sequence with 10043).
|
|
1078
|
+
let failure;
|
|
1079
|
+
try {
|
|
1080
|
+
await this.jsm.streams.deleteMessage(inboxStream(this.space), m.seq, false);
|
|
1081
|
+
}
|
|
1082
|
+
catch (e) {
|
|
1083
|
+
if (!isJetStreamMissing(e, 10043))
|
|
1084
|
+
failure = e instanceof Error ? e : new Error(String(e));
|
|
1085
|
+
}
|
|
1086
|
+
// FENCE AFTER THE DELETE, which is broker I/O. A daemon that stopped serving while it was in
|
|
1087
|
+
// flight leaves the entry to the holder: a term here would move the shared durable past an entry
|
|
1088
|
+
// the failed delete left stored, so the holder would never retry it.
|
|
1089
|
+
if (!this.plane3MayAct())
|
|
1090
|
+
return;
|
|
1091
|
+
if (failure) {
|
|
1092
|
+
if (redeliveries >= READER_MAX_REDELIVERIES) {
|
|
1093
|
+
m.term();
|
|
1094
|
+
this.emit("error", new Error(`plane-3 reader: gave up removing entry ${m.seq} for retired lifecycle ${owner}.${pr.lifecycleUid} after ${redeliveries} redeliveries: ${failure.message}`));
|
|
1095
|
+
return;
|
|
1096
|
+
}
|
|
1097
|
+
m.nak(2000);
|
|
1098
|
+
return;
|
|
1099
|
+
}
|
|
1100
|
+
// The member row that addressed this entry is one a daemon left behind when it died between
|
|
1101
|
+
// its create and its post-write ACL read (#4994). It authorizes nothing, but fan-out copies
|
|
1102
|
+
// every later message on the channel into this inbox until it goes, and this reader then
|
|
1103
|
+
// removes each copy. Remove the row where that cost is paid, by its exact key, only when it
|
|
1104
|
+
// exists: a purge of an absent key would write a marker per dropped entry.
|
|
1105
|
+
if (entry.reason === "durable-channel")
|
|
1106
|
+
await this.removeRetiredMemberRow(entry.row ?? entry.channel, owner, pr.lifecycleUid);
|
|
1107
|
+
if (!this.plane3MayAct())
|
|
1108
|
+
return;
|
|
1109
|
+
m.ack();
|
|
1110
|
+
return;
|
|
1111
|
+
}
|
|
1112
|
+
// UNKNOWN owner — the manager has not (re)hydrated this owner's ACL yet (e.g. right after a
|
|
1113
|
+
// manager PROCESS restart). This is NOT a revocation: DEFER (redeliver), never drop — an ack here
|
|
1114
|
+
// would lose at-least-once on restart (impl-review BLOCKER-2). A delayed nak + a redelivery
|
|
1115
|
+
// ceiling stops one perma-unknown owner from head-of-lining the shared reader.
|
|
1116
|
+
// (Follow-up: the manager does not yet rehydrate its managed set across a process restart — until
|
|
1117
|
+
// it does, a long-unknown owner's entries term after the ceiling; tracked, not a silent ack-drop.)
|
|
1118
|
+
if (redeliveries >= READER_MAX_REDELIVERIES) {
|
|
1119
|
+
m.term();
|
|
1120
|
+
this.emit("error", new Error(`plane-3 reader: gave up on entry for unknown owner ${owner} after ${redeliveries} redeliveries`));
|
|
1121
|
+
return;
|
|
1122
|
+
}
|
|
1123
|
+
m.nak(2000);
|
|
1124
|
+
return;
|
|
1125
|
+
}
|
|
1126
|
+
// KNOWN owner whose CURRENT ACL no longer covers the channel — a revocation/narrowing. Drop: the
|
|
1127
|
+
// entry is no longer authorized (SPEC §7 current-ACL gate before surfacing).
|
|
1128
|
+
if (!channelInAllow(acl, entry.channel)) {
|
|
1129
|
+
m.ack();
|
|
1130
|
+
return;
|
|
1131
|
+
}
|
|
1132
|
+
// INTERVAL re-auth (not a current-member boolean): a pre-leave entry (seq ≤ leaveCursor) stays
|
|
1133
|
+
// deliverable; seq > leaveCursor (or after a rejoin's newer joinCursor) is the hard cut. Any of the
|
|
1134
|
+
// lifecycle's rows that cover the channel may hold the post, whichever row addressed the copy.
|
|
1135
|
+
if (entry.reason === "durable-channel" && !(await this.lifecycleCoversPost(owner, pr.lifecycleUid, entry.channel, entry.seq))) {
|
|
1136
|
+
m.ack();
|
|
1137
|
+
return;
|
|
1138
|
+
}
|
|
1139
|
+
// COMMIT-POINT FENCE. Everything above this line is broker I/O - the ACL read and the member
|
|
1140
|
+
// re-read - so the entry check at the top of the handler is stale by now if this unit was
|
|
1141
|
+
// descheduled inside either. The transfer below is the effect that must not happen on behalf of
|
|
1142
|
+
// a shard we no longer serve; returning without acking leaves it for the holder.
|
|
1143
|
+
if (!this.plane3MayAct())
|
|
1144
|
+
return;
|
|
1145
|
+
try {
|
|
1146
|
+
// DLV has no original chat subject, so preserve the channel the trusted fan-out reader derived
|
|
1147
|
+
// from CHAT. Never let the publisher-controlled payload label choose connector attention.
|
|
1148
|
+
const frame = {
|
|
1149
|
+
version: 1,
|
|
1150
|
+
channel: entry.channel,
|
|
1151
|
+
msg: authenticatedChannelMessage(entry.msg, entry.channel),
|
|
1152
|
+
};
|
|
1153
|
+
const frameHeaders = headers();
|
|
1154
|
+
frameHeaders.set(PLANE3_FRAME_HEADER, "1");
|
|
1155
|
+
// The count read may be broker I/O, so the fence is asked again before the write.
|
|
1156
|
+
frameHeaders.set(DELIVERY_LIMIT_HEADER, await this.deliveryLimitStamp());
|
|
1157
|
+
if (!this.plane3MayAct())
|
|
1158
|
+
return;
|
|
1159
|
+
// Lifecycle-keyed subject + msgID (ours): the DLV consumer filters on the lifecycle-scoped
|
|
1160
|
+
// subject (streams.ts filter_subject), and a predecessor's transferred copy must never suppress
|
|
1161
|
+
// a successor's under the same alias (disjoint lifecycles, stream-wide dedupe).
|
|
1162
|
+
await this.js.publish(dlvSubject(this.space, pr.owner, pr.actor, pr.lifecycleUid), JSON.stringify(frame), {
|
|
1163
|
+
...this.plane3MsgId(entry, `${owner}:${pr.lifecycleUid}`),
|
|
1164
|
+
headers: frameHeaders,
|
|
1165
|
+
});
|
|
1166
|
+
}
|
|
1167
|
+
catch {
|
|
1168
|
+
// EVERY transfer failure is deliberately a retry, never a successful result. Keep the entry
|
|
1169
|
+
// pending (redeliver), bounded by the same ceiling so a poison
|
|
1170
|
+
// entry can't head-of-line the shared reader forever.
|
|
1171
|
+
if (redeliveries >= READER_MAX_REDELIVERIES) {
|
|
1172
|
+
m.term();
|
|
1173
|
+
this.emit("error", new Error(`plane-3 reader: gave up transferring ${entry.msg.id} for ${owner} after ${redeliveries} redeliveries`));
|
|
1174
|
+
return;
|
|
1175
|
+
}
|
|
1176
|
+
m.nak(2000);
|
|
1177
|
+
return;
|
|
1178
|
+
}
|
|
1179
|
+
// The ack is the last commit: the transfer above already happened, so the only question left is
|
|
1180
|
+
// whether THIS daemon may retire the source entry. If it stopped serving mid-transfer, leave the
|
|
1181
|
+
// entry pending - the successor re-transfers and stream-wide dedupe on the msgID collapses the
|
|
1182
|
+
// duplicate, which is the recoverable direction. Acking here is the unrecoverable one.
|
|
1183
|
+
if (!this.plane3MayAct())
|
|
1184
|
+
return;
|
|
1185
|
+
m.ack();
|
|
1186
|
+
}
|
|
1187
|
+
/** Agent-side: bind + pump our pre-created Plane-3 DELIVER durable (`dlv_<id>`). Every message here is
|
|
1188
|
+
* delivery-daemon-written (DLV is delivery-write-only, broker-enforced) and is a CHANNEL message by contract
|
|
1189
|
+
* (the backstop never carries DMs), so `kind=channel` is path-derived (SPEC §4) and the body is
|
|
1190
|
+
* trusted (no spoof-guard). `durable:true` — real JetStream ack, coalesced with the core-sub live
|
|
1191
|
+
* copy by `MeshAgent.ingest`. No-op when the durable isn't present (open mode / not provisioned). A read
|
|
1192
|
+
* list declared on a credential that cannot fetch the durable refuses. */
|
|
1193
|
+
async pumpDlv() {
|
|
1194
|
+
if (!this.js)
|
|
1195
|
+
return;
|
|
1196
|
+
if (!this.ownLifecycleUid)
|
|
1197
|
+
return; // no lifecycle uid — never provisioned for Plane-3 (its durable is lifecycle-keyed)
|
|
1198
|
+
if (this.allowSubscribe?.length === 0)
|
|
1199
|
+
return; // known channel-less grant: no DLV durable or bind rows exist
|
|
1200
|
+
const durable = dlvDurable(this.owner, this.actor, this.ownLifecycleUid);
|
|
1201
|
+
if (this.allowSubscribe === undefined && !this.userMode && this.nc) {
|
|
1202
|
+
// Native credential-only clients have no separate ACL option. Use the publish permissions
|
|
1203
|
+
// captured when THIS connection authenticated; a callout sentinel does not carry the grant.
|
|
1204
|
+
const permission = this.watchDeletePermissions.get(this.nc)?.();
|
|
1205
|
+
// Only an absent allow selects no binding. A declared bind with a deny still reaches the
|
|
1206
|
+
// broker and fails, so a refusal cannot masquerade as a channel-less credential.
|
|
1207
|
+
if (permission && !issuedPermitsSubject({ ...permission, deny: [] }, `$JS.API.CONSUMER.INFO.${dlvStream(this.space)}.${durable}`))
|
|
1208
|
+
return;
|
|
1209
|
+
}
|
|
1210
|
+
let consumer;
|
|
1211
|
+
try {
|
|
1212
|
+
consumer = await this.js.consumers.get(dlvStream(this.space), durable);
|
|
1213
|
+
}
|
|
1214
|
+
catch (e) {
|
|
1215
|
+
if (isJetStreamMissing(e, JetStreamApiCodes.ConsumerNotFound)) {
|
|
1216
|
+
// A channel-less credential may ask about its durable but holds no fetch row for it. A read
|
|
1217
|
+
// list declared on such a credential disagrees with the grant, so the start refuses.
|
|
1218
|
+
const issued = this.nc && this.watchDeletePermissions.get(this.nc)?.();
|
|
1219
|
+
if ((this.allowSubscribe?.length ?? 0) > 0 && issued
|
|
1220
|
+
&& !issuedPermitsSubject({ ...issued, deny: [] }, `$JS.API.CONSUMER.MSG.NEXT.${dlvStream(this.space)}.${durable}`))
|
|
1221
|
+
throw new Error(`this connection declared a read list, but its credential cannot fetch delivery durable "${durable}" for space "${this.space}"; the read list and the credential disagree`);
|
|
1222
|
+
return;
|
|
1223
|
+
}
|
|
1224
|
+
throw e; // a denied bind is not evidence that the read ACL is empty or the durable is absent
|
|
1225
|
+
}
|
|
1226
|
+
const lease = await this.readDeliveryLease(0);
|
|
1227
|
+
const liveDeliveryPlane = lease?.ready === true;
|
|
1228
|
+
if (!liveDeliveryPlane)
|
|
1229
|
+
this.emit("warning", new Error(`delivery durable "${durable}" for space "${this.space}" bound while the plane this connection reaches has no ready delivery lease, so the durable delivers nothing until a delivery daemon serves this plane. If a daemon serves this space on another plane, reconnect against it, which re-binds the durable there.`));
|
|
1230
|
+
const ackWaitMs = await ackWaitOf(consumer);
|
|
1231
|
+
const msgs = await consumer.consume();
|
|
1232
|
+
this.streamMsgs.push(msgs);
|
|
1233
|
+
void (async () => {
|
|
1234
|
+
for await (const m of msgs) {
|
|
1235
|
+
// A copy with a full store behind it means the store held its count, so older copies may have
|
|
1236
|
+
// been discarded unread (#5404); NATS gives no notice of the discard itself. The count is the
|
|
1237
|
+
// one the trusted writer stamped on this copy. Checked on every delivery, ahead of the checks
|
|
1238
|
+
// below and the connector's id dedupe, so a copy the agent also got live cannot hide it. The
|
|
1239
|
+
// connector folds repeats into one notice.
|
|
1240
|
+
let limit;
|
|
1241
|
+
try {
|
|
1242
|
+
limit = deliveryLimitOf(m.headers?.get(DELIVERY_LIMIT_HEADER));
|
|
1243
|
+
}
|
|
1244
|
+
catch (e) {
|
|
1245
|
+
this.emit("error", e);
|
|
1246
|
+
limit = null;
|
|
1247
|
+
}
|
|
1248
|
+
if (limit !== null && m.info.pending >= limit - 1)
|
|
1249
|
+
this.emit("stored-copies-full", { limit });
|
|
1250
|
+
// The header is minted only by the trusted DLV writer. A body-only discriminator is
|
|
1251
|
+
// insufficient because old Cotal envelopes permit unknown fields and could imitate it.
|
|
1252
|
+
if (m.headers?.get(PLANE3_FRAME_HEADER) !== "1") {
|
|
1253
|
+
this.emit("error", new Error("plane-3 delivery: unauthenticated or unversioned DLV entry terminated"));
|
|
1254
|
+
try {
|
|
1255
|
+
m.term();
|
|
1256
|
+
}
|
|
1257
|
+
catch { /* draining */ }
|
|
1258
|
+
continue;
|
|
1259
|
+
}
|
|
1260
|
+
let raw;
|
|
1261
|
+
try {
|
|
1262
|
+
raw = m.json();
|
|
1263
|
+
}
|
|
1264
|
+
catch (e) {
|
|
1265
|
+
this.emit("error", e);
|
|
1266
|
+
try {
|
|
1267
|
+
m.term();
|
|
1268
|
+
}
|
|
1269
|
+
catch { /* draining */ }
|
|
1270
|
+
continue;
|
|
1271
|
+
}
|
|
1272
|
+
if (!isPlane3DeliveryFrame(raw)) {
|
|
1273
|
+
this.emit("error", new Error("plane-3 delivery: malformed versioned DLV entry terminated"));
|
|
1274
|
+
try {
|
|
1275
|
+
m.term();
|
|
1276
|
+
}
|
|
1277
|
+
catch { /* draining */ }
|
|
1278
|
+
continue;
|
|
1279
|
+
}
|
|
1280
|
+
const msg = authenticatedChannelMessage(raw.msg, raw.channel);
|
|
1281
|
+
if (msg.from?.id === this.card.id) {
|
|
1282
|
+
m.ack();
|
|
1283
|
+
continue;
|
|
1284
|
+
} // own echo (defensive)
|
|
1285
|
+
this.emit("message", msg, durableDelivery(m, ackWaitMs), { historical: false, kind: "channel" });
|
|
1286
|
+
}
|
|
1287
|
+
})().catch((e) => { if (!this.stopped)
|
|
1288
|
+
this.emit("error", e); });
|
|
1289
|
+
}
|
|
1290
|
+
}
|
|
1291
|
+
//# sourceMappingURL=endpoint-delivery-plane.js.map
|