@cotal-ai/core 0.168.0 → 0.169.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-file.d.ts.map +1 -1
- package/dist/agent-file.js +12 -9
- package/dist/agent-file.js.map +1 -1
- package/dist/connector-setup.d.ts +2 -0
- package/dist/connector-setup.d.ts.map +1 -1
- package/dist/connector.d.ts +8 -11
- package/dist/connector.d.ts.map +1 -1
- package/dist/control-exchange.d.ts +114 -0
- package/dist/control-exchange.d.ts.map +1 -0
- package/dist/control-exchange.js +416 -0
- package/dist/control-exchange.js.map +1 -0
- package/dist/endpoint-call.d.ts +2 -1
- package/dist/endpoint-call.d.ts.map +1 -1
- package/dist/endpoint-call.js +14 -7
- package/dist/endpoint-call.js.map +1 -1
- package/dist/endpoint-delivery-plane.d.ts +337 -0
- package/dist/endpoint-delivery-plane.d.ts.map +1 -0
- package/dist/endpoint-delivery-plane.js +1291 -0
- package/dist/endpoint-delivery-plane.js.map +1 -0
- package/dist/endpoint-envelope.d.ts +2 -1
- package/dist/endpoint-envelope.d.ts.map +1 -1
- package/dist/endpoint-envelope.js +9 -3
- package/dist/endpoint-envelope.js.map +1 -1
- package/dist/endpoint-history.d.ts +370 -0
- package/dist/endpoint-history.d.ts.map +1 -0
- package/dist/endpoint-history.js +969 -0
- package/dist/endpoint-history.js.map +1 -0
- package/dist/endpoint-inbound.d.ts +10 -0
- package/dist/endpoint-inbound.d.ts.map +1 -0
- package/dist/endpoint-inbound.js +28 -0
- package/dist/endpoint-inbound.js.map +1 -0
- package/dist/endpoint-serve.d.ts +11 -1
- package/dist/endpoint-serve.d.ts.map +1 -1
- package/dist/endpoint-serve.js +42 -5
- package/dist/endpoint-serve.js.map +1 -1
- package/dist/endpoint-subjects.d.ts +3 -0
- package/dist/endpoint-subjects.d.ts.map +1 -1
- package/dist/endpoint-subjects.js +3 -0
- package/dist/endpoint-subjects.js.map +1 -1
- package/dist/endpoint.d.ts +64 -253
- package/dist/endpoint.d.ts.map +1 -1
- package/dist/endpoint.js +177 -2015
- package/dist/endpoint.js.map +1 -1
- package/dist/index.d.ts +2 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -1
- package/dist/index.js.map +1 -1
- package/dist/launch-material.d.ts +9 -3
- package/dist/launch-material.d.ts.map +1 -1
- package/dist/launch-material.js +22 -2
- package/dist/launch-material.js.map +1 -1
- package/dist/provision-permissions.d.ts.map +1 -1
- package/dist/provision-permissions.js +7 -1
- package/dist/provision-permissions.js.map +1 -1
- package/dist/remote-manager-authority.d.ts +1 -1
- package/dist/run-journal.d.ts +2 -0
- package/dist/run-journal.d.ts.map +1 -1
- package/dist/run-journal.js +1 -1
- package/dist/run-journal.js.map +1 -1
- package/package.json +5 -1
package/dist/endpoint.js
CHANGED
|
@@ -1,12 +1,13 @@
|
|
|
1
|
+
import { EndpointHistory } from "./endpoint-history.js";
|
|
2
|
+
export { MULTI_FILTER_BATCH, MULTI_FILTER_READ_CONCURRENCY } from "./endpoint-history.js";
|
|
1
3
|
import { eventPayloadSubject, eventPayloadStream } from "./event-payloads.js";
|
|
2
4
|
import { requestSafeReconnect } from "./request-safe-reconnect.js";
|
|
3
5
|
import { nativeBindingAfter, nativeConnectionBinding } from "./native-connection.js";
|
|
4
6
|
import { EventEmitter } from "node:events";
|
|
5
7
|
import { drainAndClose } from "./connection-cleanup.js";
|
|
6
8
|
import { randomUUID } from "node:crypto";
|
|
7
|
-
import { setTimeout as delay } from "node:timers/promises";
|
|
8
9
|
import { createConnection, isIP } from "node:net";
|
|
9
|
-
import { connect, headers, tokenAuthenticator, nanos,
|
|
10
|
+
import { connect, headers, tokenAuthenticator, nanos, AuthorizationError, PermissionViolationError, UserAuthenticationExpiredError, NoRespondersError, RequestError, TimeoutError, ClosedConnectionError, ConnectionError, } from "@nats-io/transport-node";
|
|
10
11
|
import { wsconnect } from "@nats-io/nats-core";
|
|
11
12
|
import { parseLivenessAnswer, responderFromLease, responderFromProbe, isLivenessPlane, LIVENESS_PLANES, } from "./liveness.js";
|
|
12
13
|
import { EndpointPresence } from "./endpoint-presence.js";
|
|
@@ -17,45 +18,41 @@ import { armRenewal, renewalDeadlineMs, jitteredBackoffMs } from "./renewal.js";
|
|
|
17
18
|
import { requireBrokerFloor } from "./broker-floor.js";
|
|
18
19
|
import { withDeadline } from "./deadline.js";
|
|
19
20
|
import { inspectCredHealth } from "./provision.js";
|
|
20
|
-
import {
|
|
21
|
+
import {} from "./secret-store.js";
|
|
21
22
|
import { BIND_SPLIT_REISSUES, resolveService, invokeCommand, goalFollowRefusal, submitAndFollowGoal } from "./endpoint-invoke.js";
|
|
22
23
|
import { EpEnvelopeError, respondedButUnbound, replyRefusedBeforeEffect, replyTargetUnmapped, EP_BIND_REFUSED } from "./endpoint-envelope.js";
|
|
23
24
|
import { isRepeatSafeCommand } from "./endpoint-grants.js";
|
|
24
25
|
import { assertIdToken, assertGeneration, assertRuntimeToken, endpointToken } from "./endpoint-subjects.js";
|
|
25
26
|
import { servePrincipalRoute, describeContracts } from "./endpoint-serve.js";
|
|
26
27
|
import { epPrincipalCall } from "./endpoint-call.js";
|
|
27
|
-
import { AGENT_CONTRACTS, AGENT_ENDPOINT } from "./agent-endpoint.js";
|
|
28
|
+
import { AGENT_CLUSTER, AGENT_CONTRACTS, AGENT_ENDPOINT } from "./agent-endpoint.js";
|
|
28
29
|
import { processRuntime } from "./process-runtime.js";
|
|
29
30
|
import { importNativeSubjectPermissions, issuedPermitsSubject, readAcceptedRow } from "./issued-authority.js";
|
|
30
31
|
import { REGISTRY_ENDPOINT, REGISTRY_ENDPOINTS_COMMAND } from "./registry-contract.js";
|
|
31
32
|
import { listRegistryEndpoints, registryListingCaller } from "./registry-client.js";
|
|
32
33
|
import { lastKvEntries } from "./kv-scan.js";
|
|
33
|
-
import {
|
|
34
|
-
import { isRestoredConsumer } from "./backup.js";
|
|
35
|
-
import { ARTIFACT_PART_KIND, isArtifactPart } from "./artifact.js";
|
|
34
|
+
import { isConsumerNotFound, isJetStreamMissing, isStreamNotFound } from "./jetstream-missing.js";
|
|
36
35
|
import { assertValidKind } from "./agent-file.js";
|
|
37
36
|
import { assertValidName } from "./resolve.js";
|
|
38
|
-
import { EVICT_PRINCIPALS_MAX } from "./evict.js";
|
|
39
37
|
import { nextHeartbeat } from "./presence-heartbeat.js";
|
|
40
|
-
import { anycastStoreDefinition, createSpaceStreams, dmDurableConfig, dlvDurableConfig, taskDurableConfig,
|
|
41
|
-
import { DELIVERY_LIMIT_HEADER,
|
|
42
|
-
import { jetstream, jetstreamManager, AckPolicy, DeliverPolicy, JetStreamApiCodes,
|
|
38
|
+
import { anycastStoreDefinition, createSpaceStreams, dmDurableConfig, dlvDurableConfig, taskDurableConfig, MANAGER_LEASE_TTL_MS, MANAGER_LEASE_ATTEMPT_MS, PRESENCE_STORAGE } from "./streams.js";
|
|
39
|
+
import { DELIVERY_LIMIT_HEADER, PLANE3_MAX_MSGS_PER_SUBJECT } from "./backup-config.js";
|
|
40
|
+
import { jetstream, jetstreamManager, AckPolicy, DeliverPolicy, JetStreamApiCodes, } from "@nats-io/jetstream";
|
|
43
41
|
import {} from "@nats-io/jetstream";
|
|
44
42
|
import { Kvm } from "@nats-io/kv";
|
|
45
43
|
import { Bucket, KvWatchInclude } from "@nats-io/kv/internal";
|
|
46
|
-
import { openMembersRegistry, commitMemberRevision, tombstoneMember, activateMember, readMember, deleteMember, listMembers,
|
|
47
|
-
import { openAclRegistry, readAcl, readAclForAlias,
|
|
44
|
+
import { openMembersRegistry, commitMemberRevision, tombstoneMember, activateMember, readMember, deleteMember, listMembers, } from "./members.js";
|
|
45
|
+
import { openAclRegistry, readAcl, readAclForAlias, AmbiguousAclAlias, commitAcl as writeAclRecord, reissueAcl as writeAclReissue } from "./acls.js";
|
|
48
46
|
import { EndpointDeliveryLease } from "./endpoint-delivery-lease.js";
|
|
47
|
+
import { EndpointDeliveryPlane } from "./endpoint-delivery-plane.js";
|
|
48
|
+
import { ackWaitOf, authenticatedChannelMessage, durableDelivery, isRecord } from "./endpoint-inbound.js";
|
|
49
49
|
import { openChannelRegistry, effectiveReplay, effectiveReplayWindowMs, effectiveDeliveryClass, readChannelConfig, readChannelDefaults, decodeChannelEntry, } from "./channels.js";
|
|
50
50
|
import { isCasLoss } from "./endpoint-records.js";
|
|
51
51
|
import { ANYCAST_CANCEL_OP, ANYCAST_NOTICE_KEY, ANYCAST_RECORDS_CREDENTIAL_OUTDATED, ANYCAST_STATUS_OP, ANYCAST_TAKES_OP, AnycastRequestAuthority, isAnycastSeq, } from "./anycast-requests.js";
|
|
52
|
-
import { anycastSubject, assertValidOwnerToken, assertValidRole, CHANNEL_DEFAULTS_KEY, chatStream,
|
|
52
|
+
import { anycastSubject, assertValidOwnerToken, assertValidRole, CHANNEL_DEFAULTS_KEY, chatStream, chatSubject, controlServiceSubject, livenessSubject, livenessServeFilter, CONTROL_DELIVERY, CONTROL_DELIVERY_ADMIN, dmStream, dmDurable, dlvStream, dinboxSubject, FANOUT_DURABLE, managerBucket, MANAGER_LEASE_KEY, MANAGER_RENEWAL_LEASE_KEY, managerLeaseKey, membershipInAllow, isConcreteChannel, isChannelFamily, normalizeMentions, MENTION_NOTICE_KEY, parseSubject, isPrincipalOwnerToken, assertInboxConnId, presenceBucket, channelBucket, membershipBucket, MEMBERSHIP_FEED_KEY, principalKey, parsePrincipalKey, assertLifecycleToken, mintLifecycleUid, lifecycleNameKey, memberKey, DEV_OWNER, spacePrefix, spaceWildcard, subjectMatches, taskStream, taskDurable, takeServeFilter, takeSubject, parseTakeSubject, TAKE_ROUTE, token, unicastSubject, unicastRecvFilter, } from "./subjects.js";
|
|
53
53
|
import { isPermissionDenied } from "./permission-denied.js";
|
|
54
54
|
export { isPermissionDenied };
|
|
55
55
|
export const DEFAULT_SERVER = "nats://127.0.0.1:4222";
|
|
56
|
-
const PLANE3_FRAME_HEADER = "Cotal-Delivery-Frame";
|
|
57
|
-
/** The `ctl.delivery` ops a sender calls about one of its own anycast requests (SPEC §8.1). */
|
|
58
|
-
const ANYCAST_CONTROL_OPS = new Set([ANYCAST_STATUS_OP, ANYCAST_CANCEL_OP, ANYCAST_TAKES_OP]);
|
|
59
56
|
/** Takes a daemon answers at once; one more is refused as busy, so the holder keeps its copy and
|
|
60
57
|
* asks again rather than queueing work the daemon cannot bound. */
|
|
61
58
|
const MAX_TAKES_IN_FLIGHT = 64;
|
|
@@ -63,8 +60,6 @@ const MAX_TAKES_IN_FLIGHT = 64;
|
|
|
63
60
|
const ANYCAST_SWEEP_MS = 10 * 60 * 1000;
|
|
64
61
|
/** How long a holder waits for the daemon to answer a take. */
|
|
65
62
|
const ANYCAST_TAKE_TIMEOUT_MS = 5_000;
|
|
66
|
-
/** How long the trusted writer reuses its read of the DLV count before reading it again. */
|
|
67
|
-
const DELIVERY_LIMIT_READ_MS = 5_000;
|
|
68
63
|
/** The DLV count a stored copy was written under, from its {@link DELIVERY_LIMIT_HEADER} (SPEC §8):
|
|
69
64
|
* a whole number of at least 1, null for `none`, and 1000 when the header is absent, since only a
|
|
70
65
|
* daemon from before the count was configurable writes a copy without it and its count was 1000.
|
|
@@ -80,31 +75,6 @@ export function deliveryLimitOf(value) {
|
|
|
80
75
|
}
|
|
81
76
|
/** Space joined when none is given on the CLI (the `cotal-<space>` cmux tab, etc.). */
|
|
82
77
|
export const DEFAULT_SPACE = "main";
|
|
83
|
-
/** How many channel filters one multi-filter consumer create may carry.
|
|
84
|
-
*
|
|
85
|
-
* A create names every requested channel in one request, so the request grows with the channel
|
|
86
|
-
* count and the CLIENT's request timeout is what gives way, not the broker: the create is never
|
|
87
|
-
* refused, it just does not answer. Measured on an isolated broker, one message per channel,
|
|
88
|
-
* `limit=5`: 70 filters answer in 43ms, 1,000 in 246ms, 5,000 in 5,345ms, and 10,000 does not
|
|
89
|
-
* answer at all, failing `timeout` after 5,023ms. Note 5,000 SUCCEEDED while taking longer than
|
|
90
|
-
* 10,000 took to fail, which is what says the ceiling is on the create request rather than on the
|
|
91
|
-
* read: past roughly 5s the create itself is what times out.
|
|
92
|
-
*
|
|
93
|
-
* 1,000 is chosen from that sweep rather than from the failure point: it is a fifth of the largest
|
|
94
|
-
* count that still answered, and it answers in a quarter second, so a batch stays far away from
|
|
95
|
-
* both the timeout and the response-deadline budget the dashboard has to share with its DM read.
|
|
96
|
-
* A space with the 69 chat channels this was built for is still ONE read, so the round-trip claim
|
|
97
|
-
* in #1210 is unchanged at that size. */
|
|
98
|
-
export const MULTI_FILTER_BATCH = 1_000;
|
|
99
|
-
/** How many filter batches may be in flight at once. Bounded because the point of #1210 was to stop
|
|
100
|
-
* issuing one read per channel: a space large enough to need batches must not get the fan-out back
|
|
101
|
-
* under another name. */
|
|
102
|
-
export const MULTI_FILTER_READ_CONCURRENCY = 4;
|
|
103
|
-
/** A history reader's inactive threshold. It bounds only a reader whose delete never ran (its link
|
|
104
|
-
* dropped, or its process stopped), because every read deletes its readers when it ends. Five
|
|
105
|
-
* minutes is what the nats.js ordered consumers it replaces carried, and it keeps a reader alive
|
|
106
|
-
* across the gap between pulls on a slow link. */
|
|
107
|
-
const HISTORY_READER_INACTIVE_MS = 5 * 60_000;
|
|
108
78
|
/** The longest delay `setTimeout` honors, 2^31-1 ms. */
|
|
109
79
|
const MAX_TIMER_MS = 2_147_483_647;
|
|
110
80
|
/** This endpoint's own `dm_` durable does not exist. A DM-only endpoint, and any endpoint on an
|
|
@@ -133,35 +103,6 @@ export class DmBindRefusedError extends Error {
|
|
|
133
103
|
this.name = "DmBindRefusedError";
|
|
134
104
|
}
|
|
135
105
|
}
|
|
136
|
-
/**
|
|
137
|
-
* Events: "message" (CotalMessage), "presence" (PresenceEvent), "roster" (Presence[]), "error" (Error),
|
|
138
|
-
* "connection" ({ connected: boolean }) — true on every successful (re)bind (initial start, manual
|
|
139
|
-
* reconnect, AND background self-heal), false the moment the connection drops (rebuild null window /
|
|
140
|
-
* terminal close). Lets an in-process agent track connectedness off the endpoint's own (re)binds
|
|
141
|
-
* instead of an imperative flag the self-heal path can't reach; "transport" ({ connected, server? })
|
|
142
|
-
* is the lower-level NATS socket edge, true before the full bind finishes and false during an internal
|
|
143
|
-
* nats.js reconnect without changing `connection` readiness. "stored-copies-full" ({ limit: number }): a
|
|
144
|
-
* stored copy arrived with the store at the count the copy was written under ({@link deliveryLimitOf}),
|
|
145
|
-
* so the store may have discarded older copies before this session read them; emitted for each such
|
|
146
|
-
* copy. A lowering of the count or an age set on the store removes copies without this event.
|
|
147
|
-
*
|
|
148
|
-
* Callers MUST attach an "error" listener before `start()`: async faults (incl. NATS
|
|
149
|
-
* permission denials, surfaced via `watchStatus`) are emitted as "error", and Node throws
|
|
150
|
-
* synchronously on an unhandled "error" — a missing listener turns any such fault into a
|
|
151
|
-
* process crash instead of a logged denial.
|
|
152
|
-
*/
|
|
153
|
-
/** Plane-3 trusted-reader redelivery ceiling: a dinbox entry that keeps failing re-auth-defer
|
|
154
|
-
* (unknown owner) or DELIVER transfer is `term()`d + surfaced after this many redeliveries, so one
|
|
155
|
-
* stuck/poison entry can't head-of-line the single shared reader forever. */
|
|
156
|
-
const READER_MAX_REDELIVERIES = 10;
|
|
157
|
-
/** Page size for the mediated history read when the caller names none — matches `channelHistory`'s
|
|
158
|
-
* own default, since the mediated read exists to be a drop-in for it. */
|
|
159
|
-
const READ_HISTORY_DEFAULT_LIMIT = 100;
|
|
160
|
-
/** Hard server-side ceiling on one mediated page. The caller PROPOSES a limit and the mediator
|
|
161
|
-
* decides: the reader is pooled and privileged, so an unbounded caller-chosen limit would let one
|
|
162
|
-
* request pull a channel's whole retained set through it. Clamped, not refused — a UI asking for
|
|
163
|
-
* more than a page should get a page, and `complete: false` already tells it more remains. */
|
|
164
|
-
const READ_HISTORY_MAX_LIMIT = 200;
|
|
165
106
|
/** The NEWEST prefix-from-the-end of `items` whose serialized `readHistory` reply fits `budget`
|
|
166
107
|
* bytes, order preserved. Returns `[]` when not even the newest single message fits — the caller
|
|
167
108
|
* must refuse loudly there rather than serve an empty page, which would read as "no history".
|
|
@@ -309,10 +250,52 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
309
250
|
/** Caller-owned membership watches survive a connection rebuild as INTENT. Their iterators are
|
|
310
251
|
* connection-scoped and are stopped/re-created around the epoch swap. */
|
|
311
252
|
membershipFeedWatches = new Set();
|
|
312
|
-
/** The
|
|
313
|
-
*
|
|
314
|
-
|
|
315
|
-
|
|
253
|
+
/** The plane-3 delivery daemon side of this endpoint (fan-out writer, trusted reader, control
|
|
254
|
+
* responders) and the agent's DELIVER pump. Every dependency is read when it is used. */
|
|
255
|
+
deliveryPlane = new EndpointDeliveryPlane({
|
|
256
|
+
ackWaitMs: () => this.ackWaitMs,
|
|
257
|
+
actor: () => this.actor,
|
|
258
|
+
owner: () => this.owner,
|
|
259
|
+
allowSubscribe: () => this.allowSubscribe,
|
|
260
|
+
anycastUnavailable: () => this.anycastUnavailable,
|
|
261
|
+
card: () => this.card,
|
|
262
|
+
js: () => this.js,
|
|
263
|
+
jsm: () => this.jsm,
|
|
264
|
+
nc: () => this.nc,
|
|
265
|
+
ownLifecycleUid: () => this.ownLifecycleUid,
|
|
266
|
+
residentSwap: () => this.residentSwap,
|
|
267
|
+
space: () => this.space,
|
|
268
|
+
stopped: () => this.stopped,
|
|
269
|
+
streamMsgs: () => this.streamMsgs,
|
|
270
|
+
subs: () => this.subs,
|
|
271
|
+
userMode: () => this.userMode,
|
|
272
|
+
watchDeletePermissions: () => this.watchDeletePermissions,
|
|
273
|
+
maxPayload: () => this.maxPayload,
|
|
274
|
+
aclForAlias: (...args) => this.aclForAlias(...args),
|
|
275
|
+
channelHistory: (...args) => this.channelHistory(...args),
|
|
276
|
+
deliveryClassFresh: (...args) => this.deliveryClassFresh(...args),
|
|
277
|
+
durableJoinFor: (...args) => this.durableJoinFor(...args),
|
|
278
|
+
durableLeaveFor: (...args) => this.durableLeaveFor(...args),
|
|
279
|
+
ownerMemberships: (...args) => this.ownerMemberships(...args),
|
|
280
|
+
ownsDeliveryLease: (...args) => this.ownsDeliveryLease(...args),
|
|
281
|
+
readDeliveryLease: (...args) => this.readDeliveryLease(...args),
|
|
282
|
+
readDeliveryLeaseEntry: (...args) => this.readDeliveryLeaseEntry(...args),
|
|
283
|
+
reloadCreds: (...args) => this.reloadCreds(...args),
|
|
284
|
+
serveControl: (...args) => this.serveControl(...args),
|
|
285
|
+
aclRegistry: (...args) => this.aclRegistry(...args),
|
|
286
|
+
membersRegistry: (...args) => this.membersRegistry(...args),
|
|
287
|
+
armAnycastRequests: (...args) => this.armAnycastRequests(...args),
|
|
288
|
+
deliveryAnycast: (...args) => this.deliveryAnycast(...args),
|
|
289
|
+
dropAnycastRequests: (...args) => this.dropAnycastRequests(...args),
|
|
290
|
+
manager: (...args) => this.manager(...args),
|
|
291
|
+
notLiveMsg: (...args) => this.notLiveMsg(...args),
|
|
292
|
+
plane3MsgId: (...args) => this.plane3MsgId(...args),
|
|
293
|
+
publishDinbox: (...args) => this.publishDinbox(...args),
|
|
294
|
+
resolveOwnerByName: (...args) => this.resolveOwnerByName(...args),
|
|
295
|
+
scheduleResidentSwap: (...args) => this.scheduleResidentSwap(...args),
|
|
296
|
+
sendMentionNotice: (...args) => this.sendMentionNotice(...args),
|
|
297
|
+
emit: (event, ...args) => this.emit(event, ...args),
|
|
298
|
+
});
|
|
316
299
|
/** The anycast take responder, the request records it writes, and their maintenance timer
|
|
317
300
|
* (delivery daemon, SPEC §8.1). Bound with the other Plane-3 responders and dropped with them. */
|
|
318
301
|
takeServeSub;
|
|
@@ -324,24 +307,6 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
324
307
|
anycastUnavailable;
|
|
325
308
|
anycastSweep;
|
|
326
309
|
takesInFlight = 0;
|
|
327
|
-
/** The lifecycles whose membership inventory this daemon has been asked for, as principal to uids
|
|
328
|
-
* (#4965). A durable join or leave bound to one of them is refused, so no row of a retiring lifecycle
|
|
329
|
-
* is written after its inventory read starts. An entry goes when a later inventory for the same
|
|
330
|
-
* principal finds its ACL row retired; an alias that never retires again keeps its one entry until
|
|
331
|
-
* the daemon restarts. This is the second layer of the fence: it covers a manager that does not
|
|
332
|
-
* narrow the ACL row before the inventory, on this daemon process only. The first is the ACL row
|
|
333
|
-
* each join re-reads around its write (`durableJoinFor`'s `aclFence`), which holds across daemons. */
|
|
334
|
-
retiringLifecycles = new Map();
|
|
335
|
-
/** The durable joins and leaves running now, by `<principal>/<lifecycleUid>`: the writes an
|
|
336
|
-
* inventory of that lifecycle waits for before it reads. */
|
|
337
|
-
membershipWrites = new Map();
|
|
338
|
-
/** True once {@link quiescePlane3} has stopped serving this shard pending an ownership answer.
|
|
339
|
-
* Guards {@link armPlane3} so a RECONNECT cannot silently resume serving mid-question. */
|
|
340
|
-
plane3Quiesced = false;
|
|
341
|
-
/** When set, this endpoint hosts the Plane-3 fan-out writer + trusted reader (the server-side delivery
|
|
342
|
-
* daemon). `aclFor` maps an owner id to its current read ACL (`allowSubscribe`) for the reader's
|
|
343
|
-
* re-authorization — read FRESH per entry from the durable ACL registry KV, hence async. */
|
|
344
|
-
plane3;
|
|
345
310
|
/** Local cache of the channel registry (key = channel token), kept by a KV watch or, on an
|
|
346
311
|
* endpoint that reads it on demand, by {@link readChannels}. */
|
|
347
312
|
channelConfigs = new Map();
|
|
@@ -351,9 +316,31 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
351
316
|
/** Orders {@link readChannels} results: a read that settles after a newer one was applied is dropped. */
|
|
352
317
|
channelReadGeneration = 0;
|
|
353
318
|
channelReadApplied = 0;
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
319
|
+
history = new EndpointHistory({
|
|
320
|
+
connection: () => this.nc,
|
|
321
|
+
jetstream: () => this.js,
|
|
322
|
+
manager: () => this.jsm,
|
|
323
|
+
setManager: (manager) => { this.jsm = manager; },
|
|
324
|
+
space: () => this.space,
|
|
325
|
+
owner: () => this.owner,
|
|
326
|
+
actor: () => this.actor,
|
|
327
|
+
userMode: () => this.userMode,
|
|
328
|
+
connectionId: () => this.connId,
|
|
329
|
+
card: () => this.card,
|
|
330
|
+
notLiveMsg: () => this.notLiveMsg(),
|
|
331
|
+
requireLifecycleUid: (operation) => this.requireLifecycleUid(operation),
|
|
332
|
+
requestDelivery: (op, args) => this.requestDelivery(op, args),
|
|
333
|
+
deleteConsumerIfPresent: (stream, name) => this.deleteConsumerIfPresent(stream, name),
|
|
334
|
+
joinPolicyFresh: (channel) => this.joinPolicyFresh(channel),
|
|
335
|
+
chatMayHaveLost: (subject, sinceSeq) => this.chatMayHaveLost(subject, sinceSeq),
|
|
336
|
+
emit: (event, ...args) => this.emit(event, ...args),
|
|
337
|
+
decodeHistory: historyMessageFromDelivery,
|
|
338
|
+
isRecord,
|
|
339
|
+
isUsableMessageId,
|
|
340
|
+
isFromSubjectSender,
|
|
341
|
+
authenticatedMessage,
|
|
342
|
+
isPublishPermissionDenied,
|
|
343
|
+
});
|
|
357
344
|
subs = [];
|
|
358
345
|
/** The observer feeds {@link tap} was asked for. This is INTENT, re-subscribed by {@link bindTap}
|
|
359
346
|
* on every (re)connect: a rebuild after an outage the library could not ride out closes the old
|
|
@@ -408,9 +395,6 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
408
395
|
/** `$JS.API.CONSUMER.DELETE.<stream>.<prefix>_` for each of this connection's own KV watches and
|
|
409
396
|
* running membership scans (see {@link recordOwnWatchConsumer}). */
|
|
410
397
|
ownWatchDeletePrefixes = new Set();
|
|
411
|
-
/** Per stream, the history-reader names of this connection's current reader id and the reads
|
|
412
|
-
* waiting for one (see {@link acquireHistoryReader}). */
|
|
413
|
-
historyReaderSlots = new Map();
|
|
414
398
|
/** True until the first successful connect completes its boot backfill — distinguishes first-connect
|
|
415
399
|
* (backfill the boot channels' history) from a reconnect (reopen the core-subs, no re-backfill).
|
|
416
400
|
* Persists across reconnect (NOT connection-scoped). Replaces the legacy chat-durable consumed-cursor
|
|
@@ -1676,19 +1660,19 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
1676
1660
|
// Handles that armDeliveryControl created on a previous connection: null them so a rebind
|
|
1677
1661
|
// does not carry a dead protocol's subs forward. Best-effort unsubscribe first (the sub is
|
|
1678
1662
|
// dead with its connection either way; unsubscribing an already-dead sub is a noop).
|
|
1679
|
-
if (this.deliveryServeSub) {
|
|
1663
|
+
if (this.deliveryPlane.deliveryServeSub) {
|
|
1680
1664
|
try {
|
|
1681
|
-
this.deliveryServeSub.unsubscribe();
|
|
1665
|
+
this.deliveryPlane.deliveryServeSub.unsubscribe();
|
|
1682
1666
|
}
|
|
1683
1667
|
catch { /* dead with the old connection */ }
|
|
1684
|
-
this.deliveryServeSub = undefined;
|
|
1668
|
+
this.deliveryPlane.deliveryServeSub = undefined;
|
|
1685
1669
|
}
|
|
1686
|
-
if (this.deliveryAdminServeSub) {
|
|
1670
|
+
if (this.deliveryPlane.deliveryAdminServeSub) {
|
|
1687
1671
|
try {
|
|
1688
|
-
this.deliveryAdminServeSub.unsubscribe();
|
|
1672
|
+
this.deliveryPlane.deliveryAdminServeSub.unsubscribe();
|
|
1689
1673
|
}
|
|
1690
1674
|
catch { /* dead with the old connection */ }
|
|
1691
|
-
this.deliveryAdminServeSub = undefined;
|
|
1675
|
+
this.deliveryPlane.deliveryAdminServeSub = undefined;
|
|
1692
1676
|
}
|
|
1693
1677
|
this.dropAnycastRequests();
|
|
1694
1678
|
for (const sub of this.subs) {
|
|
@@ -1963,7 +1947,14 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
1963
1947
|
throw e;
|
|
1964
1948
|
}
|
|
1965
1949
|
}
|
|
1966
|
-
/**
|
|
1950
|
+
/** Stop the endpoint and close its connection.
|
|
1951
|
+
*
|
|
1952
|
+
* Every principal serve ends first: its handlers' `signal` aborts, and stop waits for each
|
|
1953
|
+
* handler in flight until its request's deadline at most, so a handler that never settles does
|
|
1954
|
+
* not hold stop open longer than 30 s, the route's cap on a deadline. A reply that comes after
|
|
1955
|
+
* that is dropped.
|
|
1956
|
+
*
|
|
1957
|
+
* The presence epoch moves first: a bind still awaiting the broker must find it moved
|
|
1967
1958
|
* before any await below gives it a window to install a watch on a stopped endpoint
|
|
1968
1959
|
* (see {@link startPresenceWatch}). */
|
|
1969
1960
|
stop() {
|
|
@@ -1979,7 +1970,8 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
1979
1970
|
this.activeGoalFollowers.clear();
|
|
1980
1971
|
this.presenceEpoch++;
|
|
1981
1972
|
// Every principal serve ends now, so a `ready` still waiting for the broker rejects and no new
|
|
1982
|
-
// request is taken. Its handlers in flight finish below, before the connection they reply on
|
|
1973
|
+
// request is taken. Its handlers in flight finish below, before the connection they reply on
|
|
1974
|
+
// closes, or their requests' deadlines pass.
|
|
1983
1975
|
const principalStops = [...this.principalServes.values()].map((intent) => this.stopPrincipalServe(intent, false));
|
|
1984
1976
|
this.presence.cancelRebind();
|
|
1985
1977
|
// Wake a reestablishLoop sitting in backoff so it sees `stopped` and exits instead of
|
|
@@ -2206,6 +2198,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
2206
2198
|
*
|
|
2207
2199
|
* `expectedLastSubjectSeq` is a parameter because it is a header VALUE: sizing at 0 and publishing
|
|
2208
2200
|
* at 123456 differ by five bytes.
|
|
2201
|
+
* `expectedLastMsgId` is measured when given, so a caller that may send it can size for it.
|
|
2209
2202
|
*
|
|
2210
2203
|
* Residual: `ts` is re-stamped at publish, so the two differ in value — not in length until
|
|
2211
2204
|
* epoch-millis needs a 14th digit.
|
|
@@ -2223,6 +2216,10 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
2223
2216
|
const mh = headers();
|
|
2224
2217
|
mh.set("Nats-Msg-Id", opts.id);
|
|
2225
2218
|
mh.set("Nats-Expected-Last-Subject-Sequence", `${opts.expectedLastSubjectSeq}`);
|
|
2219
|
+
if (opts.expectedLastMsgId !== undefined) {
|
|
2220
|
+
assertIdToken(opts.expectedLastMsgId, "expected last msg id");
|
|
2221
|
+
mh.set("Nats-Expected-Last-Msg-Id", opts.expectedLastMsgId);
|
|
2222
|
+
}
|
|
2226
2223
|
// `encode()` is on the client's header implementation but not on the published `MsgHdrs` type,
|
|
2227
2224
|
// so it is reached through a cast rather than copied. If a client version drops it, this throws
|
|
2228
2225
|
// immediately and loudly — the calibration cell would also fail — instead of returning a number
|
|
@@ -2248,6 +2245,10 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
2248
2245
|
* means "the subject must be empty". A mismatch throws, and the throw stays classifiable by the
|
|
2249
2246
|
* already-public {@link isCasLoss} — the error is deliberately **not wrapped**, since wrapping
|
|
2250
2247
|
* would hide the `err_code` that classification reads.
|
|
2248
|
+
* - `expectedLastMsgId`, when given, becomes `Nats-Expected-Last-Msg-Id`. The broker checks it after
|
|
2249
|
+
* its dedup cache, so a fresh value nobody has published turns a publish into a probe: an id the
|
|
2250
|
+
* cache holds answers `duplicate: true` with the stored sequence, and any other id is refused
|
|
2251
|
+
* with error 10070 before anything is stored.
|
|
2251
2252
|
*
|
|
2252
2253
|
* @throws if the endpoint is not live, the channel is not concrete, `id` is malformed, `parts` is
|
|
2253
2254
|
* empty, or `expectedLastSubjectSeq` is not a non-negative safe integer.
|
|
@@ -2265,11 +2266,13 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
2265
2266
|
throw new Error(`expectedLastSubjectSeq must be a non-negative safe integer, got ${JSON.stringify(expected)}`);
|
|
2266
2267
|
if (!Array.isArray(opts.parts) || opts.parts.length === 0)
|
|
2267
2268
|
throw new Error("multicastExpecting requires at least one part");
|
|
2269
|
+
if (opts.expectedLastMsgId !== undefined)
|
|
2270
|
+
assertIdToken(opts.expectedLastMsgId, "expected last msg id");
|
|
2268
2271
|
const message = this.casEnvelope(opts);
|
|
2269
2272
|
assertPartsSerializable(message.parts);
|
|
2270
2273
|
// Publish DIRECTLY rather than through publishMsg: this path must set the expectation and read
|
|
2271
2274
|
// the ack, and publishMsg deliberately does neither.
|
|
2272
|
-
const ack = await this.js.publish(chatSubject(this.space, this.owner, this.actor, opts.channel), JSON.stringify(message), { msgID: opts.id, expect: { lastSubjectSequence: expected } });
|
|
2275
|
+
const ack = await this.js.publish(chatSubject(this.space, this.owner, this.actor, opts.channel), JSON.stringify(message), { msgID: opts.id, expect: { lastSubjectSequence: expected, ...(opts.expectedLastMsgId !== undefined ? { lastMsgID: opts.expectedLastMsgId } : {}) } });
|
|
2273
2276
|
return { message, ack: { seq: ack.seq, duplicate: ack.duplicate === true } };
|
|
2274
2277
|
}
|
|
2275
2278
|
/** Publish a payload frame on this principal's forge-locked sibling event rail. */
|
|
@@ -3797,13 +3800,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
3797
3800
|
* published by that principal: the same guarantee a live subscription to that subject gives.
|
|
3798
3801
|
* Without it the read is the channel from any sender. */
|
|
3799
3802
|
async channelHistory(channel, opts) {
|
|
3800
|
-
|
|
3801
|
-
// A publisher is one principal. A `*` in either slot would widen the read back to any sender
|
|
3802
|
-
// while the caller believes it narrowed it, so both slots are checked as concrete tokens.
|
|
3803
|
-
const subject = from === undefined
|
|
3804
|
-
? chatSubject(this.space, "*", "*", channel)
|
|
3805
|
-
: chatSubject(this.space, assertValidOwnerToken(from.owner), assertValidOwnerToken(from.actor), channel);
|
|
3806
|
-
return (await this.streamHistory(chatStream(this.space), [subject], opts?.limit ?? 100, undefined, opts?.signal)).map((r) => r.msg);
|
|
3803
|
+
return this.history.channelHistory(channel, opts);
|
|
3807
3804
|
}
|
|
3808
3805
|
/** Read one concrete channel using this lifecycle's single-filter CHAT grant (SPEC §8/§9).
|
|
3809
3806
|
* No observer authority, presence or delivery ACL mirror is required. Returns the newest page,
|
|
@@ -3811,87 +3808,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
3811
3808
|
* Reads have a ten-second budget, shortened by `signal`. Concurrent connections sharing this
|
|
3812
3809
|
* lifecycle wait for native consumer inactivity cleanup; they never delete another reader. */
|
|
3813
3810
|
async channelHistoryPage(channel, opts) {
|
|
3814
|
-
|
|
3815
|
-
if (!isConcreteChannel(channel))
|
|
3816
|
-
throw new Error("channelHistoryPage requires one concrete channel");
|
|
3817
|
-
const limit = opts?.limit ?? 50;
|
|
3818
|
-
if (!Number.isSafeInteger(limit) || limit < 1 || limit > 1000)
|
|
3819
|
-
throw new Error("channelHistoryPage limit must be a whole number from 1 to 1000");
|
|
3820
|
-
if (!this.nc || !this.js)
|
|
3821
|
-
throw new Error(this.notLiveMsg());
|
|
3822
|
-
const jsm = this.jsm ??= await jetstreamManager(this.nc);
|
|
3823
|
-
const signal = AbortSignal.any([AbortSignal.timeout(10_000), ...(opts?.signal ? [opts.signal] : [])]);
|
|
3824
|
-
signal.throwIfAborted();
|
|
3825
|
-
const stream = chatStream(this.space);
|
|
3826
|
-
const name = chatHistDurable(this.owner, this.actor, this.requireLifecycleUid("channelHistoryPage"));
|
|
3827
|
-
const subject = chatSubject(this.space, "*", "*", channel);
|
|
3828
|
-
const marker = randomUUID();
|
|
3829
|
-
let created;
|
|
3830
|
-
for (;;) {
|
|
3831
|
-
signal.throwIfAborted();
|
|
3832
|
-
try {
|
|
3833
|
-
// The pinned client's add is CREATE-only. A unique description makes even an otherwise
|
|
3834
|
-
// identical occupied config conflict. Never bind/update it or pre-delete by shared name.
|
|
3835
|
-
created = await jsm.consumers.add(stream, {
|
|
3836
|
-
name, description: marker, filter_subject: subject,
|
|
3837
|
-
ack_policy: AckPolicy.None, deliver_policy: DeliverPolicy.All,
|
|
3838
|
-
mem_storage: true, inactive_threshold: nanos(250),
|
|
3839
|
-
});
|
|
3840
|
-
break;
|
|
3841
|
-
}
|
|
3842
|
-
catch (e) {
|
|
3843
|
-
if (!(e instanceof JetStreamApiError) || e.code !== 10148)
|
|
3844
|
-
throw e; // consumer already exists
|
|
3845
|
-
await delay(50, undefined, { signal });
|
|
3846
|
-
}
|
|
3847
|
-
}
|
|
3848
|
-
signal.throwIfAborted();
|
|
3849
|
-
const owns = (info) => info.created === created.created &&
|
|
3850
|
-
info.config.description === marker && info.config.filter_subject === subject;
|
|
3851
|
-
if (!owns(created))
|
|
3852
|
-
throw new Error("channelHistoryPage: consumer ownership was not established");
|
|
3853
|
-
const consumer = await this.js.consumers.get(stream, name);
|
|
3854
|
-
const pending = created.num_pending;
|
|
3855
|
-
const out = [];
|
|
3856
|
-
let delivered = 0;
|
|
3857
|
-
if (pending > 0) {
|
|
3858
|
-
const iter = await consumer.consume({ max_messages: Math.min(pending, 32) });
|
|
3859
|
-
const stop = () => iter.stop(abortReason(signal));
|
|
3860
|
-
signal.addEventListener("abort", stop, { once: true });
|
|
3861
|
-
try {
|
|
3862
|
-
signal.throwIfAborted();
|
|
3863
|
-
for await (const m of iter) {
|
|
3864
|
-
signal.throwIfAborted();
|
|
3865
|
-
if (!subjectMatches(subject, m.subject))
|
|
3866
|
-
throw new Error("channelHistoryPage: consumer delivered another channel; ownership was lost");
|
|
3867
|
-
delivered++;
|
|
3868
|
-
const msg = historyMessageFromDelivery(m);
|
|
3869
|
-
if (msg) {
|
|
3870
|
-
out.push(msg);
|
|
3871
|
-
if (out.length > limit + 1)
|
|
3872
|
-
out.shift();
|
|
3873
|
-
}
|
|
3874
|
-
if (delivered >= pending)
|
|
3875
|
-
break;
|
|
3876
|
-
}
|
|
3877
|
-
}
|
|
3878
|
-
finally {
|
|
3879
|
-
signal.removeEventListener("abort", stop);
|
|
3880
|
-
iter.stop();
|
|
3881
|
-
}
|
|
3882
|
-
signal.throwIfAborted();
|
|
3883
|
-
if (delivered !== pending)
|
|
3884
|
-
throw new Error(`channelHistoryPage: read ${delivered} of ${pending} messages before the window was cut short`);
|
|
3885
|
-
}
|
|
3886
|
-
signal.throwIfAborted();
|
|
3887
|
-
// A legacy join/recall reader can delete/recreate this fixed name. A final ownership check
|
|
3888
|
-
// makes that interference loud. It is detection, not a fence against those legacy readers.
|
|
3889
|
-
if (!owns(await consumer.info()))
|
|
3890
|
-
throw new Error("channelHistoryPage: consumer ownership was lost");
|
|
3891
|
-
signal.throwIfAborted();
|
|
3892
|
-
// No DELETE, even after abort: a delayed owner could otherwise delete its successor between
|
|
3893
|
-
// INFO and DELETE. Native inactivity reaps this transient reader without that TOCTOU race.
|
|
3894
|
-
return { items: out.slice(-limit), complete: out.length <= limit };
|
|
3811
|
+
return this.history.channelHistoryPage(channel, opts);
|
|
3895
3812
|
}
|
|
3896
3813
|
/**
|
|
3897
3814
|
* The newest `limit` chat messages across concrete channels or one channel family, oldest-first within the page,
|
|
@@ -3922,68 +3839,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
3922
3839
|
* so the broker checks the credential's grant for that filter. No additional authority is minted.
|
|
3923
3840
|
*/
|
|
3924
3841
|
async multiChannelHistory(channels, opts) {
|
|
3925
|
-
|
|
3926
|
-
if (!isConcreteChannel(channel) && !(channels.length === 1 && isChannelFamily(channel)))
|
|
3927
|
-
throw new Error(`multiChannelHistory: "${channel}" is a wildcard channel - name one channel family alone or concrete channels without wildcards`);
|
|
3928
|
-
// `chatSubject` builds the filter through `token()`, which REWRITES rather than refuses: it
|
|
3929
|
-
// maps a character a subject may not carry to `_`, trims each segment, and drops empty ones.
|
|
3930
|
-
// So "foo/bar" would filter on `foo_bar`, ".lead" on `lead`, and "team..b" on `team.b` - the
|
|
3931
|
-
// caller names one channel and the broker returns another, which is the exact promise this
|
|
3932
|
-
// method makes ("a channel left out of the list never crosses the link") inverted. This is the
|
|
3933
|
-
// same aliasing `assertValidChannel` was written for on the policy path; the read path needs
|
|
3934
|
-
// it too, and it fails loud rather than serving a channel nobody asked for.
|
|
3935
|
-
assertValidChannel(channel);
|
|
3936
|
-
return chatSubject(this.space, "*", "*", channel);
|
|
3937
|
-
}))];
|
|
3938
|
-
// No channels is not an empty stream, but it IS an empty answer, and asking the broker for a
|
|
3939
|
-
// consumer with no filter would read the WHOLE stream instead of none of it.
|
|
3940
|
-
if (subjects.length === 0)
|
|
3941
|
-
return [];
|
|
3942
|
-
const limit = opts?.limit ?? 100;
|
|
3943
|
-
// ONE CREATE CANNOT CARRY AN UNBOUNDED FILTER LIST. The create names every subject in one
|
|
3944
|
-
// request and the client's request timeout is what gives way, so past roughly 5,000 filters the
|
|
3945
|
-
// read does not answer at all and the route loses the whole chat source: measured on an
|
|
3946
|
-
// isolated broker, 10,000 channels failed `timeout` after 5,023ms while the fan-out this
|
|
3947
|
-
// replaced still returned 2,739 messages on the same corpus. Reading in batches keeps the
|
|
3948
|
-
// single-read cost at the sizes this was built for (69 channels is one batch, so #1210's
|
|
3949
|
-
// round-trip numbers are unchanged) and degrades to a few reads instead of none above that.
|
|
3950
|
-
// `batch` is a knob rather than a constant so the batched path can be compared against the
|
|
3951
|
-
// single-create path on ONE corpus: forcing a small batch makes a space that would otherwise be
|
|
3952
|
-
// one read take many, which is the only way to assert the two select the same messages.
|
|
3953
|
-
const size = Math.max(1, Math.trunc(opts?.batch ?? MULTI_FILTER_BATCH));
|
|
3954
|
-
const batches = [];
|
|
3955
|
-
for (let i = 0; i < subjects.length; i += size)
|
|
3956
|
-
batches.push(subjects.slice(i, i + size));
|
|
3957
|
-
const pages = new Array(batches.length);
|
|
3958
|
-
let next = 0;
|
|
3959
|
-
const readBatch = async () => {
|
|
3960
|
-
for (;;) {
|
|
3961
|
-
const i = next++;
|
|
3962
|
-
if (i >= batches.length)
|
|
3963
|
-
return;
|
|
3964
|
-
pages[i] = await this.streamHistory(chatStream(this.space), batches[i], limit, undefined, opts?.signal);
|
|
3965
|
-
}
|
|
3966
|
-
};
|
|
3967
|
-
// A BATCH THAT FAILS FAILS THE READ. Returning the batches that answered would be the newest
|
|
3968
|
-
// `limit` across SOME of the channels asked for while looking like the newest across all of
|
|
3969
|
-
// them, which is a wrong page presented as a right one. Throwing keeps the caller's existing
|
|
3970
|
-
// envelope: the dashboard marks `chat` missing and says the page is partial, which is what it
|
|
3971
|
-
// already did when this method was one read.
|
|
3972
|
-
await Promise.all(Array.from({ length: Math.min(MULTI_FILTER_READ_CONCURRENCY, batches.length) }, readBatch));
|
|
3973
|
-
// MERGE ON ARRIVAL, NOT ON `ts`. Each batch returns the newest `limit` within its own subjects,
|
|
3974
|
-
// and the newest `limit` overall is a subset of their union, so re-selecting by stream sequence
|
|
3975
|
-
// reproduces exactly what one create over the whole list would have selected. Sorting by the
|
|
3976
|
-
// payload's `ts` here instead would change which messages the page holds, which is the
|
|
3977
|
-
// selection question this pull request already had to answer once.
|
|
3978
|
-
const rows = pages.flat().sort((a, b) => a.seq - b.seq).slice(-limit);
|
|
3979
|
-
return rows.map(({ seq, subject, msg }) => {
|
|
3980
|
-
const p = parseSubject(subject);
|
|
3981
|
-
// The filter set is built from chat subjects, so this cannot fire against a healthy broker.
|
|
3982
|
-
// It is here because the alternative to raising is tagging a message with a guess.
|
|
3983
|
-
if (p?.kind !== "chat")
|
|
3984
|
-
throw new Error(`multiChannelHistory: the broker delivered ${subject}, which is not a chat subject on this space`);
|
|
3985
|
-
return { channel: p.rest, seq, msg };
|
|
3986
|
-
});
|
|
3842
|
+
return this.history.multiChannelHistory(channels, opts);
|
|
3987
3843
|
}
|
|
3988
3844
|
/** Read a channel's recent history through the delivery daemon, using a provisioned ACL mirror.
|
|
3989
3845
|
* SPEC's ban on raw control-surface stream reads does not remove the single-channel CHAT grant.
|
|
@@ -4013,16 +3869,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
4013
3869
|
* different authorization model, and one handler with two authz paths is the wrong shape on the
|
|
4014
3870
|
* surface where a mistake exposes private messages). */
|
|
4015
3871
|
async readHistory(channel, opts) {
|
|
4016
|
-
|
|
4017
|
-
if (!reply.ok)
|
|
4018
|
-
throw new Error(reply.error ?? "readHistory failed");
|
|
4019
|
-
const data = reply.data;
|
|
4020
|
-
// Validate rather than coerce. A malformed reply must not be massaged into a plausible page:
|
|
4021
|
-
// defaulting `complete` would invent the very signal a caller uses to decide whether it is
|
|
4022
|
-
// looking at the start of a conversation.
|
|
4023
|
-
if (!Array.isArray(data?.items) || typeof data?.complete !== "boolean")
|
|
4024
|
-
throw new Error("readHistory: the delivery daemon returned a malformed page (expected { items, complete })");
|
|
4025
|
-
return { items: data.items, complete: data.complete };
|
|
3872
|
+
return this.history.readHistory(channel, opts);
|
|
4026
3873
|
}
|
|
4027
3874
|
/** Fetch recent DMs (any sender→any recipient) from the space's DM backlog. `signal` cancels the
|
|
4028
3875
|
* active pull and reclaims its ephemeral consumer. God-view only:
|
|
@@ -4032,15 +3879,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
4032
3879
|
* returned — a stored row missing `ts` / `space` / `parts` / `from.name` is dropped, never
|
|
4033
3880
|
* returned with that member silently `undefined`. */
|
|
4034
3881
|
async dmHistory(opts) {
|
|
4035
|
-
|
|
4036
|
-
// every inst.<recipOwner>.<recipActor>.<sndOwner>.<sndActor> DM — the whole DM subtree (god-view)
|
|
4037
|
-
return (await this.streamHistory(dmStream(this.space), [`${spacePrefix(this.space)}.inst.>`], opts?.limit ?? 100, undefined, opts?.signal)).map((r) => r.msg);
|
|
4038
|
-
}
|
|
4039
|
-
catch (e) {
|
|
4040
|
-
if (isPermissionDenied(e))
|
|
4041
|
-
return [];
|
|
4042
|
-
throw e;
|
|
4043
|
-
}
|
|
3882
|
+
return this.history.dmHistory(opts);
|
|
4044
3883
|
}
|
|
4045
3884
|
/** The newest `limit` DMs (100 by default) sent to or by any of `principals`, oldest first. Each
|
|
4046
3885
|
* principal is a concrete `<owner>.<actor>`; anything else throws, because a wildcard would widen
|
|
@@ -4099,434 +3938,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
4099
3938
|
* `before` pages toward the past: pass the `seq` of the oldest message you already have.
|
|
4100
3939
|
*/
|
|
4101
3940
|
async streamHistory(stream, subjects, limit, before, signal) {
|
|
4102
|
-
|
|
4103
|
-
throw new Error(this.notLiveMsg());
|
|
4104
|
-
signal?.throwIfAborted();
|
|
4105
|
-
// A LIMIT THAT IS NOT A FINITE NUMBER HAS NO ANSWER, AND THE SEARCH BELOW CANNOT REFUSE IT.
|
|
4106
|
-
// Every comparison against NaN is false, so `limit <= 0` does not fire for one, and neither of
|
|
4107
|
-
// the widening loop's exits can ever be true either: `page.length >= NaN` is false forever and
|
|
4108
|
-
// `start === 1` compares against a `start` that is itself NaN. The loop does not return
|
|
4109
|
-
// everything, it never returns. Measured through the dashboard on a real broker: no answer
|
|
4110
|
-
// after 30s, and the abandoned read still consuming half a core fifteen seconds after its
|
|
4111
|
-
// caller had gone, while the process kept serving every other route so nothing announced it.
|
|
4112
|
-
// `Infinity` reaches the other end of the same hole: it passes the guard, `start` collapses to
|
|
4113
|
-
// 1, and `slice(-Infinity)` is the subject's whole retained set.
|
|
4114
|
-
// A caller that computed a limit it cannot state is told so, rather than handed a process that
|
|
4115
|
-
// quietly spins. `0` and negatives keep their existing meaning, an empty page.
|
|
4116
|
-
// A COUNT OF MESSAGES IS A WHOLE NUMBER THIS SERVER CAN COUNT EXACTLY, and the check sits ABOVE
|
|
4117
|
-
// the zero check on purpose: `-Infinity <= 0` is true, so a guard placed below it would fold a
|
|
4118
|
-
// limit nobody can answer into a silent empty page. `NaN` never returns (both widening exits
|
|
4119
|
-
// compare against it and are false forever). `Infinity` and any magnitude past the safe range
|
|
4120
|
-
// collapse `start` to 1. A fraction is the quietest of the set: the page is taken with
|
|
4121
|
-
// `slice(-limit)` and slice truncates toward zero, so any limit in (0,1) becomes `slice(0)` and
|
|
4122
|
-
// hands back the ENTIRE retained history. Measured on 30 retained: 0.5 and 0.9 both returned all
|
|
4123
|
-
// 30, and 2.5 returned 2, a page nobody asked for.
|
|
4124
|
-
if (!Number.isSafeInteger(limit))
|
|
4125
|
-
throw new Error(`history limit must be a whole number of messages this server can count exactly, received ${limit}`);
|
|
4126
|
-
if (limit <= 0)
|
|
4127
|
-
return [];
|
|
4128
|
-
const nc = this.nc;
|
|
4129
|
-
let reader;
|
|
4130
|
-
try {
|
|
4131
|
-
reader = await this.acquireHistoryReader(nc, this.jsm ?? (this.jsm = await jetstreamManager(nc)), stream, signal);
|
|
4132
|
-
// THE EXACT CEILING, on the already-granted surface. A one-shot consumer with
|
|
4133
|
-
// `DeliverPolicy.Last` plus this subject's filter reports the newest MATCHING sequence, and its
|
|
4134
|
-
// `num_pending` of zero means the channel is genuinely empty. Using the STREAM's last sequence
|
|
4135
|
-
// instead (which is all `STREAM.INFO` offers) was a loose upper bound, and on a quiet channel
|
|
4136
|
-
// in a busy stream the gap between the two is the whole problem: every window near the stream
|
|
4137
|
-
// head is empty, so the search widened over and over before finding anything.
|
|
4138
|
-
//
|
|
4139
|
-
// Deliberately NOT `getMessage({ last_by_subj })`, which would be the obvious way to ask: it
|
|
4140
|
-
// needs `$JS.API.STREAM.MSG.GET`, which read credentials do not hold. That grant hole already
|
|
4141
|
-
// shipped once from this function and turned every non-admin history read into an empty list.
|
|
4142
|
-
const ceiling = before !== undefined ? before - 1 : await this.lastMatchingSeq(reader, subjects, signal);
|
|
4143
|
-
if (ceiling < 1)
|
|
4144
|
-
return [];
|
|
4145
|
-
// Widen from the exact ceiling until a window holds a full page, or until the window IS the
|
|
4146
|
-
// whole subject. Draining each attempt is bounded, and that is the point: a window only fails
|
|
4147
|
-
// when it holds FEWER than `limit` matches, so every wasted drain moves less than one page.
|
|
4148
|
-
// Geometric growth keeps the number of attempts logarithmic, so total wasted transfer is a
|
|
4149
|
-
// small multiple of a page.
|
|
4150
|
-
//
|
|
4151
|
-
// THE FIRST WINDOW IS ONE PAGE WIDE, NOT FOUR. It used to open at `limit * 4` on the reasoning
|
|
4152
|
-
// that a filtered subject is sparse inside its stream and a wider first look would land the
|
|
4153
|
-
// page in one drain. That is true for one channel of a busy stream and false for a subject
|
|
4154
|
-
// that IS most of its stream, and the second case is the one a whole-backlog read takes: the
|
|
4155
|
-
// window succeeds on the first attempt and drains four pages to keep one, because
|
|
4156
|
-
// `drainWindow` delivers everything in the window and keeps the tail. Measured on `/api/dms`
|
|
4157
|
-
// at limit 500 against a 2500-message backlog: 1,995,854 to 1,995,859 bytes moved across
|
|
4158
|
-
// four runs to return a 346,001-byte page, and 8161ms to 8753ms on that link, so all four
|
|
4159
|
-
// missed the dashboard's 8000ms deadline with nothing else on the connection (Cotal #1210).
|
|
4160
|
-
// That cost does not keep growing with the backlog, and saying it did was wrong: the old first
|
|
4161
|
-
// span was `max(limit * 4, 64)`, so at limit 500 it is 2000 messages, and ANY backlog of 2000
|
|
4162
|
-
// or more drains the same 2000-message window. The point is that the window, not the backlog,
|
|
4163
|
-
// sets the cost, and four pages to return one is already past the deadline on this link for
|
|
4164
|
-
// every such deployment. One page wide makes the
|
|
4165
|
-
// SUCCESSFUL drain
|
|
4166
|
-
// obey the same bound the failed ones already promised: it moves at most one page. A sparse
|
|
4167
|
-
// subject pays one more widening step for that, which is four round trips against a transfer
|
|
4168
|
-
// several times the size of the answer.
|
|
4169
|
-
//
|
|
4170
|
-
// A SHORT PAGE is either "the channel is exhausted" or "the window is still above the
|
|
4171
|
-
// first match". Sequence 1 is the wrong floor for the first of those: three matches at
|
|
4172
|
-
// the high end of a busy stream are exhausted as soon as the window's lower edge passes
|
|
4173
|
-
// the subject's first matching sequence, and walking on to 1 re-reads everyone else's
|
|
4174
|
-
// retained set (#840). Probe that floor only after a short drain so a dense page (one
|
|
4175
|
-
// drain, full) never pays for it. The remaining unbounded-looking case is a subject
|
|
4176
|
-
// whose FIRST match really is near sequence 1; that span is the channel's own, not the
|
|
4177
|
-
// stream's.
|
|
4178
|
-
let span = Math.max(limit, 64);
|
|
4179
|
-
let floor = 1;
|
|
4180
|
-
let floorKnown = false;
|
|
4181
|
-
for (;;) {
|
|
4182
|
-
signal?.throwIfAborted();
|
|
4183
|
-
const start = Math.max(floor, ceiling - span + 1);
|
|
4184
|
-
const page = await this.drainWindow(reader, subjects, start, ceiling, limit, signal);
|
|
4185
|
-
if (page.length >= limit)
|
|
4186
|
-
return page.slice(-limit);
|
|
4187
|
-
if (!floorKnown) {
|
|
4188
|
-
floor = Math.max(1, await this.firstMatchingSeq(reader, subjects, signal));
|
|
4189
|
-
floorKnown = true;
|
|
4190
|
-
}
|
|
4191
|
-
if (start <= floor)
|
|
4192
|
-
return page.slice(-limit);
|
|
4193
|
-
span *= 4;
|
|
4194
|
-
}
|
|
4195
|
-
}
|
|
4196
|
-
catch (e) {
|
|
4197
|
-
// NARROW. This catch is how the MSG.GET grant bug shipped: it turned a Permissions Violation
|
|
4198
|
-
// into an empty history, so every non-admin read looked exactly like a quiet channel, and the
|
|
4199
|
-
// smokes stayed green because they run as admin against a local broker. Emitting on the error
|
|
4200
|
-
// event is not enough either — callers consume the RETURN VALUE, and the dashboard renders
|
|
4201
|
-
// empty regardless of what an operator log says.
|
|
4202
|
-
//
|
|
4203
|
-
// So only one thing may still produce an empty result here: the stream does not exist (a space
|
|
4204
|
-
// with no history yet). Anything else — a denial, a timeout, a 503, a protocol or
|
|
4205
|
-
// consumer-create failure — is raised, because "no history" and "I could not read the history"
|
|
4206
|
-
// are different answers and only one of them is safe to render as an empty conversation.
|
|
4207
|
-
// `dmHistory` alone turns a denial into its documented empty god-view result.
|
|
4208
|
-
const msg = String(e?.message ?? "");
|
|
4209
|
-
if (/stream not found/i.test(msg) || e?.code === 404)
|
|
4210
|
-
return [];
|
|
4211
|
-
throw e;
|
|
4212
|
-
}
|
|
4213
|
-
finally {
|
|
4214
|
-
if (reader)
|
|
4215
|
-
this.releaseHistoryReader(reader);
|
|
4216
|
-
}
|
|
4217
|
-
}
|
|
4218
|
-
/** This connection's history-reader id, the key of its credential's reader deletes
|
|
4219
|
-
* ({@link historyReaderConsumer}). A static credential's is its connId, the nkey it signs with.
|
|
4220
|
-
* Under the auth callout the connId is the inbox nonce this client chose, so the callout keys the
|
|
4221
|
-
* deletes on {@link calloutReaderId} instead: the server's id and the client id it assigned this
|
|
4222
|
-
* connection, which the client reads from the server's INFO and does not choose. */
|
|
4223
|
-
historyReaderId(nc) {
|
|
4224
|
-
if (!this.userMode)
|
|
4225
|
-
return this.connId;
|
|
4226
|
-
const info = nc.info;
|
|
4227
|
-
if (!info)
|
|
4228
|
-
throw new Error("history: the server has not sent its INFO, so this connection has no history reader id yet");
|
|
4229
|
-
return calloutReaderId(info.server_id, info.client_id);
|
|
4230
|
-
}
|
|
4231
|
-
/** Take one of this connection's history-reader names on `stream` ({@link historyReaderConsumer})
|
|
4232
|
-
* for one read. The read holds the name for its whole walk and creates and deletes a consumer
|
|
4233
|
-
* under it at each step, so this connection keeps at most {@link HISTORY_READERS} reader consumers
|
|
4234
|
-
* on a stream (#4123). A read past that many waits for a name, and `signal` ends the wait. A
|
|
4235
|
-
* reconnect under the auth callout changes the reader id, and reads from then on take the new
|
|
4236
|
-
* id's names. */
|
|
4237
|
-
async acquireHistoryReader(nc, jsm, stream, signal) {
|
|
4238
|
-
for (;;) {
|
|
4239
|
-
signal?.throwIfAborted();
|
|
4240
|
-
const readerId = this.historyReaderId(nc);
|
|
4241
|
-
let pool = this.historyReaderSlots.get(stream);
|
|
4242
|
-
if (pool?.readerId !== readerId) {
|
|
4243
|
-
pool = { readerId, free: Array.from({ length: HISTORY_READERS }, (_, i) => HISTORY_READERS - 1 - i), held: 0, waiting: [], refusedUntil: 0 };
|
|
4244
|
-
this.historyReaderSlots.set(stream, pool);
|
|
4245
|
-
}
|
|
4246
|
-
// A read here would leave one more reader it may not delete, so it fails before creating one.
|
|
4247
|
-
if (pool.refusedUntil > Date.now())
|
|
4248
|
-
throw this.readerDeleteRefused(stream);
|
|
4249
|
-
const slot = pool.free.pop();
|
|
4250
|
-
if (slot !== undefined) {
|
|
4251
|
-
pool.held++;
|
|
4252
|
-
return { nc, js: jetstream(nc), jsm, stream, pool, slot, name: historyReaderConsumer(readerId, slot), held: true, left: false };
|
|
4253
|
-
}
|
|
4254
|
-
// No read here holds a name, so none will come back from one: every name is set aside (see
|
|
4255
|
-
// openHistoryReader and closeHistoryReader). Waiting would hang until those expire.
|
|
4256
|
-
if (pool.held === 0)
|
|
4257
|
-
throw new Error(`history: every history reader name this connection has on ${stream} is set aside, held by a consumer it did not create (another process using the same credential, or one that stopped mid-read) or by a reader it could not confirm deleted; they expire ${HISTORY_READER_INACTIVE_MS / 60_000} minutes after their last use`);
|
|
4258
|
-
const waitOn = pool;
|
|
4259
|
-
await new Promise((resolve, reject) => {
|
|
4260
|
-
const wake = () => { signal?.removeEventListener("abort", stop); resolve(); };
|
|
4261
|
-
const stop = () => { waitOn.waiting.splice(waitOn.waiting.indexOf(wake), 1); reject(abortReason(signal)); };
|
|
4262
|
-
waitOn.waiting.push(wake);
|
|
4263
|
-
signal?.addEventListener("abort", stop, { once: true });
|
|
4264
|
-
});
|
|
4265
|
-
}
|
|
4266
|
-
}
|
|
4267
|
-
/** Give a read's history-reader name back and wake one read waiting for a name. A name whose
|
|
4268
|
-
* delete failed is set aside instead. */
|
|
4269
|
-
releaseHistoryReader(reader) {
|
|
4270
|
-
if (!reader.held)
|
|
4271
|
-
return;
|
|
4272
|
-
reader.held = false;
|
|
4273
|
-
const pool = reader.pool;
|
|
4274
|
-
pool.held--;
|
|
4275
|
-
if (!reader.left) {
|
|
4276
|
-
pool.free.push(reader.slot);
|
|
4277
|
-
pool.waiting.shift()?.();
|
|
4278
|
-
return;
|
|
4279
|
-
}
|
|
4280
|
-
this.setAsideHistoryReader(pool, reader.slot);
|
|
4281
|
-
// Every waiter checks again, so one that no running read can now serve fails instead of hanging.
|
|
4282
|
-
if (pool.held === 0)
|
|
4283
|
-
for (const wake of pool.waiting.splice(0))
|
|
4284
|
-
wake();
|
|
4285
|
-
}
|
|
4286
|
-
/** Keep `slot` out of use until a consumer under it would have expired, then free it. */
|
|
4287
|
-
setAsideHistoryReader(pool, slot) {
|
|
4288
|
-
setTimeout(() => { pool.free.push(slot); pool.waiting.shift()?.(); }, HISTORY_READER_INACTIVE_MS).unref();
|
|
4289
|
-
}
|
|
4290
|
-
/** Create the reader consumer for one step of a read, under the read's name. It is a plain pull
|
|
4291
|
-
* consumer: an nats.js ordered consumer renames itself on every rebuild, and the observer and admin
|
|
4292
|
-
* credentials may delete only these exact names.
|
|
4293
|
-
*
|
|
4294
|
-
* Every create carries a fresh description. NATS answers a create whose config equals the existing
|
|
4295
|
-
* consumer's with that consumer, delivery state and all (measured on nats-server 2.15), and a read
|
|
4296
|
-
* repeats the same config on the same name. Without the description a reader left under the name
|
|
4297
|
-
* would come back as if new: a `DeliverPolicy.Last` reader that already delivered its message
|
|
4298
|
-
* reports nothing pending, which reads as an empty channel. With it, a create on a name in use is
|
|
4299
|
-
* refused, and a consumer that does not carry this step's description is never used.
|
|
4300
|
-
*
|
|
4301
|
-
* A consumer already under the name is never deleted, whoever left it: another holder of this
|
|
4302
|
-
* credential, a process that stopped mid-read, or this endpoint when a delete's answer was lost and
|
|
4303
|
-
* the name was taken again since. The read moves to a free name, and the taken one stays out of use
|
|
4304
|
-
* until such a consumer would have expired. */
|
|
4305
|
-
async openHistoryReader(reader, config) {
|
|
4306
|
-
// A single filter must ride the filter-bearing API subject so scoped read grants apply.
|
|
4307
|
-
if (config.filter_subjects?.length === 1) {
|
|
4308
|
-
const { filter_subjects, ...rest } = config;
|
|
4309
|
-
config = { ...rest, filter_subject: filter_subjects[0] };
|
|
4310
|
-
}
|
|
4311
|
-
const pool = reader.pool;
|
|
4312
|
-
for (;;) {
|
|
4313
|
-
// A reconnect under the auth callout moved this connection to new reader names, and its
|
|
4314
|
-
// credential may no longer delete this one. Creating under it would leave the reader behind.
|
|
4315
|
-
if (this.historyReaderId(reader.nc) !== pool.readerId)
|
|
4316
|
-
throw new Error(`history: the connection was re-established during the read on ${reader.stream}, so its history reader names changed; the read is not served`);
|
|
4317
|
-
const description = `cotal history read ${randomUUID()}`;
|
|
4318
|
-
let info;
|
|
4319
|
-
try {
|
|
4320
|
-
info = await reader.jsm.consumers.add(reader.stream, {
|
|
4321
|
-
...config,
|
|
4322
|
-
name: reader.name,
|
|
4323
|
-
description,
|
|
4324
|
-
ack_policy: AckPolicy.None,
|
|
4325
|
-
max_deliver: 1,
|
|
4326
|
-
mem_storage: true,
|
|
4327
|
-
num_replicas: 1,
|
|
4328
|
-
inactive_threshold: nanos(HISTORY_READER_INACTIVE_MS),
|
|
4329
|
-
});
|
|
4330
|
-
}
|
|
4331
|
-
catch (e) {
|
|
4332
|
-
if (!isConsumerAlreadyExists(e))
|
|
4333
|
-
throw e;
|
|
4334
|
-
}
|
|
4335
|
-
if (info?.config.description === description)
|
|
4336
|
-
return reader.js.consumers.getConsumerFromInfo(info);
|
|
4337
|
-
this.setAsideHistoryReader(pool, reader.slot);
|
|
4338
|
-
const next = pool.free.pop();
|
|
4339
|
-
if (next === undefined) {
|
|
4340
|
-
// The read ends holding no name. Every waiter checks again, so one that no running read can
|
|
4341
|
-
// now serve fails instead of hanging.
|
|
4342
|
-
reader.held = false;
|
|
4343
|
-
pool.held--;
|
|
4344
|
-
for (const wake of pool.waiting.splice(0))
|
|
4345
|
-
wake();
|
|
4346
|
-
throw new Error(`history: reader ${reader.name} on ${reader.stream} is held by a consumer this read did not create, and no other reader name is free`);
|
|
4347
|
-
}
|
|
4348
|
-
reader.slot = next;
|
|
4349
|
-
reader.name = historyReaderConsumer(pool.readerId, next);
|
|
4350
|
-
}
|
|
4351
|
-
}
|
|
4352
|
-
/** Delete the read's reader consumer; already gone is the same outcome. A failed delete may still
|
|
4353
|
-
* have run with its answer lost, and the name may be taken again before this endpoint could tell,
|
|
4354
|
-
* so the name is set aside when the read gives it back rather than deleted again later. A refused
|
|
4355
|
-
* delete means the credential was minted before the observer and admin profiles held their reader
|
|
4356
|
-
* deletes. Each read would then leave a reader for the inactive threshold, which is how a polling
|
|
4357
|
-
* dashboard filled the stream's consumer limit (#4123), so the read fails and says to remint
|
|
4358
|
-
* rather than leak quietly, and reads on the stream fail at once until that reader expires. */
|
|
4359
|
-
async closeHistoryReader(reader) {
|
|
4360
|
-
try {
|
|
4361
|
-
await reader.jsm.consumers.delete(reader.stream, reader.name);
|
|
4362
|
-
}
|
|
4363
|
-
catch (e) {
|
|
4364
|
-
if (isConsumerNotFound(e))
|
|
4365
|
-
return;
|
|
4366
|
-
reader.left = true;
|
|
4367
|
-
// No `cause`: a permission-shaped cause would let streamHistory read this as the admin-only
|
|
4368
|
-
// DM denial and return an empty page.
|
|
4369
|
-
if (isPublishPermissionDenied(e)) {
|
|
4370
|
-
if (this.historyReaderId(reader.nc) !== reader.pool.readerId)
|
|
4371
|
-
throw new Error(`history: the connection was re-established during the read on ${reader.stream}, so it may no longer delete its reader ${reader.name}, which expires on its own; the read is not served`);
|
|
4372
|
-
reader.pool.refusedUntil = Date.now() + HISTORY_READER_INACTIVE_MS;
|
|
4373
|
-
throw this.readerDeleteRefused(reader.stream);
|
|
4374
|
-
}
|
|
4375
|
-
throw e;
|
|
4376
|
-
}
|
|
4377
|
-
}
|
|
4378
|
-
readerDeleteRefused(stream) {
|
|
4379
|
-
return new Error(`history: this credential may not delete its own history readers on ${stream}; it predates the observer and admin reader-delete grant, so remint it`);
|
|
4380
|
-
}
|
|
4381
|
-
/** The oldest stream sequence matching `subject`, or 0 when the subject has no messages.
|
|
4382
|
-
* Mirror of {@link lastMatchingSeq}: same CREATE/INFO/NEXT/DELETE surface, `DeliverPolicy.All`
|
|
4383
|
-
* instead of `Last`, first delivered seq instead of last. Read credentials already hold this. */
|
|
4384
|
-
async firstMatchingSeq(reader, subjects, signal) {
|
|
4385
|
-
signal?.throwIfAborted();
|
|
4386
|
-
const consumer = await this.openHistoryReader(reader, {
|
|
4387
|
-
filter_subjects: subjects,
|
|
4388
|
-
deliver_policy: DeliverPolicy.All,
|
|
4389
|
-
});
|
|
4390
|
-
try {
|
|
4391
|
-
if ((await consumer.info(true)).num_pending === 0)
|
|
4392
|
-
return 0;
|
|
4393
|
-
signal?.throwIfAborted();
|
|
4394
|
-
const iter = await consumer.fetch({ max_messages: 1 });
|
|
4395
|
-
const stop = () => iter.stop(abortReason(signal));
|
|
4396
|
-
signal?.addEventListener("abort", stop, { once: true });
|
|
4397
|
-
try {
|
|
4398
|
-
signal?.throwIfAborted();
|
|
4399
|
-
for await (const m of iter)
|
|
4400
|
-
return m.seq;
|
|
4401
|
-
}
|
|
4402
|
-
finally {
|
|
4403
|
-
signal?.removeEventListener("abort", stop);
|
|
4404
|
-
iter.stop();
|
|
4405
|
-
}
|
|
4406
|
-
throw new Error(`history: the broker reported messages on ${subjectLabel(subjects)} but delivered none - the read was cut short, not empty`);
|
|
4407
|
-
}
|
|
4408
|
-
finally {
|
|
4409
|
-
await this.closeHistoryReader(reader);
|
|
4410
|
-
}
|
|
4411
|
-
}
|
|
4412
|
-
/** The newest stream sequence matching `subject`, or 0 when the subject has no messages.
|
|
4413
|
-
*
|
|
4414
|
-
* One reader consumer at `DeliverPolicy.Last` with this subject's filter: its `num_pending`
|
|
4415
|
-
* (available from the create, before anything is delivered) is 0 for an empty subject, and
|
|
4416
|
-
* otherwise one message carries the sequence. Same CREATE/INFO/NEXT/DELETE surface `drainWindow`
|
|
4417
|
-
* already uses, so no broker authority is added. */
|
|
4418
|
-
async lastMatchingSeq(reader, subjects, signal) {
|
|
4419
|
-
signal?.throwIfAborted();
|
|
4420
|
-
const consumer = await this.openHistoryReader(reader, {
|
|
4421
|
-
filter_subjects: subjects,
|
|
4422
|
-
deliver_policy: DeliverPolicy.Last,
|
|
4423
|
-
});
|
|
4424
|
-
try {
|
|
4425
|
-
// Bind-time zero is the ONLY thing that means "this subject has no messages".
|
|
4426
|
-
if ((await consumer.info(true)).num_pending === 0)
|
|
4427
|
-
return 0;
|
|
4428
|
-
signal?.throwIfAborted();
|
|
4429
|
-
const iter = await consumer.fetch({ max_messages: 1 });
|
|
4430
|
-
const stop = () => iter.stop(abortReason(signal));
|
|
4431
|
-
signal?.addEventListener("abort", stop, { once: true });
|
|
4432
|
-
try {
|
|
4433
|
-
signal?.throwIfAborted();
|
|
4434
|
-
for await (const m of iter)
|
|
4435
|
-
return m.seq;
|
|
4436
|
-
}
|
|
4437
|
-
finally {
|
|
4438
|
-
signal?.removeEventListener("abort", stop);
|
|
4439
|
-
iter.stop();
|
|
4440
|
-
}
|
|
4441
|
-
// Bind said a message was pending and none arrived. The pinned client's pull iterator ends
|
|
4442
|
-
// CLEANLY when the connection closes ("we don't propagate the error here"), so this is what a
|
|
4443
|
-
// dropped link looks like from here. Returning 0 would make the caller report an empty
|
|
4444
|
-
// channel, which is the same "could not read means no history" lie the narrowed catch above
|
|
4445
|
-
// exists to stop.
|
|
4446
|
-
throw new Error(`history: the broker reported messages on ${subjectLabel(subjects)} but delivered none - the read was cut short, not empty`);
|
|
4447
|
-
}
|
|
4448
|
-
finally {
|
|
4449
|
-
await this.closeHistoryReader(reader);
|
|
4450
|
-
}
|
|
4451
|
-
}
|
|
4452
|
-
/** Drain every message matching `subject` with sequence in `[start, ceiling]`, oldest-first.
|
|
4453
|
-
* One reader consumer, one batched pull — `AckPolicy.None`, so no per-message ack
|
|
4454
|
-
* round trip. Fetches exactly the pending count so it returns as soon as the window is
|
|
4455
|
-
* delivered rather than blocking for the pull's full expiry. */
|
|
4456
|
-
async drainWindow(reader, subjects, start, ceiling, limit, signal) {
|
|
4457
|
-
signal?.throwIfAborted();
|
|
4458
|
-
const out = [];
|
|
4459
|
-
const consumer = await this.openHistoryReader(reader, {
|
|
4460
|
-
filter_subjects: subjects,
|
|
4461
|
-
deliver_policy: DeliverPolicy.StartSequence,
|
|
4462
|
-
opt_start_seq: start,
|
|
4463
|
-
});
|
|
4464
|
-
try {
|
|
4465
|
-
// A freshly created consumer already carries its ConsumerInfo, so read the CACHED copy: the
|
|
4466
|
-
// explicit uncached `info()` this used to call was a round trip for data we already had.
|
|
4467
|
-
const pending = (await consumer.info(true)).num_pending;
|
|
4468
|
-
if (pending === 0)
|
|
4469
|
-
return out;
|
|
4470
|
-
signal?.throwIfAborted();
|
|
4471
|
-
// Keep only a small rolling buffer in flight. Stopping a client iterator cannot unsend bytes the
|
|
4472
|
-
// broker already committed to that pull request, so fetching the whole page let an aborted history
|
|
4473
|
-
// read keep filling a constrained shared connection and starve the next dashboard poll. `consume`
|
|
4474
|
-
// replenishes this bounded buffer as it is read, so large pages still complete without committing
|
|
4475
|
-
// all of their bytes to the connection up front.
|
|
4476
|
-
const iter = await consumer.consume({ max_messages: Math.min(pending, 32) });
|
|
4477
|
-
const stop = () => iter.stop(abortReason(signal));
|
|
4478
|
-
signal?.addEventListener("abort", stop, { once: true });
|
|
4479
|
-
// PROVE THE WINDOW COMPLETED. The pull iterator ends cleanly on a dropped connection, so a
|
|
4480
|
-
// close after three of ten deliveries would otherwise return a convincing three-message page.
|
|
4481
|
-
// The window is done when we have reached its upper bound or consumed everything bind said
|
|
4482
|
-
// was pending; anything else is a cut-short read and must say so.
|
|
4483
|
-
let delivered = 0;
|
|
4484
|
-
let complete = false;
|
|
4485
|
-
try {
|
|
4486
|
-
signal?.throwIfAborted();
|
|
4487
|
-
for await (const m of iter) {
|
|
4488
|
-
delivered++;
|
|
4489
|
-
// A plain consumer does not rebuild itself the way an ordered one did, so a gap in its
|
|
4490
|
-
// delivery sequence is a message this page would silently lack.
|
|
4491
|
-
if (m.info.deliverySequence !== delivered)
|
|
4492
|
-
throw new Error(`history: delivery ${m.info.deliverySequence} arrived where ${delivered} was due on ${subjectLabel(subjects)} - the window has a gap, so the page is not served`);
|
|
4493
|
-
if (m.seq >= ceiling) { // reached the page's upper bound
|
|
4494
|
-
if (m.seq === ceiling) {
|
|
4495
|
-
const msg = historyMessageFromDelivery(m);
|
|
4496
|
-
if (msg)
|
|
4497
|
-
out.push({ seq: m.seq, subject: m.subject, msg });
|
|
4498
|
-
}
|
|
4499
|
-
complete = true;
|
|
4500
|
-
break;
|
|
4501
|
-
}
|
|
4502
|
-
const msg = historyMessageFromDelivery(m);
|
|
4503
|
-
if (msg) {
|
|
4504
|
-
out.push({ seq: m.seq, subject: m.subject, msg });
|
|
4505
|
-
if (out.length > limit)
|
|
4506
|
-
out.shift();
|
|
4507
|
-
}
|
|
4508
|
-
if (delivered >= pending) {
|
|
4509
|
-
complete = true;
|
|
4510
|
-
break;
|
|
4511
|
-
}
|
|
4512
|
-
}
|
|
4513
|
-
}
|
|
4514
|
-
finally {
|
|
4515
|
-
signal?.removeEventListener("abort", stop);
|
|
4516
|
-
iter.stop();
|
|
4517
|
-
}
|
|
4518
|
-
if (!complete)
|
|
4519
|
-
throw new Error(`history: read ${delivered} of ${pending} messages on ${subjectLabel(subjects)} before the stream ended early - the window was cut short, not empty`);
|
|
4520
|
-
return out;
|
|
4521
|
-
}
|
|
4522
|
-
finally {
|
|
4523
|
-
// DELETE THE READER. Left alone it would stay for its inactive threshold, and the widening
|
|
4524
|
-
// search can run several steps per call while a polling dashboard repeats every call on each
|
|
4525
|
-
// poll. The observer and admin profiles used to hold no delete at all (#691), so a dashboard
|
|
4526
|
-
// polling every ten seconds filled the stream's consumer limit and every later read failed
|
|
4527
|
-
// (#4123). Each reader now goes when its step ends.
|
|
4528
|
-
await this.closeHistoryReader(reader);
|
|
4529
|
-
}
|
|
3941
|
+
return this.history.streamHistory(stream, subjects, limit, before, signal);
|
|
4530
3942
|
}
|
|
4531
3943
|
// ---- internals -----------------------------------------------------------
|
|
4532
3944
|
/**
|
|
@@ -4575,9 +3987,10 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
4575
3987
|
continue;
|
|
4576
3988
|
if (s.error instanceof PermissionViolationError && this.chatSubRefusals.has(s.error))
|
|
4577
3989
|
continue;
|
|
4578
|
-
// A principal-route
|
|
4579
|
-
// (SPEC 13.21)
|
|
4580
|
-
|
|
3990
|
+
// A refused principal-route request publish or subscription belongs to the call that throws
|
|
3991
|
+
// it or the serve that ends on it (SPEC 13.21). A refused reply publish has no owner, so it
|
|
3992
|
+
// is reported below.
|
|
3993
|
+
if (s.error instanceof PermissionViolationError && this.ownedPrincipalRouteRefusal(s.error))
|
|
4581
3994
|
continue;
|
|
4582
3995
|
if (s.error instanceof PermissionViolationError && handedToRequest.has(s.error))
|
|
4583
3996
|
continue;
|
|
@@ -5426,7 +4839,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
5426
4839
|
/** Privileged durable-LEAVE write: tombstone the membership at `leaveCursor = frontier` so the
|
|
5427
4840
|
* backstop denies `seq > leaveCursor` while a pre-leave entry stays deliverable (SPEC §7 interval). */
|
|
5428
4841
|
async durableLeaveFor(owner, channel, lifecycleUid, expectedGeneration) {
|
|
5429
|
-
if (!this.plane3)
|
|
4842
|
+
if (!this.deliveryPlane.plane3)
|
|
5430
4843
|
return; // not a Plane-3 host — no membership to tombstone
|
|
5431
4844
|
const kv = await this.membersRegistry();
|
|
5432
4845
|
// expectedGeneration (captured by the agent at durableJoin) refuses a stale leave from tombstoning
|
|
@@ -5518,302 +4931,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
5518
4931
|
* {@link armPlane3} on EVERY (re)connect — a reconnect drains the old connection, so re-binding both
|
|
5519
4932
|
* is required, not optional (the responder would otherwise be lost on a broker blip). */
|
|
5520
4933
|
async startPlane3(aclFor, opts = {}) {
|
|
5521
|
-
|
|
5522
|
-
throw new Error(this.notLiveMsg());
|
|
5523
|
-
this.plane3 = { aclFor, reloadMembershipCreds: opts.reloadMembershipCreds, evictPrincipal: opts.evictPrincipal, evictPrincipals: opts.evictPrincipals, planeConnLiveness: opts.planeConnLiveness, principalLiveness: opts.principalLiveness, reloadStoreIdentity: opts.reloadStoreIdentity, onDeliveryCredsAdopted: opts.onDeliveryCredsAdopted, startupComplete: opts.startupComplete };
|
|
5524
|
-
await this.armPlane3();
|
|
5525
|
-
}
|
|
5526
|
-
/** Whether an ALREADY-DISPATCHED unit of Plane-3 work may still take effect.
|
|
5527
|
-
*
|
|
5528
|
-
* Unsubscribing stops NEW work; it cannot recall work already in flight. A handler that entered
|
|
5529
|
-
* before {@link quiescePlane3} and awaited broker I/O inside it resumes AFTER the freeze, and a
|
|
5530
|
-
* reviewer traced the ordering that makes that a split rather than a latency blip: the loser
|
|
5531
|
-
* accepts a unit and awaits, the loser is descheduled, the successor acquires the shard and flips
|
|
5532
|
-
* its lease READY, then the loser resumes and answers or acks for a shard it no longer holds.
|
|
5533
|
-
* Subscription counts and parked-pull readings cannot see it, because the effect is the reply and
|
|
5534
|
-
* the ack rather than the binding.
|
|
5535
|
-
*
|
|
5536
|
-
* So each work path re-asks HERE, at the point of effect, after its awaits and before it acts.
|
|
5537
|
-
* The lease keeps one ROW, not one SERVER; this is the half that keeps one server. An unacked
|
|
5538
|
-
* message is not lost by refusing: a stopped consumer redelivers it to whoever holds the shard
|
|
5539
|
-
* next, which is the same property quiescing already relies on. */
|
|
5540
|
-
plane3MayAct() {
|
|
5541
|
-
return !this.plane3Quiesced;
|
|
5542
|
-
}
|
|
5543
|
-
/** Serve one runtime durable-membership control request (the server-side delivery daemon). The caller
|
|
5544
|
-
* id is the authenticated subject sender ({@link serveControl} fail-closes on a mismatch). Validation
|
|
5545
|
-
* is against the durable ACL registry — the SAME KV the reader re-auths against (single source of
|
|
5546
|
-
* truth, no in-memory ledger to drift). */
|
|
5547
|
-
async handleDeliveryControl(req) {
|
|
5548
|
-
// FENCE: entered before a quiesce, resuming after it. Answering now would put a second server on
|
|
5549
|
-
// this shard's control rail while the winner is already READY.
|
|
5550
|
-
if (!this.plane3MayAct())
|
|
5551
|
-
return { ok: false, error: "delivery: this daemon is not serving this shard (it is re-checking ownership); retry" };
|
|
5552
|
-
const caller = req.from.id;
|
|
5553
|
-
const args = req.args ?? {};
|
|
5554
|
-
if (req.op === "durableJoin")
|
|
5555
|
-
return this.deliveryJoin(caller, args);
|
|
5556
|
-
if (req.op === "durableLeave")
|
|
5557
|
-
return this.deliveryLeave(caller, args);
|
|
5558
|
-
if (req.op === "readHistory")
|
|
5559
|
-
return this.deliveryReadHistory(caller, args);
|
|
5560
|
-
if (req.op === ANYCAST_TAKES_OP)
|
|
5561
|
-
return this.anycastUnavailable ? { ok: false, error: `delivery: ${this.anycastUnavailable}` } : { ok: true, data: { takes: true } };
|
|
5562
|
-
if (req.op === ANYCAST_STATUS_OP || req.op === ANYCAST_CANCEL_OP)
|
|
5563
|
-
return this.deliveryAnycast(req.op, caller, args);
|
|
5564
|
-
if (req.op === "listMemberships") {
|
|
5565
|
-
if (typeof args.lifecycleUid !== "string")
|
|
5566
|
-
return { ok: false, error: "listMemberships: the caller's lifecycleUid is required (membership rows are lifecycle-keyed, SPEC 13.1)" };
|
|
5567
|
-
let uid;
|
|
5568
|
-
try {
|
|
5569
|
-
uid = assertLifecycleToken(args.lifecycleUid);
|
|
5570
|
-
}
|
|
5571
|
-
catch (e) {
|
|
5572
|
-
return { ok: false, error: e.message };
|
|
5573
|
-
}
|
|
5574
|
-
let memberships;
|
|
5575
|
-
if (args.channel !== undefined) {
|
|
5576
|
-
const channel = this.checkDurableChannelArg(args, "listMemberships", { family: true });
|
|
5577
|
-
if (typeof channel !== "string")
|
|
5578
|
-
return channel;
|
|
5579
|
-
memberships = (await listMembers(await this.membersRegistry(), { owner: caller }))
|
|
5580
|
-
.filter((r) => r.channel === channel && r.leaveCursor === undefined)
|
|
5581
|
-
.map((r) => ({ channel: r.channel, generation: r.generation, activated: r.activated === true, lifecycleUid: r.lifecycleUid }));
|
|
5582
|
-
}
|
|
5583
|
-
else {
|
|
5584
|
-
memberships = await this.ownerMemberships(caller, uid);
|
|
5585
|
-
}
|
|
5586
|
-
// FENCE AFTER THE AWAIT, which is the one that matters. The entry check above cannot catch the
|
|
5587
|
-
// case this exists for: a request admitted WHILE SERVING, then descheduled inside its broker
|
|
5588
|
-
// read, resuming after a successor has taken the shard and reached READY. Re-ask at the point
|
|
5589
|
-
// of effect, so the answer is never served by a daemon that has stood down.
|
|
5590
|
-
if (!this.plane3MayAct())
|
|
5591
|
-
return { ok: false, error: "delivery: this daemon stopped serving this shard while the request was in flight; retry" };
|
|
5592
|
-
return { ok: true, data: { memberships } };
|
|
5593
|
-
}
|
|
5594
|
-
return { ok: false, error: `op "${req.op}" not supported on the delivery control service` };
|
|
5595
|
-
}
|
|
5596
|
-
/** Validate the channel ARG shape only: non-blank, valid, concrete (NO ACL check, that is op-specific).
|
|
5597
|
-
* `family` also admits a channel family such as `review.>` (a membership join or leave, #5404).
|
|
5598
|
-
* Returns the channel on success or a ControlReply error to short-circuit. */
|
|
5599
|
-
checkDurableChannelArg(args, op, opts = {}) {
|
|
5600
|
-
const channel = typeof args.channel === "string" ? args.channel.trim() : "";
|
|
5601
|
-
if (!channel)
|
|
5602
|
-
return { ok: false, error: `${op}: channel must be a non-blank string` };
|
|
5603
|
-
try {
|
|
5604
|
-
assertValidChannel(channel);
|
|
5605
|
-
}
|
|
5606
|
-
catch (e) {
|
|
5607
|
-
return { ok: false, error: e.message };
|
|
5608
|
-
}
|
|
5609
|
-
if (opts.family && isChannelFamily(channel))
|
|
5610
|
-
return channel;
|
|
5611
|
-
if (!isConcreteChannel(channel))
|
|
5612
|
-
return {
|
|
5613
|
-
ok: false,
|
|
5614
|
-
error: opts.family
|
|
5615
|
-
? `${op}: "${channel}" stays live only: a durable membership takes a concrete channel or a family such as review.>, never the root > or a pattern with *`
|
|
5616
|
-
: `${op}: "${channel}" must be a concrete channel (durable membership is per-concrete-channel, not wildcard)`,
|
|
5617
|
-
};
|
|
5618
|
-
return channel;
|
|
5619
|
-
}
|
|
5620
|
-
/** JOIN requires the channel be within the caller's CURRENT read ACL (you can't durable-subscribe a
|
|
5621
|
-
* channel you may not read). A family must lie inside one ACL entry, so every channel under it is
|
|
5622
|
-
* readable. */
|
|
5623
|
-
async deliveryJoin(caller, args) {
|
|
5624
|
-
const channel = this.checkDurableChannelArg(args, "durableJoin", { family: true });
|
|
5625
|
-
if (typeof channel !== "string")
|
|
5626
|
-
return channel; // a ControlReply error
|
|
5627
|
-
// The caller arrives as an ALIAS (the control subject carries owner+actor, never a uid): resolve
|
|
5628
|
-
// its single live lifecycle-keyed ACL row SERVER-SIDE — the trusted registry, never a caller
|
|
5629
|
-
// assertion, decides which lifecycle joins. Two live rows (a reservation breach / unfinished
|
|
5630
|
-
// teardown) refuse loudly rather than binding a membership to a guessed lifecycle.
|
|
5631
|
-
let acl;
|
|
5632
|
-
try {
|
|
5633
|
-
acl = await this.aclForAlias(caller);
|
|
5634
|
-
}
|
|
5635
|
-
catch (e) {
|
|
5636
|
-
return { ok: false, error: e.message };
|
|
5637
|
-
}
|
|
5638
|
-
if (acl === undefined)
|
|
5639
|
-
return { ok: false, error: `durableJoin: no read ACL on record for ${caller} (not provisioned for durable delivery)` };
|
|
5640
|
-
if (!membershipInAllow(acl.allowSubscribe, channel))
|
|
5641
|
-
return { ok: false, error: `channel "${channel}" is not within your read ACL [${acl.allowSubscribe.join(", ")}]` };
|
|
5642
|
-
const uid = acl.lifecycleUid;
|
|
5643
|
-
// The lookup above may be stale by the time the row is written (a retirement can fence and purge
|
|
5644
|
-
// this lifecycle meanwhile), so the write re-checks the exact ACL row itself (`aclFence`).
|
|
5645
|
-
try {
|
|
5646
|
-
return { ok: true, data: await this.membershipWrite(caller, uid, () => this.durableJoinFor(caller, channel, uid, { aclFence: true })) };
|
|
5647
|
-
}
|
|
5648
|
-
catch (e) {
|
|
5649
|
-
return { ok: false, error: e.message };
|
|
5650
|
-
}
|
|
5651
|
-
}
|
|
5652
|
-
/** Whether `lifecycleUid`'s exact ACL row still allows reads, so an inventory of it came from a
|
|
5653
|
-
* caller that did not fence. A row narrowed to no reads, or retired or absent, is fenced. A read
|
|
5654
|
-
* that fails answers true: the caller keeps its in-memory mark, which only ever refuses. */
|
|
5655
|
-
async lifecycleAclUnfenced(principal, lifecycleUid) {
|
|
5656
|
-
try {
|
|
5657
|
-
const row = await readAcl(await this.aclRegistry(), principal, lifecycleUid);
|
|
5658
|
-
return row !== undefined && row.record.allowSubscribe.length > 0;
|
|
5659
|
-
}
|
|
5660
|
-
catch {
|
|
5661
|
-
return true;
|
|
5662
|
-
}
|
|
5663
|
-
}
|
|
5664
|
-
/** Run one durable join or leave that writes `principal`'s rows of `lifecycleUid`, refused once that
|
|
5665
|
-
* lifecycle's inventory has been asked for ({@link retiringLifecycles}). The check and the
|
|
5666
|
-
* registration have no await between them, so an inventory either finds this write running and
|
|
5667
|
-
* waits for it, or this write finds the inventory's mark and is refused. */
|
|
5668
|
-
async membershipWrite(principal, lifecycleUid, write) {
|
|
5669
|
-
if (this.retiringLifecycles.get(principal)?.has(lifecycleUid))
|
|
5670
|
-
throw new Error(`lifecycle ${lifecycleUid} of ${principal} is retiring: its membership inventory was requested, so its durable memberships no longer change`);
|
|
5671
|
-
const key = `${principal}/${lifecycleUid}`;
|
|
5672
|
-
const running = write();
|
|
5673
|
-
let writes = this.membershipWrites.get(key);
|
|
5674
|
-
if (writes === undefined)
|
|
5675
|
-
this.membershipWrites.set(key, (writes = new Set()));
|
|
5676
|
-
writes.add(running);
|
|
5677
|
-
try {
|
|
5678
|
-
return await running;
|
|
5679
|
-
}
|
|
5680
|
-
finally {
|
|
5681
|
-
writes.delete(running);
|
|
5682
|
-
if (writes.size === 0)
|
|
5683
|
-
this.membershipWrites.delete(key);
|
|
5684
|
-
}
|
|
5685
|
-
}
|
|
5686
|
-
/** LEAVE must NOT require current-ACL coverage. Leave fires precisely when the ACL was narrowed/revoked
|
|
5687
|
-
* (a refused live sub → {@link closeRefusedMembership}); gating the tombstone on the current ACL would
|
|
5688
|
-
* loop forever and leave the SPEC §7 boundary open (the membership could resume if the ACL is later
|
|
5689
|
-
* restored). The guards are: authenticated caller (serveControl), concrete channel, a finite generation
|
|
5690
|
-
* (the join epoch — without it a stale/replayed leave could tombstone a newer rejoin), and an EXISTING
|
|
5691
|
-
* own membership; `durableLeaveFor` → `tombstoneMember` then enforces the generation match. */
|
|
5692
|
-
async deliveryLeave(caller, args) {
|
|
5693
|
-
const channel = this.checkDurableChannelArg(args, "durableLeave", { family: true });
|
|
5694
|
-
if (typeof channel !== "string")
|
|
5695
|
-
return channel; // a ControlReply error
|
|
5696
|
-
if (typeof args.generation !== "number" || !Number.isFinite(args.generation))
|
|
5697
|
-
return { ok: false, error: "durableLeave: a finite generation is required (fail-closed stale-leave guard)" };
|
|
5698
|
-
// The LEAVE carries the leaver's own lifecycleUid: leave must work precisely when the ACL row was
|
|
5699
|
-
// narrowed or already purged (see the method doc), so the alias→row resolution join uses is not
|
|
5700
|
-
// available here. The uid is caller-asserted but harmless to lie about: the member key it selects
|
|
5701
|
-
// is confined to the AUTHENTICATED caller's own alias (`<channel>/<caller>.<uid>`), so the worst a
|
|
5702
|
-
// false uid reaches is the caller's own retired incarnation's row — legitimate cleanup — and the
|
|
5703
|
-
// generation guard still applies.
|
|
5704
|
-
if (typeof args.lifecycleUid !== "string")
|
|
5705
|
-
return { ok: false, error: "durableLeave: the leaving incarnation's lifecycleUid is required (membership rows are lifecycle-keyed, SPEC 13.1)" };
|
|
5706
|
-
let uid;
|
|
5707
|
-
try {
|
|
5708
|
-
uid = assertLifecycleToken(args.lifecycleUid);
|
|
5709
|
-
}
|
|
5710
|
-
catch (e) {
|
|
5711
|
-
return { ok: false, error: e.message };
|
|
5712
|
-
}
|
|
5713
|
-
const existing = await readMember(await this.membersRegistry(), channel, caller, uid);
|
|
5714
|
-
if (!existing)
|
|
5715
|
-
return { ok: true, data: { channel, alreadyLeft: true } }; // nothing to tombstone — idempotent
|
|
5716
|
-
const generation = args.generation;
|
|
5717
|
-
try {
|
|
5718
|
-
await this.membershipWrite(caller, uid, () => this.durableLeaveFor(caller, channel, uid, generation));
|
|
5719
|
-
}
|
|
5720
|
-
catch (e) {
|
|
5721
|
-
return { ok: false, error: e.message };
|
|
5722
|
-
}
|
|
5723
|
-
return { ok: true, data: { channel } };
|
|
5724
|
-
}
|
|
5725
|
-
/** Serve one MEDIATED HISTORY READ (`readHistory`, the client side is {@link readHistory}). The daemon
|
|
5726
|
-
* holds the consumer so the caller does not have to: this is a privilege reduction, not a new
|
|
5727
|
-
* capability, and it is the shape SPEC's "Mediated reads (normative)" rule asks for.
|
|
5728
|
-
*
|
|
5729
|
-
* THE ONE INVARIANT THIS METHOD EXISTS FOR: authorization is read FRESH on every call and never
|
|
5730
|
-
* cached across calls. A consumer pins its authorization when it is created, so a revoked caller
|
|
5731
|
-
* keeps being served by a consumer it already holds; re-reading here is what makes a revocation
|
|
5732
|
-
* stop the very NEXT read. Cache this and the verb loses its only advantage over the raw consumer
|
|
5733
|
-
* path.
|
|
5734
|
-
*
|
|
5735
|
-
* AUTHORITY (SPEC §9.6): "current read ACL" is the effective broker-accepted credential. The
|
|
5736
|
-
* durable registry is a live mirror of that credential, not a second grant source. History
|
|
5737
|
-
* authorizes against `allowSubscribe ∩ issuedAllowSubscribe` — the live row (so revocation via
|
|
5738
|
-
* plain commitAcl still stops the next read) intersected with the mint-time ceiling (so a
|
|
5739
|
-
* registry widen without a remint cannot grant what the JWT does not). Raising the ceiling is
|
|
5740
|
-
* {@link reissueAcl}, which is what provision does when it bakes the list into the JWT.
|
|
5741
|
-
*
|
|
5742
|
-
* The caller arrives as an alias (the control subject carries owner+actor, never a uid) and is the
|
|
5743
|
-
* AUTHENTICATED subject sender — `serveControl` has already fail-closed on any payload that names a
|
|
5744
|
-
* different principal, so `caller` cannot be self-asserted. */
|
|
5745
|
-
async deliveryReadHistory(caller, args) {
|
|
5746
|
-
const channel = this.checkDurableChannelArg(args, "readHistory");
|
|
5747
|
-
if (typeof channel !== "string")
|
|
5748
|
-
return channel; // a ControlReply error
|
|
5749
|
-
// The caller PROPOSES a limit; the mediator decides. Reject a nonsense limit loudly rather than
|
|
5750
|
-
// silently substituting a default — a caller asking for -1 has a bug and should hear about it.
|
|
5751
|
-
const asked = args.limit === undefined ? READ_HISTORY_DEFAULT_LIMIT : args.limit;
|
|
5752
|
-
if (typeof asked !== "number" || !Number.isInteger(asked) || asked < 1)
|
|
5753
|
-
return { ok: false, error: `readHistory: limit must be a positive integer (got ${JSON.stringify(args.limit)})` };
|
|
5754
|
-
const limit = Math.min(asked, READ_HISTORY_MAX_LIMIT);
|
|
5755
|
-
// FRESH per call — see the method doc. Same registry, same alias resolution, and the same loud
|
|
5756
|
-
// refusal on two live rows that `durableJoin` uses: a stale lifecycle must never authorize its
|
|
5757
|
-
// successor's read.
|
|
5758
|
-
//
|
|
5759
|
-
// SPEC §9.6: "current read ACL" is the effective broker-accepted credential. The registry row is
|
|
5760
|
-
// a live mirror and can be rewritten by plain commitAcl without a remint. Authorizing history
|
|
5761
|
-
// from the row ALONE let a widen grant reads channelHistory still broker-denies — a new
|
|
5762
|
-
// capability wearing a privilege-reduction label (panel BLOCKING at 914fd7b0). History therefore
|
|
5763
|
-
// requires the channel in BOTH the live row AND the mint-time ceiling (`issuedAllowSubscribe`).
|
|
5764
|
-
// Revocation still works: narrowing allowSubscribe stops the next read. Raising the ceiling
|
|
5765
|
-
// requires reissueAcl (provision/remint), which is the same act that bakes the list into the JWT.
|
|
5766
|
-
let acl;
|
|
5767
|
-
try {
|
|
5768
|
-
acl = await this.aclForAlias(caller);
|
|
5769
|
-
}
|
|
5770
|
-
catch (e) {
|
|
5771
|
-
return { ok: false, error: e.message };
|
|
5772
|
-
}
|
|
5773
|
-
if (acl === undefined)
|
|
5774
|
-
return { ok: false, error: `readHistory: no read ACL on record for ${caller} - not permitted` };
|
|
5775
|
-
const inLive = channelInAllow(acl.allowSubscribe, channel);
|
|
5776
|
-
const inIssued = channelInAllow(acl.issuedAllowSubscribe, channel);
|
|
5777
|
-
if (!inLive || !inIssued) {
|
|
5778
|
-
// Name the live list in the error — that is what operators edit and what revocation clears.
|
|
5779
|
-
// A ceiling miss (registry widened past the credential) is the same refusal shape as a live
|
|
5780
|
-
// miss so a caller cannot probe which half failed.
|
|
5781
|
-
return {
|
|
5782
|
-
ok: false,
|
|
5783
|
-
error: `readHistory: channel "${channel}" is not within your read ACL [${acl.allowSubscribe.join(", ")}] - refused`,
|
|
5784
|
-
};
|
|
5785
|
-
}
|
|
5786
|
-
// One message MORE than the page is the completeness probe: if the backlog hands back limit+1,
|
|
5787
|
-
// something older exists behind the page and `complete` is false. Cheaper and more honest than a
|
|
5788
|
-
// separate count, which could race the page it describes.
|
|
5789
|
-
let page;
|
|
5790
|
-
try {
|
|
5791
|
-
page = await this.channelHistory(channel, { limit: limit + 1 });
|
|
5792
|
-
}
|
|
5793
|
-
catch (e) {
|
|
5794
|
-
// Surface the read failure. `streamHistory` deliberately raises rather than returning empty on a
|
|
5795
|
-
// denial or a cut-short window, and that distinction must survive the rail: an error here would
|
|
5796
|
-
// otherwise reach the caller as an empty page and render as "no history".
|
|
5797
|
-
return { ok: false, error: `readHistory: ${e.message}` };
|
|
5798
|
-
}
|
|
5799
|
-
const reachedStart = page.length <= limit;
|
|
5800
|
-
const wanted = reachedStart ? page : page.slice(-limit);
|
|
5801
|
-
// A page that cannot be SENT is not a page. The reply rides one NATS message, so `limit` alone is
|
|
5802
|
-
// the wrong bound: 200 large messages serialize past `max_payload`, and `serveControl` can then only
|
|
5803
|
-
// answer with a refusal in place of the page.
|
|
5804
|
-
//
|
|
5805
|
-
// So bound by BYTES too, keeping the NEWEST that fit — which needs no new vocabulary, because
|
|
5806
|
-
// `complete: false` already means "older history remains behind this page". Trimming here is the
|
|
5807
|
-
// documented truncation signal doing its job, not a silent degradation.
|
|
5808
|
-
const fitted = fitHistoryPage(wanted, this.maxPayload);
|
|
5809
|
-
// `wanted.length > 0` is load-bearing, not defensive: an EMPTY channel also fits nothing, and
|
|
5810
|
-
// without this guard a channel nobody has posted to was refused with "the newest message exceeds
|
|
5811
|
-
// the payload budget" — a confident, entirely wrong explanation for a legitimately empty result.
|
|
5812
|
-
// Genuine emptiness is `{ items: [], complete: true }`; only a message too large to ever send is
|
|
5813
|
-
// the error.
|
|
5814
|
-
if (wanted.length > 0 && fitted.length === 0)
|
|
5815
|
-
return { ok: false, error: `readHistory: the newest message on "${channel}" alone exceeds the broker payload budget (${this.maxPayload} bytes) - refused loudly rather than served as an empty page` };
|
|
5816
|
-
return { ok: true, data: { items: fitted, complete: reachedStart && fitted.length === wanted.length } };
|
|
4934
|
+
return this.deliveryPlane.startPlane3(aclFor, opts);
|
|
5817
4935
|
}
|
|
5818
4936
|
/** Stop serving Plane-3 WITHOUT tearing down the connection, so a daemon that has just learned its
|
|
5819
4937
|
* lease may no longer be its own can stop acting on the shard while it finds out for certain.
|
|
@@ -5835,37 +4953,13 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
5835
4953
|
* Deliberately not `stop()`: the connection, the lease KV handles and the control rails must stay
|
|
5836
4954
|
* up, because the daemon still has to ask the broker who owns the shard. */
|
|
5837
4955
|
async quiescePlane3() {
|
|
5838
|
-
this.
|
|
5839
|
-
if (this.deliveryServeSub) {
|
|
5840
|
-
try {
|
|
5841
|
-
this.deliveryServeSub.unsubscribe();
|
|
5842
|
-
}
|
|
5843
|
-
catch { /* already dead */ }
|
|
5844
|
-
this.deliveryServeSub = undefined;
|
|
5845
|
-
}
|
|
5846
|
-
if (this.deliveryAdminServeSub) {
|
|
5847
|
-
try {
|
|
5848
|
-
this.deliveryAdminServeSub.unsubscribe();
|
|
5849
|
-
}
|
|
5850
|
-
catch { /* already dead */ }
|
|
5851
|
-
this.deliveryAdminServeSub = undefined;
|
|
5852
|
-
}
|
|
5853
|
-
this.dropAnycastRequests();
|
|
5854
|
-
// Stopping the consumers ends the `for await` loops that drive fan-out and the reader. In-flight
|
|
5855
|
-
// messages are NOT acked by a stopped consumer, so they redeliver to whoever holds the shard
|
|
5856
|
-
// next: quiescing loses no message, it only stops this process from claiming them.
|
|
5857
|
-
for (const msgs of this.streamMsgs.splice(0)) {
|
|
5858
|
-
try {
|
|
5859
|
-
msgs.stop();
|
|
5860
|
-
}
|
|
5861
|
-
catch { /* already draining */ }
|
|
5862
|
-
}
|
|
4956
|
+
return this.deliveryPlane.quiescePlane3();
|
|
5863
4957
|
}
|
|
5864
4958
|
/** Whether Plane-3 is currently quiesced: unbound, and refusing to re-bind until ownership is proven.
|
|
5865
4959
|
* Read-only. The flag is the daemon's own answer to "am I serving this shard", so a cell that asserts
|
|
5866
4960
|
* a recovery worked can check the endpoint's state rather than inferring it from a log line. */
|
|
5867
4961
|
plane3IsQuiesced() {
|
|
5868
|
-
return this.
|
|
4962
|
+
return this.deliveryPlane.plane3IsQuiesced();
|
|
5869
4963
|
}
|
|
5870
4964
|
/** Resume serving Plane-3 after {@link quiescePlane3}, once ownership has been re-established.
|
|
5871
4965
|
* Idempotent, and a no-op when the daemon was never quiesced.
|
|
@@ -5878,80 +4972,19 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
5878
4972
|
* That is a readiness lie surviving a transient broker error, the daemon claims a responder it does
|
|
5879
4973
|
* not have, which is the #1318 outage wearing the readiness flag instead of the exit path. */
|
|
5880
4974
|
async rearmPlane3() {
|
|
5881
|
-
|
|
5882
|
-
|
|
5883
|
-
|
|
5884
|
-
|
|
5885
|
-
|
|
5886
|
-
this.plane3Quiesced = false;
|
|
5887
|
-
try {
|
|
5888
|
-
await this.armPlane3();
|
|
5889
|
-
}
|
|
5890
|
-
catch (e) {
|
|
5891
|
-
this.plane3Quiesced = true;
|
|
5892
|
-
// Drop whatever DID bind. Leaving half-bound responders up would answer control requests for a
|
|
5893
|
-
// shard this process has just declared itself not to be serving.
|
|
5894
|
-
this.quiescePlane3();
|
|
5895
|
-
throw e;
|
|
5896
|
-
}
|
|
4975
|
+
return this.deliveryPlane.rearmPlane3();
|
|
4976
|
+
}
|
|
4977
|
+
/** Forwards to the delivery plane's {@link EndpointDeliveryPlane.plane3MayAct}. */
|
|
4978
|
+
plane3MayAct() {
|
|
4979
|
+
return this.deliveryPlane.plane3MayAct();
|
|
5897
4980
|
}
|
|
5898
|
-
/**
|
|
5899
|
-
* cursor. Called by {@link startPlane3} once AND by {@link connectAndBind} on every (re)connect, so
|
|
5900
|
-
* the delivery daemon's reconnect RE-ARMS the backstop + the ctl.delivery responder. Without this, a broker blip would silently kill
|
|
5901
|
-
* the loops while `durableJoinFor` kept reporting `durable:true` (the impl-review's BLOCKER-1). No-op
|
|
5902
|
-
* unless this endpoint hosts Plane-3 (`this.plane3` set). */
|
|
4981
|
+
/** Forwards to the delivery plane's {@link EndpointDeliveryPlane.armPlane3}. */
|
|
5903
4982
|
async armPlane3() {
|
|
5904
|
-
|
|
5905
|
-
|
|
5906
|
-
|
|
5907
|
-
|
|
5908
|
-
|
|
5909
|
-
if (this.plane3Quiesced)
|
|
5910
|
-
return;
|
|
5911
|
-
await this.manager(); // the manager runs consume:false, so this.jsm is lazy — ensure it
|
|
5912
|
-
this.armDeliveryControl();
|
|
5913
|
-
await this.armAnycastRequests();
|
|
5914
|
-
await this.runFanout();
|
|
5915
|
-
await this.runReader();
|
|
5916
|
-
}
|
|
5917
|
-
/** (Re)register the `ctl.delivery` control responder on the CURRENT connection. A reconnect drains the
|
|
5918
|
-
* old connection (the old sub is dead and `clearConnectionScoped` leaves caller-owned subs alone), so
|
|
5919
|
-
* this MUST run on every arm — otherwise durable join/leave/list silently lose their responder after a
|
|
5920
|
-
* broker blip. The stale sub is dropped (unsubscribed + removed from `this.subs`) before re-creating.
|
|
5921
|
-
* `boundReply` is essential here: the daemon holds a wildcard reply-publish grant, so the serve path
|
|
5922
|
-
* must reject any reply target outside the authenticated sender's own subtree (confused-deputy fix). */
|
|
5923
|
-
armDeliveryControl() {
|
|
5924
|
-
if (this.deliveryServeSub) {
|
|
5925
|
-
try {
|
|
5926
|
-
this.deliveryServeSub.unsubscribe();
|
|
5927
|
-
}
|
|
5928
|
-
catch { /* dead with the old connection */ }
|
|
5929
|
-
const i = this.subs.indexOf(this.deliveryServeSub);
|
|
5930
|
-
if (i >= 0)
|
|
5931
|
-
this.subs.splice(i, 1);
|
|
5932
|
-
}
|
|
5933
|
-
// A request's status and cancel touch only the TASK queue and the anycast records, so they do
|
|
5934
|
-
// not wait behind a channel join on the in-order chain.
|
|
5935
|
-
this.deliveryServeSub = this.serveControl(CONTROL_DELIVERY, (req) => this.handleDeliveryControl(req), {
|
|
5936
|
-
boundReply: true,
|
|
5937
|
-
concurrent: (req) => ANYCAST_CONTROL_OPS.has(req.op),
|
|
5938
|
-
});
|
|
5939
|
-
if (this.deliveryAdminServeSub) {
|
|
5940
|
-
try {
|
|
5941
|
-
this.deliveryAdminServeSub.unsubscribe();
|
|
5942
|
-
}
|
|
5943
|
-
catch { /* dead with the old connection */ }
|
|
5944
|
-
const i = this.subs.indexOf(this.deliveryAdminServeSub);
|
|
5945
|
-
if (i >= 0)
|
|
5946
|
-
this.subs.splice(i, 1);
|
|
5947
|
-
}
|
|
5948
|
-
// The membership inventory is read-only and shares nothing with the other admin commands, so it
|
|
5949
|
-
// does not queue behind them. In a mass stop each retirement's inventory followed every other
|
|
5950
|
-
// seat's `evictPrincipal` on this rail and missed its 5 s deadline (#4965).
|
|
5951
|
-
this.deliveryAdminServeSub = this.serveControl(CONTROL_DELIVERY_ADMIN, (req) => this.handleDeliveryAdmin(req), {
|
|
5952
|
-
boundReply: true,
|
|
5953
|
-
concurrent: (req) => req.op === "lifecycleMemberships",
|
|
5954
|
-
});
|
|
4983
|
+
return this.deliveryPlane.armPlane3();
|
|
4984
|
+
}
|
|
4985
|
+
/** Forwards to the delivery plane's {@link EndpointDeliveryPlane.pumpDlv}. */
|
|
4986
|
+
async pumpDlv() {
|
|
4987
|
+
return this.deliveryPlane.pumpDlv();
|
|
5955
4988
|
}
|
|
5956
4989
|
/** (Re)bind the anycast take responder and the request records maintenance on the current
|
|
5957
4990
|
* connection (SPEC §8.1). The records bucket is created here as well as at space setup, so a mesh
|
|
@@ -6339,23 +5372,28 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
6339
5372
|
return "refused";
|
|
6340
5373
|
}
|
|
6341
5374
|
/**
|
|
6342
|
-
* Serve this process's
|
|
6343
|
-
* set about this principal, addressed by its own principal, lifecycle UID
|
|
6344
|
-
*
|
|
5375
|
+
* Serve this process's `agent` endpoint on the principal route (SPEC 13.21, 13.22): the
|
|
5376
|
+
* `AGENT_CLUSTER` command set about this principal, addressed by its own principal, lifecycle UID
|
|
5377
|
+
* and runtime id, with no registration. `handlers` serves the cluster's commands with
|
|
5378
|
+
* `AGENT_CONTRACTS`; `describe` is built in.
|
|
6345
5379
|
*
|
|
6346
5380
|
* Throws on an open mesh and on a static credential, where no row names who may call, without a
|
|
6347
|
-
* lifecycle UID, before the endpoint has started,
|
|
6348
|
-
* cluster
|
|
6349
|
-
* subscribes this process's own subject
|
|
5381
|
+
* lifecycle UID, before the endpoint has started, while it already serves `agent`, for an
|
|
5382
|
+
* `endpoint` or `cluster` field, which this entry point does not take, and for a set of handlers
|
|
5383
|
+
* that is not the cluster's commands with its contracts. It subscribes this process's own subject
|
|
5384
|
+
* with no queue group.
|
|
6350
5385
|
*
|
|
6351
5386
|
* `ready` is a positive check: it resolves only after the broker accepted the subscription
|
|
6352
5387
|
* (subscribe, flush, and no permission violation for that subject), and rejects with the
|
|
6353
5388
|
* refusal, which a credential minted before the rows existed gets. The serve is INTENT: every
|
|
6354
5389
|
* replaced connection subscribes again and runs the check again, and a refusal there ends the
|
|
6355
|
-
* serve, resolves `failed` and clears the presence field. `stop` unsubscribes
|
|
6356
|
-
* handlers in flight.
|
|
5390
|
+
* serve, resolves `failed` and clears the presence field. `stop` unsubscribes, aborts the
|
|
5391
|
+
* handlers' `signal`, and waits for each handler in flight until its request's deadline at most.
|
|
6357
5392
|
*/
|
|
6358
|
-
servePrincipal(
|
|
5393
|
+
servePrincipal(spec) {
|
|
5394
|
+
if ("endpoint" in spec || "cluster" in spec)
|
|
5395
|
+
throw new Error(`servePrincipal serves only the "${AGENT_ENDPOINT}" endpoint with AGENT_CLUSTER and takes handlers alone; an endpoint or cluster is refused, never ignored (SPEC 13.22)`);
|
|
5396
|
+
const serve = { endpoint: AGENT_ENDPOINT, cluster: AGENT_CLUSTER, handlers: spec.handlers };
|
|
6359
5397
|
if (!this.authed)
|
|
6360
5398
|
throw new Error("the principal route is served only on an authenticated mesh: on an open mesh no row says who may call (SPEC 13.21)");
|
|
6361
5399
|
if (!this.userMode)
|
|
@@ -6384,6 +5422,10 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
6384
5422
|
const handle = this.bindPrincipalHandle(intent, nc);
|
|
6385
5423
|
this.principalServes.set(serve.endpoint, intent);
|
|
6386
5424
|
principalServers.set(serverKey, intent);
|
|
5425
|
+
// A fresh `seq` per serve: the last serve may have ended with prompts open, and the pair
|
|
5426
|
+
// `(runtime.id, seq)` it showed then must not name this serve's empty list (SPEC §6).
|
|
5427
|
+
if (serve.endpoint === AGENT_ENDPOINT)
|
|
5428
|
+
this.promptsPresence = { open: 0, seq: this.nextPromptsSeq() };
|
|
6387
5429
|
void this.armPrincipalServe(intent, nc, handle);
|
|
6388
5430
|
return { ready, failed, stop: () => this.stopPrincipalServe(intent) };
|
|
6389
5431
|
}
|
|
@@ -6391,15 +5433,25 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
6391
5433
|
bindPrincipalHandle(intent, nc) {
|
|
6392
5434
|
return servePrincipalRoute(nc, this.space, intent.self, intent.serve);
|
|
6393
5435
|
}
|
|
6394
|
-
/** A
|
|
6395
|
-
|
|
5436
|
+
/** A principal-route refusal a call or a serve reports itself: a request publish, a serve
|
|
5437
|
+
* subscription or a reply read. Only a reply publish is left, which no call or serve awaits. */
|
|
5438
|
+
ownedPrincipalRouteRefusal(err) {
|
|
6396
5439
|
const p = `${spacePrefix(this.space)}.ep.`;
|
|
6397
|
-
|
|
5440
|
+
if (err.operation === "publish")
|
|
5441
|
+
return err.subject.startsWith(`${p}prin.`);
|
|
5442
|
+
return err.subject.startsWith(`${p}prin.`) || err.subject.startsWith(`${p}prin-reply.`);
|
|
6398
5443
|
}
|
|
6399
5444
|
/** Track one binding: a refusal ends the serve, and an accepted subscription on the current
|
|
6400
5445
|
* connection shows the presence field. */
|
|
6401
5446
|
async armPrincipalServe(intent, nc, handle) {
|
|
6402
5447
|
intent.binding = { nc, handle };
|
|
5448
|
+
// Every earlier binding is on a replaced connection: retire it once its handlers finish.
|
|
5449
|
+
for (const old of intent.handles)
|
|
5450
|
+
void old.stop().finally(() => {
|
|
5451
|
+
const i = intent.handles.indexOf(old);
|
|
5452
|
+
if (i >= 0)
|
|
5453
|
+
intent.handles.splice(i, 1);
|
|
5454
|
+
});
|
|
6403
5455
|
intent.handles.push(handle);
|
|
6404
5456
|
intent.readyOn = undefined;
|
|
6405
5457
|
void handle.refused.then((err) => {
|
|
@@ -6487,7 +5539,8 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
6487
5539
|
if (principalServers.get(intent.serverKey) === intent)
|
|
6488
5540
|
principalServers.delete(intent.serverKey);
|
|
6489
5541
|
}
|
|
6490
|
-
/** Stop serving `intent` and wait for its handlers in flight
|
|
5542
|
+
/** Stop serving `intent` and wait for its handlers in flight, each until its request's deadline
|
|
5543
|
+
* at most. The ending is synchronous, so a
|
|
6491
5544
|
* `ready` still waiting for the broker rejects before the caller's next step. `republish` writes
|
|
6492
5545
|
* presence without the `prompts` field; the endpoint's own stop writes it offline instead. */
|
|
6493
5546
|
async stopPrincipalServe(intent, republish = true) {
|
|
@@ -6510,8 +5563,8 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
6510
5563
|
* prompts it holds `open` or `sent`, and a counter that goes up by one each time a prompt opens or
|
|
6511
5564
|
* changes state. The field shows only while the `agent` endpoint is served and the broker accepted
|
|
6512
5565
|
* its subscription on the current connection. Until this is called it shows `open: 0` with the
|
|
6513
|
-
*
|
|
6514
|
-
* not served, and a RangeError for a `seq` below one this runtime already used.
|
|
5566
|
+
* fresh `seq` the serve took from {@link nextPromptsSeq} when it started. Throws when the `agent`
|
|
5567
|
+
* endpoint is not served, and a RangeError for a `seq` below one this runtime already used.
|
|
6515
5568
|
*/
|
|
6516
5569
|
async setPromptsPresence(prompts) {
|
|
6517
5570
|
for (const k of ["open", "seq"])
|
|
@@ -6544,7 +5597,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
6544
5597
|
const intent = this.principalServes.get(AGENT_ENDPOINT);
|
|
6545
5598
|
if (!intent || intent.ended || !this.nc || intent.readyOn !== this.nc)
|
|
6546
5599
|
return undefined;
|
|
6547
|
-
return this.promptsPresence
|
|
5600
|
+
return this.promptsPresence;
|
|
6548
5601
|
}
|
|
6549
5602
|
/**
|
|
6550
5603
|
* Call one command of a principal's own endpoint on the principal route (SPEC 13.21), as this
|
|
@@ -6573,671 +5626,6 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
6573
5626
|
...(call.args !== undefined ? { args: call.args } : {}),
|
|
6574
5627
|
}, { deadlineMs: call.deadlineMs, ...(call.signal ? { signal: call.signal } : {}) });
|
|
6575
5628
|
}
|
|
6576
|
-
/** Serve one PRIVILEGED delivery-admin request (the D5 rail-split). The cred layer is the caller
|
|
6577
|
-
* boundary — only the supervisor profile can publish here — and `serveControl`'s sender check +
|
|
6578
|
-
* bounded reply still apply on top. `reloadCreds` is the class-2 renewal ADOPTION step: re-read
|
|
6579
|
-
* the renewal-owner-re-signed creds file, pin, swap the live connection, reconnect the membership
|
|
6580
|
-
* feed's rw connection, and reply with proof (identities + the adopted JWT windows) — or a
|
|
6581
|
-
* structured failure (e.g. the file was never re-signed), never a silent partial. */
|
|
6582
|
-
async handleDeliveryAdmin(req) {
|
|
6583
|
-
// A reload's resident swap reconnects the connection every reply here rides. A caller's next
|
|
6584
|
-
// request, such as a manager's eviction right after its reload, is answered once that
|
|
6585
|
-
// connection is back, not while it is down.
|
|
6586
|
-
await this.residentSwap;
|
|
6587
|
-
// FENCE: same reason as the runtime rail above.
|
|
6588
|
-
if (!this.plane3MayAct())
|
|
6589
|
-
return { ok: false, error: "delivery: this daemon is not serving this shard (it is re-checking ownership); retry" };
|
|
6590
|
-
if (req.op === "reloadCreds") {
|
|
6591
|
-
// NOT BEFORE START-UP IS DONE (#2304). Both halves below would run: the delivery half commits
|
|
6592
|
-
// and schedules `nc.reconnect()`, which lands underneath start-up awaits still in flight on
|
|
6593
|
-
// this connection (the lease watch's consumer create times out and the daemon exits). Refuse
|
|
6594
|
-
// before either half acts, so nothing is adopted and the renewal owner records why.
|
|
6595
|
-
if (this.plane3?.startupComplete?.() === false)
|
|
6596
|
-
return { ok: false, error: "delivery: this daemon has not finished starting; nothing adopted - the next renewal pass or the daemon's 75% re-read adopts the re-signed creds" };
|
|
6597
|
-
// The renewal owner's EXPECTED-generation tokens (SHA-256 of each JWT it re-signed), per
|
|
6598
|
-
// component. A missing entry means "no expectation" (the passive backstop still adopts).
|
|
6599
|
-
const expected = (req.args?.expected ?? {});
|
|
6600
|
-
// Prove BOTH components INDEPENDENTLY (non-short-circuit): one failing must neither hide the
|
|
6601
|
-
// other's outcome nor be masked by it. Top-level `ok` is false if EITHER proof failed, so the
|
|
6602
|
-
// manager can never record a green adoption while a component was refused.
|
|
6603
|
-
// Each component's outcome claims only what is PROVEN: `brokerAccepted` (the preflight) carries
|
|
6604
|
-
// the accepted generation's window; `residentSwap` records that the wire swap is best-effort and
|
|
6605
|
-
// self-healing, NOT witnessed. Neither claims verified resident reauth (that would need a
|
|
6606
|
-
// generation-tied reconnect witness on both connections).
|
|
6607
|
-
const [delivery, membership] = await Promise.all([
|
|
6608
|
-
this.reloadCreds(expected.delivery).then((brokerAccepted) => ({ ok: true, brokerAccepted, residentSwap: "scheduled" }), (e) => ({ ok: false, error: e.message })),
|
|
6609
|
-
this.plane3?.reloadMembershipCreds
|
|
6610
|
-
? this.plane3.reloadMembershipCreds(expected.membership).then((result) => ({ ok: true, ...result }), (e) => ({ ok: false, error: e.message }))
|
|
6611
|
-
: Promise.resolve({ ok: true, skipped: "no-membership-hook" }),
|
|
6612
|
-
]);
|
|
6613
|
-
const failures = [
|
|
6614
|
-
delivery.ok ? undefined : `delivery: ${delivery.error}`,
|
|
6615
|
-
membership.ok ? undefined : `membership: ${membership.error}`,
|
|
6616
|
-
].filter((m) => m !== undefined);
|
|
6617
|
-
// Arm the resident wire swap ONLY now — after BOTH proofs settled, right before the reply is
|
|
6618
|
-
// returned+responded — so a slow membership proof can never let the delivery reconnect strand
|
|
6619
|
-
// this reply. Only when delivery actually adopted a new candidate (currentCreds was updated).
|
|
6620
|
-
if (delivery.ok) {
|
|
6621
|
-
this.plane3?.onDeliveryCredsAdopted?.();
|
|
6622
|
-
this.scheduleResidentSwap();
|
|
6623
|
-
}
|
|
6624
|
-
return failures.length
|
|
6625
|
-
? { ok: false, error: failures.join("; "), data: { delivery, membership } }
|
|
6626
|
-
: { ok: true, data: { delivery, membership } };
|
|
6627
|
-
}
|
|
6628
|
-
if (req.op === "evictPrincipal") {
|
|
6629
|
-
// The LIVE-EVICTION executor: force-drop a denied principal's connections.
|
|
6630
|
-
// Composition-root hook because the $SYS observer/evictor creds live outside this endpoint's
|
|
6631
|
-
// trust boundary; absent hook = a daemon build without the executor, refused loudly.
|
|
6632
|
-
if (!this.plane3?.evictPrincipal)
|
|
6633
|
-
return { ok: false, error: "evictPrincipal: no eviction executor wired on this daemon" };
|
|
6634
|
-
const principal = typeof req.args?.principal === "string" ? req.args.principal.trim() : "";
|
|
6635
|
-
if (!principal)
|
|
6636
|
-
return { ok: false, error: "evictPrincipal: a principal (owner.actor dot-form) is required" };
|
|
6637
|
-
try {
|
|
6638
|
-
return { ok: true, data: await this.plane3.evictPrincipal(principal) };
|
|
6639
|
-
}
|
|
6640
|
-
catch (e) {
|
|
6641
|
-
return { ok: false, error: e.message };
|
|
6642
|
-
}
|
|
6643
|
-
}
|
|
6644
|
-
if (req.op === "evictPrincipals") {
|
|
6645
|
-
// The same executor over a SET: one shared sweep instead of one request and one scan per
|
|
6646
|
-
// principal. The set is bounded so one request stays inside one sweep's work.
|
|
6647
|
-
if (!this.plane3?.evictPrincipals)
|
|
6648
|
-
return { ok: false, error: "evictPrincipals: no batch eviction executor wired on this daemon" };
|
|
6649
|
-
const raw = req.args?.principals;
|
|
6650
|
-
const principals = Array.isArray(raw) ? raw.map((p) => (typeof p === "string" ? p.trim() : "")) : [];
|
|
6651
|
-
if (principals.length === 0 || principals.some((p) => !p))
|
|
6652
|
-
return { ok: false, error: "evictPrincipals: principals must be a non-empty array of owner.actor dot-form principals" };
|
|
6653
|
-
if (principals.length > EVICT_PRINCIPALS_MAX || new Set(principals).size !== principals.length)
|
|
6654
|
-
return { ok: false, error: `evictPrincipals: principals must be distinct and at most ${EVICT_PRINCIPALS_MAX}` };
|
|
6655
|
-
try {
|
|
6656
|
-
return { ok: true, data: await this.plane3.evictPrincipals(principals) };
|
|
6657
|
-
}
|
|
6658
|
-
catch (e) {
|
|
6659
|
-
return { ok: false, error: e.message };
|
|
6660
|
-
}
|
|
6661
|
-
}
|
|
6662
|
-
if (req.op === "planeConnLiveness") {
|
|
6663
|
-
// The plane-claim liveness oracle (#29 HIGH 3): a CLOSED read-only verb — two claimed
|
|
6664
|
-
// scanner tuples in, two bound verdicts + sweep completeness out. The executor hook owns
|
|
6665
|
-
// the closed query validation (it holds the $SYS observer cred outside this trust boundary);
|
|
6666
|
-
// absent hook = a daemon build without the oracle, refused loudly (the auth plane treats
|
|
6667
|
-
// that refusal as UNKNOWN and never reclaims over it).
|
|
6668
|
-
if (!this.plane3?.planeConnLiveness)
|
|
6669
|
-
return { ok: false, error: "planeConnLiveness: no plane-liveness oracle wired on this daemon" };
|
|
6670
|
-
try {
|
|
6671
|
-
return { ok: true, data: await this.plane3.planeConnLiveness(req.args?.query) };
|
|
6672
|
-
}
|
|
6673
|
-
catch (e) {
|
|
6674
|
-
return { ok: false, error: e.message };
|
|
6675
|
-
}
|
|
6676
|
-
}
|
|
6677
|
-
if (req.op === "lifecycleMemberships") {
|
|
6678
|
-
// The terminal teardown's membership INVENTORY: a complete, read-only, lifecycle-exact listing
|
|
6679
|
-
// of one principal's durable membership rows (tombstones included) from the trusted daemon that
|
|
6680
|
-
// owns the members bucket. The caller deletes only the exact keys it gets back, through its
|
|
6681
|
-
// target-pinned deprovisioner grant; this verb deletes nothing and returns no other lifecycle.
|
|
6682
|
-
const principal = typeof req.args?.principal === "string" ? req.args.principal.trim() : "";
|
|
6683
|
-
const uid = typeof req.args?.lifecycleUid === "string" ? req.args.lifecycleUid : "";
|
|
6684
|
-
if (!parsePrincipalKey(principal))
|
|
6685
|
-
return { ok: false, error: "lifecycleMemberships: a principal (owner.actor dot-form) is required" };
|
|
6686
|
-
try {
|
|
6687
|
-
assertLifecycleToken(uid);
|
|
6688
|
-
}
|
|
6689
|
-
catch (e) {
|
|
6690
|
-
return { ok: false, error: `lifecycleMemberships: ${e.message}` };
|
|
6691
|
-
}
|
|
6692
|
-
// A retirement fences the lifecycle's ACL row before it asks (fenceLifecycleAcl), and every join
|
|
6693
|
-
// re-reads that row around the row it creates, so a row written after this read begins removes
|
|
6694
|
-
// itself, on whichever daemon wrote it. For a manager that does not fence, this daemon's own
|
|
6695
|
-
// memberships of the lifecycle stop changing here (#4965): a join or leave bound to it is refused
|
|
6696
|
-
// from now on, and the read starts after the ones already running have finished.
|
|
6697
|
-
let marked = this.retiringLifecycles.get(principal);
|
|
6698
|
-
if (marked === undefined)
|
|
6699
|
-
this.retiringLifecycles.set(principal, (marked = new Set()));
|
|
6700
|
-
// An earlier lifecycle of this principal whose ACL row is retired can bind no join or leave any
|
|
6701
|
-
// more, so its mark goes. A failed read keeps the mark, which only ever refuses.
|
|
6702
|
-
for (const earlier of [...marked])
|
|
6703
|
-
if (earlier !== uid && await aclRetired(await this.aclRegistry(), principal, earlier).catch(() => false))
|
|
6704
|
-
marked.delete(earlier);
|
|
6705
|
-
// A mark is kept only for an inventory whose caller did not fence (#4996): a lifecycle whose ACL
|
|
6706
|
-
// row is narrowed to no reads, or already retired, needs none, since every join re-reads that
|
|
6707
|
-
// row around its write. A read that fails proves nothing, so the mark is kept, which only refuses.
|
|
6708
|
-
if (await this.lifecycleAclUnfenced(principal, uid))
|
|
6709
|
-
marked.add(uid);
|
|
6710
|
-
else if (marked.size === 0)
|
|
6711
|
-
this.retiringLifecycles.delete(principal);
|
|
6712
|
-
await Promise.allSettled(this.membershipWrites.get(`${principal}/${uid}`) ?? []);
|
|
6713
|
-
try {
|
|
6714
|
-
const channels = await listLifecycleMemberChannels(await this.membersRegistry(), principal, uid);
|
|
6715
|
-
return { ok: true, data: { complete: true, channels } };
|
|
6716
|
-
}
|
|
6717
|
-
catch (e) {
|
|
6718
|
-
return { ok: false, error: `lifecycleMemberships: the inventory read did not complete (${e.message})` };
|
|
6719
|
-
}
|
|
6720
|
-
}
|
|
6721
|
-
if (req.op === "principalLiveness") {
|
|
6722
|
-
// The freeze-holder liveness probe (#391): the READ-ONLY half of `evictPrincipal`. A repair
|
|
6723
|
-
// that must REFUSE while the holder is alive cannot use eviction as its own precheck — that
|
|
6724
|
-
// kills the holder before anything can refuse on its behalf. Same executor-hook shape as the
|
|
6725
|
-
// verbs above (the $SYS observer cred lives outside this trust boundary); absent hook =
|
|
6726
|
-
// refused loudly, and the caller maps a refusal to UNKNOWN and never repairs over it.
|
|
6727
|
-
if (!this.plane3?.principalLiveness)
|
|
6728
|
-
return { ok: false, error: "principalLiveness: no liveness oracle wired on this daemon" };
|
|
6729
|
-
const principal = typeof req.args?.principal === "string" ? req.args.principal.trim() : "";
|
|
6730
|
-
if (!principal)
|
|
6731
|
-
return { ok: false, error: "principalLiveness: a principal (owner.actor dot-form) is required" };
|
|
6732
|
-
try {
|
|
6733
|
-
return { ok: true, data: await this.plane3.principalLiveness(principal) };
|
|
6734
|
-
}
|
|
6735
|
-
catch (e) {
|
|
6736
|
-
return { ok: false, error: e.message };
|
|
6737
|
-
}
|
|
6738
|
-
}
|
|
6739
|
-
if (req.op === "reloadStoreIdentity") {
|
|
6740
|
-
// Proof that this daemon reloads standing creds from ONE named store. The manager
|
|
6741
|
-
// compares it to its remint store before every renewal pass. Absent hook
|
|
6742
|
-
// is a daemon that cannot name its store, which is itself a divergent composition.
|
|
6743
|
-
if (!this.plane3?.reloadStoreIdentity)
|
|
6744
|
-
return { ok: false, error: "reloadStoreIdentity: this daemon did not name the SecretStore it reloads from" };
|
|
6745
|
-
try {
|
|
6746
|
-
const identity = this.plane3.reloadStoreIdentity();
|
|
6747
|
-
// WHO IS ANSWERING is part of the answer. This rail is queue-grouped, so the broker hands
|
|
6748
|
-
// the request to any bound responder, and every process with a `delivery` credential for
|
|
6749
|
-
// the space can bind it, while only the DELIVERY LEASE HOLDER actually reloads the standing
|
|
6750
|
-
// credentials. A bare store identity therefore lets a second responder's store stand in for
|
|
6751
|
-
// the reloading process's, and the manager's remint decision is then made about a process
|
|
6752
|
-
// that reloads nothing. So the reply carries this responder's wire identity and whether it
|
|
6753
|
-
// holds the lease, and the caller requires the binding (#1694).
|
|
6754
|
-
//
|
|
6755
|
-
// The lease read is the LIVE row, not a cached claim: a holder that lost the shard must not
|
|
6756
|
-
// keep asserting it. A read that fails is reported as `false` rather than thrown, because
|
|
6757
|
-
// "I cannot prove I hold the shard" is what a non-holder's answer says, and it sends the
|
|
6758
|
-
// caller down the same fail-closed path rather than inventing an authority.
|
|
6759
|
-
let holdsDeliveryLease = false;
|
|
6760
|
-
try {
|
|
6761
|
-
const own = await this.readDeliveryLeaseEntry(0);
|
|
6762
|
-
holdsDeliveryLease = own !== undefined && this.ownsDeliveryLease(own.info);
|
|
6763
|
-
}
|
|
6764
|
-
catch {
|
|
6765
|
-
holdsDeliveryLease = false;
|
|
6766
|
-
}
|
|
6767
|
-
// Round-trip through the closed parser so a hook cannot smuggle extra fields onto the rail.
|
|
6768
|
-
return { ok: true, data: { identity: parseSecretStoreIdentity(identity), responder: this.card.id, holdsDeliveryLease } };
|
|
6769
|
-
}
|
|
6770
|
-
catch (e) {
|
|
6771
|
-
return { ok: false, error: e.message };
|
|
6772
|
-
}
|
|
6773
|
-
}
|
|
6774
|
-
return { ok: false, error: `op "${req.op}" not supported on the delivery admin service` };
|
|
6775
|
-
}
|
|
6776
|
-
/** Create one of the daemon's Plane-3 durables. A named create is idempotent for the same config,
|
|
6777
|
-
* and a denial is never taken for existence. The one conflict accepted is this durable as a full
|
|
6778
|
-
* restore recreated it, starting after its recorded ACK floor ({@link isRestoredConsumer}); every
|
|
6779
|
-
* other failure propagates. */
|
|
6780
|
-
async addPlane3Durable(stream, config) {
|
|
6781
|
-
try {
|
|
6782
|
-
await this.jsm.consumers.add(stream, config);
|
|
6783
|
-
}
|
|
6784
|
-
catch (e) {
|
|
6785
|
-
if (!isConsumerAlreadyExists(e) || !isRestoredConsumer(this.space, stream, await this.jsm.consumers.info(stream, config.durable_name), config))
|
|
6786
|
-
throw e;
|
|
6787
|
-
}
|
|
6788
|
-
}
|
|
6789
|
-
/** Fan-out loop: bind the privileged `fanout` durable on CHAT and route each message (routing only —
|
|
6790
|
-
* the trusted reader is the auth gate). */
|
|
6791
|
-
async runFanout() {
|
|
6792
|
-
if (!this.js || !this.jsm)
|
|
6793
|
-
return;
|
|
6794
|
-
await this.addPlane3Durable(chatStream(this.space), fanoutDurableConfig(this.space, { ackWaitMs: this.ackWaitMs }));
|
|
6795
|
-
const consumer = await this.js.consumers.get(chatStream(this.space), FANOUT_DURABLE);
|
|
6796
|
-
const msgs = await consumer.consume();
|
|
6797
|
-
this.streamMsgs.push(msgs);
|
|
6798
|
-
void (async () => {
|
|
6799
|
-
for await (const m of msgs) {
|
|
6800
|
-
try {
|
|
6801
|
-
await this.fanOutMessage(m);
|
|
6802
|
-
}
|
|
6803
|
-
catch (e) {
|
|
6804
|
-
if (!this.stopped)
|
|
6805
|
-
this.emit("error", e);
|
|
6806
|
-
try {
|
|
6807
|
-
m.nak();
|
|
6808
|
-
}
|
|
6809
|
-
catch { /* draining */ }
|
|
6810
|
-
}
|
|
6811
|
-
}
|
|
6812
|
-
})().catch((e) => { if (!this.stopped)
|
|
6813
|
-
this.emit("error", e); });
|
|
6814
|
-
}
|
|
6815
|
-
/** Route ONE chat message to eligible owners' mixed inboxes. `durable` channel → its `durable-active`
|
|
6816
|
-
* members within interval; `live` channel → `@mention` targets authorized to read it (ACL only).
|
|
6817
|
-
* On either class, a mention of an agent whose read ACL excludes the channel is reported to the
|
|
6818
|
-
* sender with a mention notice. Members KV is scanned FRESH per message (no cache — red-team
|
|
6819
|
-
* BLOCKER-1 catch-up correctness). */
|
|
6820
|
-
async fanOutMessage(m) {
|
|
6821
|
-
// FENCE, and NOT followed by an ack: this unit was dispatched before the quiesce and is resuming
|
|
6822
|
-
// after it, so writing the fan-out now would put entries in member inboxes on behalf of a shard
|
|
6823
|
-
// this daemon no longer serves. Returning WITHOUT acking is the whole point - a stopped consumer
|
|
6824
|
-
// redelivers to whoever holds the shard next, so refusing here loses no message, while acking it
|
|
6825
|
-
// would consume the successor's work.
|
|
6826
|
-
if (!this.plane3MayAct())
|
|
6827
|
-
return;
|
|
6828
|
-
const parsed = parseSubject(m.subject);
|
|
6829
|
-
if (!parsed || parsed.kind !== "chat") {
|
|
6830
|
-
m.ack();
|
|
6831
|
-
return;
|
|
6832
|
-
}
|
|
6833
|
-
const channel = parsed.rest;
|
|
6834
|
-
let msg;
|
|
6835
|
-
try {
|
|
6836
|
-
msg = m.json();
|
|
6837
|
-
}
|
|
6838
|
-
catch {
|
|
6839
|
-
m.ack();
|
|
6840
|
-
return;
|
|
6841
|
-
}
|
|
6842
|
-
if (!isFromSubjectSender(msg, parsed)) {
|
|
6843
|
-
m.ack();
|
|
6844
|
-
return;
|
|
6845
|
-
} // authenticity (SPEC §5)
|
|
6846
|
-
const seq = m.seq;
|
|
6847
|
-
const normalizedMsg = authenticatedChannelMessage(msg, channel);
|
|
6848
|
-
// SECOND FENCE, AT THE COMMIT POINT. The entry check above cannot cover this unit's own awaits:
|
|
6849
|
-
// the class read and the member scan below both hit the broker, and a daemon descheduled inside
|
|
6850
|
-
// them can resume after a successor owns the shard. Re-ask immediately before the first WRITE,
|
|
6851
|
-
// and again return without acking so the entry redelivers rather than being consumed here.
|
|
6852
|
-
if (!this.plane3MayAct())
|
|
6853
|
-
return;
|
|
6854
|
-
const durable = (await this.deliveryClassFresh(channel)) === "durable";
|
|
6855
|
-
// Each mention is resolved once. A live channel delivers to the agents among them that may read
|
|
6856
|
-
// it, and on either class a mention of an agent that may not read the channel is reported to the
|
|
6857
|
-
// sender below, so a post never claims a reader it cannot have.
|
|
6858
|
-
const mentioned = [];
|
|
6859
|
-
for (const name of msg.mentions ?? []) {
|
|
6860
|
-
const owner = this.resolveOwnerByName(name);
|
|
6861
|
-
if (!owner || owner === msg.from.id)
|
|
6862
|
-
continue;
|
|
6863
|
-
// A mention arrives as an ALIAS (the roster names no lifecycle): resolve its single live ACL
|
|
6864
|
-
// row for BOTH the read-authorization and the lifecycle to address. Ambiguity (two live rows)
|
|
6865
|
-
// refuses THIS copy loudly rather than guessing a lifecycle.
|
|
6866
|
-
let row;
|
|
6867
|
-
try {
|
|
6868
|
-
row = await this.aclForAlias(owner);
|
|
6869
|
-
}
|
|
6870
|
-
catch (e) {
|
|
6871
|
-
this.emit("error", e);
|
|
6872
|
-
continue;
|
|
6873
|
-
}
|
|
6874
|
-
if (row)
|
|
6875
|
-
mentioned.push({ name, owner, row });
|
|
6876
|
-
}
|
|
6877
|
-
if (durable) {
|
|
6878
|
-
// The channel's own rows and every family row above it (`review.>`). A lifecycle holding two of
|
|
6879
|
-
// them gets two publishes and keeps one copy: the dedupe id is per message and lifecycle (#5404).
|
|
6880
|
-
for (const rec of await listMembers(await this.membersRegistry(), { covering: channel })) {
|
|
6881
|
-
if (rec.owner === msg.from.id)
|
|
6882
|
-
continue; // never backstop the sender's own post
|
|
6883
|
-
if (!durableEligible(rec, seq))
|
|
6884
|
-
continue; // routing fast-filter (reader re-checks)
|
|
6885
|
-
// Address the member row's RECORDED lifecycle: a retired row (tombstone pending) routes to the
|
|
6886
|
-
// retired lifecycle's inbox, never the alias's new occupant (SPEC 13.1 cross-plane scoping).
|
|
6887
|
-
// Store the AUTHENTICATED-channel copy (main's normalization): the durable frame validates
|
|
6888
|
-
// msg.channel === frame.channel, and payload to/toService are stripped.
|
|
6889
|
-
// Per-copy, because the scan above and every publish below is a broker round trip: a fan-out
|
|
6890
|
-
// to fifty members can straddle a quiesce in its middle. Stopping part-way is safe where
|
|
6891
|
-
// stopping late is not - the entry is never acked here, so the successor redelivers it and
|
|
6892
|
-
// re-sends the whole set; publishDinbox is idempotent per (owner, lifecycle, msgID).
|
|
6893
|
-
if (!this.plane3MayAct())
|
|
6894
|
-
return;
|
|
6895
|
-
await this.publishDinbox(rec.owner, rec.lifecycleUid, {
|
|
6896
|
-
msg: normalizedMsg, channel, seq, reason: "durable-channel", generation: rec.generation,
|
|
6897
|
-
...(rec.channel !== channel ? { row: rec.channel } : {}),
|
|
6898
|
-
});
|
|
6899
|
-
}
|
|
6900
|
-
}
|
|
6901
|
-
else {
|
|
6902
|
-
for (const { owner, row } of mentioned) {
|
|
6903
|
-
if (!channelInAllow(row.allowSubscribe, channel))
|
|
6904
|
-
continue; // @mention can't bypass the read ACL
|
|
6905
|
-
if (!this.plane3MayAct())
|
|
6906
|
-
return; // same rule on the live-mention path
|
|
6907
|
-
await this.publishDinbox(owner, row.lifecycleUid, { msg: normalizedMsg, channel, seq, reason: "live-mention", generation: 0 });
|
|
6908
|
-
}
|
|
6909
|
-
}
|
|
6910
|
-
const unreadable = mentioned.filter(({ row }) => !channelInAllow(row.allowSubscribe, channel)).map(({ name }) => name);
|
|
6911
|
-
if (unreadable.length) {
|
|
6912
|
-
if (!this.plane3MayAct())
|
|
6913
|
-
return;
|
|
6914
|
-
await this.sendMentionNotice(msg, channel, seq, unreadable);
|
|
6915
|
-
}
|
|
6916
|
-
// FINAL FENCE, BEFORE THE ACK ITSELF, which is the real commit: acking tells the stream this
|
|
6917
|
-
// shard's work is done. A daemon that stopped serving mid-fan-out must not make that claim, or
|
|
6918
|
-
// the successor never sees the entry and the message is lost outright rather than duplicated.
|
|
6919
|
-
if (!this.plane3MayAct())
|
|
6920
|
-
return;
|
|
6921
|
-
m.ack();
|
|
6922
|
-
}
|
|
6923
|
-
deliveryLimitRead;
|
|
6924
|
-
/** The DLV count the trusted writer stamps on a copy (SPEC §8): the stream's `max_msgs_per_subject`,
|
|
6925
|
-
* read at most {@link DELIVERY_LIMIT_READ_MS} before the write, as a whole number or `none`. */
|
|
6926
|
-
async deliveryLimitStamp() {
|
|
6927
|
-
const at = Date.now();
|
|
6928
|
-
if (this.deliveryLimitRead && at - this.deliveryLimitRead.at < DELIVERY_LIMIT_READ_MS)
|
|
6929
|
-
return this.deliveryLimitRead.stamp;
|
|
6930
|
-
const count = (await this.jsm.streams.info(dlvStream(this.space))).config.max_msgs_per_subject;
|
|
6931
|
-
const stamp = count === HISTORY_NO_COUNT || count < 1 ? "none" : String(count);
|
|
6932
|
-
this.deliveryLimitRead = { at, stamp };
|
|
6933
|
-
return stamp;
|
|
6934
|
-
}
|
|
6935
|
-
/** Trusted-reader loop: bind the single privileged `reader` durable over `dinbox.>` and re-authorize
|
|
6936
|
-
* + transfer each entry. */
|
|
6937
|
-
async runReader() {
|
|
6938
|
-
if (!this.js || !this.jsm)
|
|
6939
|
-
return;
|
|
6940
|
-
await this.addPlane3Durable(inboxStream(this.space), inboxReaderConfig(this.space, { ackWaitMs: this.ackWaitMs }));
|
|
6941
|
-
const consumer = await this.js.consumers.get(inboxStream(this.space), INBOX_READER_DURABLE);
|
|
6942
|
-
const msgs = await consumer.consume();
|
|
6943
|
-
this.streamMsgs.push(msgs);
|
|
6944
|
-
void (async () => {
|
|
6945
|
-
for await (const m of msgs) {
|
|
6946
|
-
try {
|
|
6947
|
-
await this.readerHandle(m);
|
|
6948
|
-
}
|
|
6949
|
-
catch (e) {
|
|
6950
|
-
if (!this.stopped)
|
|
6951
|
-
this.emit("error", e);
|
|
6952
|
-
try {
|
|
6953
|
-
m.nak();
|
|
6954
|
-
}
|
|
6955
|
-
catch { /* draining */ }
|
|
6956
|
-
}
|
|
6957
|
-
}
|
|
6958
|
-
})().catch((e) => { if (!this.stopped)
|
|
6959
|
-
this.emit("error", e); });
|
|
6960
|
-
}
|
|
6961
|
-
/** Whether one of the lifecycle's rows covering concrete `channel`, the channel's own or a family
|
|
6962
|
-
* above it, is eligible for the post at `seq`. One exact read per row, the channel's own first. */
|
|
6963
|
-
async lifecycleCoversPost(owner, lifecycleUid, channel, seq) {
|
|
6964
|
-
const kv = await this.membersRegistry();
|
|
6965
|
-
for (const row of [channel, ...familiesCovering(channel).reverse()]) {
|
|
6966
|
-
const rec = await readMember(kv, row, owner, lifecycleUid);
|
|
6967
|
-
if (rec && durableEligible(rec.record, seq))
|
|
6968
|
-
return true;
|
|
6969
|
-
}
|
|
6970
|
-
return false;
|
|
6971
|
-
}
|
|
6972
|
-
/** Remove a retired lifecycle's member row for `channel`, when one exists. Best effort: the row
|
|
6973
|
-
* authorizes nothing (its ACL row is retired), and the next entry fan-out writes against it
|
|
6974
|
-
* brings the reader back here. */
|
|
6975
|
-
async removeRetiredMemberRow(channel, owner, lifecycleUid) {
|
|
6976
|
-
try {
|
|
6977
|
-
const kv = await this.membersRegistry();
|
|
6978
|
-
if (await readMember(kv, channel, owner, lifecycleUid) === undefined)
|
|
6979
|
-
return;
|
|
6980
|
-
await deleteMember(kv, channel, owner, lifecycleUid);
|
|
6981
|
-
}
|
|
6982
|
-
catch (e) {
|
|
6983
|
-
this.emit("warning", new Error(`plane-3 reader: could not remove retired lifecycle ${owner}.${lifecycleUid}'s "${channel}" member row: ${e.message}`));
|
|
6984
|
-
}
|
|
6985
|
-
}
|
|
6986
|
-
/** Re-authorize ONE mixed-inbox entry and transfer it to the owner's DELIVER store. Deny (drop) on a
|
|
6987
|
-
* revoked/narrowed ACL or out-of-interval seq; on transfer success, ack the mixed entry (durability
|
|
6988
|
-
* has moved to DLV — an §8 equivalent per-member at-least-once mechanism). The agent acks DLV. */
|
|
6989
|
-
async readerHandle(m) {
|
|
6990
|
-
// FENCE, same rule as the fan-out writer: no effect and no ack once this daemon has stopped
|
|
6991
|
-
// serving the shard. The entry redelivers to the holder rather than being consumed here.
|
|
6992
|
-
if (!this.plane3MayAct())
|
|
6993
|
-
return;
|
|
6994
|
-
const pr = parseDinboxPrincipal(m.subject);
|
|
6995
|
-
if (!pr) {
|
|
6996
|
-
m.ack();
|
|
6997
|
-
return;
|
|
6998
|
-
} // unparseable subject (incl. a pre-cut 5-segment form) — not a real entry
|
|
6999
|
-
const owner = `${pr.owner}.${pr.actor}`; // the member principal dot-form (acl/member keys, msgID)
|
|
7000
|
-
let entry;
|
|
7001
|
-
try {
|
|
7002
|
-
entry = m.json();
|
|
7003
|
-
}
|
|
7004
|
-
catch {
|
|
7005
|
-
m.ack();
|
|
7006
|
-
return;
|
|
7007
|
-
} // undecodable — drop
|
|
7008
|
-
if (!isRecord(entry)) {
|
|
7009
|
-
m.ack();
|
|
7010
|
-
return;
|
|
7011
|
-
} // non-object envelope — permanently invalid
|
|
7012
|
-
const redeliveries = m.info?.deliveryCount ?? 1; // JsMsg delivery attempts (1 on first delivery)
|
|
7013
|
-
// Lifecycle-exact ACL re-auth (SPEC 13.1): the entry was addressed to pr.lifecycleUid's inbox, so
|
|
7014
|
-
// the row read is that lifecycle's exact key — a retired lifecycle's purged row reads as unknown
|
|
7015
|
-
// and its residual entries are removed below, never authorized against the successor's row.
|
|
7016
|
-
const acl = await this.plane3?.aclFor(owner, pr.lifecycleUid);
|
|
7017
|
-
if (acl === undefined) {
|
|
7018
|
-
// RETIRED lifecycle: its exact ACL key carries retirement's tombstone, so no attempt can ever
|
|
7019
|
-
// deliver this entry. Drop it AND remove it from the store. Deferring it to the ceiling and
|
|
7020
|
-
// keeping it would make every reader that starts from a fresh cursor pay the same sweep again.
|
|
7021
|
-
// An ACL row that is merely absent proves nothing and still defers below.
|
|
7022
|
-
if (await aclRetired(await this.aclRegistry(), owner, pr.lifecycleUid)) {
|
|
7023
|
-
if (!this.plane3MayAct())
|
|
7024
|
-
return;
|
|
7025
|
-
// Remove BEFORE the ack. The ack ends the reader's work on this entry, so acking first and
|
|
7026
|
-
// then failing the delete would keep the entry with nothing left to remove it. A failed delete
|
|
7027
|
-
// stays pending and retries, bounded by the same ceiling; an entry the stream no longer holds
|
|
7028
|
-
// is already removed (the broker answers a delete of a missing sequence with 10043).
|
|
7029
|
-
let failure;
|
|
7030
|
-
try {
|
|
7031
|
-
await this.jsm.streams.deleteMessage(inboxStream(this.space), m.seq, false);
|
|
7032
|
-
}
|
|
7033
|
-
catch (e) {
|
|
7034
|
-
if (!isJetStreamMissing(e, 10043))
|
|
7035
|
-
failure = e instanceof Error ? e : new Error(String(e));
|
|
7036
|
-
}
|
|
7037
|
-
// FENCE AFTER THE DELETE, which is broker I/O. A daemon that stopped serving while it was in
|
|
7038
|
-
// flight leaves the entry to the holder: a term here would move the shared durable past an entry
|
|
7039
|
-
// the failed delete left stored, so the holder would never retry it.
|
|
7040
|
-
if (!this.plane3MayAct())
|
|
7041
|
-
return;
|
|
7042
|
-
if (failure) {
|
|
7043
|
-
if (redeliveries >= READER_MAX_REDELIVERIES) {
|
|
7044
|
-
m.term();
|
|
7045
|
-
this.emit("error", new Error(`plane-3 reader: gave up removing entry ${m.seq} for retired lifecycle ${owner}.${pr.lifecycleUid} after ${redeliveries} redeliveries: ${failure.message}`));
|
|
7046
|
-
return;
|
|
7047
|
-
}
|
|
7048
|
-
m.nak(2000);
|
|
7049
|
-
return;
|
|
7050
|
-
}
|
|
7051
|
-
// The member row that addressed this entry is one a daemon left behind when it died between
|
|
7052
|
-
// its create and its post-write ACL read (#4994). It authorizes nothing, but fan-out copies
|
|
7053
|
-
// every later message on the channel into this inbox until it goes, and this reader then
|
|
7054
|
-
// removes each copy. Remove the row where that cost is paid, by its exact key, only when it
|
|
7055
|
-
// exists: a purge of an absent key would write a marker per dropped entry.
|
|
7056
|
-
if (entry.reason === "durable-channel")
|
|
7057
|
-
await this.removeRetiredMemberRow(entry.row ?? entry.channel, owner, pr.lifecycleUid);
|
|
7058
|
-
if (!this.plane3MayAct())
|
|
7059
|
-
return;
|
|
7060
|
-
m.ack();
|
|
7061
|
-
return;
|
|
7062
|
-
}
|
|
7063
|
-
// UNKNOWN owner — the manager has not (re)hydrated this owner's ACL yet (e.g. right after a
|
|
7064
|
-
// manager PROCESS restart). This is NOT a revocation: DEFER (redeliver), never drop — an ack here
|
|
7065
|
-
// would lose at-least-once on restart (impl-review BLOCKER-2). A delayed nak + a redelivery
|
|
7066
|
-
// ceiling stops one perma-unknown owner from head-of-lining the shared reader.
|
|
7067
|
-
// (Follow-up: the manager does not yet rehydrate its managed set across a process restart — until
|
|
7068
|
-
// it does, a long-unknown owner's entries term after the ceiling; tracked, not a silent ack-drop.)
|
|
7069
|
-
if (redeliveries >= READER_MAX_REDELIVERIES) {
|
|
7070
|
-
m.term();
|
|
7071
|
-
this.emit("error", new Error(`plane-3 reader: gave up on entry for unknown owner ${owner} after ${redeliveries} redeliveries`));
|
|
7072
|
-
return;
|
|
7073
|
-
}
|
|
7074
|
-
m.nak(2000);
|
|
7075
|
-
return;
|
|
7076
|
-
}
|
|
7077
|
-
// KNOWN owner whose CURRENT ACL no longer covers the channel — a revocation/narrowing. Drop: the
|
|
7078
|
-
// entry is no longer authorized (SPEC §7 current-ACL gate before surfacing).
|
|
7079
|
-
if (!channelInAllow(acl, entry.channel)) {
|
|
7080
|
-
m.ack();
|
|
7081
|
-
return;
|
|
7082
|
-
}
|
|
7083
|
-
// INTERVAL re-auth (not a current-member boolean): a pre-leave entry (seq ≤ leaveCursor) stays
|
|
7084
|
-
// deliverable; seq > leaveCursor (or after a rejoin's newer joinCursor) is the hard cut. Any of the
|
|
7085
|
-
// lifecycle's rows that cover the channel may hold the post, whichever row addressed the copy.
|
|
7086
|
-
if (entry.reason === "durable-channel" && !(await this.lifecycleCoversPost(owner, pr.lifecycleUid, entry.channel, entry.seq))) {
|
|
7087
|
-
m.ack();
|
|
7088
|
-
return;
|
|
7089
|
-
}
|
|
7090
|
-
// COMMIT-POINT FENCE. Everything above this line is broker I/O - the ACL read and the member
|
|
7091
|
-
// re-read - so the entry check at the top of the handler is stale by now if this unit was
|
|
7092
|
-
// descheduled inside either. The transfer below is the effect that must not happen on behalf of
|
|
7093
|
-
// a shard we no longer serve; returning without acking leaves it for the holder.
|
|
7094
|
-
if (!this.plane3MayAct())
|
|
7095
|
-
return;
|
|
7096
|
-
try {
|
|
7097
|
-
// DLV has no original chat subject, so preserve the channel the trusted fan-out reader derived
|
|
7098
|
-
// from CHAT. Never let the publisher-controlled payload label choose connector attention.
|
|
7099
|
-
const frame = {
|
|
7100
|
-
version: 1,
|
|
7101
|
-
channel: entry.channel,
|
|
7102
|
-
msg: authenticatedChannelMessage(entry.msg, entry.channel),
|
|
7103
|
-
};
|
|
7104
|
-
const frameHeaders = headers();
|
|
7105
|
-
frameHeaders.set(PLANE3_FRAME_HEADER, "1");
|
|
7106
|
-
// The count read may be broker I/O, so the fence is asked again before the write.
|
|
7107
|
-
frameHeaders.set(DELIVERY_LIMIT_HEADER, await this.deliveryLimitStamp());
|
|
7108
|
-
if (!this.plane3MayAct())
|
|
7109
|
-
return;
|
|
7110
|
-
// Lifecycle-keyed subject + msgID (ours): the DLV consumer filters on the lifecycle-scoped
|
|
7111
|
-
// subject (streams.ts filter_subject), and a predecessor's transferred copy must never suppress
|
|
7112
|
-
// a successor's under the same alias (disjoint lifecycles, stream-wide dedupe).
|
|
7113
|
-
await this.js.publish(dlvSubject(this.space, pr.owner, pr.actor, pr.lifecycleUid), JSON.stringify(frame), {
|
|
7114
|
-
...this.plane3MsgId(entry, `${owner}:${pr.lifecycleUid}`),
|
|
7115
|
-
headers: frameHeaders,
|
|
7116
|
-
});
|
|
7117
|
-
}
|
|
7118
|
-
catch {
|
|
7119
|
-
// EVERY transfer failure is deliberately a retry, never a successful result. Keep the entry
|
|
7120
|
-
// pending (redeliver), bounded by the same ceiling so a poison
|
|
7121
|
-
// entry can't head-of-line the shared reader forever.
|
|
7122
|
-
if (redeliveries >= READER_MAX_REDELIVERIES) {
|
|
7123
|
-
m.term();
|
|
7124
|
-
this.emit("error", new Error(`plane-3 reader: gave up transferring ${entry.msg.id} for ${owner} after ${redeliveries} redeliveries`));
|
|
7125
|
-
return;
|
|
7126
|
-
}
|
|
7127
|
-
m.nak(2000);
|
|
7128
|
-
return;
|
|
7129
|
-
}
|
|
7130
|
-
// The ack is the last commit: the transfer above already happened, so the only question left is
|
|
7131
|
-
// whether THIS daemon may retire the source entry. If it stopped serving mid-transfer, leave the
|
|
7132
|
-
// entry pending - the successor re-transfers and stream-wide dedupe on the msgID collapses the
|
|
7133
|
-
// duplicate, which is the recoverable direction. Acking here is the unrecoverable one.
|
|
7134
|
-
if (!this.plane3MayAct())
|
|
7135
|
-
return;
|
|
7136
|
-
m.ack();
|
|
7137
|
-
}
|
|
7138
|
-
/** Agent-side: bind + pump our pre-created Plane-3 DELIVER durable (`dlv_<id>`). Every message here is
|
|
7139
|
-
* delivery-daemon-written (DLV is delivery-write-only, broker-enforced) and is a CHANNEL message by contract
|
|
7140
|
-
* (the backstop never carries DMs), so `kind=channel` is path-derived (SPEC §4) and the body is
|
|
7141
|
-
* trusted (no spoof-guard). `durable:true` — real JetStream ack, coalesced with the core-sub live
|
|
7142
|
-
* copy by `MeshAgent.ingest`. No-op when the durable isn't present (open mode / not provisioned). A read
|
|
7143
|
-
* list declared on a credential that cannot fetch the durable refuses. */
|
|
7144
|
-
async pumpDlv() {
|
|
7145
|
-
if (!this.js)
|
|
7146
|
-
return;
|
|
7147
|
-
if (!this.ownLifecycleUid)
|
|
7148
|
-
return; // no lifecycle uid — never provisioned for Plane-3 (its durable is lifecycle-keyed)
|
|
7149
|
-
if (this.allowSubscribe?.length === 0)
|
|
7150
|
-
return; // known channel-less grant: no DLV durable or bind rows exist
|
|
7151
|
-
const durable = dlvDurable(this.owner, this.actor, this.ownLifecycleUid);
|
|
7152
|
-
if (this.allowSubscribe === undefined && !this.userMode && this.nc) {
|
|
7153
|
-
// Native credential-only clients have no separate ACL option. Use the publish permissions
|
|
7154
|
-
// captured when THIS connection authenticated; a callout sentinel does not carry the grant.
|
|
7155
|
-
const permission = this.watchDeletePermissions.get(this.nc)?.();
|
|
7156
|
-
// Only an absent allow selects no binding. A declared bind with a deny still reaches the
|
|
7157
|
-
// broker and fails, so a refusal cannot masquerade as a channel-less credential.
|
|
7158
|
-
if (permission && !issuedPermitsSubject({ ...permission, deny: [] }, `$JS.API.CONSUMER.INFO.${dlvStream(this.space)}.${durable}`))
|
|
7159
|
-
return;
|
|
7160
|
-
}
|
|
7161
|
-
let consumer;
|
|
7162
|
-
try {
|
|
7163
|
-
consumer = await this.js.consumers.get(dlvStream(this.space), durable);
|
|
7164
|
-
}
|
|
7165
|
-
catch (e) {
|
|
7166
|
-
if (isJetStreamMissing(e, JetStreamApiCodes.ConsumerNotFound)) {
|
|
7167
|
-
// A channel-less credential may ask about its durable but holds no fetch row for it. A read
|
|
7168
|
-
// list declared on such a credential disagrees with the grant, so the start refuses.
|
|
7169
|
-
const issued = this.nc && this.watchDeletePermissions.get(this.nc)?.();
|
|
7170
|
-
if ((this.allowSubscribe?.length ?? 0) > 0 && issued
|
|
7171
|
-
&& !issuedPermitsSubject({ ...issued, deny: [] }, `$JS.API.CONSUMER.MSG.NEXT.${dlvStream(this.space)}.${durable}`))
|
|
7172
|
-
throw new Error(`this connection declared a read list, but its credential cannot fetch delivery durable "${durable}" for space "${this.space}"; the read list and the credential disagree`);
|
|
7173
|
-
return;
|
|
7174
|
-
}
|
|
7175
|
-
throw e; // a denied bind is not evidence that the read ACL is empty or the durable is absent
|
|
7176
|
-
}
|
|
7177
|
-
const lease = await this.readDeliveryLease(0);
|
|
7178
|
-
const liveDeliveryPlane = lease?.ready === true;
|
|
7179
|
-
if (!liveDeliveryPlane)
|
|
7180
|
-
this.emit("warning", new Error(`delivery durable "${durable}" for space "${this.space}" bound while the plane this connection reaches has no ready delivery lease, so the durable delivers nothing until a delivery daemon serves this plane. If a daemon serves this space on another plane, reconnect against it, which re-binds the durable there.`));
|
|
7181
|
-
const ackWaitMs = await ackWaitOf(consumer);
|
|
7182
|
-
const msgs = await consumer.consume();
|
|
7183
|
-
this.streamMsgs.push(msgs);
|
|
7184
|
-
void (async () => {
|
|
7185
|
-
for await (const m of msgs) {
|
|
7186
|
-
// A copy with a full store behind it means the store held its count, so older copies may have
|
|
7187
|
-
// been discarded unread (#5404); NATS gives no notice of the discard itself. The count is the
|
|
7188
|
-
// one the trusted writer stamped on this copy. Checked on every delivery, ahead of the checks
|
|
7189
|
-
// below and the connector's id dedupe, so a copy the agent also got live cannot hide it. The
|
|
7190
|
-
// connector folds repeats into one notice.
|
|
7191
|
-
let limit;
|
|
7192
|
-
try {
|
|
7193
|
-
limit = deliveryLimitOf(m.headers?.get(DELIVERY_LIMIT_HEADER));
|
|
7194
|
-
}
|
|
7195
|
-
catch (e) {
|
|
7196
|
-
this.emit("error", e);
|
|
7197
|
-
limit = null;
|
|
7198
|
-
}
|
|
7199
|
-
if (limit !== null && m.info.pending >= limit - 1)
|
|
7200
|
-
this.emit("stored-copies-full", { limit });
|
|
7201
|
-
// The header is minted only by the trusted DLV writer. A body-only discriminator is
|
|
7202
|
-
// insufficient because old Cotal envelopes permit unknown fields and could imitate it.
|
|
7203
|
-
if (m.headers?.get(PLANE3_FRAME_HEADER) !== "1") {
|
|
7204
|
-
this.emit("error", new Error("plane-3 delivery: unauthenticated or unversioned DLV entry terminated"));
|
|
7205
|
-
try {
|
|
7206
|
-
m.term();
|
|
7207
|
-
}
|
|
7208
|
-
catch { /* draining */ }
|
|
7209
|
-
continue;
|
|
7210
|
-
}
|
|
7211
|
-
let raw;
|
|
7212
|
-
try {
|
|
7213
|
-
raw = m.json();
|
|
7214
|
-
}
|
|
7215
|
-
catch (e) {
|
|
7216
|
-
this.emit("error", e);
|
|
7217
|
-
try {
|
|
7218
|
-
m.term();
|
|
7219
|
-
}
|
|
7220
|
-
catch { /* draining */ }
|
|
7221
|
-
continue;
|
|
7222
|
-
}
|
|
7223
|
-
if (!isPlane3DeliveryFrame(raw)) {
|
|
7224
|
-
this.emit("error", new Error("plane-3 delivery: malformed versioned DLV entry terminated"));
|
|
7225
|
-
try {
|
|
7226
|
-
m.term();
|
|
7227
|
-
}
|
|
7228
|
-
catch { /* draining */ }
|
|
7229
|
-
continue;
|
|
7230
|
-
}
|
|
7231
|
-
const msg = authenticatedChannelMessage(raw.msg, raw.channel);
|
|
7232
|
-
if (msg.from?.id === this.card.id) {
|
|
7233
|
-
m.ack();
|
|
7234
|
-
continue;
|
|
7235
|
-
} // own echo (defensive)
|
|
7236
|
-
this.emit("message", msg, durableDelivery(m, ackWaitMs), { historical: false, kind: "channel" });
|
|
7237
|
-
}
|
|
7238
|
-
})().catch((e) => { if (!this.stopped)
|
|
7239
|
-
this.emit("error", e); });
|
|
7240
|
-
}
|
|
7241
5629
|
/** Agent-side: request a Plane-3 durable backstop for a channel via the server-side delivery daemon (ctl.delivery). Throws
|
|
7242
5630
|
* when no privileged writer is present (open / no delivery daemon). 30s timeout — activation catch-up may
|
|
7243
5631
|
* run before the reply (the window is small, but a busy channel can take more than the 5s default). */
|
|
@@ -7937,60 +6325,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
7937
6325
|
* calls must be serial — every reader here awaits to completion, so they are.
|
|
7938
6326
|
*/
|
|
7939
6327
|
async collectHistory(subject, start, opts = {}) {
|
|
7940
|
-
|
|
7941
|
-
// (recall + join-backfill + drop-marker can race in-process) would delete/recreate the consumer
|
|
7942
|
-
// under each other and cross-feed results. The chain makes the "serial callers" assumption true.
|
|
7943
|
-
const run = this.histLock.then(() => this.collectHistoryInner(subject, start, opts));
|
|
7944
|
-
this.histLock = run.catch(() => { }); // keep the chain alive on error
|
|
7945
|
-
return run;
|
|
7946
|
-
}
|
|
7947
|
-
async collectHistoryInner(subject, start, opts = {}) {
|
|
7948
|
-
if (!this.jsm || !this.js)
|
|
7949
|
-
throw new Error(this.notLiveMsg());
|
|
7950
|
-
const stream = chatStream(this.space);
|
|
7951
|
-
const name = chatHistDurable(this.owner, this.actor, this.requireLifecycleUid("chat history reads"));
|
|
7952
|
-
const out = [];
|
|
7953
|
-
// Clear any consumer leaked by a crashed prior read before re-creating it with THIS read's
|
|
7954
|
-
// single filter (the read ACL is enforced at create — see the doc above).
|
|
7955
|
-
await this.deleteConsumerIfPresent(stream, name);
|
|
7956
|
-
await this.jsm.consumers.add(stream, {
|
|
7957
|
-
name,
|
|
7958
|
-
filter_subject: subject,
|
|
7959
|
-
ack_policy: AckPolicy.None,
|
|
7960
|
-
mem_storage: true,
|
|
7961
|
-
inactive_threshold: nanos(30_000),
|
|
7962
|
-
...("time" in start
|
|
7963
|
-
? { deliver_policy: DeliverPolicy.StartTime, opt_start_time: start.time.toISOString() }
|
|
7964
|
-
: { deliver_policy: DeliverPolicy.StartSequence, opt_start_seq: start.seq }),
|
|
7965
|
-
});
|
|
7966
|
-
try {
|
|
7967
|
-
const consumer = await this.js.consumers.get(stream, name);
|
|
7968
|
-
let pending = (await consumer.info()).num_pending;
|
|
7969
|
-
while (pending > 0) {
|
|
7970
|
-
const want = Math.min(pending, 256);
|
|
7971
|
-
const iter = await consumer.fetch({ max_messages: want, expires: 5_000 });
|
|
7972
|
-
let got = 0;
|
|
7973
|
-
for await (const m of iter) {
|
|
7974
|
-
got++;
|
|
7975
|
-
if (opts.untilSeq !== undefined && m.seq > opts.untilSeq)
|
|
7976
|
-
return out; // crossed the frontier
|
|
7977
|
-
// Belt-and-suspenders over the lock: only keep messages on the requested channel subject
|
|
7978
|
-
// (the consumer's filter already bounds this; guards against any stale-consumer edge).
|
|
7979
|
-
if (!subjectMatches(subject, m.subject))
|
|
7980
|
-
continue;
|
|
7981
|
-
out.push(m);
|
|
7982
|
-
if (opts.limit !== undefined && out.length >= opts.limit)
|
|
7983
|
-
return out;
|
|
7984
|
-
}
|
|
7985
|
-
if (got < want)
|
|
7986
|
-
break; // drained early
|
|
7987
|
-
pending -= got;
|
|
7988
|
-
}
|
|
7989
|
-
}
|
|
7990
|
-
finally {
|
|
7991
|
-
await this.deleteConsumerIfPresent(stream, name);
|
|
7992
|
-
}
|
|
7993
|
-
return out;
|
|
6328
|
+
return this.history.collectHistory(subject, start, opts);
|
|
7994
6329
|
}
|
|
7995
6330
|
/** Read a channel's retained history up to `upToSeq` (the join frontier) and emit each message
|
|
7996
6331
|
* as a `historical` "message" event. `sinceMs` bounds how far back via a native consumer
|
|
@@ -8004,44 +6339,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
8004
6339
|
* after it, which is exactly the catch-up. New messages (`seq > upToSeq`) are skipped — the live
|
|
8005
6340
|
* tail owns them. Reads through the contained {@link collectHistory}. */
|
|
8006
6341
|
async backfillChannel(channel, upToSeq, sinceMs, floorSeq) {
|
|
8007
|
-
|
|
8008
|
-
return 0;
|
|
8009
|
-
const subject = chatSubject(this.space, "*", "*", channel);
|
|
8010
|
-
// A window that reaches back before 1970 covers the whole retained window (#7323). Its start time
|
|
8011
|
-
// would be a date JetStream does not read, or not a valid date at all, and the read replayed nothing.
|
|
8012
|
-
const since = sinceMs === undefined ? undefined : Date.now() - sinceMs;
|
|
8013
|
-
const start = floorSeq !== undefined ? { seq: floorSeq + 1 } : since === undefined || !(since > 0) ? { seq: 1 } : { time: new Date(since) };
|
|
8014
|
-
let msgs;
|
|
8015
|
-
try {
|
|
8016
|
-
msgs = await this.collectHistory(subject, start, { untilSeq: upToSeq });
|
|
8017
|
-
}
|
|
8018
|
-
catch (e) {
|
|
8019
|
-
this.emit("error", e);
|
|
8020
|
-
if (isPermissionDenied(e))
|
|
8021
|
-
throw e;
|
|
8022
|
-
return 0;
|
|
8023
|
-
}
|
|
8024
|
-
const noop = { ack: () => { }, nak: () => { }, durable: false };
|
|
8025
|
-
let n = 0;
|
|
8026
|
-
for (const sm of msgs) {
|
|
8027
|
-
let msg;
|
|
8028
|
-
try {
|
|
8029
|
-
msg = sm.json();
|
|
8030
|
-
}
|
|
8031
|
-
catch {
|
|
8032
|
-
continue; // skip undecodable
|
|
8033
|
-
}
|
|
8034
|
-
// Same authenticity guard as the tail; skip our own echoes in history.
|
|
8035
|
-
if (!isRecord(msg) || !isUsableMessageId(msg.id))
|
|
8036
|
-
continue; // malformed envelope (SPEC sec 5) — history skips
|
|
8037
|
-
const parsed = parseSubject(sm.subject);
|
|
8038
|
-
if (!parsed || !isFromSubjectSender(msg, parsed) || msg.from.id === this.card.id)
|
|
8039
|
-
continue;
|
|
8040
|
-
// Backfill only ever reads the chat stream, so the authenticated class is always "channel".
|
|
8041
|
-
this.emit("message", authenticatedMessage(msg, parsed), noop, { historical: true, kind: "channel" });
|
|
8042
|
-
n++;
|
|
8043
|
-
}
|
|
8044
|
-
return n;
|
|
6342
|
+
return this.history.backfillChannel(channel, upToSeq, sinceMs, floorSeq);
|
|
8045
6343
|
}
|
|
8046
6344
|
/**
|
|
8047
6345
|
* Replay-gated pull of a channel's retained ambient from `sinceSeq` (exclusive) forward — the
|
|
@@ -8064,55 +6362,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
8064
6362
|
* the window was empty and complete.
|
|
8065
6363
|
*/
|
|
8066
6364
|
async recallChannel(channel, sinceSeq) {
|
|
8067
|
-
|
|
8068
|
-
throw new Error(this.notLiveMsg());
|
|
8069
|
-
if (!isConcreteChannel(channel))
|
|
8070
|
-
return { messages: [], seqs: [], dropped: false, unanswered: true };
|
|
8071
|
-
const policy = await this.joinPolicyFresh(channel);
|
|
8072
|
-
if (!policy.replay)
|
|
8073
|
-
return { messages: [], seqs: [], dropped: true, unanswered: true };
|
|
8074
|
-
const subject = chatSubject(this.space, "*", "*", channel);
|
|
8075
|
-
let raw;
|
|
8076
|
-
try {
|
|
8077
|
-
raw = await this.collectHistory(subject, { seq: sinceSeq + 1 });
|
|
8078
|
-
}
|
|
8079
|
-
catch (e) {
|
|
8080
|
-
this.emit("error", e);
|
|
8081
|
-
if (isPermissionDenied(e))
|
|
8082
|
-
throw e;
|
|
8083
|
-
return { messages: [], seqs: [], dropped: true, unanswered: true };
|
|
8084
|
-
}
|
|
8085
|
-
const collected = [];
|
|
8086
|
-
const seqs = [];
|
|
8087
|
-
for (const sm of raw) {
|
|
8088
|
-
let msg;
|
|
8089
|
-
try {
|
|
8090
|
-
msg = sm.json();
|
|
8091
|
-
}
|
|
8092
|
-
catch {
|
|
8093
|
-
continue; // skip undecodable
|
|
8094
|
-
}
|
|
8095
|
-
// Same authenticity guard as the tail/backfill; skip our own echoes.
|
|
8096
|
-
if (!isRecord(msg) || !isUsableMessageId(msg.id))
|
|
8097
|
-
continue; // malformed envelope (SPEC sec 5) — recall skips
|
|
8098
|
-
const parsed = parseSubject(sm.subject);
|
|
8099
|
-
if (!parsed || !isFromSubjectSender(msg, parsed) || msg.from.id === this.card.id)
|
|
8100
|
-
continue;
|
|
8101
|
-
collected.push(authenticatedMessage(msg, parsed));
|
|
8102
|
-
seqs.push(sm.seq);
|
|
8103
|
-
}
|
|
8104
|
-
let dropped;
|
|
8105
|
-
try {
|
|
8106
|
-
dropped = await this.chatMayHaveLost(subject, sinceSeq);
|
|
8107
|
-
}
|
|
8108
|
-
catch (e) {
|
|
8109
|
-
// Completeness could not be read, so the read is reported as unanswered: nothing settles on it.
|
|
8110
|
-
this.emit("error", e);
|
|
8111
|
-
if (isPermissionDenied(e))
|
|
8112
|
-
throw e;
|
|
8113
|
-
return { messages: [], seqs: [], dropped: true, unanswered: true };
|
|
8114
|
-
}
|
|
8115
|
-
return { messages: collected, seqs, dropped, unanswered: false };
|
|
6365
|
+
return this.history.recallChannel(channel, sinceSeq);
|
|
8116
6366
|
}
|
|
8117
6367
|
/** Whether the chat stream's limits may have removed posts on `subject` after `sinceSeq`, read from
|
|
8118
6368
|
* the stream's own config rather than a compiled default. Throws when it cannot read, so no caller
|
|
@@ -8227,12 +6477,6 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
8227
6477
|
return this.presence.sweep();
|
|
8228
6478
|
}
|
|
8229
6479
|
}
|
|
8230
|
-
/** What a history read failure NAMES when it could not finish. One filter subject is the useful
|
|
8231
|
-
* thing to print; a set of sixty-nine of them is a wall of text in a message a human has to read,
|
|
8232
|
-
* so a set says its size and the stream it was read from instead. */
|
|
8233
|
-
function subjectLabel(subjects) {
|
|
8234
|
-
return subjects.length === 1 ? subjects[0] : `${subjects.length} filtered subjects`;
|
|
8235
|
-
}
|
|
8236
6480
|
/** A usable delivery-message id (#624): a string, possibly empty (the never-a-key case), but
|
|
8237
6481
|
* never absent and never a non-string. An absent or non-string id is a malformed envelope under
|
|
8238
6482
|
* SPEC sec 5; each delivery pump handles it per its own class (durable term, live drop, history
|
|
@@ -8279,12 +6523,6 @@ function authenticatedMessage(msg, parsed) {
|
|
|
8279
6523
|
return authenticatedAnycastMessage(msg, parsed.rest);
|
|
8280
6524
|
return msg;
|
|
8281
6525
|
}
|
|
8282
|
-
function authenticatedChannelMessage(msg, channel) {
|
|
8283
|
-
if (msg.channel === channel && msg.to === undefined && msg.toService === undefined)
|
|
8284
|
-
return msg;
|
|
8285
|
-
const { to: _to, toService: _toService, ...base } = msg;
|
|
8286
|
-
return { ...base, channel };
|
|
8287
|
-
}
|
|
8288
6526
|
function authenticatedDmMessage(msg, to) {
|
|
8289
6527
|
if (msg.to === to && msg.channel === undefined && msg.toService === undefined)
|
|
8290
6528
|
return msg;
|
|
@@ -8297,23 +6535,6 @@ function authenticatedAnycastMessage(msg, toService) {
|
|
|
8297
6535
|
const { to: _to, channel: _channel, ...base } = msg;
|
|
8298
6536
|
return { ...base, toService };
|
|
8299
6537
|
}
|
|
8300
|
-
/** The ack wait the broker reports for a bound consumer: how long a copy it delivers stays with
|
|
8301
|
-
* its reader without an ack. */
|
|
8302
|
-
async function ackWaitOf(consumer) {
|
|
8303
|
-
const { config } = await consumer.info(true);
|
|
8304
|
-
if (config.ack_wait === undefined)
|
|
8305
|
-
throw new Error(`consumer ${config.durable_name} reports no ack wait`);
|
|
8306
|
-
return millis(config.ack_wait);
|
|
8307
|
-
}
|
|
8308
|
-
/** A JetStream copy's {@link Delivery}, carrying the broker's own redelivery state for it. */
|
|
8309
|
-
function durableDelivery(m, ackWaitMs) {
|
|
8310
|
-
return {
|
|
8311
|
-
ack: () => m.ack(),
|
|
8312
|
-
nak: () => m.nak(),
|
|
8313
|
-
durable: true,
|
|
8314
|
-
redelivery: { deliveryCount: m.info.deliveryCount, ackWaitMs, working: () => m.working() },
|
|
8315
|
-
};
|
|
8316
|
-
}
|
|
8317
6538
|
/** A DM-only copy's delivery. An ack, nak or in-progress signal on a closed connection is refused
|
|
8318
6539
|
* with a throw instead of being dropped silently: the copy stays unacked and the broker redelivers it. */
|
|
8319
6540
|
function dmOnlyDelivery(m, ackWaitMs, nc) {
|
|
@@ -8406,59 +6627,6 @@ function isReadableMessagePart(value) {
|
|
|
8406
6627
|
function isHistoryDrainEnvelope(value) {
|
|
8407
6628
|
return isRecord(value) && isUsableMessageId(value.id) && isRecord(value.from);
|
|
8408
6629
|
}
|
|
8409
|
-
function isPlane3DeliveryFrame(value) {
|
|
8410
|
-
if (!isRecord(value))
|
|
8411
|
-
return false;
|
|
8412
|
-
const keys = Object.keys(value);
|
|
8413
|
-
if (keys.length !== 3 || !keys.includes("version") || !keys.includes("channel") || !keys.includes("msg"))
|
|
8414
|
-
return false;
|
|
8415
|
-
if (value.version !== 1 || typeof value.channel !== "string" || !isConcreteChannel(value.channel))
|
|
8416
|
-
return false;
|
|
8417
|
-
return isCotalMessage(value.msg) && value.msg.channel === value.channel;
|
|
8418
|
-
}
|
|
8419
|
-
function isCotalMessage(value) {
|
|
8420
|
-
if (!isRecord(value))
|
|
8421
|
-
return false;
|
|
8422
|
-
const routes = ["channel", "to", "toService"].filter((key) => key in value);
|
|
8423
|
-
if (routes.length !== 1 || typeof value[routes[0]] !== "string")
|
|
8424
|
-
return false;
|
|
8425
|
-
if (typeof value.id !== "string" || typeof value.ts !== "number" || !Number.isFinite(value.ts) ||
|
|
8426
|
-
typeof value.space !== "string" || !isEndpointRef(value.from) || !Array.isArray(value.parts) ||
|
|
8427
|
-
!value.parts.every(isMessagePart))
|
|
8428
|
-
return false;
|
|
8429
|
-
if (value.mentions !== undefined && (!Array.isArray(value.mentions) || !value.mentions.every((name) => typeof name === "string")))
|
|
8430
|
-
return false;
|
|
8431
|
-
if (value.replyTo !== undefined && typeof value.replyTo !== "string")
|
|
8432
|
-
return false;
|
|
8433
|
-
if (value.contextId !== undefined && typeof value.contextId !== "string")
|
|
8434
|
-
return false;
|
|
8435
|
-
return true;
|
|
8436
|
-
}
|
|
8437
|
-
function isEndpointRef(value) {
|
|
8438
|
-
return isRecord(value) && typeof value.id === "string" && typeof value.name === "string" &&
|
|
8439
|
-
(value.role === undefined || typeof value.role === "string");
|
|
8440
|
-
}
|
|
8441
|
-
/** A conformant message part: the three CORE kinds, or a reverse-DNS extension kind (SPEC §5).
|
|
8442
|
-
*
|
|
8443
|
-
* Reached from {@link isCotalMessage}, which gates the Plane-3 delivery frame — so a core kind
|
|
8444
|
-
* missing an arm here is not a schema nicety: the durable backstop DROPS every message carrying
|
|
8445
|
-
* it, silently, and the drop surfaces nowhere near the feature that added the part. `artifact`
|
|
8446
|
-
* was exactly that case before it was added (a bare kind has no dot, so it fell through to the
|
|
8447
|
-
* extension regex and failed). */
|
|
8448
|
-
function isMessagePart(value) {
|
|
8449
|
-
if (!isRecord(value) || typeof value.kind !== "string")
|
|
8450
|
-
return false;
|
|
8451
|
-
if (value.kind === "text")
|
|
8452
|
-
return typeof value.text === "string";
|
|
8453
|
-
if (value.kind === "data")
|
|
8454
|
-
return Object.prototype.hasOwnProperty.call(value, "data");
|
|
8455
|
-
if (value.kind === ARTIFACT_PART_KIND)
|
|
8456
|
-
return isArtifactPart(value);
|
|
8457
|
-
return /^[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)+$/.test(value.kind);
|
|
8458
|
-
}
|
|
8459
|
-
function isRecord(value) {
|
|
8460
|
-
return !!value && typeof value === "object" && !Array.isArray(value);
|
|
8461
|
-
}
|
|
8462
6630
|
/** A `data` part value the wire can carry faithfully: a JSON value at EVERY depth (SPEC §5's
|
|
8463
6631
|
* `<any JSON value>`). `JSON.stringify` does not reject what it cannot represent — it silently
|
|
8464
6632
|
* rewrites: `undefined` (and a function-valued member) drops the key, `NaN`/`Infinity` store as
|
|
@@ -8647,12 +6815,6 @@ function membershipConsumerReleased(err) {
|
|
|
8647
6815
|
return (isJetStreamMissing(err, JetStreamApiCodes.ConsumerNotFound, JetStreamApiCodes.StreamNotFound)
|
|
8648
6816
|
|| isPublishPermissionDenied(err));
|
|
8649
6817
|
}
|
|
8650
|
-
/** The signal's exact abort reason, or the platform-standard AbortError when none was supplied. */
|
|
8651
|
-
function abortReason(signal) {
|
|
8652
|
-
return signal?.reason instanceof Error
|
|
8653
|
-
? signal.reason
|
|
8654
|
-
: new DOMException("This operation was aborted", "AbortError");
|
|
8655
|
-
}
|
|
8656
6818
|
/** True ONLY for a denial on a **publish** — the single case that proves the message was never
|
|
8657
6819
|
* ACCEPTED or stored. (Not "never reached the server": the server necessarily received enough of
|
|
8658
6820
|
* it to reject it. The distinction matters precisely here, because this helper exists to separate
|