@cotal-ai/core 0.168.0 → 0.169.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/dist/agent-file.d.ts.map +1 -1
  2. package/dist/agent-file.js +12 -9
  3. package/dist/agent-file.js.map +1 -1
  4. package/dist/connector-setup.d.ts +2 -0
  5. package/dist/connector-setup.d.ts.map +1 -1
  6. package/dist/connector.d.ts +8 -11
  7. package/dist/connector.d.ts.map +1 -1
  8. package/dist/control-exchange.d.ts +114 -0
  9. package/dist/control-exchange.d.ts.map +1 -0
  10. package/dist/control-exchange.js +416 -0
  11. package/dist/control-exchange.js.map +1 -0
  12. package/dist/endpoint-call.d.ts +2 -1
  13. package/dist/endpoint-call.d.ts.map +1 -1
  14. package/dist/endpoint-call.js +14 -7
  15. package/dist/endpoint-call.js.map +1 -1
  16. package/dist/endpoint-delivery-plane.d.ts +337 -0
  17. package/dist/endpoint-delivery-plane.d.ts.map +1 -0
  18. package/dist/endpoint-delivery-plane.js +1291 -0
  19. package/dist/endpoint-delivery-plane.js.map +1 -0
  20. package/dist/endpoint-envelope.d.ts +2 -1
  21. package/dist/endpoint-envelope.d.ts.map +1 -1
  22. package/dist/endpoint-envelope.js +9 -3
  23. package/dist/endpoint-envelope.js.map +1 -1
  24. package/dist/endpoint-history.d.ts +370 -0
  25. package/dist/endpoint-history.d.ts.map +1 -0
  26. package/dist/endpoint-history.js +969 -0
  27. package/dist/endpoint-history.js.map +1 -0
  28. package/dist/endpoint-inbound.d.ts +10 -0
  29. package/dist/endpoint-inbound.d.ts.map +1 -0
  30. package/dist/endpoint-inbound.js +28 -0
  31. package/dist/endpoint-inbound.js.map +1 -0
  32. package/dist/endpoint-serve.d.ts +11 -1
  33. package/dist/endpoint-serve.d.ts.map +1 -1
  34. package/dist/endpoint-serve.js +42 -5
  35. package/dist/endpoint-serve.js.map +1 -1
  36. package/dist/endpoint-subjects.d.ts +3 -0
  37. package/dist/endpoint-subjects.d.ts.map +1 -1
  38. package/dist/endpoint-subjects.js +3 -0
  39. package/dist/endpoint-subjects.js.map +1 -1
  40. package/dist/endpoint.d.ts +64 -253
  41. package/dist/endpoint.d.ts.map +1 -1
  42. package/dist/endpoint.js +177 -2015
  43. package/dist/endpoint.js.map +1 -1
  44. package/dist/index.d.ts +2 -1
  45. package/dist/index.d.ts.map +1 -1
  46. package/dist/index.js +4 -1
  47. package/dist/index.js.map +1 -1
  48. package/dist/launch-material.d.ts +9 -3
  49. package/dist/launch-material.d.ts.map +1 -1
  50. package/dist/launch-material.js +22 -2
  51. package/dist/launch-material.js.map +1 -1
  52. package/dist/provision-permissions.d.ts.map +1 -1
  53. package/dist/provision-permissions.js +7 -1
  54. package/dist/provision-permissions.js.map +1 -1
  55. package/dist/remote-manager-authority.d.ts +1 -1
  56. package/dist/run-journal.d.ts +2 -0
  57. package/dist/run-journal.d.ts.map +1 -1
  58. package/dist/run-journal.js +1 -1
  59. package/dist/run-journal.js.map +1 -1
  60. package/package.json +5 -1
package/dist/endpoint.js CHANGED
@@ -1,12 +1,13 @@
1
+ import { EndpointHistory } from "./endpoint-history.js";
2
+ export { MULTI_FILTER_BATCH, MULTI_FILTER_READ_CONCURRENCY } from "./endpoint-history.js";
1
3
  import { eventPayloadSubject, eventPayloadStream } from "./event-payloads.js";
2
4
  import { requestSafeReconnect } from "./request-safe-reconnect.js";
3
5
  import { nativeBindingAfter, nativeConnectionBinding } from "./native-connection.js";
4
6
  import { EventEmitter } from "node:events";
5
7
  import { drainAndClose } from "./connection-cleanup.js";
6
8
  import { randomUUID } from "node:crypto";
7
- import { setTimeout as delay } from "node:timers/promises";
8
9
  import { createConnection, isIP } from "node:net";
9
- import { connect, headers, tokenAuthenticator, nanos, millis, AuthorizationError, PermissionViolationError, UserAuthenticationExpiredError, NoRespondersError, RequestError, TimeoutError, ClosedConnectionError, ConnectionError, } from "@nats-io/transport-node";
10
+ import { connect, headers, tokenAuthenticator, nanos, AuthorizationError, PermissionViolationError, UserAuthenticationExpiredError, NoRespondersError, RequestError, TimeoutError, ClosedConnectionError, ConnectionError, } from "@nats-io/transport-node";
10
11
  import { wsconnect } from "@nats-io/nats-core";
11
12
  import { parseLivenessAnswer, responderFromLease, responderFromProbe, isLivenessPlane, LIVENESS_PLANES, } from "./liveness.js";
12
13
  import { EndpointPresence } from "./endpoint-presence.js";
@@ -17,45 +18,41 @@ import { armRenewal, renewalDeadlineMs, jitteredBackoffMs } from "./renewal.js";
17
18
  import { requireBrokerFloor } from "./broker-floor.js";
18
19
  import { withDeadline } from "./deadline.js";
19
20
  import { inspectCredHealth } from "./provision.js";
20
- import { parseSecretStoreIdentity, } from "./secret-store.js";
21
+ import {} from "./secret-store.js";
21
22
  import { BIND_SPLIT_REISSUES, resolveService, invokeCommand, goalFollowRefusal, submitAndFollowGoal } from "./endpoint-invoke.js";
22
23
  import { EpEnvelopeError, respondedButUnbound, replyRefusedBeforeEffect, replyTargetUnmapped, EP_BIND_REFUSED } from "./endpoint-envelope.js";
23
24
  import { isRepeatSafeCommand } from "./endpoint-grants.js";
24
25
  import { assertIdToken, assertGeneration, assertRuntimeToken, endpointToken } from "./endpoint-subjects.js";
25
26
  import { servePrincipalRoute, describeContracts } from "./endpoint-serve.js";
26
27
  import { epPrincipalCall } from "./endpoint-call.js";
27
- import { AGENT_CONTRACTS, AGENT_ENDPOINT } from "./agent-endpoint.js";
28
+ import { AGENT_CLUSTER, AGENT_CONTRACTS, AGENT_ENDPOINT } from "./agent-endpoint.js";
28
29
  import { processRuntime } from "./process-runtime.js";
29
30
  import { importNativeSubjectPermissions, issuedPermitsSubject, readAcceptedRow } from "./issued-authority.js";
30
31
  import { REGISTRY_ENDPOINT, REGISTRY_ENDPOINTS_COMMAND } from "./registry-contract.js";
31
32
  import { listRegistryEndpoints, registryListingCaller } from "./registry-client.js";
32
33
  import { lastKvEntries } from "./kv-scan.js";
33
- import { isConsumerAlreadyExists, isConsumerNotFound, isJetStreamMissing, isStreamNotFound } from "./jetstream-missing.js";
34
- import { isRestoredConsumer } from "./backup.js";
35
- import { ARTIFACT_PART_KIND, isArtifactPart } from "./artifact.js";
34
+ import { isConsumerNotFound, isJetStreamMissing, isStreamNotFound } from "./jetstream-missing.js";
36
35
  import { assertValidKind } from "./agent-file.js";
37
36
  import { assertValidName } from "./resolve.js";
38
- import { EVICT_PRINCIPALS_MAX } from "./evict.js";
39
37
  import { nextHeartbeat } from "./presence-heartbeat.js";
40
- import { anycastStoreDefinition, createSpaceStreams, dmDurableConfig, dlvDurableConfig, taskDurableConfig, fanoutDurableConfig, inboxReaderConfig, MANAGER_LEASE_TTL_MS, MANAGER_LEASE_ATTEMPT_MS, PRESENCE_STORAGE } from "./streams.js";
41
- import { DELIVERY_LIMIT_HEADER, HISTORY_NO_COUNT, PLANE3_MAX_MSGS_PER_SUBJECT } from "./backup-config.js";
42
- import { jetstream, jetstreamManager, AckPolicy, DeliverPolicy, JetStreamApiCodes, JetStreamApiError, } from "@nats-io/jetstream";
38
+ import { anycastStoreDefinition, createSpaceStreams, dmDurableConfig, dlvDurableConfig, taskDurableConfig, MANAGER_LEASE_TTL_MS, MANAGER_LEASE_ATTEMPT_MS, PRESENCE_STORAGE } from "./streams.js";
39
+ import { DELIVERY_LIMIT_HEADER, PLANE3_MAX_MSGS_PER_SUBJECT } from "./backup-config.js";
40
+ import { jetstream, jetstreamManager, AckPolicy, DeliverPolicy, JetStreamApiCodes, } from "@nats-io/jetstream";
43
41
  import {} from "@nats-io/jetstream";
44
42
  import { Kvm } from "@nats-io/kv";
45
43
  import { Bucket, KvWatchInclude } from "@nats-io/kv/internal";
46
- import { openMembersRegistry, commitMemberRevision, tombstoneMember, activateMember, readMember, deleteMember, listMembers, listLifecycleMemberChannels, durableEligible, } from "./members.js";
47
- import { openAclRegistry, readAcl, readAclForAlias, aclRetired, AmbiguousAclAlias, commitAcl as writeAclRecord, reissueAcl as writeAclReissue } from "./acls.js";
44
+ import { openMembersRegistry, commitMemberRevision, tombstoneMember, activateMember, readMember, deleteMember, listMembers, } from "./members.js";
45
+ import { openAclRegistry, readAcl, readAclForAlias, AmbiguousAclAlias, commitAcl as writeAclRecord, reissueAcl as writeAclReissue } from "./acls.js";
48
46
  import { EndpointDeliveryLease } from "./endpoint-delivery-lease.js";
47
+ import { EndpointDeliveryPlane } from "./endpoint-delivery-plane.js";
48
+ import { ackWaitOf, authenticatedChannelMessage, durableDelivery, isRecord } from "./endpoint-inbound.js";
49
49
  import { openChannelRegistry, effectiveReplay, effectiveReplayWindowMs, effectiveDeliveryClass, readChannelConfig, readChannelDefaults, decodeChannelEntry, } from "./channels.js";
50
50
  import { isCasLoss } from "./endpoint-records.js";
51
51
  import { ANYCAST_CANCEL_OP, ANYCAST_NOTICE_KEY, ANYCAST_RECORDS_CREDENTIAL_OUTDATED, ANYCAST_STATUS_OP, ANYCAST_TAKES_OP, AnycastRequestAuthority, isAnycastSeq, } from "./anycast-requests.js";
52
- import { anycastSubject, assertValidOwnerToken, assertValidRole, CHANNEL_DEFAULTS_KEY, chatStream, chatHistDurable, chatSubject, historyReaderConsumer, calloutReaderId, HISTORY_READERS, controlServiceSubject, livenessSubject, livenessServeFilter, CONTROL_DELIVERY, CONTROL_DELIVERY_ADMIN, dmStream, dmDurable, dlvStream, dlvDurable, dlvSubject, dinboxSubject, inboxStream, parseDinboxPrincipal, FANOUT_DURABLE, INBOX_READER_DURABLE, managerBucket, MANAGER_LEASE_KEY, MANAGER_RENEWAL_LEASE_KEY, managerLeaseKey, assertValidChannel, channelInAllow, membershipInAllow, isConcreteChannel, isChannelFamily, familiesCovering, normalizeMentions, MENTION_NOTICE_KEY, parseSubject, isPrincipalOwnerToken, assertInboxConnId, presenceBucket, channelBucket, membershipBucket, MEMBERSHIP_FEED_KEY, principalKey, parsePrincipalKey, assertLifecycleToken, mintLifecycleUid, lifecycleNameKey, memberKey, DEV_OWNER, spacePrefix, spaceWildcard, subjectMatches, taskStream, taskDurable, takeServeFilter, takeSubject, parseTakeSubject, TAKE_ROUTE, token, unicastSubject, unicastRecvFilter, } from "./subjects.js";
52
+ import { anycastSubject, assertValidOwnerToken, assertValidRole, CHANNEL_DEFAULTS_KEY, chatStream, chatSubject, controlServiceSubject, livenessSubject, livenessServeFilter, CONTROL_DELIVERY, CONTROL_DELIVERY_ADMIN, dmStream, dmDurable, dlvStream, dinboxSubject, FANOUT_DURABLE, managerBucket, MANAGER_LEASE_KEY, MANAGER_RENEWAL_LEASE_KEY, managerLeaseKey, membershipInAllow, isConcreteChannel, isChannelFamily, normalizeMentions, MENTION_NOTICE_KEY, parseSubject, isPrincipalOwnerToken, assertInboxConnId, presenceBucket, channelBucket, membershipBucket, MEMBERSHIP_FEED_KEY, principalKey, parsePrincipalKey, assertLifecycleToken, mintLifecycleUid, lifecycleNameKey, memberKey, DEV_OWNER, spacePrefix, spaceWildcard, subjectMatches, taskStream, taskDurable, takeServeFilter, takeSubject, parseTakeSubject, TAKE_ROUTE, token, unicastSubject, unicastRecvFilter, } from "./subjects.js";
53
53
  import { isPermissionDenied } from "./permission-denied.js";
54
54
  export { isPermissionDenied };
55
55
  export const DEFAULT_SERVER = "nats://127.0.0.1:4222";
56
- const PLANE3_FRAME_HEADER = "Cotal-Delivery-Frame";
57
- /** The `ctl.delivery` ops a sender calls about one of its own anycast requests (SPEC §8.1). */
58
- const ANYCAST_CONTROL_OPS = new Set([ANYCAST_STATUS_OP, ANYCAST_CANCEL_OP, ANYCAST_TAKES_OP]);
59
56
  /** Takes a daemon answers at once; one more is refused as busy, so the holder keeps its copy and
60
57
  * asks again rather than queueing work the daemon cannot bound. */
61
58
  const MAX_TAKES_IN_FLIGHT = 64;
@@ -63,8 +60,6 @@ const MAX_TAKES_IN_FLIGHT = 64;
63
60
  const ANYCAST_SWEEP_MS = 10 * 60 * 1000;
64
61
  /** How long a holder waits for the daemon to answer a take. */
65
62
  const ANYCAST_TAKE_TIMEOUT_MS = 5_000;
66
- /** How long the trusted writer reuses its read of the DLV count before reading it again. */
67
- const DELIVERY_LIMIT_READ_MS = 5_000;
68
63
  /** The DLV count a stored copy was written under, from its {@link DELIVERY_LIMIT_HEADER} (SPEC §8):
69
64
  * a whole number of at least 1, null for `none`, and 1000 when the header is absent, since only a
70
65
  * daemon from before the count was configurable writes a copy without it and its count was 1000.
@@ -80,31 +75,6 @@ export function deliveryLimitOf(value) {
80
75
  }
81
76
  /** Space joined when none is given on the CLI (the `cotal-<space>` cmux tab, etc.). */
82
77
  export const DEFAULT_SPACE = "main";
83
- /** How many channel filters one multi-filter consumer create may carry.
84
- *
85
- * A create names every requested channel in one request, so the request grows with the channel
86
- * count and the CLIENT's request timeout is what gives way, not the broker: the create is never
87
- * refused, it just does not answer. Measured on an isolated broker, one message per channel,
88
- * `limit=5`: 70 filters answer in 43ms, 1,000 in 246ms, 5,000 in 5,345ms, and 10,000 does not
89
- * answer at all, failing `timeout` after 5,023ms. Note 5,000 SUCCEEDED while taking longer than
90
- * 10,000 took to fail, which is what says the ceiling is on the create request rather than on the
91
- * read: past roughly 5s the create itself is what times out.
92
- *
93
- * 1,000 is chosen from that sweep rather than from the failure point: it is a fifth of the largest
94
- * count that still answered, and it answers in a quarter second, so a batch stays far away from
95
- * both the timeout and the response-deadline budget the dashboard has to share with its DM read.
96
- * A space with the 69 chat channels this was built for is still ONE read, so the round-trip claim
97
- * in #1210 is unchanged at that size. */
98
- export const MULTI_FILTER_BATCH = 1_000;
99
- /** How many filter batches may be in flight at once. Bounded because the point of #1210 was to stop
100
- * issuing one read per channel: a space large enough to need batches must not get the fan-out back
101
- * under another name. */
102
- export const MULTI_FILTER_READ_CONCURRENCY = 4;
103
- /** A history reader's inactive threshold. It bounds only a reader whose delete never ran (its link
104
- * dropped, or its process stopped), because every read deletes its readers when it ends. Five
105
- * minutes is what the nats.js ordered consumers it replaces carried, and it keeps a reader alive
106
- * across the gap between pulls on a slow link. */
107
- const HISTORY_READER_INACTIVE_MS = 5 * 60_000;
108
78
  /** The longest delay `setTimeout` honors, 2^31-1 ms. */
109
79
  const MAX_TIMER_MS = 2_147_483_647;
110
80
  /** This endpoint's own `dm_` durable does not exist. A DM-only endpoint, and any endpoint on an
@@ -133,35 +103,6 @@ export class DmBindRefusedError extends Error {
133
103
  this.name = "DmBindRefusedError";
134
104
  }
135
105
  }
136
- /**
137
- * Events: "message" (CotalMessage), "presence" (PresenceEvent), "roster" (Presence[]), "error" (Error),
138
- * "connection" ({ connected: boolean }) — true on every successful (re)bind (initial start, manual
139
- * reconnect, AND background self-heal), false the moment the connection drops (rebuild null window /
140
- * terminal close). Lets an in-process agent track connectedness off the endpoint's own (re)binds
141
- * instead of an imperative flag the self-heal path can't reach; "transport" ({ connected, server? })
142
- * is the lower-level NATS socket edge, true before the full bind finishes and false during an internal
143
- * nats.js reconnect without changing `connection` readiness. "stored-copies-full" ({ limit: number }): a
144
- * stored copy arrived with the store at the count the copy was written under ({@link deliveryLimitOf}),
145
- * so the store may have discarded older copies before this session read them; emitted for each such
146
- * copy. A lowering of the count or an age set on the store removes copies without this event.
147
- *
148
- * Callers MUST attach an "error" listener before `start()`: async faults (incl. NATS
149
- * permission denials, surfaced via `watchStatus`) are emitted as "error", and Node throws
150
- * synchronously on an unhandled "error" — a missing listener turns any such fault into a
151
- * process crash instead of a logged denial.
152
- */
153
- /** Plane-3 trusted-reader redelivery ceiling: a dinbox entry that keeps failing re-auth-defer
154
- * (unknown owner) or DELIVER transfer is `term()`d + surfaced after this many redeliveries, so one
155
- * stuck/poison entry can't head-of-line the single shared reader forever. */
156
- const READER_MAX_REDELIVERIES = 10;
157
- /** Page size for the mediated history read when the caller names none — matches `channelHistory`'s
158
- * own default, since the mediated read exists to be a drop-in for it. */
159
- const READ_HISTORY_DEFAULT_LIMIT = 100;
160
- /** Hard server-side ceiling on one mediated page. The caller PROPOSES a limit and the mediator
161
- * decides: the reader is pooled and privileged, so an unbounded caller-chosen limit would let one
162
- * request pull a channel's whole retained set through it. Clamped, not refused — a UI asking for
163
- * more than a page should get a page, and `complete: false` already tells it more remains. */
164
- const READ_HISTORY_MAX_LIMIT = 200;
165
106
  /** The NEWEST prefix-from-the-end of `items` whose serialized `readHistory` reply fits `budget`
166
107
  * bytes, order preserved. Returns `[]` when not even the newest single message fits — the caller
167
108
  * must refuse loudly there rather than serve an empty page, which would read as "no history".
@@ -309,10 +250,52 @@ export class CotalEndpoint extends EventEmitter {
309
250
  /** Caller-owned membership watches survive a connection rebuild as INTENT. Their iterators are
310
251
  * connection-scoped and are stopped/re-created around the epoch swap. */
311
252
  membershipFeedWatches = new Set();
312
- /** The live `ctl.delivery` serve subscription (delivery daemon) — re-created on every (re)connect by
313
- * {@link armDeliveryControl}; tracked so the stale one is dropped on reconnect. */
314
- deliveryServeSub;
315
- deliveryAdminServeSub;
253
+ /** The plane-3 delivery daemon side of this endpoint (fan-out writer, trusted reader, control
254
+ * responders) and the agent's DELIVER pump. Every dependency is read when it is used. */
255
+ deliveryPlane = new EndpointDeliveryPlane({
256
+ ackWaitMs: () => this.ackWaitMs,
257
+ actor: () => this.actor,
258
+ owner: () => this.owner,
259
+ allowSubscribe: () => this.allowSubscribe,
260
+ anycastUnavailable: () => this.anycastUnavailable,
261
+ card: () => this.card,
262
+ js: () => this.js,
263
+ jsm: () => this.jsm,
264
+ nc: () => this.nc,
265
+ ownLifecycleUid: () => this.ownLifecycleUid,
266
+ residentSwap: () => this.residentSwap,
267
+ space: () => this.space,
268
+ stopped: () => this.stopped,
269
+ streamMsgs: () => this.streamMsgs,
270
+ subs: () => this.subs,
271
+ userMode: () => this.userMode,
272
+ watchDeletePermissions: () => this.watchDeletePermissions,
273
+ maxPayload: () => this.maxPayload,
274
+ aclForAlias: (...args) => this.aclForAlias(...args),
275
+ channelHistory: (...args) => this.channelHistory(...args),
276
+ deliveryClassFresh: (...args) => this.deliveryClassFresh(...args),
277
+ durableJoinFor: (...args) => this.durableJoinFor(...args),
278
+ durableLeaveFor: (...args) => this.durableLeaveFor(...args),
279
+ ownerMemberships: (...args) => this.ownerMemberships(...args),
280
+ ownsDeliveryLease: (...args) => this.ownsDeliveryLease(...args),
281
+ readDeliveryLease: (...args) => this.readDeliveryLease(...args),
282
+ readDeliveryLeaseEntry: (...args) => this.readDeliveryLeaseEntry(...args),
283
+ reloadCreds: (...args) => this.reloadCreds(...args),
284
+ serveControl: (...args) => this.serveControl(...args),
285
+ aclRegistry: (...args) => this.aclRegistry(...args),
286
+ membersRegistry: (...args) => this.membersRegistry(...args),
287
+ armAnycastRequests: (...args) => this.armAnycastRequests(...args),
288
+ deliveryAnycast: (...args) => this.deliveryAnycast(...args),
289
+ dropAnycastRequests: (...args) => this.dropAnycastRequests(...args),
290
+ manager: (...args) => this.manager(...args),
291
+ notLiveMsg: (...args) => this.notLiveMsg(...args),
292
+ plane3MsgId: (...args) => this.plane3MsgId(...args),
293
+ publishDinbox: (...args) => this.publishDinbox(...args),
294
+ resolveOwnerByName: (...args) => this.resolveOwnerByName(...args),
295
+ scheduleResidentSwap: (...args) => this.scheduleResidentSwap(...args),
296
+ sendMentionNotice: (...args) => this.sendMentionNotice(...args),
297
+ emit: (event, ...args) => this.emit(event, ...args),
298
+ });
316
299
  /** The anycast take responder, the request records it writes, and their maintenance timer
317
300
  * (delivery daemon, SPEC §8.1). Bound with the other Plane-3 responders and dropped with them. */
318
301
  takeServeSub;
@@ -324,24 +307,6 @@ export class CotalEndpoint extends EventEmitter {
324
307
  anycastUnavailable;
325
308
  anycastSweep;
326
309
  takesInFlight = 0;
327
- /** The lifecycles whose membership inventory this daemon has been asked for, as principal to uids
328
- * (#4965). A durable join or leave bound to one of them is refused, so no row of a retiring lifecycle
329
- * is written after its inventory read starts. An entry goes when a later inventory for the same
330
- * principal finds its ACL row retired; an alias that never retires again keeps its one entry until
331
- * the daemon restarts. This is the second layer of the fence: it covers a manager that does not
332
- * narrow the ACL row before the inventory, on this daemon process only. The first is the ACL row
333
- * each join re-reads around its write (`durableJoinFor`'s `aclFence`), which holds across daemons. */
334
- retiringLifecycles = new Map();
335
- /** The durable joins and leaves running now, by `<principal>/<lifecycleUid>`: the writes an
336
- * inventory of that lifecycle waits for before it reads. */
337
- membershipWrites = new Map();
338
- /** True once {@link quiescePlane3} has stopped serving this shard pending an ownership answer.
339
- * Guards {@link armPlane3} so a RECONNECT cannot silently resume serving mid-question. */
340
- plane3Quiesced = false;
341
- /** When set, this endpoint hosts the Plane-3 fan-out writer + trusted reader (the server-side delivery
342
- * daemon). `aclFor` maps an owner id to its current read ACL (`allowSubscribe`) for the reader's
343
- * re-authorization — read FRESH per entry from the durable ACL registry KV, hence async. */
344
- plane3;
345
310
  /** Local cache of the channel registry (key = channel token), kept by a KV watch or, on an
346
311
  * endpoint that reads it on demand, by {@link readChannels}. */
347
312
  channelConfigs = new Map();
@@ -351,9 +316,31 @@ export class CotalEndpoint extends EventEmitter {
351
316
  /** Orders {@link readChannels} results: a read that settles after a newer one was applied is dropped. */
352
317
  channelReadGeneration = 0;
353
318
  channelReadApplied = 0;
354
- /** Serializes history reads ({@link collectHistory}): they share the fixed per-instance
355
- * `chathist_<id>` consumer, so overlapping reads would delete/recreate it under one another. */
356
- histLock = Promise.resolve();
319
+ history = new EndpointHistory({
320
+ connection: () => this.nc,
321
+ jetstream: () => this.js,
322
+ manager: () => this.jsm,
323
+ setManager: (manager) => { this.jsm = manager; },
324
+ space: () => this.space,
325
+ owner: () => this.owner,
326
+ actor: () => this.actor,
327
+ userMode: () => this.userMode,
328
+ connectionId: () => this.connId,
329
+ card: () => this.card,
330
+ notLiveMsg: () => this.notLiveMsg(),
331
+ requireLifecycleUid: (operation) => this.requireLifecycleUid(operation),
332
+ requestDelivery: (op, args) => this.requestDelivery(op, args),
333
+ deleteConsumerIfPresent: (stream, name) => this.deleteConsumerIfPresent(stream, name),
334
+ joinPolicyFresh: (channel) => this.joinPolicyFresh(channel),
335
+ chatMayHaveLost: (subject, sinceSeq) => this.chatMayHaveLost(subject, sinceSeq),
336
+ emit: (event, ...args) => this.emit(event, ...args),
337
+ decodeHistory: historyMessageFromDelivery,
338
+ isRecord,
339
+ isUsableMessageId,
340
+ isFromSubjectSender,
341
+ authenticatedMessage,
342
+ isPublishPermissionDenied,
343
+ });
357
344
  subs = [];
358
345
  /** The observer feeds {@link tap} was asked for. This is INTENT, re-subscribed by {@link bindTap}
359
346
  * on every (re)connect: a rebuild after an outage the library could not ride out closes the old
@@ -408,9 +395,6 @@ export class CotalEndpoint extends EventEmitter {
408
395
  /** `$JS.API.CONSUMER.DELETE.<stream>.<prefix>_` for each of this connection's own KV watches and
409
396
  * running membership scans (see {@link recordOwnWatchConsumer}). */
410
397
  ownWatchDeletePrefixes = new Set();
411
- /** Per stream, the history-reader names of this connection's current reader id and the reads
412
- * waiting for one (see {@link acquireHistoryReader}). */
413
- historyReaderSlots = new Map();
414
398
  /** True until the first successful connect completes its boot backfill — distinguishes first-connect
415
399
  * (backfill the boot channels' history) from a reconnect (reopen the core-subs, no re-backfill).
416
400
  * Persists across reconnect (NOT connection-scoped). Replaces the legacy chat-durable consumed-cursor
@@ -1676,19 +1660,19 @@ export class CotalEndpoint extends EventEmitter {
1676
1660
  // Handles that armDeliveryControl created on a previous connection: null them so a rebind
1677
1661
  // does not carry a dead protocol's subs forward. Best-effort unsubscribe first (the sub is
1678
1662
  // dead with its connection either way; unsubscribing an already-dead sub is a noop).
1679
- if (this.deliveryServeSub) {
1663
+ if (this.deliveryPlane.deliveryServeSub) {
1680
1664
  try {
1681
- this.deliveryServeSub.unsubscribe();
1665
+ this.deliveryPlane.deliveryServeSub.unsubscribe();
1682
1666
  }
1683
1667
  catch { /* dead with the old connection */ }
1684
- this.deliveryServeSub = undefined;
1668
+ this.deliveryPlane.deliveryServeSub = undefined;
1685
1669
  }
1686
- if (this.deliveryAdminServeSub) {
1670
+ if (this.deliveryPlane.deliveryAdminServeSub) {
1687
1671
  try {
1688
- this.deliveryAdminServeSub.unsubscribe();
1672
+ this.deliveryPlane.deliveryAdminServeSub.unsubscribe();
1689
1673
  }
1690
1674
  catch { /* dead with the old connection */ }
1691
- this.deliveryAdminServeSub = undefined;
1675
+ this.deliveryPlane.deliveryAdminServeSub = undefined;
1692
1676
  }
1693
1677
  this.dropAnycastRequests();
1694
1678
  for (const sub of this.subs) {
@@ -1963,7 +1947,14 @@ export class CotalEndpoint extends EventEmitter {
1963
1947
  throw e;
1964
1948
  }
1965
1949
  }
1966
- /** The presence epoch moves first: a bind still awaiting the broker must find it moved
1950
+ /** Stop the endpoint and close its connection.
1951
+ *
1952
+ * Every principal serve ends first: its handlers' `signal` aborts, and stop waits for each
1953
+ * handler in flight until its request's deadline at most, so a handler that never settles does
1954
+ * not hold stop open longer than 30 s, the route's cap on a deadline. A reply that comes after
1955
+ * that is dropped.
1956
+ *
1957
+ * The presence epoch moves first: a bind still awaiting the broker must find it moved
1967
1958
  * before any await below gives it a window to install a watch on a stopped endpoint
1968
1959
  * (see {@link startPresenceWatch}). */
1969
1960
  stop() {
@@ -1979,7 +1970,8 @@ export class CotalEndpoint extends EventEmitter {
1979
1970
  this.activeGoalFollowers.clear();
1980
1971
  this.presenceEpoch++;
1981
1972
  // Every principal serve ends now, so a `ready` still waiting for the broker rejects and no new
1982
- // request is taken. Its handlers in flight finish below, before the connection they reply on closes.
1973
+ // request is taken. Its handlers in flight finish below, before the connection they reply on
1974
+ // closes, or their requests' deadlines pass.
1983
1975
  const principalStops = [...this.principalServes.values()].map((intent) => this.stopPrincipalServe(intent, false));
1984
1976
  this.presence.cancelRebind();
1985
1977
  // Wake a reestablishLoop sitting in backoff so it sees `stopped` and exits instead of
@@ -2206,6 +2198,7 @@ export class CotalEndpoint extends EventEmitter {
2206
2198
  *
2207
2199
  * `expectedLastSubjectSeq` is a parameter because it is a header VALUE: sizing at 0 and publishing
2208
2200
  * at 123456 differ by five bytes.
2201
+ * `expectedLastMsgId` is measured when given, so a caller that may send it can size for it.
2209
2202
  *
2210
2203
  * Residual: `ts` is re-stamped at publish, so the two differ in value — not in length until
2211
2204
  * epoch-millis needs a 14th digit.
@@ -2223,6 +2216,10 @@ export class CotalEndpoint extends EventEmitter {
2223
2216
  const mh = headers();
2224
2217
  mh.set("Nats-Msg-Id", opts.id);
2225
2218
  mh.set("Nats-Expected-Last-Subject-Sequence", `${opts.expectedLastSubjectSeq}`);
2219
+ if (opts.expectedLastMsgId !== undefined) {
2220
+ assertIdToken(opts.expectedLastMsgId, "expected last msg id");
2221
+ mh.set("Nats-Expected-Last-Msg-Id", opts.expectedLastMsgId);
2222
+ }
2226
2223
  // `encode()` is on the client's header implementation but not on the published `MsgHdrs` type,
2227
2224
  // so it is reached through a cast rather than copied. If a client version drops it, this throws
2228
2225
  // immediately and loudly — the calibration cell would also fail — instead of returning a number
@@ -2248,6 +2245,10 @@ export class CotalEndpoint extends EventEmitter {
2248
2245
  * means "the subject must be empty". A mismatch throws, and the throw stays classifiable by the
2249
2246
  * already-public {@link isCasLoss} — the error is deliberately **not wrapped**, since wrapping
2250
2247
  * would hide the `err_code` that classification reads.
2248
+ * - `expectedLastMsgId`, when given, becomes `Nats-Expected-Last-Msg-Id`. The broker checks it after
2249
+ * its dedup cache, so a fresh value nobody has published turns a publish into a probe: an id the
2250
+ * cache holds answers `duplicate: true` with the stored sequence, and any other id is refused
2251
+ * with error 10070 before anything is stored.
2251
2252
  *
2252
2253
  * @throws if the endpoint is not live, the channel is not concrete, `id` is malformed, `parts` is
2253
2254
  * empty, or `expectedLastSubjectSeq` is not a non-negative safe integer.
@@ -2265,11 +2266,13 @@ export class CotalEndpoint extends EventEmitter {
2265
2266
  throw new Error(`expectedLastSubjectSeq must be a non-negative safe integer, got ${JSON.stringify(expected)}`);
2266
2267
  if (!Array.isArray(opts.parts) || opts.parts.length === 0)
2267
2268
  throw new Error("multicastExpecting requires at least one part");
2269
+ if (opts.expectedLastMsgId !== undefined)
2270
+ assertIdToken(opts.expectedLastMsgId, "expected last msg id");
2268
2271
  const message = this.casEnvelope(opts);
2269
2272
  assertPartsSerializable(message.parts);
2270
2273
  // Publish DIRECTLY rather than through publishMsg: this path must set the expectation and read
2271
2274
  // the ack, and publishMsg deliberately does neither.
2272
- const ack = await this.js.publish(chatSubject(this.space, this.owner, this.actor, opts.channel), JSON.stringify(message), { msgID: opts.id, expect: { lastSubjectSequence: expected } });
2275
+ const ack = await this.js.publish(chatSubject(this.space, this.owner, this.actor, opts.channel), JSON.stringify(message), { msgID: opts.id, expect: { lastSubjectSequence: expected, ...(opts.expectedLastMsgId !== undefined ? { lastMsgID: opts.expectedLastMsgId } : {}) } });
2273
2276
  return { message, ack: { seq: ack.seq, duplicate: ack.duplicate === true } };
2274
2277
  }
2275
2278
  /** Publish a payload frame on this principal's forge-locked sibling event rail. */
@@ -3797,13 +3800,7 @@ export class CotalEndpoint extends EventEmitter {
3797
3800
  * published by that principal: the same guarantee a live subscription to that subject gives.
3798
3801
  * Without it the read is the channel from any sender. */
3799
3802
  async channelHistory(channel, opts) {
3800
- const from = opts?.publisher;
3801
- // A publisher is one principal. A `*` in either slot would widen the read back to any sender
3802
- // while the caller believes it narrowed it, so both slots are checked as concrete tokens.
3803
- const subject = from === undefined
3804
- ? chatSubject(this.space, "*", "*", channel)
3805
- : chatSubject(this.space, assertValidOwnerToken(from.owner), assertValidOwnerToken(from.actor), channel);
3806
- return (await this.streamHistory(chatStream(this.space), [subject], opts?.limit ?? 100, undefined, opts?.signal)).map((r) => r.msg);
3803
+ return this.history.channelHistory(channel, opts);
3807
3804
  }
3808
3805
  /** Read one concrete channel using this lifecycle's single-filter CHAT grant (SPEC §8/§9).
3809
3806
  * No observer authority, presence or delivery ACL mirror is required. Returns the newest page,
@@ -3811,87 +3808,7 @@ export class CotalEndpoint extends EventEmitter {
3811
3808
  * Reads have a ten-second budget, shortened by `signal`. Concurrent connections sharing this
3812
3809
  * lifecycle wait for native consumer inactivity cleanup; they never delete another reader. */
3813
3810
  async channelHistoryPage(channel, opts) {
3814
- assertValidChannel(channel);
3815
- if (!isConcreteChannel(channel))
3816
- throw new Error("channelHistoryPage requires one concrete channel");
3817
- const limit = opts?.limit ?? 50;
3818
- if (!Number.isSafeInteger(limit) || limit < 1 || limit > 1000)
3819
- throw new Error("channelHistoryPage limit must be a whole number from 1 to 1000");
3820
- if (!this.nc || !this.js)
3821
- throw new Error(this.notLiveMsg());
3822
- const jsm = this.jsm ??= await jetstreamManager(this.nc);
3823
- const signal = AbortSignal.any([AbortSignal.timeout(10_000), ...(opts?.signal ? [opts.signal] : [])]);
3824
- signal.throwIfAborted();
3825
- const stream = chatStream(this.space);
3826
- const name = chatHistDurable(this.owner, this.actor, this.requireLifecycleUid("channelHistoryPage"));
3827
- const subject = chatSubject(this.space, "*", "*", channel);
3828
- const marker = randomUUID();
3829
- let created;
3830
- for (;;) {
3831
- signal.throwIfAborted();
3832
- try {
3833
- // The pinned client's add is CREATE-only. A unique description makes even an otherwise
3834
- // identical occupied config conflict. Never bind/update it or pre-delete by shared name.
3835
- created = await jsm.consumers.add(stream, {
3836
- name, description: marker, filter_subject: subject,
3837
- ack_policy: AckPolicy.None, deliver_policy: DeliverPolicy.All,
3838
- mem_storage: true, inactive_threshold: nanos(250),
3839
- });
3840
- break;
3841
- }
3842
- catch (e) {
3843
- if (!(e instanceof JetStreamApiError) || e.code !== 10148)
3844
- throw e; // consumer already exists
3845
- await delay(50, undefined, { signal });
3846
- }
3847
- }
3848
- signal.throwIfAborted();
3849
- const owns = (info) => info.created === created.created &&
3850
- info.config.description === marker && info.config.filter_subject === subject;
3851
- if (!owns(created))
3852
- throw new Error("channelHistoryPage: consumer ownership was not established");
3853
- const consumer = await this.js.consumers.get(stream, name);
3854
- const pending = created.num_pending;
3855
- const out = [];
3856
- let delivered = 0;
3857
- if (pending > 0) {
3858
- const iter = await consumer.consume({ max_messages: Math.min(pending, 32) });
3859
- const stop = () => iter.stop(abortReason(signal));
3860
- signal.addEventListener("abort", stop, { once: true });
3861
- try {
3862
- signal.throwIfAborted();
3863
- for await (const m of iter) {
3864
- signal.throwIfAborted();
3865
- if (!subjectMatches(subject, m.subject))
3866
- throw new Error("channelHistoryPage: consumer delivered another channel; ownership was lost");
3867
- delivered++;
3868
- const msg = historyMessageFromDelivery(m);
3869
- if (msg) {
3870
- out.push(msg);
3871
- if (out.length > limit + 1)
3872
- out.shift();
3873
- }
3874
- if (delivered >= pending)
3875
- break;
3876
- }
3877
- }
3878
- finally {
3879
- signal.removeEventListener("abort", stop);
3880
- iter.stop();
3881
- }
3882
- signal.throwIfAborted();
3883
- if (delivered !== pending)
3884
- throw new Error(`channelHistoryPage: read ${delivered} of ${pending} messages before the window was cut short`);
3885
- }
3886
- signal.throwIfAborted();
3887
- // A legacy join/recall reader can delete/recreate this fixed name. A final ownership check
3888
- // makes that interference loud. It is detection, not a fence against those legacy readers.
3889
- if (!owns(await consumer.info()))
3890
- throw new Error("channelHistoryPage: consumer ownership was lost");
3891
- signal.throwIfAborted();
3892
- // No DELETE, even after abort: a delayed owner could otherwise delete its successor between
3893
- // INFO and DELETE. Native inactivity reaps this transient reader without that TOCTOU race.
3894
- return { items: out.slice(-limit), complete: out.length <= limit };
3811
+ return this.history.channelHistoryPage(channel, opts);
3895
3812
  }
3896
3813
  /**
3897
3814
  * The newest `limit` chat messages across concrete channels or one channel family, oldest-first within the page,
@@ -3922,68 +3839,7 @@ export class CotalEndpoint extends EventEmitter {
3922
3839
  * so the broker checks the credential's grant for that filter. No additional authority is minted.
3923
3840
  */
3924
3841
  async multiChannelHistory(channels, opts) {
3925
- const subjects = [...new Set(channels.map((channel) => {
3926
- if (!isConcreteChannel(channel) && !(channels.length === 1 && isChannelFamily(channel)))
3927
- throw new Error(`multiChannelHistory: "${channel}" is a wildcard channel - name one channel family alone or concrete channels without wildcards`);
3928
- // `chatSubject` builds the filter through `token()`, which REWRITES rather than refuses: it
3929
- // maps a character a subject may not carry to `_`, trims each segment, and drops empty ones.
3930
- // So "foo/bar" would filter on `foo_bar`, ".lead" on `lead`, and "team..b" on `team.b` - the
3931
- // caller names one channel and the broker returns another, which is the exact promise this
3932
- // method makes ("a channel left out of the list never crosses the link") inverted. This is the
3933
- // same aliasing `assertValidChannel` was written for on the policy path; the read path needs
3934
- // it too, and it fails loud rather than serving a channel nobody asked for.
3935
- assertValidChannel(channel);
3936
- return chatSubject(this.space, "*", "*", channel);
3937
- }))];
3938
- // No channels is not an empty stream, but it IS an empty answer, and asking the broker for a
3939
- // consumer with no filter would read the WHOLE stream instead of none of it.
3940
- if (subjects.length === 0)
3941
- return [];
3942
- const limit = opts?.limit ?? 100;
3943
- // ONE CREATE CANNOT CARRY AN UNBOUNDED FILTER LIST. The create names every subject in one
3944
- // request and the client's request timeout is what gives way, so past roughly 5,000 filters the
3945
- // read does not answer at all and the route loses the whole chat source: measured on an
3946
- // isolated broker, 10,000 channels failed `timeout` after 5,023ms while the fan-out this
3947
- // replaced still returned 2,739 messages on the same corpus. Reading in batches keeps the
3948
- // single-read cost at the sizes this was built for (69 channels is one batch, so #1210's
3949
- // round-trip numbers are unchanged) and degrades to a few reads instead of none above that.
3950
- // `batch` is a knob rather than a constant so the batched path can be compared against the
3951
- // single-create path on ONE corpus: forcing a small batch makes a space that would otherwise be
3952
- // one read take many, which is the only way to assert the two select the same messages.
3953
- const size = Math.max(1, Math.trunc(opts?.batch ?? MULTI_FILTER_BATCH));
3954
- const batches = [];
3955
- for (let i = 0; i < subjects.length; i += size)
3956
- batches.push(subjects.slice(i, i + size));
3957
- const pages = new Array(batches.length);
3958
- let next = 0;
3959
- const readBatch = async () => {
3960
- for (;;) {
3961
- const i = next++;
3962
- if (i >= batches.length)
3963
- return;
3964
- pages[i] = await this.streamHistory(chatStream(this.space), batches[i], limit, undefined, opts?.signal);
3965
- }
3966
- };
3967
- // A BATCH THAT FAILS FAILS THE READ. Returning the batches that answered would be the newest
3968
- // `limit` across SOME of the channels asked for while looking like the newest across all of
3969
- // them, which is a wrong page presented as a right one. Throwing keeps the caller's existing
3970
- // envelope: the dashboard marks `chat` missing and says the page is partial, which is what it
3971
- // already did when this method was one read.
3972
- await Promise.all(Array.from({ length: Math.min(MULTI_FILTER_READ_CONCURRENCY, batches.length) }, readBatch));
3973
- // MERGE ON ARRIVAL, NOT ON `ts`. Each batch returns the newest `limit` within its own subjects,
3974
- // and the newest `limit` overall is a subset of their union, so re-selecting by stream sequence
3975
- // reproduces exactly what one create over the whole list would have selected. Sorting by the
3976
- // payload's `ts` here instead would change which messages the page holds, which is the
3977
- // selection question this pull request already had to answer once.
3978
- const rows = pages.flat().sort((a, b) => a.seq - b.seq).slice(-limit);
3979
- return rows.map(({ seq, subject, msg }) => {
3980
- const p = parseSubject(subject);
3981
- // The filter set is built from chat subjects, so this cannot fire against a healthy broker.
3982
- // It is here because the alternative to raising is tagging a message with a guess.
3983
- if (p?.kind !== "chat")
3984
- throw new Error(`multiChannelHistory: the broker delivered ${subject}, which is not a chat subject on this space`);
3985
- return { channel: p.rest, seq, msg };
3986
- });
3842
+ return this.history.multiChannelHistory(channels, opts);
3987
3843
  }
3988
3844
  /** Read a channel's recent history through the delivery daemon, using a provisioned ACL mirror.
3989
3845
  * SPEC's ban on raw control-surface stream reads does not remove the single-channel CHAT grant.
@@ -4013,16 +3869,7 @@ export class CotalEndpoint extends EventEmitter {
4013
3869
  * different authorization model, and one handler with two authz paths is the wrong shape on the
4014
3870
  * surface where a mistake exposes private messages). */
4015
3871
  async readHistory(channel, opts) {
4016
- const reply = await this.requestDelivery("readHistory", { channel, limit: opts?.limit });
4017
- if (!reply.ok)
4018
- throw new Error(reply.error ?? "readHistory failed");
4019
- const data = reply.data;
4020
- // Validate rather than coerce. A malformed reply must not be massaged into a plausible page:
4021
- // defaulting `complete` would invent the very signal a caller uses to decide whether it is
4022
- // looking at the start of a conversation.
4023
- if (!Array.isArray(data?.items) || typeof data?.complete !== "boolean")
4024
- throw new Error("readHistory: the delivery daemon returned a malformed page (expected { items, complete })");
4025
- return { items: data.items, complete: data.complete };
3872
+ return this.history.readHistory(channel, opts);
4026
3873
  }
4027
3874
  /** Fetch recent DMs (any sender→any recipient) from the space's DM backlog. `signal` cancels the
4028
3875
  * active pull and reclaims its ephemeral consumer. God-view only:
@@ -4032,15 +3879,7 @@ export class CotalEndpoint extends EventEmitter {
4032
3879
  * returned — a stored row missing `ts` / `space` / `parts` / `from.name` is dropped, never
4033
3880
  * returned with that member silently `undefined`. */
4034
3881
  async dmHistory(opts) {
4035
- try {
4036
- // every inst.<recipOwner>.<recipActor>.<sndOwner>.<sndActor> DM — the whole DM subtree (god-view)
4037
- return (await this.streamHistory(dmStream(this.space), [`${spacePrefix(this.space)}.inst.>`], opts?.limit ?? 100, undefined, opts?.signal)).map((r) => r.msg);
4038
- }
4039
- catch (e) {
4040
- if (isPermissionDenied(e))
4041
- return [];
4042
- throw e;
4043
- }
3882
+ return this.history.dmHistory(opts);
4044
3883
  }
4045
3884
  /** The newest `limit` DMs (100 by default) sent to or by any of `principals`, oldest first. Each
4046
3885
  * principal is a concrete `<owner>.<actor>`; anything else throws, because a wildcard would widen
@@ -4099,434 +3938,7 @@ export class CotalEndpoint extends EventEmitter {
4099
3938
  * `before` pages toward the past: pass the `seq` of the oldest message you already have.
4100
3939
  */
4101
3940
  async streamHistory(stream, subjects, limit, before, signal) {
4102
- if (!this.nc)
4103
- throw new Error(this.notLiveMsg());
4104
- signal?.throwIfAborted();
4105
- // A LIMIT THAT IS NOT A FINITE NUMBER HAS NO ANSWER, AND THE SEARCH BELOW CANNOT REFUSE IT.
4106
- // Every comparison against NaN is false, so `limit <= 0` does not fire for one, and neither of
4107
- // the widening loop's exits can ever be true either: `page.length >= NaN` is false forever and
4108
- // `start === 1` compares against a `start` that is itself NaN. The loop does not return
4109
- // everything, it never returns. Measured through the dashboard on a real broker: no answer
4110
- // after 30s, and the abandoned read still consuming half a core fifteen seconds after its
4111
- // caller had gone, while the process kept serving every other route so nothing announced it.
4112
- // `Infinity` reaches the other end of the same hole: it passes the guard, `start` collapses to
4113
- // 1, and `slice(-Infinity)` is the subject's whole retained set.
4114
- // A caller that computed a limit it cannot state is told so, rather than handed a process that
4115
- // quietly spins. `0` and negatives keep their existing meaning, an empty page.
4116
- // A COUNT OF MESSAGES IS A WHOLE NUMBER THIS SERVER CAN COUNT EXACTLY, and the check sits ABOVE
4117
- // the zero check on purpose: `-Infinity <= 0` is true, so a guard placed below it would fold a
4118
- // limit nobody can answer into a silent empty page. `NaN` never returns (both widening exits
4119
- // compare against it and are false forever). `Infinity` and any magnitude past the safe range
4120
- // collapse `start` to 1. A fraction is the quietest of the set: the page is taken with
4121
- // `slice(-limit)` and slice truncates toward zero, so any limit in (0,1) becomes `slice(0)` and
4122
- // hands back the ENTIRE retained history. Measured on 30 retained: 0.5 and 0.9 both returned all
4123
- // 30, and 2.5 returned 2, a page nobody asked for.
4124
- if (!Number.isSafeInteger(limit))
4125
- throw new Error(`history limit must be a whole number of messages this server can count exactly, received ${limit}`);
4126
- if (limit <= 0)
4127
- return [];
4128
- const nc = this.nc;
4129
- let reader;
4130
- try {
4131
- reader = await this.acquireHistoryReader(nc, this.jsm ?? (this.jsm = await jetstreamManager(nc)), stream, signal);
4132
- // THE EXACT CEILING, on the already-granted surface. A one-shot consumer with
4133
- // `DeliverPolicy.Last` plus this subject's filter reports the newest MATCHING sequence, and its
4134
- // `num_pending` of zero means the channel is genuinely empty. Using the STREAM's last sequence
4135
- // instead (which is all `STREAM.INFO` offers) was a loose upper bound, and on a quiet channel
4136
- // in a busy stream the gap between the two is the whole problem: every window near the stream
4137
- // head is empty, so the search widened over and over before finding anything.
4138
- //
4139
- // Deliberately NOT `getMessage({ last_by_subj })`, which would be the obvious way to ask: it
4140
- // needs `$JS.API.STREAM.MSG.GET`, which read credentials do not hold. That grant hole already
4141
- // shipped once from this function and turned every non-admin history read into an empty list.
4142
- const ceiling = before !== undefined ? before - 1 : await this.lastMatchingSeq(reader, subjects, signal);
4143
- if (ceiling < 1)
4144
- return [];
4145
- // Widen from the exact ceiling until a window holds a full page, or until the window IS the
4146
- // whole subject. Draining each attempt is bounded, and that is the point: a window only fails
4147
- // when it holds FEWER than `limit` matches, so every wasted drain moves less than one page.
4148
- // Geometric growth keeps the number of attempts logarithmic, so total wasted transfer is a
4149
- // small multiple of a page.
4150
- //
4151
- // THE FIRST WINDOW IS ONE PAGE WIDE, NOT FOUR. It used to open at `limit * 4` on the reasoning
4152
- // that a filtered subject is sparse inside its stream and a wider first look would land the
4153
- // page in one drain. That is true for one channel of a busy stream and false for a subject
4154
- // that IS most of its stream, and the second case is the one a whole-backlog read takes: the
4155
- // window succeeds on the first attempt and drains four pages to keep one, because
4156
- // `drainWindow` delivers everything in the window and keeps the tail. Measured on `/api/dms`
4157
- // at limit 500 against a 2500-message backlog: 1,995,854 to 1,995,859 bytes moved across
4158
- // four runs to return a 346,001-byte page, and 8161ms to 8753ms on that link, so all four
4159
- // missed the dashboard's 8000ms deadline with nothing else on the connection (Cotal #1210).
4160
- // That cost does not keep growing with the backlog, and saying it did was wrong: the old first
4161
- // span was `max(limit * 4, 64)`, so at limit 500 it is 2000 messages, and ANY backlog of 2000
4162
- // or more drains the same 2000-message window. The point is that the window, not the backlog,
4163
- // sets the cost, and four pages to return one is already past the deadline on this link for
4164
- // every such deployment. One page wide makes the
4165
- // SUCCESSFUL drain
4166
- // obey the same bound the failed ones already promised: it moves at most one page. A sparse
4167
- // subject pays one more widening step for that, which is four round trips against a transfer
4168
- // several times the size of the answer.
4169
- //
4170
- // A SHORT PAGE is either "the channel is exhausted" or "the window is still above the
4171
- // first match". Sequence 1 is the wrong floor for the first of those: three matches at
4172
- // the high end of a busy stream are exhausted as soon as the window's lower edge passes
4173
- // the subject's first matching sequence, and walking on to 1 re-reads everyone else's
4174
- // retained set (#840). Probe that floor only after a short drain so a dense page (one
4175
- // drain, full) never pays for it. The remaining unbounded-looking case is a subject
4176
- // whose FIRST match really is near sequence 1; that span is the channel's own, not the
4177
- // stream's.
4178
- let span = Math.max(limit, 64);
4179
- let floor = 1;
4180
- let floorKnown = false;
4181
- for (;;) {
4182
- signal?.throwIfAborted();
4183
- const start = Math.max(floor, ceiling - span + 1);
4184
- const page = await this.drainWindow(reader, subjects, start, ceiling, limit, signal);
4185
- if (page.length >= limit)
4186
- return page.slice(-limit);
4187
- if (!floorKnown) {
4188
- floor = Math.max(1, await this.firstMatchingSeq(reader, subjects, signal));
4189
- floorKnown = true;
4190
- }
4191
- if (start <= floor)
4192
- return page.slice(-limit);
4193
- span *= 4;
4194
- }
4195
- }
4196
- catch (e) {
4197
- // NARROW. This catch is how the MSG.GET grant bug shipped: it turned a Permissions Violation
4198
- // into an empty history, so every non-admin read looked exactly like a quiet channel, and the
4199
- // smokes stayed green because they run as admin against a local broker. Emitting on the error
4200
- // event is not enough either — callers consume the RETURN VALUE, and the dashboard renders
4201
- // empty regardless of what an operator log says.
4202
- //
4203
- // So only one thing may still produce an empty result here: the stream does not exist (a space
4204
- // with no history yet). Anything else — a denial, a timeout, a 503, a protocol or
4205
- // consumer-create failure — is raised, because "no history" and "I could not read the history"
4206
- // are different answers and only one of them is safe to render as an empty conversation.
4207
- // `dmHistory` alone turns a denial into its documented empty god-view result.
4208
- const msg = String(e?.message ?? "");
4209
- if (/stream not found/i.test(msg) || e?.code === 404)
4210
- return [];
4211
- throw e;
4212
- }
4213
- finally {
4214
- if (reader)
4215
- this.releaseHistoryReader(reader);
4216
- }
4217
- }
4218
- /** This connection's history-reader id, the key of its credential's reader deletes
4219
- * ({@link historyReaderConsumer}). A static credential's is its connId, the nkey it signs with.
4220
- * Under the auth callout the connId is the inbox nonce this client chose, so the callout keys the
4221
- * deletes on {@link calloutReaderId} instead: the server's id and the client id it assigned this
4222
- * connection, which the client reads from the server's INFO and does not choose. */
4223
- historyReaderId(nc) {
4224
- if (!this.userMode)
4225
- return this.connId;
4226
- const info = nc.info;
4227
- if (!info)
4228
- throw new Error("history: the server has not sent its INFO, so this connection has no history reader id yet");
4229
- return calloutReaderId(info.server_id, info.client_id);
4230
- }
4231
- /** Take one of this connection's history-reader names on `stream` ({@link historyReaderConsumer})
4232
- * for one read. The read holds the name for its whole walk and creates and deletes a consumer
4233
- * under it at each step, so this connection keeps at most {@link HISTORY_READERS} reader consumers
4234
- * on a stream (#4123). A read past that many waits for a name, and `signal` ends the wait. A
4235
- * reconnect under the auth callout changes the reader id, and reads from then on take the new
4236
- * id's names. */
4237
- async acquireHistoryReader(nc, jsm, stream, signal) {
4238
- for (;;) {
4239
- signal?.throwIfAborted();
4240
- const readerId = this.historyReaderId(nc);
4241
- let pool = this.historyReaderSlots.get(stream);
4242
- if (pool?.readerId !== readerId) {
4243
- pool = { readerId, free: Array.from({ length: HISTORY_READERS }, (_, i) => HISTORY_READERS - 1 - i), held: 0, waiting: [], refusedUntil: 0 };
4244
- this.historyReaderSlots.set(stream, pool);
4245
- }
4246
- // A read here would leave one more reader it may not delete, so it fails before creating one.
4247
- if (pool.refusedUntil > Date.now())
4248
- throw this.readerDeleteRefused(stream);
4249
- const slot = pool.free.pop();
4250
- if (slot !== undefined) {
4251
- pool.held++;
4252
- return { nc, js: jetstream(nc), jsm, stream, pool, slot, name: historyReaderConsumer(readerId, slot), held: true, left: false };
4253
- }
4254
- // No read here holds a name, so none will come back from one: every name is set aside (see
4255
- // openHistoryReader and closeHistoryReader). Waiting would hang until those expire.
4256
- if (pool.held === 0)
4257
- throw new Error(`history: every history reader name this connection has on ${stream} is set aside, held by a consumer it did not create (another process using the same credential, or one that stopped mid-read) or by a reader it could not confirm deleted; they expire ${HISTORY_READER_INACTIVE_MS / 60_000} minutes after their last use`);
4258
- const waitOn = pool;
4259
- await new Promise((resolve, reject) => {
4260
- const wake = () => { signal?.removeEventListener("abort", stop); resolve(); };
4261
- const stop = () => { waitOn.waiting.splice(waitOn.waiting.indexOf(wake), 1); reject(abortReason(signal)); };
4262
- waitOn.waiting.push(wake);
4263
- signal?.addEventListener("abort", stop, { once: true });
4264
- });
4265
- }
4266
- }
4267
- /** Give a read's history-reader name back and wake one read waiting for a name. A name whose
4268
- * delete failed is set aside instead. */
4269
- releaseHistoryReader(reader) {
4270
- if (!reader.held)
4271
- return;
4272
- reader.held = false;
4273
- const pool = reader.pool;
4274
- pool.held--;
4275
- if (!reader.left) {
4276
- pool.free.push(reader.slot);
4277
- pool.waiting.shift()?.();
4278
- return;
4279
- }
4280
- this.setAsideHistoryReader(pool, reader.slot);
4281
- // Every waiter checks again, so one that no running read can now serve fails instead of hanging.
4282
- if (pool.held === 0)
4283
- for (const wake of pool.waiting.splice(0))
4284
- wake();
4285
- }
4286
- /** Keep `slot` out of use until a consumer under it would have expired, then free it. */
4287
- setAsideHistoryReader(pool, slot) {
4288
- setTimeout(() => { pool.free.push(slot); pool.waiting.shift()?.(); }, HISTORY_READER_INACTIVE_MS).unref();
4289
- }
4290
- /** Create the reader consumer for one step of a read, under the read's name. It is a plain pull
4291
- * consumer: an nats.js ordered consumer renames itself on every rebuild, and the observer and admin
4292
- * credentials may delete only these exact names.
4293
- *
4294
- * Every create carries a fresh description. NATS answers a create whose config equals the existing
4295
- * consumer's with that consumer, delivery state and all (measured on nats-server 2.15), and a read
4296
- * repeats the same config on the same name. Without the description a reader left under the name
4297
- * would come back as if new: a `DeliverPolicy.Last` reader that already delivered its message
4298
- * reports nothing pending, which reads as an empty channel. With it, a create on a name in use is
4299
- * refused, and a consumer that does not carry this step's description is never used.
4300
- *
4301
- * A consumer already under the name is never deleted, whoever left it: another holder of this
4302
- * credential, a process that stopped mid-read, or this endpoint when a delete's answer was lost and
4303
- * the name was taken again since. The read moves to a free name, and the taken one stays out of use
4304
- * until such a consumer would have expired. */
4305
- async openHistoryReader(reader, config) {
4306
- // A single filter must ride the filter-bearing API subject so scoped read grants apply.
4307
- if (config.filter_subjects?.length === 1) {
4308
- const { filter_subjects, ...rest } = config;
4309
- config = { ...rest, filter_subject: filter_subjects[0] };
4310
- }
4311
- const pool = reader.pool;
4312
- for (;;) {
4313
- // A reconnect under the auth callout moved this connection to new reader names, and its
4314
- // credential may no longer delete this one. Creating under it would leave the reader behind.
4315
- if (this.historyReaderId(reader.nc) !== pool.readerId)
4316
- throw new Error(`history: the connection was re-established during the read on ${reader.stream}, so its history reader names changed; the read is not served`);
4317
- const description = `cotal history read ${randomUUID()}`;
4318
- let info;
4319
- try {
4320
- info = await reader.jsm.consumers.add(reader.stream, {
4321
- ...config,
4322
- name: reader.name,
4323
- description,
4324
- ack_policy: AckPolicy.None,
4325
- max_deliver: 1,
4326
- mem_storage: true,
4327
- num_replicas: 1,
4328
- inactive_threshold: nanos(HISTORY_READER_INACTIVE_MS),
4329
- });
4330
- }
4331
- catch (e) {
4332
- if (!isConsumerAlreadyExists(e))
4333
- throw e;
4334
- }
4335
- if (info?.config.description === description)
4336
- return reader.js.consumers.getConsumerFromInfo(info);
4337
- this.setAsideHistoryReader(pool, reader.slot);
4338
- const next = pool.free.pop();
4339
- if (next === undefined) {
4340
- // The read ends holding no name. Every waiter checks again, so one that no running read can
4341
- // now serve fails instead of hanging.
4342
- reader.held = false;
4343
- pool.held--;
4344
- for (const wake of pool.waiting.splice(0))
4345
- wake();
4346
- throw new Error(`history: reader ${reader.name} on ${reader.stream} is held by a consumer this read did not create, and no other reader name is free`);
4347
- }
4348
- reader.slot = next;
4349
- reader.name = historyReaderConsumer(pool.readerId, next);
4350
- }
4351
- }
4352
- /** Delete the read's reader consumer; already gone is the same outcome. A failed delete may still
4353
- * have run with its answer lost, and the name may be taken again before this endpoint could tell,
4354
- * so the name is set aside when the read gives it back rather than deleted again later. A refused
4355
- * delete means the credential was minted before the observer and admin profiles held their reader
4356
- * deletes. Each read would then leave a reader for the inactive threshold, which is how a polling
4357
- * dashboard filled the stream's consumer limit (#4123), so the read fails and says to remint
4358
- * rather than leak quietly, and reads on the stream fail at once until that reader expires. */
4359
- async closeHistoryReader(reader) {
4360
- try {
4361
- await reader.jsm.consumers.delete(reader.stream, reader.name);
4362
- }
4363
- catch (e) {
4364
- if (isConsumerNotFound(e))
4365
- return;
4366
- reader.left = true;
4367
- // No `cause`: a permission-shaped cause would let streamHistory read this as the admin-only
4368
- // DM denial and return an empty page.
4369
- if (isPublishPermissionDenied(e)) {
4370
- if (this.historyReaderId(reader.nc) !== reader.pool.readerId)
4371
- throw new Error(`history: the connection was re-established during the read on ${reader.stream}, so it may no longer delete its reader ${reader.name}, which expires on its own; the read is not served`);
4372
- reader.pool.refusedUntil = Date.now() + HISTORY_READER_INACTIVE_MS;
4373
- throw this.readerDeleteRefused(reader.stream);
4374
- }
4375
- throw e;
4376
- }
4377
- }
4378
- readerDeleteRefused(stream) {
4379
- return new Error(`history: this credential may not delete its own history readers on ${stream}; it predates the observer and admin reader-delete grant, so remint it`);
4380
- }
4381
- /** The oldest stream sequence matching `subject`, or 0 when the subject has no messages.
4382
- * Mirror of {@link lastMatchingSeq}: same CREATE/INFO/NEXT/DELETE surface, `DeliverPolicy.All`
4383
- * instead of `Last`, first delivered seq instead of last. Read credentials already hold this. */
4384
- async firstMatchingSeq(reader, subjects, signal) {
4385
- signal?.throwIfAborted();
4386
- const consumer = await this.openHistoryReader(reader, {
4387
- filter_subjects: subjects,
4388
- deliver_policy: DeliverPolicy.All,
4389
- });
4390
- try {
4391
- if ((await consumer.info(true)).num_pending === 0)
4392
- return 0;
4393
- signal?.throwIfAborted();
4394
- const iter = await consumer.fetch({ max_messages: 1 });
4395
- const stop = () => iter.stop(abortReason(signal));
4396
- signal?.addEventListener("abort", stop, { once: true });
4397
- try {
4398
- signal?.throwIfAborted();
4399
- for await (const m of iter)
4400
- return m.seq;
4401
- }
4402
- finally {
4403
- signal?.removeEventListener("abort", stop);
4404
- iter.stop();
4405
- }
4406
- throw new Error(`history: the broker reported messages on ${subjectLabel(subjects)} but delivered none - the read was cut short, not empty`);
4407
- }
4408
- finally {
4409
- await this.closeHistoryReader(reader);
4410
- }
4411
- }
4412
- /** The newest stream sequence matching `subject`, or 0 when the subject has no messages.
4413
- *
4414
- * One reader consumer at `DeliverPolicy.Last` with this subject's filter: its `num_pending`
4415
- * (available from the create, before anything is delivered) is 0 for an empty subject, and
4416
- * otherwise one message carries the sequence. Same CREATE/INFO/NEXT/DELETE surface `drainWindow`
4417
- * already uses, so no broker authority is added. */
4418
- async lastMatchingSeq(reader, subjects, signal) {
4419
- signal?.throwIfAborted();
4420
- const consumer = await this.openHistoryReader(reader, {
4421
- filter_subjects: subjects,
4422
- deliver_policy: DeliverPolicy.Last,
4423
- });
4424
- try {
4425
- // Bind-time zero is the ONLY thing that means "this subject has no messages".
4426
- if ((await consumer.info(true)).num_pending === 0)
4427
- return 0;
4428
- signal?.throwIfAborted();
4429
- const iter = await consumer.fetch({ max_messages: 1 });
4430
- const stop = () => iter.stop(abortReason(signal));
4431
- signal?.addEventListener("abort", stop, { once: true });
4432
- try {
4433
- signal?.throwIfAborted();
4434
- for await (const m of iter)
4435
- return m.seq;
4436
- }
4437
- finally {
4438
- signal?.removeEventListener("abort", stop);
4439
- iter.stop();
4440
- }
4441
- // Bind said a message was pending and none arrived. The pinned client's pull iterator ends
4442
- // CLEANLY when the connection closes ("we don't propagate the error here"), so this is what a
4443
- // dropped link looks like from here. Returning 0 would make the caller report an empty
4444
- // channel, which is the same "could not read means no history" lie the narrowed catch above
4445
- // exists to stop.
4446
- throw new Error(`history: the broker reported messages on ${subjectLabel(subjects)} but delivered none - the read was cut short, not empty`);
4447
- }
4448
- finally {
4449
- await this.closeHistoryReader(reader);
4450
- }
4451
- }
4452
- /** Drain every message matching `subject` with sequence in `[start, ceiling]`, oldest-first.
4453
- * One reader consumer, one batched pull — `AckPolicy.None`, so no per-message ack
4454
- * round trip. Fetches exactly the pending count so it returns as soon as the window is
4455
- * delivered rather than blocking for the pull's full expiry. */
4456
- async drainWindow(reader, subjects, start, ceiling, limit, signal) {
4457
- signal?.throwIfAborted();
4458
- const out = [];
4459
- const consumer = await this.openHistoryReader(reader, {
4460
- filter_subjects: subjects,
4461
- deliver_policy: DeliverPolicy.StartSequence,
4462
- opt_start_seq: start,
4463
- });
4464
- try {
4465
- // A freshly created consumer already carries its ConsumerInfo, so read the CACHED copy: the
4466
- // explicit uncached `info()` this used to call was a round trip for data we already had.
4467
- const pending = (await consumer.info(true)).num_pending;
4468
- if (pending === 0)
4469
- return out;
4470
- signal?.throwIfAborted();
4471
- // Keep only a small rolling buffer in flight. Stopping a client iterator cannot unsend bytes the
4472
- // broker already committed to that pull request, so fetching the whole page let an aborted history
4473
- // read keep filling a constrained shared connection and starve the next dashboard poll. `consume`
4474
- // replenishes this bounded buffer as it is read, so large pages still complete without committing
4475
- // all of their bytes to the connection up front.
4476
- const iter = await consumer.consume({ max_messages: Math.min(pending, 32) });
4477
- const stop = () => iter.stop(abortReason(signal));
4478
- signal?.addEventListener("abort", stop, { once: true });
4479
- // PROVE THE WINDOW COMPLETED. The pull iterator ends cleanly on a dropped connection, so a
4480
- // close after three of ten deliveries would otherwise return a convincing three-message page.
4481
- // The window is done when we have reached its upper bound or consumed everything bind said
4482
- // was pending; anything else is a cut-short read and must say so.
4483
- let delivered = 0;
4484
- let complete = false;
4485
- try {
4486
- signal?.throwIfAborted();
4487
- for await (const m of iter) {
4488
- delivered++;
4489
- // A plain consumer does not rebuild itself the way an ordered one did, so a gap in its
4490
- // delivery sequence is a message this page would silently lack.
4491
- if (m.info.deliverySequence !== delivered)
4492
- throw new Error(`history: delivery ${m.info.deliverySequence} arrived where ${delivered} was due on ${subjectLabel(subjects)} - the window has a gap, so the page is not served`);
4493
- if (m.seq >= ceiling) { // reached the page's upper bound
4494
- if (m.seq === ceiling) {
4495
- const msg = historyMessageFromDelivery(m);
4496
- if (msg)
4497
- out.push({ seq: m.seq, subject: m.subject, msg });
4498
- }
4499
- complete = true;
4500
- break;
4501
- }
4502
- const msg = historyMessageFromDelivery(m);
4503
- if (msg) {
4504
- out.push({ seq: m.seq, subject: m.subject, msg });
4505
- if (out.length > limit)
4506
- out.shift();
4507
- }
4508
- if (delivered >= pending) {
4509
- complete = true;
4510
- break;
4511
- }
4512
- }
4513
- }
4514
- finally {
4515
- signal?.removeEventListener("abort", stop);
4516
- iter.stop();
4517
- }
4518
- if (!complete)
4519
- throw new Error(`history: read ${delivered} of ${pending} messages on ${subjectLabel(subjects)} before the stream ended early - the window was cut short, not empty`);
4520
- return out;
4521
- }
4522
- finally {
4523
- // DELETE THE READER. Left alone it would stay for its inactive threshold, and the widening
4524
- // search can run several steps per call while a polling dashboard repeats every call on each
4525
- // poll. The observer and admin profiles used to hold no delete at all (#691), so a dashboard
4526
- // polling every ten seconds filled the stream's consumer limit and every later read failed
4527
- // (#4123). Each reader now goes when its step ends.
4528
- await this.closeHistoryReader(reader);
4529
- }
3941
+ return this.history.streamHistory(stream, subjects, limit, before, signal);
4530
3942
  }
4531
3943
  // ---- internals -----------------------------------------------------------
4532
3944
  /**
@@ -4575,9 +3987,10 @@ export class CotalEndpoint extends EventEmitter {
4575
3987
  continue;
4576
3988
  if (s.error instanceof PermissionViolationError && this.chatSubRefusals.has(s.error))
4577
3989
  continue;
4578
- // A principal-route refusal belongs to the call that throws it or the serve that ends on it
4579
- // (SPEC 13.21); nothing else on this private connection publishes or subscribes there.
4580
- if (s.error instanceof PermissionViolationError && this.isPrincipalRouteSubject(s.error.subject))
3990
+ // A refused principal-route request publish or subscription belongs to the call that throws
3991
+ // it or the serve that ends on it (SPEC 13.21). A refused reply publish has no owner, so it
3992
+ // is reported below.
3993
+ if (s.error instanceof PermissionViolationError && this.ownedPrincipalRouteRefusal(s.error))
4581
3994
  continue;
4582
3995
  if (s.error instanceof PermissionViolationError && handedToRequest.has(s.error))
4583
3996
  continue;
@@ -5426,7 +4839,7 @@ export class CotalEndpoint extends EventEmitter {
5426
4839
  /** Privileged durable-LEAVE write: tombstone the membership at `leaveCursor = frontier` so the
5427
4840
  * backstop denies `seq > leaveCursor` while a pre-leave entry stays deliverable (SPEC §7 interval). */
5428
4841
  async durableLeaveFor(owner, channel, lifecycleUid, expectedGeneration) {
5429
- if (!this.plane3)
4842
+ if (!this.deliveryPlane.plane3)
5430
4843
  return; // not a Plane-3 host — no membership to tombstone
5431
4844
  const kv = await this.membersRegistry();
5432
4845
  // expectedGeneration (captured by the agent at durableJoin) refuses a stale leave from tombstoning
@@ -5518,302 +4931,7 @@ export class CotalEndpoint extends EventEmitter {
5518
4931
  * {@link armPlane3} on EVERY (re)connect — a reconnect drains the old connection, so re-binding both
5519
4932
  * is required, not optional (the responder would otherwise be lost on a broker blip). */
5520
4933
  async startPlane3(aclFor, opts = {}) {
5521
- if (!this.js)
5522
- throw new Error(this.notLiveMsg());
5523
- this.plane3 = { aclFor, reloadMembershipCreds: opts.reloadMembershipCreds, evictPrincipal: opts.evictPrincipal, evictPrincipals: opts.evictPrincipals, planeConnLiveness: opts.planeConnLiveness, principalLiveness: opts.principalLiveness, reloadStoreIdentity: opts.reloadStoreIdentity, onDeliveryCredsAdopted: opts.onDeliveryCredsAdopted, startupComplete: opts.startupComplete };
5524
- await this.armPlane3();
5525
- }
5526
- /** Whether an ALREADY-DISPATCHED unit of Plane-3 work may still take effect.
5527
- *
5528
- * Unsubscribing stops NEW work; it cannot recall work already in flight. A handler that entered
5529
- * before {@link quiescePlane3} and awaited broker I/O inside it resumes AFTER the freeze, and a
5530
- * reviewer traced the ordering that makes that a split rather than a latency blip: the loser
5531
- * accepts a unit and awaits, the loser is descheduled, the successor acquires the shard and flips
5532
- * its lease READY, then the loser resumes and answers or acks for a shard it no longer holds.
5533
- * Subscription counts and parked-pull readings cannot see it, because the effect is the reply and
5534
- * the ack rather than the binding.
5535
- *
5536
- * So each work path re-asks HERE, at the point of effect, after its awaits and before it acts.
5537
- * The lease keeps one ROW, not one SERVER; this is the half that keeps one server. An unacked
5538
- * message is not lost by refusing: a stopped consumer redelivers it to whoever holds the shard
5539
- * next, which is the same property quiescing already relies on. */
5540
- plane3MayAct() {
5541
- return !this.plane3Quiesced;
5542
- }
5543
- /** Serve one runtime durable-membership control request (the server-side delivery daemon). The caller
5544
- * id is the authenticated subject sender ({@link serveControl} fail-closes on a mismatch). Validation
5545
- * is against the durable ACL registry — the SAME KV the reader re-auths against (single source of
5546
- * truth, no in-memory ledger to drift). */
5547
- async handleDeliveryControl(req) {
5548
- // FENCE: entered before a quiesce, resuming after it. Answering now would put a second server on
5549
- // this shard's control rail while the winner is already READY.
5550
- if (!this.plane3MayAct())
5551
- return { ok: false, error: "delivery: this daemon is not serving this shard (it is re-checking ownership); retry" };
5552
- const caller = req.from.id;
5553
- const args = req.args ?? {};
5554
- if (req.op === "durableJoin")
5555
- return this.deliveryJoin(caller, args);
5556
- if (req.op === "durableLeave")
5557
- return this.deliveryLeave(caller, args);
5558
- if (req.op === "readHistory")
5559
- return this.deliveryReadHistory(caller, args);
5560
- if (req.op === ANYCAST_TAKES_OP)
5561
- return this.anycastUnavailable ? { ok: false, error: `delivery: ${this.anycastUnavailable}` } : { ok: true, data: { takes: true } };
5562
- if (req.op === ANYCAST_STATUS_OP || req.op === ANYCAST_CANCEL_OP)
5563
- return this.deliveryAnycast(req.op, caller, args);
5564
- if (req.op === "listMemberships") {
5565
- if (typeof args.lifecycleUid !== "string")
5566
- return { ok: false, error: "listMemberships: the caller's lifecycleUid is required (membership rows are lifecycle-keyed, SPEC 13.1)" };
5567
- let uid;
5568
- try {
5569
- uid = assertLifecycleToken(args.lifecycleUid);
5570
- }
5571
- catch (e) {
5572
- return { ok: false, error: e.message };
5573
- }
5574
- let memberships;
5575
- if (args.channel !== undefined) {
5576
- const channel = this.checkDurableChannelArg(args, "listMemberships", { family: true });
5577
- if (typeof channel !== "string")
5578
- return channel;
5579
- memberships = (await listMembers(await this.membersRegistry(), { owner: caller }))
5580
- .filter((r) => r.channel === channel && r.leaveCursor === undefined)
5581
- .map((r) => ({ channel: r.channel, generation: r.generation, activated: r.activated === true, lifecycleUid: r.lifecycleUid }));
5582
- }
5583
- else {
5584
- memberships = await this.ownerMemberships(caller, uid);
5585
- }
5586
- // FENCE AFTER THE AWAIT, which is the one that matters. The entry check above cannot catch the
5587
- // case this exists for: a request admitted WHILE SERVING, then descheduled inside its broker
5588
- // read, resuming after a successor has taken the shard and reached READY. Re-ask at the point
5589
- // of effect, so the answer is never served by a daemon that has stood down.
5590
- if (!this.plane3MayAct())
5591
- return { ok: false, error: "delivery: this daemon stopped serving this shard while the request was in flight; retry" };
5592
- return { ok: true, data: { memberships } };
5593
- }
5594
- return { ok: false, error: `op "${req.op}" not supported on the delivery control service` };
5595
- }
5596
- /** Validate the channel ARG shape only: non-blank, valid, concrete (NO ACL check, that is op-specific).
5597
- * `family` also admits a channel family such as `review.>` (a membership join or leave, #5404).
5598
- * Returns the channel on success or a ControlReply error to short-circuit. */
5599
- checkDurableChannelArg(args, op, opts = {}) {
5600
- const channel = typeof args.channel === "string" ? args.channel.trim() : "";
5601
- if (!channel)
5602
- return { ok: false, error: `${op}: channel must be a non-blank string` };
5603
- try {
5604
- assertValidChannel(channel);
5605
- }
5606
- catch (e) {
5607
- return { ok: false, error: e.message };
5608
- }
5609
- if (opts.family && isChannelFamily(channel))
5610
- return channel;
5611
- if (!isConcreteChannel(channel))
5612
- return {
5613
- ok: false,
5614
- error: opts.family
5615
- ? `${op}: "${channel}" stays live only: a durable membership takes a concrete channel or a family such as review.>, never the root > or a pattern with *`
5616
- : `${op}: "${channel}" must be a concrete channel (durable membership is per-concrete-channel, not wildcard)`,
5617
- };
5618
- return channel;
5619
- }
5620
- /** JOIN requires the channel be within the caller's CURRENT read ACL (you can't durable-subscribe a
5621
- * channel you may not read). A family must lie inside one ACL entry, so every channel under it is
5622
- * readable. */
5623
- async deliveryJoin(caller, args) {
5624
- const channel = this.checkDurableChannelArg(args, "durableJoin", { family: true });
5625
- if (typeof channel !== "string")
5626
- return channel; // a ControlReply error
5627
- // The caller arrives as an ALIAS (the control subject carries owner+actor, never a uid): resolve
5628
- // its single live lifecycle-keyed ACL row SERVER-SIDE — the trusted registry, never a caller
5629
- // assertion, decides which lifecycle joins. Two live rows (a reservation breach / unfinished
5630
- // teardown) refuse loudly rather than binding a membership to a guessed lifecycle.
5631
- let acl;
5632
- try {
5633
- acl = await this.aclForAlias(caller);
5634
- }
5635
- catch (e) {
5636
- return { ok: false, error: e.message };
5637
- }
5638
- if (acl === undefined)
5639
- return { ok: false, error: `durableJoin: no read ACL on record for ${caller} (not provisioned for durable delivery)` };
5640
- if (!membershipInAllow(acl.allowSubscribe, channel))
5641
- return { ok: false, error: `channel "${channel}" is not within your read ACL [${acl.allowSubscribe.join(", ")}]` };
5642
- const uid = acl.lifecycleUid;
5643
- // The lookup above may be stale by the time the row is written (a retirement can fence and purge
5644
- // this lifecycle meanwhile), so the write re-checks the exact ACL row itself (`aclFence`).
5645
- try {
5646
- return { ok: true, data: await this.membershipWrite(caller, uid, () => this.durableJoinFor(caller, channel, uid, { aclFence: true })) };
5647
- }
5648
- catch (e) {
5649
- return { ok: false, error: e.message };
5650
- }
5651
- }
5652
- /** Whether `lifecycleUid`'s exact ACL row still allows reads, so an inventory of it came from a
5653
- * caller that did not fence. A row narrowed to no reads, or retired or absent, is fenced. A read
5654
- * that fails answers true: the caller keeps its in-memory mark, which only ever refuses. */
5655
- async lifecycleAclUnfenced(principal, lifecycleUid) {
5656
- try {
5657
- const row = await readAcl(await this.aclRegistry(), principal, lifecycleUid);
5658
- return row !== undefined && row.record.allowSubscribe.length > 0;
5659
- }
5660
- catch {
5661
- return true;
5662
- }
5663
- }
5664
- /** Run one durable join or leave that writes `principal`'s rows of `lifecycleUid`, refused once that
5665
- * lifecycle's inventory has been asked for ({@link retiringLifecycles}). The check and the
5666
- * registration have no await between them, so an inventory either finds this write running and
5667
- * waits for it, or this write finds the inventory's mark and is refused. */
5668
- async membershipWrite(principal, lifecycleUid, write) {
5669
- if (this.retiringLifecycles.get(principal)?.has(lifecycleUid))
5670
- throw new Error(`lifecycle ${lifecycleUid} of ${principal} is retiring: its membership inventory was requested, so its durable memberships no longer change`);
5671
- const key = `${principal}/${lifecycleUid}`;
5672
- const running = write();
5673
- let writes = this.membershipWrites.get(key);
5674
- if (writes === undefined)
5675
- this.membershipWrites.set(key, (writes = new Set()));
5676
- writes.add(running);
5677
- try {
5678
- return await running;
5679
- }
5680
- finally {
5681
- writes.delete(running);
5682
- if (writes.size === 0)
5683
- this.membershipWrites.delete(key);
5684
- }
5685
- }
5686
- /** LEAVE must NOT require current-ACL coverage. Leave fires precisely when the ACL was narrowed/revoked
5687
- * (a refused live sub → {@link closeRefusedMembership}); gating the tombstone on the current ACL would
5688
- * loop forever and leave the SPEC §7 boundary open (the membership could resume if the ACL is later
5689
- * restored). The guards are: authenticated caller (serveControl), concrete channel, a finite generation
5690
- * (the join epoch — without it a stale/replayed leave could tombstone a newer rejoin), and an EXISTING
5691
- * own membership; `durableLeaveFor` → `tombstoneMember` then enforces the generation match. */
5692
- async deliveryLeave(caller, args) {
5693
- const channel = this.checkDurableChannelArg(args, "durableLeave", { family: true });
5694
- if (typeof channel !== "string")
5695
- return channel; // a ControlReply error
5696
- if (typeof args.generation !== "number" || !Number.isFinite(args.generation))
5697
- return { ok: false, error: "durableLeave: a finite generation is required (fail-closed stale-leave guard)" };
5698
- // The LEAVE carries the leaver's own lifecycleUid: leave must work precisely when the ACL row was
5699
- // narrowed or already purged (see the method doc), so the alias→row resolution join uses is not
5700
- // available here. The uid is caller-asserted but harmless to lie about: the member key it selects
5701
- // is confined to the AUTHENTICATED caller's own alias (`<channel>/<caller>.<uid>`), so the worst a
5702
- // false uid reaches is the caller's own retired incarnation's row — legitimate cleanup — and the
5703
- // generation guard still applies.
5704
- if (typeof args.lifecycleUid !== "string")
5705
- return { ok: false, error: "durableLeave: the leaving incarnation's lifecycleUid is required (membership rows are lifecycle-keyed, SPEC 13.1)" };
5706
- let uid;
5707
- try {
5708
- uid = assertLifecycleToken(args.lifecycleUid);
5709
- }
5710
- catch (e) {
5711
- return { ok: false, error: e.message };
5712
- }
5713
- const existing = await readMember(await this.membersRegistry(), channel, caller, uid);
5714
- if (!existing)
5715
- return { ok: true, data: { channel, alreadyLeft: true } }; // nothing to tombstone — idempotent
5716
- const generation = args.generation;
5717
- try {
5718
- await this.membershipWrite(caller, uid, () => this.durableLeaveFor(caller, channel, uid, generation));
5719
- }
5720
- catch (e) {
5721
- return { ok: false, error: e.message };
5722
- }
5723
- return { ok: true, data: { channel } };
5724
- }
5725
- /** Serve one MEDIATED HISTORY READ (`readHistory`, the client side is {@link readHistory}). The daemon
5726
- * holds the consumer so the caller does not have to: this is a privilege reduction, not a new
5727
- * capability, and it is the shape SPEC's "Mediated reads (normative)" rule asks for.
5728
- *
5729
- * THE ONE INVARIANT THIS METHOD EXISTS FOR: authorization is read FRESH on every call and never
5730
- * cached across calls. A consumer pins its authorization when it is created, so a revoked caller
5731
- * keeps being served by a consumer it already holds; re-reading here is what makes a revocation
5732
- * stop the very NEXT read. Cache this and the verb loses its only advantage over the raw consumer
5733
- * path.
5734
- *
5735
- * AUTHORITY (SPEC §9.6): "current read ACL" is the effective broker-accepted credential. The
5736
- * durable registry is a live mirror of that credential, not a second grant source. History
5737
- * authorizes against `allowSubscribe ∩ issuedAllowSubscribe` — the live row (so revocation via
5738
- * plain commitAcl still stops the next read) intersected with the mint-time ceiling (so a
5739
- * registry widen without a remint cannot grant what the JWT does not). Raising the ceiling is
5740
- * {@link reissueAcl}, which is what provision does when it bakes the list into the JWT.
5741
- *
5742
- * The caller arrives as an alias (the control subject carries owner+actor, never a uid) and is the
5743
- * AUTHENTICATED subject sender — `serveControl` has already fail-closed on any payload that names a
5744
- * different principal, so `caller` cannot be self-asserted. */
5745
- async deliveryReadHistory(caller, args) {
5746
- const channel = this.checkDurableChannelArg(args, "readHistory");
5747
- if (typeof channel !== "string")
5748
- return channel; // a ControlReply error
5749
- // The caller PROPOSES a limit; the mediator decides. Reject a nonsense limit loudly rather than
5750
- // silently substituting a default — a caller asking for -1 has a bug and should hear about it.
5751
- const asked = args.limit === undefined ? READ_HISTORY_DEFAULT_LIMIT : args.limit;
5752
- if (typeof asked !== "number" || !Number.isInteger(asked) || asked < 1)
5753
- return { ok: false, error: `readHistory: limit must be a positive integer (got ${JSON.stringify(args.limit)})` };
5754
- const limit = Math.min(asked, READ_HISTORY_MAX_LIMIT);
5755
- // FRESH per call — see the method doc. Same registry, same alias resolution, and the same loud
5756
- // refusal on two live rows that `durableJoin` uses: a stale lifecycle must never authorize its
5757
- // successor's read.
5758
- //
5759
- // SPEC §9.6: "current read ACL" is the effective broker-accepted credential. The registry row is
5760
- // a live mirror and can be rewritten by plain commitAcl without a remint. Authorizing history
5761
- // from the row ALONE let a widen grant reads channelHistory still broker-denies — a new
5762
- // capability wearing a privilege-reduction label (panel BLOCKING at 914fd7b0). History therefore
5763
- // requires the channel in BOTH the live row AND the mint-time ceiling (`issuedAllowSubscribe`).
5764
- // Revocation still works: narrowing allowSubscribe stops the next read. Raising the ceiling
5765
- // requires reissueAcl (provision/remint), which is the same act that bakes the list into the JWT.
5766
- let acl;
5767
- try {
5768
- acl = await this.aclForAlias(caller);
5769
- }
5770
- catch (e) {
5771
- return { ok: false, error: e.message };
5772
- }
5773
- if (acl === undefined)
5774
- return { ok: false, error: `readHistory: no read ACL on record for ${caller} - not permitted` };
5775
- const inLive = channelInAllow(acl.allowSubscribe, channel);
5776
- const inIssued = channelInAllow(acl.issuedAllowSubscribe, channel);
5777
- if (!inLive || !inIssued) {
5778
- // Name the live list in the error — that is what operators edit and what revocation clears.
5779
- // A ceiling miss (registry widened past the credential) is the same refusal shape as a live
5780
- // miss so a caller cannot probe which half failed.
5781
- return {
5782
- ok: false,
5783
- error: `readHistory: channel "${channel}" is not within your read ACL [${acl.allowSubscribe.join(", ")}] - refused`,
5784
- };
5785
- }
5786
- // One message MORE than the page is the completeness probe: if the backlog hands back limit+1,
5787
- // something older exists behind the page and `complete` is false. Cheaper and more honest than a
5788
- // separate count, which could race the page it describes.
5789
- let page;
5790
- try {
5791
- page = await this.channelHistory(channel, { limit: limit + 1 });
5792
- }
5793
- catch (e) {
5794
- // Surface the read failure. `streamHistory` deliberately raises rather than returning empty on a
5795
- // denial or a cut-short window, and that distinction must survive the rail: an error here would
5796
- // otherwise reach the caller as an empty page and render as "no history".
5797
- return { ok: false, error: `readHistory: ${e.message}` };
5798
- }
5799
- const reachedStart = page.length <= limit;
5800
- const wanted = reachedStart ? page : page.slice(-limit);
5801
- // A page that cannot be SENT is not a page. The reply rides one NATS message, so `limit` alone is
5802
- // the wrong bound: 200 large messages serialize past `max_payload`, and `serveControl` can then only
5803
- // answer with a refusal in place of the page.
5804
- //
5805
- // So bound by BYTES too, keeping the NEWEST that fit — which needs no new vocabulary, because
5806
- // `complete: false` already means "older history remains behind this page". Trimming here is the
5807
- // documented truncation signal doing its job, not a silent degradation.
5808
- const fitted = fitHistoryPage(wanted, this.maxPayload);
5809
- // `wanted.length > 0` is load-bearing, not defensive: an EMPTY channel also fits nothing, and
5810
- // without this guard a channel nobody has posted to was refused with "the newest message exceeds
5811
- // the payload budget" — a confident, entirely wrong explanation for a legitimately empty result.
5812
- // Genuine emptiness is `{ items: [], complete: true }`; only a message too large to ever send is
5813
- // the error.
5814
- if (wanted.length > 0 && fitted.length === 0)
5815
- return { ok: false, error: `readHistory: the newest message on "${channel}" alone exceeds the broker payload budget (${this.maxPayload} bytes) - refused loudly rather than served as an empty page` };
5816
- return { ok: true, data: { items: fitted, complete: reachedStart && fitted.length === wanted.length } };
4934
+ return this.deliveryPlane.startPlane3(aclFor, opts);
5817
4935
  }
5818
4936
  /** Stop serving Plane-3 WITHOUT tearing down the connection, so a daemon that has just learned its
5819
4937
  * lease may no longer be its own can stop acting on the shard while it finds out for certain.
@@ -5835,37 +4953,13 @@ export class CotalEndpoint extends EventEmitter {
5835
4953
  * Deliberately not `stop()`: the connection, the lease KV handles and the control rails must stay
5836
4954
  * up, because the daemon still has to ask the broker who owns the shard. */
5837
4955
  async quiescePlane3() {
5838
- this.plane3Quiesced = true;
5839
- if (this.deliveryServeSub) {
5840
- try {
5841
- this.deliveryServeSub.unsubscribe();
5842
- }
5843
- catch { /* already dead */ }
5844
- this.deliveryServeSub = undefined;
5845
- }
5846
- if (this.deliveryAdminServeSub) {
5847
- try {
5848
- this.deliveryAdminServeSub.unsubscribe();
5849
- }
5850
- catch { /* already dead */ }
5851
- this.deliveryAdminServeSub = undefined;
5852
- }
5853
- this.dropAnycastRequests();
5854
- // Stopping the consumers ends the `for await` loops that drive fan-out and the reader. In-flight
5855
- // messages are NOT acked by a stopped consumer, so they redeliver to whoever holds the shard
5856
- // next: quiescing loses no message, it only stops this process from claiming them.
5857
- for (const msgs of this.streamMsgs.splice(0)) {
5858
- try {
5859
- msgs.stop();
5860
- }
5861
- catch { /* already draining */ }
5862
- }
4956
+ return this.deliveryPlane.quiescePlane3();
5863
4957
  }
5864
4958
  /** Whether Plane-3 is currently quiesced: unbound, and refusing to re-bind until ownership is proven.
5865
4959
  * Read-only. The flag is the daemon's own answer to "am I serving this shard", so a cell that asserts
5866
4960
  * a recovery worked can check the endpoint's state rather than inferring it from a log line. */
5867
4961
  plane3IsQuiesced() {
5868
- return this.plane3Quiesced;
4962
+ return this.deliveryPlane.plane3IsQuiesced();
5869
4963
  }
5870
4964
  /** Resume serving Plane-3 after {@link quiescePlane3}, once ownership has been re-established.
5871
4965
  * Idempotent, and a no-op when the daemon was never quiesced.
@@ -5878,80 +4972,19 @@ export class CotalEndpoint extends EventEmitter {
5878
4972
  * That is a readiness lie surviving a transient broker error, the daemon claims a responder it does
5879
4973
  * not have, which is the #1318 outage wearing the readiness flag instead of the exit path. */
5880
4974
  async rearmPlane3() {
5881
- if (!this.plane3Quiesced)
5882
- return;
5883
- // `armPlane3` refuses while quiesced (a reconnect must not resume mid-question), so the flag is
5884
- // lowered for the attempt and RAISED AGAIN on failure. A partial bind leaves the endpoint quiesced,
5885
- // which is the honest state: not serving, and a later retry will attempt the whole sequence again.
5886
- this.plane3Quiesced = false;
5887
- try {
5888
- await this.armPlane3();
5889
- }
5890
- catch (e) {
5891
- this.plane3Quiesced = true;
5892
- // Drop whatever DID bind. Leaving half-bound responders up would answer control requests for a
5893
- // shard this process has just declared itself not to be serving.
5894
- this.quiescePlane3();
5895
- throw e;
5896
- }
4975
+ return this.deliveryPlane.rearmPlane3();
4976
+ }
4977
+ /** Forwards to the delivery plane's {@link EndpointDeliveryPlane.plane3MayAct}. */
4978
+ plane3MayAct() {
4979
+ return this.deliveryPlane.plane3MayAct();
5897
4980
  }
5898
- /** (Re)bind the Plane-3 fan-out writer + trusted reader. Idempotent — the durables resume from their
5899
- * cursor. Called by {@link startPlane3} once AND by {@link connectAndBind} on every (re)connect, so
5900
- * the delivery daemon's reconnect RE-ARMS the backstop + the ctl.delivery responder. Without this, a broker blip would silently kill
5901
- * the loops while `durableJoinFor` kept reporting `durable:true` (the impl-review's BLOCKER-1). No-op
5902
- * unless this endpoint hosts Plane-3 (`this.plane3` set). */
4981
+ /** Forwards to the delivery plane's {@link EndpointDeliveryPlane.armPlane3}. */
5903
4982
  async armPlane3() {
5904
- if (!this.plane3 || !this.js)
5905
- return;
5906
- // A quiesced endpoint must not be re-armed by a RECONNECT: the reconnect path calls this too,
5907
- // and silently resuming there would restore exactly the double-serving this guards against
5908
- // while the ownership question is still open.
5909
- if (this.plane3Quiesced)
5910
- return;
5911
- await this.manager(); // the manager runs consume:false, so this.jsm is lazy — ensure it
5912
- this.armDeliveryControl();
5913
- await this.armAnycastRequests();
5914
- await this.runFanout();
5915
- await this.runReader();
5916
- }
5917
- /** (Re)register the `ctl.delivery` control responder on the CURRENT connection. A reconnect drains the
5918
- * old connection (the old sub is dead and `clearConnectionScoped` leaves caller-owned subs alone), so
5919
- * this MUST run on every arm — otherwise durable join/leave/list silently lose their responder after a
5920
- * broker blip. The stale sub is dropped (unsubscribed + removed from `this.subs`) before re-creating.
5921
- * `boundReply` is essential here: the daemon holds a wildcard reply-publish grant, so the serve path
5922
- * must reject any reply target outside the authenticated sender's own subtree (confused-deputy fix). */
5923
- armDeliveryControl() {
5924
- if (this.deliveryServeSub) {
5925
- try {
5926
- this.deliveryServeSub.unsubscribe();
5927
- }
5928
- catch { /* dead with the old connection */ }
5929
- const i = this.subs.indexOf(this.deliveryServeSub);
5930
- if (i >= 0)
5931
- this.subs.splice(i, 1);
5932
- }
5933
- // A request's status and cancel touch only the TASK queue and the anycast records, so they do
5934
- // not wait behind a channel join on the in-order chain.
5935
- this.deliveryServeSub = this.serveControl(CONTROL_DELIVERY, (req) => this.handleDeliveryControl(req), {
5936
- boundReply: true,
5937
- concurrent: (req) => ANYCAST_CONTROL_OPS.has(req.op),
5938
- });
5939
- if (this.deliveryAdminServeSub) {
5940
- try {
5941
- this.deliveryAdminServeSub.unsubscribe();
5942
- }
5943
- catch { /* dead with the old connection */ }
5944
- const i = this.subs.indexOf(this.deliveryAdminServeSub);
5945
- if (i >= 0)
5946
- this.subs.splice(i, 1);
5947
- }
5948
- // The membership inventory is read-only and shares nothing with the other admin commands, so it
5949
- // does not queue behind them. In a mass stop each retirement's inventory followed every other
5950
- // seat's `evictPrincipal` on this rail and missed its 5 s deadline (#4965).
5951
- this.deliveryAdminServeSub = this.serveControl(CONTROL_DELIVERY_ADMIN, (req) => this.handleDeliveryAdmin(req), {
5952
- boundReply: true,
5953
- concurrent: (req) => req.op === "lifecycleMemberships",
5954
- });
4983
+ return this.deliveryPlane.armPlane3();
4984
+ }
4985
+ /** Forwards to the delivery plane's {@link EndpointDeliveryPlane.pumpDlv}. */
4986
+ async pumpDlv() {
4987
+ return this.deliveryPlane.pumpDlv();
5955
4988
  }
5956
4989
  /** (Re)bind the anycast take responder and the request records maintenance on the current
5957
4990
  * connection (SPEC §8.1). The records bucket is created here as well as at space setup, so a mesh
@@ -6339,23 +5372,28 @@ export class CotalEndpoint extends EventEmitter {
6339
5372
  return "refused";
6340
5373
  }
6341
5374
  /**
6342
- * Serve this process's own endpoint on the principal route (SPEC 13.21): a core-defined command
6343
- * set about this principal, addressed by its own principal, lifecycle UID and runtime id, with no
6344
- * registration. Revision 1 defines one endpoint, `agent` (SPEC 13.22).
5375
+ * Serve this process's `agent` endpoint on the principal route (SPEC 13.21, 13.22): the
5376
+ * `AGENT_CLUSTER` command set about this principal, addressed by its own principal, lifecycle UID
5377
+ * and runtime id, with no registration. `handlers` serves the cluster's commands with
5378
+ * `AGENT_CONTRACTS`; `describe` is built in.
6345
5379
  *
6346
5380
  * Throws on an open mesh and on a static credential, where no row names who may call, without a
6347
- * lifecycle UID, before the endpoint has started, for an endpoint it already serves, and for a
6348
- * cluster with a command that declares no `authorities` (see {@link servePrincipalRoute}). It
6349
- * subscribes this process's own subject with no queue group.
5381
+ * lifecycle UID, before the endpoint has started, while it already serves `agent`, for an
5382
+ * `endpoint` or `cluster` field, which this entry point does not take, and for a set of handlers
5383
+ * that is not the cluster's commands with its contracts. It subscribes this process's own subject
5384
+ * with no queue group.
6350
5385
  *
6351
5386
  * `ready` is a positive check: it resolves only after the broker accepted the subscription
6352
5387
  * (subscribe, flush, and no permission violation for that subject), and rejects with the
6353
5388
  * refusal, which a credential minted before the rows existed gets. The serve is INTENT: every
6354
5389
  * replaced connection subscribes again and runs the check again, and a refusal there ends the
6355
- * serve, resolves `failed` and clears the presence field. `stop` unsubscribes and waits for the
6356
- * handlers in flight.
5390
+ * serve, resolves `failed` and clears the presence field. `stop` unsubscribes, aborts the
5391
+ * handlers' `signal`, and waits for each handler in flight until its request's deadline at most.
6357
5392
  */
6358
- servePrincipal(serve) {
5393
+ servePrincipal(spec) {
5394
+ if ("endpoint" in spec || "cluster" in spec)
5395
+ throw new Error(`servePrincipal serves only the "${AGENT_ENDPOINT}" endpoint with AGENT_CLUSTER and takes handlers alone; an endpoint or cluster is refused, never ignored (SPEC 13.22)`);
5396
+ const serve = { endpoint: AGENT_ENDPOINT, cluster: AGENT_CLUSTER, handlers: spec.handlers };
6359
5397
  if (!this.authed)
6360
5398
  throw new Error("the principal route is served only on an authenticated mesh: on an open mesh no row says who may call (SPEC 13.21)");
6361
5399
  if (!this.userMode)
@@ -6384,6 +5422,10 @@ export class CotalEndpoint extends EventEmitter {
6384
5422
  const handle = this.bindPrincipalHandle(intent, nc);
6385
5423
  this.principalServes.set(serve.endpoint, intent);
6386
5424
  principalServers.set(serverKey, intent);
5425
+ // A fresh `seq` per serve: the last serve may have ended with prompts open, and the pair
5426
+ // `(runtime.id, seq)` it showed then must not name this serve's empty list (SPEC §6).
5427
+ if (serve.endpoint === AGENT_ENDPOINT)
5428
+ this.promptsPresence = { open: 0, seq: this.nextPromptsSeq() };
6387
5429
  void this.armPrincipalServe(intent, nc, handle);
6388
5430
  return { ready, failed, stop: () => this.stopPrincipalServe(intent) };
6389
5431
  }
@@ -6391,15 +5433,25 @@ export class CotalEndpoint extends EventEmitter {
6391
5433
  bindPrincipalHandle(intent, nc) {
6392
5434
  return servePrincipalRoute(nc, this.space, intent.self, intent.serve);
6393
5435
  }
6394
- /** A request or reply subject (or filter) of the principal route in this space. */
6395
- isPrincipalRouteSubject(subject) {
5436
+ /** A principal-route refusal a call or a serve reports itself: a request publish, a serve
5437
+ * subscription or a reply read. Only a reply publish is left, which no call or serve awaits. */
5438
+ ownedPrincipalRouteRefusal(err) {
6396
5439
  const p = `${spacePrefix(this.space)}.ep.`;
6397
- return subject.startsWith(`${p}prin.`) || subject.startsWith(`${p}prin-reply.`);
5440
+ if (err.operation === "publish")
5441
+ return err.subject.startsWith(`${p}prin.`);
5442
+ return err.subject.startsWith(`${p}prin.`) || err.subject.startsWith(`${p}prin-reply.`);
6398
5443
  }
6399
5444
  /** Track one binding: a refusal ends the serve, and an accepted subscription on the current
6400
5445
  * connection shows the presence field. */
6401
5446
  async armPrincipalServe(intent, nc, handle) {
6402
5447
  intent.binding = { nc, handle };
5448
+ // Every earlier binding is on a replaced connection: retire it once its handlers finish.
5449
+ for (const old of intent.handles)
5450
+ void old.stop().finally(() => {
5451
+ const i = intent.handles.indexOf(old);
5452
+ if (i >= 0)
5453
+ intent.handles.splice(i, 1);
5454
+ });
6403
5455
  intent.handles.push(handle);
6404
5456
  intent.readyOn = undefined;
6405
5457
  void handle.refused.then((err) => {
@@ -6487,7 +5539,8 @@ export class CotalEndpoint extends EventEmitter {
6487
5539
  if (principalServers.get(intent.serverKey) === intent)
6488
5540
  principalServers.delete(intent.serverKey);
6489
5541
  }
6490
- /** Stop serving `intent` and wait for its handlers in flight. The ending is synchronous, so a
5542
+ /** Stop serving `intent` and wait for its handlers in flight, each until its request's deadline
5543
+ * at most. The ending is synchronous, so a
6491
5544
  * `ready` still waiting for the broker rejects before the caller's next step. `republish` writes
6492
5545
  * presence without the `prompts` field; the endpoint's own stop writes it offline instead. */
6493
5546
  async stopPrincipalServe(intent, republish = true) {
@@ -6510,8 +5563,8 @@ export class CotalEndpoint extends EventEmitter {
6510
5563
  * prompts it holds `open` or `sent`, and a counter that goes up by one each time a prompt opens or
6511
5564
  * changes state. The field shows only while the `agent` endpoint is served and the broker accepted
6512
5565
  * its subscription on the current connection. Until this is called it shows `open: 0` with the
6513
- * highest `seq` this process used under its runtime id, or 0. Throws when the `agent` endpoint is
6514
- * not served, and a RangeError for a `seq` below one this runtime already used.
5566
+ * fresh `seq` the serve took from {@link nextPromptsSeq} when it started. Throws when the `agent`
5567
+ * endpoint is not served, and a RangeError for a `seq` below one this runtime already used.
6515
5568
  */
6516
5569
  async setPromptsPresence(prompts) {
6517
5570
  for (const k of ["open", "seq"])
@@ -6544,7 +5597,7 @@ export class CotalEndpoint extends EventEmitter {
6544
5597
  const intent = this.principalServes.get(AGENT_ENDPOINT);
6545
5598
  if (!intent || intent.ended || !this.nc || intent.readyOn !== this.nc)
6546
5599
  return undefined;
6547
- return this.promptsPresence ?? { open: 0, seq: promptsSeqs.get(intent.self.runtimeId) ?? 0 };
5600
+ return this.promptsPresence;
6548
5601
  }
6549
5602
  /**
6550
5603
  * Call one command of a principal's own endpoint on the principal route (SPEC 13.21), as this
@@ -6573,671 +5626,6 @@ export class CotalEndpoint extends EventEmitter {
6573
5626
  ...(call.args !== undefined ? { args: call.args } : {}),
6574
5627
  }, { deadlineMs: call.deadlineMs, ...(call.signal ? { signal: call.signal } : {}) });
6575
5628
  }
6576
- /** Serve one PRIVILEGED delivery-admin request (the D5 rail-split). The cred layer is the caller
6577
- * boundary — only the supervisor profile can publish here — and `serveControl`'s sender check +
6578
- * bounded reply still apply on top. `reloadCreds` is the class-2 renewal ADOPTION step: re-read
6579
- * the renewal-owner-re-signed creds file, pin, swap the live connection, reconnect the membership
6580
- * feed's rw connection, and reply with proof (identities + the adopted JWT windows) — or a
6581
- * structured failure (e.g. the file was never re-signed), never a silent partial. */
6582
- async handleDeliveryAdmin(req) {
6583
- // A reload's resident swap reconnects the connection every reply here rides. A caller's next
6584
- // request, such as a manager's eviction right after its reload, is answered once that
6585
- // connection is back, not while it is down.
6586
- await this.residentSwap;
6587
- // FENCE: same reason as the runtime rail above.
6588
- if (!this.plane3MayAct())
6589
- return { ok: false, error: "delivery: this daemon is not serving this shard (it is re-checking ownership); retry" };
6590
- if (req.op === "reloadCreds") {
6591
- // NOT BEFORE START-UP IS DONE (#2304). Both halves below would run: the delivery half commits
6592
- // and schedules `nc.reconnect()`, which lands underneath start-up awaits still in flight on
6593
- // this connection (the lease watch's consumer create times out and the daemon exits). Refuse
6594
- // before either half acts, so nothing is adopted and the renewal owner records why.
6595
- if (this.plane3?.startupComplete?.() === false)
6596
- return { ok: false, error: "delivery: this daemon has not finished starting; nothing adopted - the next renewal pass or the daemon's 75% re-read adopts the re-signed creds" };
6597
- // The renewal owner's EXPECTED-generation tokens (SHA-256 of each JWT it re-signed), per
6598
- // component. A missing entry means "no expectation" (the passive backstop still adopts).
6599
- const expected = (req.args?.expected ?? {});
6600
- // Prove BOTH components INDEPENDENTLY (non-short-circuit): one failing must neither hide the
6601
- // other's outcome nor be masked by it. Top-level `ok` is false if EITHER proof failed, so the
6602
- // manager can never record a green adoption while a component was refused.
6603
- // Each component's outcome claims only what is PROVEN: `brokerAccepted` (the preflight) carries
6604
- // the accepted generation's window; `residentSwap` records that the wire swap is best-effort and
6605
- // self-healing, NOT witnessed. Neither claims verified resident reauth (that would need a
6606
- // generation-tied reconnect witness on both connections).
6607
- const [delivery, membership] = await Promise.all([
6608
- this.reloadCreds(expected.delivery).then((brokerAccepted) => ({ ok: true, brokerAccepted, residentSwap: "scheduled" }), (e) => ({ ok: false, error: e.message })),
6609
- this.plane3?.reloadMembershipCreds
6610
- ? this.plane3.reloadMembershipCreds(expected.membership).then((result) => ({ ok: true, ...result }), (e) => ({ ok: false, error: e.message }))
6611
- : Promise.resolve({ ok: true, skipped: "no-membership-hook" }),
6612
- ]);
6613
- const failures = [
6614
- delivery.ok ? undefined : `delivery: ${delivery.error}`,
6615
- membership.ok ? undefined : `membership: ${membership.error}`,
6616
- ].filter((m) => m !== undefined);
6617
- // Arm the resident wire swap ONLY now — after BOTH proofs settled, right before the reply is
6618
- // returned+responded — so a slow membership proof can never let the delivery reconnect strand
6619
- // this reply. Only when delivery actually adopted a new candidate (currentCreds was updated).
6620
- if (delivery.ok) {
6621
- this.plane3?.onDeliveryCredsAdopted?.();
6622
- this.scheduleResidentSwap();
6623
- }
6624
- return failures.length
6625
- ? { ok: false, error: failures.join("; "), data: { delivery, membership } }
6626
- : { ok: true, data: { delivery, membership } };
6627
- }
6628
- if (req.op === "evictPrincipal") {
6629
- // The LIVE-EVICTION executor: force-drop a denied principal's connections.
6630
- // Composition-root hook because the $SYS observer/evictor creds live outside this endpoint's
6631
- // trust boundary; absent hook = a daemon build without the executor, refused loudly.
6632
- if (!this.plane3?.evictPrincipal)
6633
- return { ok: false, error: "evictPrincipal: no eviction executor wired on this daemon" };
6634
- const principal = typeof req.args?.principal === "string" ? req.args.principal.trim() : "";
6635
- if (!principal)
6636
- return { ok: false, error: "evictPrincipal: a principal (owner.actor dot-form) is required" };
6637
- try {
6638
- return { ok: true, data: await this.plane3.evictPrincipal(principal) };
6639
- }
6640
- catch (e) {
6641
- return { ok: false, error: e.message };
6642
- }
6643
- }
6644
- if (req.op === "evictPrincipals") {
6645
- // The same executor over a SET: one shared sweep instead of one request and one scan per
6646
- // principal. The set is bounded so one request stays inside one sweep's work.
6647
- if (!this.plane3?.evictPrincipals)
6648
- return { ok: false, error: "evictPrincipals: no batch eviction executor wired on this daemon" };
6649
- const raw = req.args?.principals;
6650
- const principals = Array.isArray(raw) ? raw.map((p) => (typeof p === "string" ? p.trim() : "")) : [];
6651
- if (principals.length === 0 || principals.some((p) => !p))
6652
- return { ok: false, error: "evictPrincipals: principals must be a non-empty array of owner.actor dot-form principals" };
6653
- if (principals.length > EVICT_PRINCIPALS_MAX || new Set(principals).size !== principals.length)
6654
- return { ok: false, error: `evictPrincipals: principals must be distinct and at most ${EVICT_PRINCIPALS_MAX}` };
6655
- try {
6656
- return { ok: true, data: await this.plane3.evictPrincipals(principals) };
6657
- }
6658
- catch (e) {
6659
- return { ok: false, error: e.message };
6660
- }
6661
- }
6662
- if (req.op === "planeConnLiveness") {
6663
- // The plane-claim liveness oracle (#29 HIGH 3): a CLOSED read-only verb — two claimed
6664
- // scanner tuples in, two bound verdicts + sweep completeness out. The executor hook owns
6665
- // the closed query validation (it holds the $SYS observer cred outside this trust boundary);
6666
- // absent hook = a daemon build without the oracle, refused loudly (the auth plane treats
6667
- // that refusal as UNKNOWN and never reclaims over it).
6668
- if (!this.plane3?.planeConnLiveness)
6669
- return { ok: false, error: "planeConnLiveness: no plane-liveness oracle wired on this daemon" };
6670
- try {
6671
- return { ok: true, data: await this.plane3.planeConnLiveness(req.args?.query) };
6672
- }
6673
- catch (e) {
6674
- return { ok: false, error: e.message };
6675
- }
6676
- }
6677
- if (req.op === "lifecycleMemberships") {
6678
- // The terminal teardown's membership INVENTORY: a complete, read-only, lifecycle-exact listing
6679
- // of one principal's durable membership rows (tombstones included) from the trusted daemon that
6680
- // owns the members bucket. The caller deletes only the exact keys it gets back, through its
6681
- // target-pinned deprovisioner grant; this verb deletes nothing and returns no other lifecycle.
6682
- const principal = typeof req.args?.principal === "string" ? req.args.principal.trim() : "";
6683
- const uid = typeof req.args?.lifecycleUid === "string" ? req.args.lifecycleUid : "";
6684
- if (!parsePrincipalKey(principal))
6685
- return { ok: false, error: "lifecycleMemberships: a principal (owner.actor dot-form) is required" };
6686
- try {
6687
- assertLifecycleToken(uid);
6688
- }
6689
- catch (e) {
6690
- return { ok: false, error: `lifecycleMemberships: ${e.message}` };
6691
- }
6692
- // A retirement fences the lifecycle's ACL row before it asks (fenceLifecycleAcl), and every join
6693
- // re-reads that row around the row it creates, so a row written after this read begins removes
6694
- // itself, on whichever daemon wrote it. For a manager that does not fence, this daemon's own
6695
- // memberships of the lifecycle stop changing here (#4965): a join or leave bound to it is refused
6696
- // from now on, and the read starts after the ones already running have finished.
6697
- let marked = this.retiringLifecycles.get(principal);
6698
- if (marked === undefined)
6699
- this.retiringLifecycles.set(principal, (marked = new Set()));
6700
- // An earlier lifecycle of this principal whose ACL row is retired can bind no join or leave any
6701
- // more, so its mark goes. A failed read keeps the mark, which only ever refuses.
6702
- for (const earlier of [...marked])
6703
- if (earlier !== uid && await aclRetired(await this.aclRegistry(), principal, earlier).catch(() => false))
6704
- marked.delete(earlier);
6705
- // A mark is kept only for an inventory whose caller did not fence (#4996): a lifecycle whose ACL
6706
- // row is narrowed to no reads, or already retired, needs none, since every join re-reads that
6707
- // row around its write. A read that fails proves nothing, so the mark is kept, which only refuses.
6708
- if (await this.lifecycleAclUnfenced(principal, uid))
6709
- marked.add(uid);
6710
- else if (marked.size === 0)
6711
- this.retiringLifecycles.delete(principal);
6712
- await Promise.allSettled(this.membershipWrites.get(`${principal}/${uid}`) ?? []);
6713
- try {
6714
- const channels = await listLifecycleMemberChannels(await this.membersRegistry(), principal, uid);
6715
- return { ok: true, data: { complete: true, channels } };
6716
- }
6717
- catch (e) {
6718
- return { ok: false, error: `lifecycleMemberships: the inventory read did not complete (${e.message})` };
6719
- }
6720
- }
6721
- if (req.op === "principalLiveness") {
6722
- // The freeze-holder liveness probe (#391): the READ-ONLY half of `evictPrincipal`. A repair
6723
- // that must REFUSE while the holder is alive cannot use eviction as its own precheck — that
6724
- // kills the holder before anything can refuse on its behalf. Same executor-hook shape as the
6725
- // verbs above (the $SYS observer cred lives outside this trust boundary); absent hook =
6726
- // refused loudly, and the caller maps a refusal to UNKNOWN and never repairs over it.
6727
- if (!this.plane3?.principalLiveness)
6728
- return { ok: false, error: "principalLiveness: no liveness oracle wired on this daemon" };
6729
- const principal = typeof req.args?.principal === "string" ? req.args.principal.trim() : "";
6730
- if (!principal)
6731
- return { ok: false, error: "principalLiveness: a principal (owner.actor dot-form) is required" };
6732
- try {
6733
- return { ok: true, data: await this.plane3.principalLiveness(principal) };
6734
- }
6735
- catch (e) {
6736
- return { ok: false, error: e.message };
6737
- }
6738
- }
6739
- if (req.op === "reloadStoreIdentity") {
6740
- // Proof that this daemon reloads standing creds from ONE named store. The manager
6741
- // compares it to its remint store before every renewal pass. Absent hook
6742
- // is a daemon that cannot name its store, which is itself a divergent composition.
6743
- if (!this.plane3?.reloadStoreIdentity)
6744
- return { ok: false, error: "reloadStoreIdentity: this daemon did not name the SecretStore it reloads from" };
6745
- try {
6746
- const identity = this.plane3.reloadStoreIdentity();
6747
- // WHO IS ANSWERING is part of the answer. This rail is queue-grouped, so the broker hands
6748
- // the request to any bound responder, and every process with a `delivery` credential for
6749
- // the space can bind it, while only the DELIVERY LEASE HOLDER actually reloads the standing
6750
- // credentials. A bare store identity therefore lets a second responder's store stand in for
6751
- // the reloading process's, and the manager's remint decision is then made about a process
6752
- // that reloads nothing. So the reply carries this responder's wire identity and whether it
6753
- // holds the lease, and the caller requires the binding (#1694).
6754
- //
6755
- // The lease read is the LIVE row, not a cached claim: a holder that lost the shard must not
6756
- // keep asserting it. A read that fails is reported as `false` rather than thrown, because
6757
- // "I cannot prove I hold the shard" is what a non-holder's answer says, and it sends the
6758
- // caller down the same fail-closed path rather than inventing an authority.
6759
- let holdsDeliveryLease = false;
6760
- try {
6761
- const own = await this.readDeliveryLeaseEntry(0);
6762
- holdsDeliveryLease = own !== undefined && this.ownsDeliveryLease(own.info);
6763
- }
6764
- catch {
6765
- holdsDeliveryLease = false;
6766
- }
6767
- // Round-trip through the closed parser so a hook cannot smuggle extra fields onto the rail.
6768
- return { ok: true, data: { identity: parseSecretStoreIdentity(identity), responder: this.card.id, holdsDeliveryLease } };
6769
- }
6770
- catch (e) {
6771
- return { ok: false, error: e.message };
6772
- }
6773
- }
6774
- return { ok: false, error: `op "${req.op}" not supported on the delivery admin service` };
6775
- }
6776
- /** Create one of the daemon's Plane-3 durables. A named create is idempotent for the same config,
6777
- * and a denial is never taken for existence. The one conflict accepted is this durable as a full
6778
- * restore recreated it, starting after its recorded ACK floor ({@link isRestoredConsumer}); every
6779
- * other failure propagates. */
6780
- async addPlane3Durable(stream, config) {
6781
- try {
6782
- await this.jsm.consumers.add(stream, config);
6783
- }
6784
- catch (e) {
6785
- if (!isConsumerAlreadyExists(e) || !isRestoredConsumer(this.space, stream, await this.jsm.consumers.info(stream, config.durable_name), config))
6786
- throw e;
6787
- }
6788
- }
6789
- /** Fan-out loop: bind the privileged `fanout` durable on CHAT and route each message (routing only —
6790
- * the trusted reader is the auth gate). */
6791
- async runFanout() {
6792
- if (!this.js || !this.jsm)
6793
- return;
6794
- await this.addPlane3Durable(chatStream(this.space), fanoutDurableConfig(this.space, { ackWaitMs: this.ackWaitMs }));
6795
- const consumer = await this.js.consumers.get(chatStream(this.space), FANOUT_DURABLE);
6796
- const msgs = await consumer.consume();
6797
- this.streamMsgs.push(msgs);
6798
- void (async () => {
6799
- for await (const m of msgs) {
6800
- try {
6801
- await this.fanOutMessage(m);
6802
- }
6803
- catch (e) {
6804
- if (!this.stopped)
6805
- this.emit("error", e);
6806
- try {
6807
- m.nak();
6808
- }
6809
- catch { /* draining */ }
6810
- }
6811
- }
6812
- })().catch((e) => { if (!this.stopped)
6813
- this.emit("error", e); });
6814
- }
6815
- /** Route ONE chat message to eligible owners' mixed inboxes. `durable` channel → its `durable-active`
6816
- * members within interval; `live` channel → `@mention` targets authorized to read it (ACL only).
6817
- * On either class, a mention of an agent whose read ACL excludes the channel is reported to the
6818
- * sender with a mention notice. Members KV is scanned FRESH per message (no cache — red-team
6819
- * BLOCKER-1 catch-up correctness). */
6820
- async fanOutMessage(m) {
6821
- // FENCE, and NOT followed by an ack: this unit was dispatched before the quiesce and is resuming
6822
- // after it, so writing the fan-out now would put entries in member inboxes on behalf of a shard
6823
- // this daemon no longer serves. Returning WITHOUT acking is the whole point - a stopped consumer
6824
- // redelivers to whoever holds the shard next, so refusing here loses no message, while acking it
6825
- // would consume the successor's work.
6826
- if (!this.plane3MayAct())
6827
- return;
6828
- const parsed = parseSubject(m.subject);
6829
- if (!parsed || parsed.kind !== "chat") {
6830
- m.ack();
6831
- return;
6832
- }
6833
- const channel = parsed.rest;
6834
- let msg;
6835
- try {
6836
- msg = m.json();
6837
- }
6838
- catch {
6839
- m.ack();
6840
- return;
6841
- }
6842
- if (!isFromSubjectSender(msg, parsed)) {
6843
- m.ack();
6844
- return;
6845
- } // authenticity (SPEC §5)
6846
- const seq = m.seq;
6847
- const normalizedMsg = authenticatedChannelMessage(msg, channel);
6848
- // SECOND FENCE, AT THE COMMIT POINT. The entry check above cannot cover this unit's own awaits:
6849
- // the class read and the member scan below both hit the broker, and a daemon descheduled inside
6850
- // them can resume after a successor owns the shard. Re-ask immediately before the first WRITE,
6851
- // and again return without acking so the entry redelivers rather than being consumed here.
6852
- if (!this.plane3MayAct())
6853
- return;
6854
- const durable = (await this.deliveryClassFresh(channel)) === "durable";
6855
- // Each mention is resolved once. A live channel delivers to the agents among them that may read
6856
- // it, and on either class a mention of an agent that may not read the channel is reported to the
6857
- // sender below, so a post never claims a reader it cannot have.
6858
- const mentioned = [];
6859
- for (const name of msg.mentions ?? []) {
6860
- const owner = this.resolveOwnerByName(name);
6861
- if (!owner || owner === msg.from.id)
6862
- continue;
6863
- // A mention arrives as an ALIAS (the roster names no lifecycle): resolve its single live ACL
6864
- // row for BOTH the read-authorization and the lifecycle to address. Ambiguity (two live rows)
6865
- // refuses THIS copy loudly rather than guessing a lifecycle.
6866
- let row;
6867
- try {
6868
- row = await this.aclForAlias(owner);
6869
- }
6870
- catch (e) {
6871
- this.emit("error", e);
6872
- continue;
6873
- }
6874
- if (row)
6875
- mentioned.push({ name, owner, row });
6876
- }
6877
- if (durable) {
6878
- // The channel's own rows and every family row above it (`review.>`). A lifecycle holding two of
6879
- // them gets two publishes and keeps one copy: the dedupe id is per message and lifecycle (#5404).
6880
- for (const rec of await listMembers(await this.membersRegistry(), { covering: channel })) {
6881
- if (rec.owner === msg.from.id)
6882
- continue; // never backstop the sender's own post
6883
- if (!durableEligible(rec, seq))
6884
- continue; // routing fast-filter (reader re-checks)
6885
- // Address the member row's RECORDED lifecycle: a retired row (tombstone pending) routes to the
6886
- // retired lifecycle's inbox, never the alias's new occupant (SPEC 13.1 cross-plane scoping).
6887
- // Store the AUTHENTICATED-channel copy (main's normalization): the durable frame validates
6888
- // msg.channel === frame.channel, and payload to/toService are stripped.
6889
- // Per-copy, because the scan above and every publish below is a broker round trip: a fan-out
6890
- // to fifty members can straddle a quiesce in its middle. Stopping part-way is safe where
6891
- // stopping late is not - the entry is never acked here, so the successor redelivers it and
6892
- // re-sends the whole set; publishDinbox is idempotent per (owner, lifecycle, msgID).
6893
- if (!this.plane3MayAct())
6894
- return;
6895
- await this.publishDinbox(rec.owner, rec.lifecycleUid, {
6896
- msg: normalizedMsg, channel, seq, reason: "durable-channel", generation: rec.generation,
6897
- ...(rec.channel !== channel ? { row: rec.channel } : {}),
6898
- });
6899
- }
6900
- }
6901
- else {
6902
- for (const { owner, row } of mentioned) {
6903
- if (!channelInAllow(row.allowSubscribe, channel))
6904
- continue; // @mention can't bypass the read ACL
6905
- if (!this.plane3MayAct())
6906
- return; // same rule on the live-mention path
6907
- await this.publishDinbox(owner, row.lifecycleUid, { msg: normalizedMsg, channel, seq, reason: "live-mention", generation: 0 });
6908
- }
6909
- }
6910
- const unreadable = mentioned.filter(({ row }) => !channelInAllow(row.allowSubscribe, channel)).map(({ name }) => name);
6911
- if (unreadable.length) {
6912
- if (!this.plane3MayAct())
6913
- return;
6914
- await this.sendMentionNotice(msg, channel, seq, unreadable);
6915
- }
6916
- // FINAL FENCE, BEFORE THE ACK ITSELF, which is the real commit: acking tells the stream this
6917
- // shard's work is done. A daemon that stopped serving mid-fan-out must not make that claim, or
6918
- // the successor never sees the entry and the message is lost outright rather than duplicated.
6919
- if (!this.plane3MayAct())
6920
- return;
6921
- m.ack();
6922
- }
6923
- deliveryLimitRead;
6924
- /** The DLV count the trusted writer stamps on a copy (SPEC §8): the stream's `max_msgs_per_subject`,
6925
- * read at most {@link DELIVERY_LIMIT_READ_MS} before the write, as a whole number or `none`. */
6926
- async deliveryLimitStamp() {
6927
- const at = Date.now();
6928
- if (this.deliveryLimitRead && at - this.deliveryLimitRead.at < DELIVERY_LIMIT_READ_MS)
6929
- return this.deliveryLimitRead.stamp;
6930
- const count = (await this.jsm.streams.info(dlvStream(this.space))).config.max_msgs_per_subject;
6931
- const stamp = count === HISTORY_NO_COUNT || count < 1 ? "none" : String(count);
6932
- this.deliveryLimitRead = { at, stamp };
6933
- return stamp;
6934
- }
6935
- /** Trusted-reader loop: bind the single privileged `reader` durable over `dinbox.>` and re-authorize
6936
- * + transfer each entry. */
6937
- async runReader() {
6938
- if (!this.js || !this.jsm)
6939
- return;
6940
- await this.addPlane3Durable(inboxStream(this.space), inboxReaderConfig(this.space, { ackWaitMs: this.ackWaitMs }));
6941
- const consumer = await this.js.consumers.get(inboxStream(this.space), INBOX_READER_DURABLE);
6942
- const msgs = await consumer.consume();
6943
- this.streamMsgs.push(msgs);
6944
- void (async () => {
6945
- for await (const m of msgs) {
6946
- try {
6947
- await this.readerHandle(m);
6948
- }
6949
- catch (e) {
6950
- if (!this.stopped)
6951
- this.emit("error", e);
6952
- try {
6953
- m.nak();
6954
- }
6955
- catch { /* draining */ }
6956
- }
6957
- }
6958
- })().catch((e) => { if (!this.stopped)
6959
- this.emit("error", e); });
6960
- }
6961
- /** Whether one of the lifecycle's rows covering concrete `channel`, the channel's own or a family
6962
- * above it, is eligible for the post at `seq`. One exact read per row, the channel's own first. */
6963
- async lifecycleCoversPost(owner, lifecycleUid, channel, seq) {
6964
- const kv = await this.membersRegistry();
6965
- for (const row of [channel, ...familiesCovering(channel).reverse()]) {
6966
- const rec = await readMember(kv, row, owner, lifecycleUid);
6967
- if (rec && durableEligible(rec.record, seq))
6968
- return true;
6969
- }
6970
- return false;
6971
- }
6972
- /** Remove a retired lifecycle's member row for `channel`, when one exists. Best effort: the row
6973
- * authorizes nothing (its ACL row is retired), and the next entry fan-out writes against it
6974
- * brings the reader back here. */
6975
- async removeRetiredMemberRow(channel, owner, lifecycleUid) {
6976
- try {
6977
- const kv = await this.membersRegistry();
6978
- if (await readMember(kv, channel, owner, lifecycleUid) === undefined)
6979
- return;
6980
- await deleteMember(kv, channel, owner, lifecycleUid);
6981
- }
6982
- catch (e) {
6983
- this.emit("warning", new Error(`plane-3 reader: could not remove retired lifecycle ${owner}.${lifecycleUid}'s "${channel}" member row: ${e.message}`));
6984
- }
6985
- }
6986
- /** Re-authorize ONE mixed-inbox entry and transfer it to the owner's DELIVER store. Deny (drop) on a
6987
- * revoked/narrowed ACL or out-of-interval seq; on transfer success, ack the mixed entry (durability
6988
- * has moved to DLV — an §8 equivalent per-member at-least-once mechanism). The agent acks DLV. */
6989
- async readerHandle(m) {
6990
- // FENCE, same rule as the fan-out writer: no effect and no ack once this daemon has stopped
6991
- // serving the shard. The entry redelivers to the holder rather than being consumed here.
6992
- if (!this.plane3MayAct())
6993
- return;
6994
- const pr = parseDinboxPrincipal(m.subject);
6995
- if (!pr) {
6996
- m.ack();
6997
- return;
6998
- } // unparseable subject (incl. a pre-cut 5-segment form) — not a real entry
6999
- const owner = `${pr.owner}.${pr.actor}`; // the member principal dot-form (acl/member keys, msgID)
7000
- let entry;
7001
- try {
7002
- entry = m.json();
7003
- }
7004
- catch {
7005
- m.ack();
7006
- return;
7007
- } // undecodable — drop
7008
- if (!isRecord(entry)) {
7009
- m.ack();
7010
- return;
7011
- } // non-object envelope — permanently invalid
7012
- const redeliveries = m.info?.deliveryCount ?? 1; // JsMsg delivery attempts (1 on first delivery)
7013
- // Lifecycle-exact ACL re-auth (SPEC 13.1): the entry was addressed to pr.lifecycleUid's inbox, so
7014
- // the row read is that lifecycle's exact key — a retired lifecycle's purged row reads as unknown
7015
- // and its residual entries are removed below, never authorized against the successor's row.
7016
- const acl = await this.plane3?.aclFor(owner, pr.lifecycleUid);
7017
- if (acl === undefined) {
7018
- // RETIRED lifecycle: its exact ACL key carries retirement's tombstone, so no attempt can ever
7019
- // deliver this entry. Drop it AND remove it from the store. Deferring it to the ceiling and
7020
- // keeping it would make every reader that starts from a fresh cursor pay the same sweep again.
7021
- // An ACL row that is merely absent proves nothing and still defers below.
7022
- if (await aclRetired(await this.aclRegistry(), owner, pr.lifecycleUid)) {
7023
- if (!this.plane3MayAct())
7024
- return;
7025
- // Remove BEFORE the ack. The ack ends the reader's work on this entry, so acking first and
7026
- // then failing the delete would keep the entry with nothing left to remove it. A failed delete
7027
- // stays pending and retries, bounded by the same ceiling; an entry the stream no longer holds
7028
- // is already removed (the broker answers a delete of a missing sequence with 10043).
7029
- let failure;
7030
- try {
7031
- await this.jsm.streams.deleteMessage(inboxStream(this.space), m.seq, false);
7032
- }
7033
- catch (e) {
7034
- if (!isJetStreamMissing(e, 10043))
7035
- failure = e instanceof Error ? e : new Error(String(e));
7036
- }
7037
- // FENCE AFTER THE DELETE, which is broker I/O. A daemon that stopped serving while it was in
7038
- // flight leaves the entry to the holder: a term here would move the shared durable past an entry
7039
- // the failed delete left stored, so the holder would never retry it.
7040
- if (!this.plane3MayAct())
7041
- return;
7042
- if (failure) {
7043
- if (redeliveries >= READER_MAX_REDELIVERIES) {
7044
- m.term();
7045
- this.emit("error", new Error(`plane-3 reader: gave up removing entry ${m.seq} for retired lifecycle ${owner}.${pr.lifecycleUid} after ${redeliveries} redeliveries: ${failure.message}`));
7046
- return;
7047
- }
7048
- m.nak(2000);
7049
- return;
7050
- }
7051
- // The member row that addressed this entry is one a daemon left behind when it died between
7052
- // its create and its post-write ACL read (#4994). It authorizes nothing, but fan-out copies
7053
- // every later message on the channel into this inbox until it goes, and this reader then
7054
- // removes each copy. Remove the row where that cost is paid, by its exact key, only when it
7055
- // exists: a purge of an absent key would write a marker per dropped entry.
7056
- if (entry.reason === "durable-channel")
7057
- await this.removeRetiredMemberRow(entry.row ?? entry.channel, owner, pr.lifecycleUid);
7058
- if (!this.plane3MayAct())
7059
- return;
7060
- m.ack();
7061
- return;
7062
- }
7063
- // UNKNOWN owner — the manager has not (re)hydrated this owner's ACL yet (e.g. right after a
7064
- // manager PROCESS restart). This is NOT a revocation: DEFER (redeliver), never drop — an ack here
7065
- // would lose at-least-once on restart (impl-review BLOCKER-2). A delayed nak + a redelivery
7066
- // ceiling stops one perma-unknown owner from head-of-lining the shared reader.
7067
- // (Follow-up: the manager does not yet rehydrate its managed set across a process restart — until
7068
- // it does, a long-unknown owner's entries term after the ceiling; tracked, not a silent ack-drop.)
7069
- if (redeliveries >= READER_MAX_REDELIVERIES) {
7070
- m.term();
7071
- this.emit("error", new Error(`plane-3 reader: gave up on entry for unknown owner ${owner} after ${redeliveries} redeliveries`));
7072
- return;
7073
- }
7074
- m.nak(2000);
7075
- return;
7076
- }
7077
- // KNOWN owner whose CURRENT ACL no longer covers the channel — a revocation/narrowing. Drop: the
7078
- // entry is no longer authorized (SPEC §7 current-ACL gate before surfacing).
7079
- if (!channelInAllow(acl, entry.channel)) {
7080
- m.ack();
7081
- return;
7082
- }
7083
- // INTERVAL re-auth (not a current-member boolean): a pre-leave entry (seq ≤ leaveCursor) stays
7084
- // deliverable; seq > leaveCursor (or after a rejoin's newer joinCursor) is the hard cut. Any of the
7085
- // lifecycle's rows that cover the channel may hold the post, whichever row addressed the copy.
7086
- if (entry.reason === "durable-channel" && !(await this.lifecycleCoversPost(owner, pr.lifecycleUid, entry.channel, entry.seq))) {
7087
- m.ack();
7088
- return;
7089
- }
7090
- // COMMIT-POINT FENCE. Everything above this line is broker I/O - the ACL read and the member
7091
- // re-read - so the entry check at the top of the handler is stale by now if this unit was
7092
- // descheduled inside either. The transfer below is the effect that must not happen on behalf of
7093
- // a shard we no longer serve; returning without acking leaves it for the holder.
7094
- if (!this.plane3MayAct())
7095
- return;
7096
- try {
7097
- // DLV has no original chat subject, so preserve the channel the trusted fan-out reader derived
7098
- // from CHAT. Never let the publisher-controlled payload label choose connector attention.
7099
- const frame = {
7100
- version: 1,
7101
- channel: entry.channel,
7102
- msg: authenticatedChannelMessage(entry.msg, entry.channel),
7103
- };
7104
- const frameHeaders = headers();
7105
- frameHeaders.set(PLANE3_FRAME_HEADER, "1");
7106
- // The count read may be broker I/O, so the fence is asked again before the write.
7107
- frameHeaders.set(DELIVERY_LIMIT_HEADER, await this.deliveryLimitStamp());
7108
- if (!this.plane3MayAct())
7109
- return;
7110
- // Lifecycle-keyed subject + msgID (ours): the DLV consumer filters on the lifecycle-scoped
7111
- // subject (streams.ts filter_subject), and a predecessor's transferred copy must never suppress
7112
- // a successor's under the same alias (disjoint lifecycles, stream-wide dedupe).
7113
- await this.js.publish(dlvSubject(this.space, pr.owner, pr.actor, pr.lifecycleUid), JSON.stringify(frame), {
7114
- ...this.plane3MsgId(entry, `${owner}:${pr.lifecycleUid}`),
7115
- headers: frameHeaders,
7116
- });
7117
- }
7118
- catch {
7119
- // EVERY transfer failure is deliberately a retry, never a successful result. Keep the entry
7120
- // pending (redeliver), bounded by the same ceiling so a poison
7121
- // entry can't head-of-line the shared reader forever.
7122
- if (redeliveries >= READER_MAX_REDELIVERIES) {
7123
- m.term();
7124
- this.emit("error", new Error(`plane-3 reader: gave up transferring ${entry.msg.id} for ${owner} after ${redeliveries} redeliveries`));
7125
- return;
7126
- }
7127
- m.nak(2000);
7128
- return;
7129
- }
7130
- // The ack is the last commit: the transfer above already happened, so the only question left is
7131
- // whether THIS daemon may retire the source entry. If it stopped serving mid-transfer, leave the
7132
- // entry pending - the successor re-transfers and stream-wide dedupe on the msgID collapses the
7133
- // duplicate, which is the recoverable direction. Acking here is the unrecoverable one.
7134
- if (!this.plane3MayAct())
7135
- return;
7136
- m.ack();
7137
- }
7138
- /** Agent-side: bind + pump our pre-created Plane-3 DELIVER durable (`dlv_<id>`). Every message here is
7139
- * delivery-daemon-written (DLV is delivery-write-only, broker-enforced) and is a CHANNEL message by contract
7140
- * (the backstop never carries DMs), so `kind=channel` is path-derived (SPEC §4) and the body is
7141
- * trusted (no spoof-guard). `durable:true` — real JetStream ack, coalesced with the core-sub live
7142
- * copy by `MeshAgent.ingest`. No-op when the durable isn't present (open mode / not provisioned). A read
7143
- * list declared on a credential that cannot fetch the durable refuses. */
7144
- async pumpDlv() {
7145
- if (!this.js)
7146
- return;
7147
- if (!this.ownLifecycleUid)
7148
- return; // no lifecycle uid — never provisioned for Plane-3 (its durable is lifecycle-keyed)
7149
- if (this.allowSubscribe?.length === 0)
7150
- return; // known channel-less grant: no DLV durable or bind rows exist
7151
- const durable = dlvDurable(this.owner, this.actor, this.ownLifecycleUid);
7152
- if (this.allowSubscribe === undefined && !this.userMode && this.nc) {
7153
- // Native credential-only clients have no separate ACL option. Use the publish permissions
7154
- // captured when THIS connection authenticated; a callout sentinel does not carry the grant.
7155
- const permission = this.watchDeletePermissions.get(this.nc)?.();
7156
- // Only an absent allow selects no binding. A declared bind with a deny still reaches the
7157
- // broker and fails, so a refusal cannot masquerade as a channel-less credential.
7158
- if (permission && !issuedPermitsSubject({ ...permission, deny: [] }, `$JS.API.CONSUMER.INFO.${dlvStream(this.space)}.${durable}`))
7159
- return;
7160
- }
7161
- let consumer;
7162
- try {
7163
- consumer = await this.js.consumers.get(dlvStream(this.space), durable);
7164
- }
7165
- catch (e) {
7166
- if (isJetStreamMissing(e, JetStreamApiCodes.ConsumerNotFound)) {
7167
- // A channel-less credential may ask about its durable but holds no fetch row for it. A read
7168
- // list declared on such a credential disagrees with the grant, so the start refuses.
7169
- const issued = this.nc && this.watchDeletePermissions.get(this.nc)?.();
7170
- if ((this.allowSubscribe?.length ?? 0) > 0 && issued
7171
- && !issuedPermitsSubject({ ...issued, deny: [] }, `$JS.API.CONSUMER.MSG.NEXT.${dlvStream(this.space)}.${durable}`))
7172
- throw new Error(`this connection declared a read list, but its credential cannot fetch delivery durable "${durable}" for space "${this.space}"; the read list and the credential disagree`);
7173
- return;
7174
- }
7175
- throw e; // a denied bind is not evidence that the read ACL is empty or the durable is absent
7176
- }
7177
- const lease = await this.readDeliveryLease(0);
7178
- const liveDeliveryPlane = lease?.ready === true;
7179
- if (!liveDeliveryPlane)
7180
- this.emit("warning", new Error(`delivery durable "${durable}" for space "${this.space}" bound while the plane this connection reaches has no ready delivery lease, so the durable delivers nothing until a delivery daemon serves this plane. If a daemon serves this space on another plane, reconnect against it, which re-binds the durable there.`));
7181
- const ackWaitMs = await ackWaitOf(consumer);
7182
- const msgs = await consumer.consume();
7183
- this.streamMsgs.push(msgs);
7184
- void (async () => {
7185
- for await (const m of msgs) {
7186
- // A copy with a full store behind it means the store held its count, so older copies may have
7187
- // been discarded unread (#5404); NATS gives no notice of the discard itself. The count is the
7188
- // one the trusted writer stamped on this copy. Checked on every delivery, ahead of the checks
7189
- // below and the connector's id dedupe, so a copy the agent also got live cannot hide it. The
7190
- // connector folds repeats into one notice.
7191
- let limit;
7192
- try {
7193
- limit = deliveryLimitOf(m.headers?.get(DELIVERY_LIMIT_HEADER));
7194
- }
7195
- catch (e) {
7196
- this.emit("error", e);
7197
- limit = null;
7198
- }
7199
- if (limit !== null && m.info.pending >= limit - 1)
7200
- this.emit("stored-copies-full", { limit });
7201
- // The header is minted only by the trusted DLV writer. A body-only discriminator is
7202
- // insufficient because old Cotal envelopes permit unknown fields and could imitate it.
7203
- if (m.headers?.get(PLANE3_FRAME_HEADER) !== "1") {
7204
- this.emit("error", new Error("plane-3 delivery: unauthenticated or unversioned DLV entry terminated"));
7205
- try {
7206
- m.term();
7207
- }
7208
- catch { /* draining */ }
7209
- continue;
7210
- }
7211
- let raw;
7212
- try {
7213
- raw = m.json();
7214
- }
7215
- catch (e) {
7216
- this.emit("error", e);
7217
- try {
7218
- m.term();
7219
- }
7220
- catch { /* draining */ }
7221
- continue;
7222
- }
7223
- if (!isPlane3DeliveryFrame(raw)) {
7224
- this.emit("error", new Error("plane-3 delivery: malformed versioned DLV entry terminated"));
7225
- try {
7226
- m.term();
7227
- }
7228
- catch { /* draining */ }
7229
- continue;
7230
- }
7231
- const msg = authenticatedChannelMessage(raw.msg, raw.channel);
7232
- if (msg.from?.id === this.card.id) {
7233
- m.ack();
7234
- continue;
7235
- } // own echo (defensive)
7236
- this.emit("message", msg, durableDelivery(m, ackWaitMs), { historical: false, kind: "channel" });
7237
- }
7238
- })().catch((e) => { if (!this.stopped)
7239
- this.emit("error", e); });
7240
- }
7241
5629
  /** Agent-side: request a Plane-3 durable backstop for a channel via the server-side delivery daemon (ctl.delivery). Throws
7242
5630
  * when no privileged writer is present (open / no delivery daemon). 30s timeout — activation catch-up may
7243
5631
  * run before the reply (the window is small, but a busy channel can take more than the 5s default). */
@@ -7937,60 +6325,7 @@ export class CotalEndpoint extends EventEmitter {
7937
6325
  * calls must be serial — every reader here awaits to completion, so they are.
7938
6326
  */
7939
6327
  async collectHistory(subject, start, opts = {}) {
7940
- // Serialize on the per-instance lock: the fixed `chathist_<id>` name means two concurrent reads
7941
- // (recall + join-backfill + drop-marker can race in-process) would delete/recreate the consumer
7942
- // under each other and cross-feed results. The chain makes the "serial callers" assumption true.
7943
- const run = this.histLock.then(() => this.collectHistoryInner(subject, start, opts));
7944
- this.histLock = run.catch(() => { }); // keep the chain alive on error
7945
- return run;
7946
- }
7947
- async collectHistoryInner(subject, start, opts = {}) {
7948
- if (!this.jsm || !this.js)
7949
- throw new Error(this.notLiveMsg());
7950
- const stream = chatStream(this.space);
7951
- const name = chatHistDurable(this.owner, this.actor, this.requireLifecycleUid("chat history reads"));
7952
- const out = [];
7953
- // Clear any consumer leaked by a crashed prior read before re-creating it with THIS read's
7954
- // single filter (the read ACL is enforced at create — see the doc above).
7955
- await this.deleteConsumerIfPresent(stream, name);
7956
- await this.jsm.consumers.add(stream, {
7957
- name,
7958
- filter_subject: subject,
7959
- ack_policy: AckPolicy.None,
7960
- mem_storage: true,
7961
- inactive_threshold: nanos(30_000),
7962
- ...("time" in start
7963
- ? { deliver_policy: DeliverPolicy.StartTime, opt_start_time: start.time.toISOString() }
7964
- : { deliver_policy: DeliverPolicy.StartSequence, opt_start_seq: start.seq }),
7965
- });
7966
- try {
7967
- const consumer = await this.js.consumers.get(stream, name);
7968
- let pending = (await consumer.info()).num_pending;
7969
- while (pending > 0) {
7970
- const want = Math.min(pending, 256);
7971
- const iter = await consumer.fetch({ max_messages: want, expires: 5_000 });
7972
- let got = 0;
7973
- for await (const m of iter) {
7974
- got++;
7975
- if (opts.untilSeq !== undefined && m.seq > opts.untilSeq)
7976
- return out; // crossed the frontier
7977
- // Belt-and-suspenders over the lock: only keep messages on the requested channel subject
7978
- // (the consumer's filter already bounds this; guards against any stale-consumer edge).
7979
- if (!subjectMatches(subject, m.subject))
7980
- continue;
7981
- out.push(m);
7982
- if (opts.limit !== undefined && out.length >= opts.limit)
7983
- return out;
7984
- }
7985
- if (got < want)
7986
- break; // drained early
7987
- pending -= got;
7988
- }
7989
- }
7990
- finally {
7991
- await this.deleteConsumerIfPresent(stream, name);
7992
- }
7993
- return out;
6328
+ return this.history.collectHistory(subject, start, opts);
7994
6329
  }
7995
6330
  /** Read a channel's retained history up to `upToSeq` (the join frontier) and emit each message
7996
6331
  * as a `historical` "message" event. `sinceMs` bounds how far back via a native consumer
@@ -8004,44 +6339,7 @@ export class CotalEndpoint extends EventEmitter {
8004
6339
  * after it, which is exactly the catch-up. New messages (`seq > upToSeq`) are skipped — the live
8005
6340
  * tail owns them. Reads through the contained {@link collectHistory}. */
8006
6341
  async backfillChannel(channel, upToSeq, sinceMs, floorSeq) {
8007
- if (floorSeq !== undefined && floorSeq >= upToSeq)
8008
- return 0;
8009
- const subject = chatSubject(this.space, "*", "*", channel);
8010
- // A window that reaches back before 1970 covers the whole retained window (#7323). Its start time
8011
- // would be a date JetStream does not read, or not a valid date at all, and the read replayed nothing.
8012
- const since = sinceMs === undefined ? undefined : Date.now() - sinceMs;
8013
- const start = floorSeq !== undefined ? { seq: floorSeq + 1 } : since === undefined || !(since > 0) ? { seq: 1 } : { time: new Date(since) };
8014
- let msgs;
8015
- try {
8016
- msgs = await this.collectHistory(subject, start, { untilSeq: upToSeq });
8017
- }
8018
- catch (e) {
8019
- this.emit("error", e);
8020
- if (isPermissionDenied(e))
8021
- throw e;
8022
- return 0;
8023
- }
8024
- const noop = { ack: () => { }, nak: () => { }, durable: false };
8025
- let n = 0;
8026
- for (const sm of msgs) {
8027
- let msg;
8028
- try {
8029
- msg = sm.json();
8030
- }
8031
- catch {
8032
- continue; // skip undecodable
8033
- }
8034
- // Same authenticity guard as the tail; skip our own echoes in history.
8035
- if (!isRecord(msg) || !isUsableMessageId(msg.id))
8036
- continue; // malformed envelope (SPEC sec 5) — history skips
8037
- const parsed = parseSubject(sm.subject);
8038
- if (!parsed || !isFromSubjectSender(msg, parsed) || msg.from.id === this.card.id)
8039
- continue;
8040
- // Backfill only ever reads the chat stream, so the authenticated class is always "channel".
8041
- this.emit("message", authenticatedMessage(msg, parsed), noop, { historical: true, kind: "channel" });
8042
- n++;
8043
- }
8044
- return n;
6342
+ return this.history.backfillChannel(channel, upToSeq, sinceMs, floorSeq);
8045
6343
  }
8046
6344
  /**
8047
6345
  * Replay-gated pull of a channel's retained ambient from `sinceSeq` (exclusive) forward — the
@@ -8064,55 +6362,7 @@ export class CotalEndpoint extends EventEmitter {
8064
6362
  * the window was empty and complete.
8065
6363
  */
8066
6364
  async recallChannel(channel, sinceSeq) {
8067
- if (!this.jsm)
8068
- throw new Error(this.notLiveMsg());
8069
- if (!isConcreteChannel(channel))
8070
- return { messages: [], seqs: [], dropped: false, unanswered: true };
8071
- const policy = await this.joinPolicyFresh(channel);
8072
- if (!policy.replay)
8073
- return { messages: [], seqs: [], dropped: true, unanswered: true };
8074
- const subject = chatSubject(this.space, "*", "*", channel);
8075
- let raw;
8076
- try {
8077
- raw = await this.collectHistory(subject, { seq: sinceSeq + 1 });
8078
- }
8079
- catch (e) {
8080
- this.emit("error", e);
8081
- if (isPermissionDenied(e))
8082
- throw e;
8083
- return { messages: [], seqs: [], dropped: true, unanswered: true };
8084
- }
8085
- const collected = [];
8086
- const seqs = [];
8087
- for (const sm of raw) {
8088
- let msg;
8089
- try {
8090
- msg = sm.json();
8091
- }
8092
- catch {
8093
- continue; // skip undecodable
8094
- }
8095
- // Same authenticity guard as the tail/backfill; skip our own echoes.
8096
- if (!isRecord(msg) || !isUsableMessageId(msg.id))
8097
- continue; // malformed envelope (SPEC sec 5) — recall skips
8098
- const parsed = parseSubject(sm.subject);
8099
- if (!parsed || !isFromSubjectSender(msg, parsed) || msg.from.id === this.card.id)
8100
- continue;
8101
- collected.push(authenticatedMessage(msg, parsed));
8102
- seqs.push(sm.seq);
8103
- }
8104
- let dropped;
8105
- try {
8106
- dropped = await this.chatMayHaveLost(subject, sinceSeq);
8107
- }
8108
- catch (e) {
8109
- // Completeness could not be read, so the read is reported as unanswered: nothing settles on it.
8110
- this.emit("error", e);
8111
- if (isPermissionDenied(e))
8112
- throw e;
8113
- return { messages: [], seqs: [], dropped: true, unanswered: true };
8114
- }
8115
- return { messages: collected, seqs, dropped, unanswered: false };
6365
+ return this.history.recallChannel(channel, sinceSeq);
8116
6366
  }
8117
6367
  /** Whether the chat stream's limits may have removed posts on `subject` after `sinceSeq`, read from
8118
6368
  * the stream's own config rather than a compiled default. Throws when it cannot read, so no caller
@@ -8227,12 +6477,6 @@ export class CotalEndpoint extends EventEmitter {
8227
6477
  return this.presence.sweep();
8228
6478
  }
8229
6479
  }
8230
- /** What a history read failure NAMES when it could not finish. One filter subject is the useful
8231
- * thing to print; a set of sixty-nine of them is a wall of text in a message a human has to read,
8232
- * so a set says its size and the stream it was read from instead. */
8233
- function subjectLabel(subjects) {
8234
- return subjects.length === 1 ? subjects[0] : `${subjects.length} filtered subjects`;
8235
- }
8236
6480
  /** A usable delivery-message id (#624): a string, possibly empty (the never-a-key case), but
8237
6481
  * never absent and never a non-string. An absent or non-string id is a malformed envelope under
8238
6482
  * SPEC sec 5; each delivery pump handles it per its own class (durable term, live drop, history
@@ -8279,12 +6523,6 @@ function authenticatedMessage(msg, parsed) {
8279
6523
  return authenticatedAnycastMessage(msg, parsed.rest);
8280
6524
  return msg;
8281
6525
  }
8282
- function authenticatedChannelMessage(msg, channel) {
8283
- if (msg.channel === channel && msg.to === undefined && msg.toService === undefined)
8284
- return msg;
8285
- const { to: _to, toService: _toService, ...base } = msg;
8286
- return { ...base, channel };
8287
- }
8288
6526
  function authenticatedDmMessage(msg, to) {
8289
6527
  if (msg.to === to && msg.channel === undefined && msg.toService === undefined)
8290
6528
  return msg;
@@ -8297,23 +6535,6 @@ function authenticatedAnycastMessage(msg, toService) {
8297
6535
  const { to: _to, channel: _channel, ...base } = msg;
8298
6536
  return { ...base, toService };
8299
6537
  }
8300
- /** The ack wait the broker reports for a bound consumer: how long a copy it delivers stays with
8301
- * its reader without an ack. */
8302
- async function ackWaitOf(consumer) {
8303
- const { config } = await consumer.info(true);
8304
- if (config.ack_wait === undefined)
8305
- throw new Error(`consumer ${config.durable_name} reports no ack wait`);
8306
- return millis(config.ack_wait);
8307
- }
8308
- /** A JetStream copy's {@link Delivery}, carrying the broker's own redelivery state for it. */
8309
- function durableDelivery(m, ackWaitMs) {
8310
- return {
8311
- ack: () => m.ack(),
8312
- nak: () => m.nak(),
8313
- durable: true,
8314
- redelivery: { deliveryCount: m.info.deliveryCount, ackWaitMs, working: () => m.working() },
8315
- };
8316
- }
8317
6538
  /** A DM-only copy's delivery. An ack, nak or in-progress signal on a closed connection is refused
8318
6539
  * with a throw instead of being dropped silently: the copy stays unacked and the broker redelivers it. */
8319
6540
  function dmOnlyDelivery(m, ackWaitMs, nc) {
@@ -8406,59 +6627,6 @@ function isReadableMessagePart(value) {
8406
6627
  function isHistoryDrainEnvelope(value) {
8407
6628
  return isRecord(value) && isUsableMessageId(value.id) && isRecord(value.from);
8408
6629
  }
8409
- function isPlane3DeliveryFrame(value) {
8410
- if (!isRecord(value))
8411
- return false;
8412
- const keys = Object.keys(value);
8413
- if (keys.length !== 3 || !keys.includes("version") || !keys.includes("channel") || !keys.includes("msg"))
8414
- return false;
8415
- if (value.version !== 1 || typeof value.channel !== "string" || !isConcreteChannel(value.channel))
8416
- return false;
8417
- return isCotalMessage(value.msg) && value.msg.channel === value.channel;
8418
- }
8419
- function isCotalMessage(value) {
8420
- if (!isRecord(value))
8421
- return false;
8422
- const routes = ["channel", "to", "toService"].filter((key) => key in value);
8423
- if (routes.length !== 1 || typeof value[routes[0]] !== "string")
8424
- return false;
8425
- if (typeof value.id !== "string" || typeof value.ts !== "number" || !Number.isFinite(value.ts) ||
8426
- typeof value.space !== "string" || !isEndpointRef(value.from) || !Array.isArray(value.parts) ||
8427
- !value.parts.every(isMessagePart))
8428
- return false;
8429
- if (value.mentions !== undefined && (!Array.isArray(value.mentions) || !value.mentions.every((name) => typeof name === "string")))
8430
- return false;
8431
- if (value.replyTo !== undefined && typeof value.replyTo !== "string")
8432
- return false;
8433
- if (value.contextId !== undefined && typeof value.contextId !== "string")
8434
- return false;
8435
- return true;
8436
- }
8437
- function isEndpointRef(value) {
8438
- return isRecord(value) && typeof value.id === "string" && typeof value.name === "string" &&
8439
- (value.role === undefined || typeof value.role === "string");
8440
- }
8441
- /** A conformant message part: the three CORE kinds, or a reverse-DNS extension kind (SPEC §5).
8442
- *
8443
- * Reached from {@link isCotalMessage}, which gates the Plane-3 delivery frame — so a core kind
8444
- * missing an arm here is not a schema nicety: the durable backstop DROPS every message carrying
8445
- * it, silently, and the drop surfaces nowhere near the feature that added the part. `artifact`
8446
- * was exactly that case before it was added (a bare kind has no dot, so it fell through to the
8447
- * extension regex and failed). */
8448
- function isMessagePart(value) {
8449
- if (!isRecord(value) || typeof value.kind !== "string")
8450
- return false;
8451
- if (value.kind === "text")
8452
- return typeof value.text === "string";
8453
- if (value.kind === "data")
8454
- return Object.prototype.hasOwnProperty.call(value, "data");
8455
- if (value.kind === ARTIFACT_PART_KIND)
8456
- return isArtifactPart(value);
8457
- return /^[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)+$/.test(value.kind);
8458
- }
8459
- function isRecord(value) {
8460
- return !!value && typeof value === "object" && !Array.isArray(value);
8461
- }
8462
6630
  /** A `data` part value the wire can carry faithfully: a JSON value at EVERY depth (SPEC §5's
8463
6631
  * `<any JSON value>`). `JSON.stringify` does not reject what it cannot represent — it silently
8464
6632
  * rewrites: `undefined` (and a function-valued member) drops the key, `NaN`/`Infinity` store as
@@ -8647,12 +6815,6 @@ function membershipConsumerReleased(err) {
8647
6815
  return (isJetStreamMissing(err, JetStreamApiCodes.ConsumerNotFound, JetStreamApiCodes.StreamNotFound)
8648
6816
  || isPublishPermissionDenied(err));
8649
6817
  }
8650
- /** The signal's exact abort reason, or the platform-standard AbortError when none was supplied. */
8651
- function abortReason(signal) {
8652
- return signal?.reason instanceof Error
8653
- ? signal.reason
8654
- : new DOMException("This operation was aborted", "AbortError");
8655
- }
8656
6818
  /** True ONLY for a denial on a **publish** — the single case that proves the message was never
8657
6819
  * ACCEPTED or stored. (Not "never reached the server": the server necessarily received enough of
8658
6820
  * it to reject it. The distinction matters precisely here, because this helper exists to separate