@cotal-ai/core 0.167.0 → 0.169.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. package/dist/agent-file.d.ts.map +1 -1
  2. package/dist/agent-file.js +12 -9
  3. package/dist/agent-file.js.map +1 -1
  4. package/dist/connector-setup.d.ts +2 -0
  5. package/dist/connector-setup.d.ts.map +1 -1
  6. package/dist/connector.d.ts +8 -11
  7. package/dist/connector.d.ts.map +1 -1
  8. package/dist/control-exchange.d.ts +114 -0
  9. package/dist/control-exchange.d.ts.map +1 -0
  10. package/dist/control-exchange.js +416 -0
  11. package/dist/control-exchange.js.map +1 -0
  12. package/dist/endpoint-call.d.ts +6 -3
  13. package/dist/endpoint-call.d.ts.map +1 -1
  14. package/dist/endpoint-call.js +30 -14
  15. package/dist/endpoint-call.js.map +1 -1
  16. package/dist/endpoint-delivery-plane.d.ts +337 -0
  17. package/dist/endpoint-delivery-plane.d.ts.map +1 -0
  18. package/dist/endpoint-delivery-plane.js +1291 -0
  19. package/dist/endpoint-delivery-plane.js.map +1 -0
  20. package/dist/endpoint-envelope.d.ts +2 -1
  21. package/dist/endpoint-envelope.d.ts.map +1 -1
  22. package/dist/endpoint-envelope.js +9 -3
  23. package/dist/endpoint-envelope.js.map +1 -1
  24. package/dist/endpoint-history.d.ts +370 -0
  25. package/dist/endpoint-history.d.ts.map +1 -0
  26. package/dist/endpoint-history.js +969 -0
  27. package/dist/endpoint-history.js.map +1 -0
  28. package/dist/endpoint-inbound.d.ts +10 -0
  29. package/dist/endpoint-inbound.d.ts.map +1 -0
  30. package/dist/endpoint-inbound.js +28 -0
  31. package/dist/endpoint-inbound.js.map +1 -0
  32. package/dist/endpoint-provenance.d.ts +11 -0
  33. package/dist/endpoint-provenance.d.ts.map +1 -0
  34. package/dist/endpoint-provenance.js +20 -0
  35. package/dist/endpoint-provenance.js.map +1 -0
  36. package/dist/endpoint-serve.d.ts +11 -1
  37. package/dist/endpoint-serve.d.ts.map +1 -1
  38. package/dist/endpoint-serve.js +43 -14
  39. package/dist/endpoint-serve.js.map +1 -1
  40. package/dist/endpoint-subjects.d.ts +3 -0
  41. package/dist/endpoint-subjects.d.ts.map +1 -1
  42. package/dist/endpoint-subjects.js +3 -0
  43. package/dist/endpoint-subjects.js.map +1 -1
  44. package/dist/endpoint.d.ts +80 -252
  45. package/dist/endpoint.d.ts.map +1 -1
  46. package/dist/endpoint.js +248 -2027
  47. package/dist/endpoint.js.map +1 -1
  48. package/dist/index.d.ts +2 -1
  49. package/dist/index.d.ts.map +1 -1
  50. package/dist/index.js +4 -1
  51. package/dist/index.js.map +1 -1
  52. package/dist/launch-material.d.ts +9 -3
  53. package/dist/launch-material.d.ts.map +1 -1
  54. package/dist/launch-material.js +22 -2
  55. package/dist/launch-material.js.map +1 -1
  56. package/dist/provision-jwt.d.ts +518 -0
  57. package/dist/provision-jwt.d.ts.map +1 -0
  58. package/dist/provision-jwt.js +641 -0
  59. package/dist/provision-jwt.js.map +1 -0
  60. package/dist/provision-permissions.d.ts +72 -0
  61. package/dist/provision-permissions.d.ts.map +1 -0
  62. package/dist/provision-permissions.js +2015 -0
  63. package/dist/provision-permissions.js.map +1 -0
  64. package/dist/provision-resources.d.ts +66 -0
  65. package/dist/provision-resources.d.ts.map +1 -0
  66. package/dist/provision-resources.js +61 -0
  67. package/dist/provision-resources.js.map +1 -0
  68. package/dist/provision-server-config.d.ts +115 -0
  69. package/dist/provision-server-config.d.ts.map +1 -0
  70. package/dist/provision-server-config.js +169 -0
  71. package/dist/provision-server-config.js.map +1 -0
  72. package/dist/provision.d.ts +19 -707
  73. package/dist/provision.d.ts.map +1 -1
  74. package/dist/provision.js +4 -2860
  75. package/dist/provision.js.map +1 -1
  76. package/dist/remote-manager-authority.d.ts +1 -1
  77. package/dist/run-journal.d.ts +2 -0
  78. package/dist/run-journal.d.ts.map +1 -1
  79. package/dist/run-journal.js +1 -1
  80. package/dist/run-journal.js.map +1 -1
  81. package/package.json +5 -1
package/dist/endpoint.js CHANGED
@@ -1,12 +1,13 @@
1
+ import { EndpointHistory } from "./endpoint-history.js";
2
+ export { MULTI_FILTER_BATCH, MULTI_FILTER_READ_CONCURRENCY } from "./endpoint-history.js";
1
3
  import { eventPayloadSubject, eventPayloadStream } from "./event-payloads.js";
2
4
  import { requestSafeReconnect } from "./request-safe-reconnect.js";
3
5
  import { nativeBindingAfter, nativeConnectionBinding } from "./native-connection.js";
4
6
  import { EventEmitter } from "node:events";
5
7
  import { drainAndClose } from "./connection-cleanup.js";
6
8
  import { randomUUID } from "node:crypto";
7
- import { setTimeout as delay } from "node:timers/promises";
8
9
  import { createConnection, isIP } from "node:net";
9
- import { connect, headers, tokenAuthenticator, nanos, millis, AuthorizationError, PermissionViolationError, UserAuthenticationExpiredError, NoRespondersError, RequestError, TimeoutError, ClosedConnectionError, ConnectionError, } from "@nats-io/transport-node";
10
+ import { connect, headers, tokenAuthenticator, nanos, AuthorizationError, PermissionViolationError, UserAuthenticationExpiredError, NoRespondersError, RequestError, TimeoutError, ClosedConnectionError, ConnectionError, } from "@nats-io/transport-node";
10
11
  import { wsconnect } from "@nats-io/nats-core";
11
12
  import { parseLivenessAnswer, responderFromLease, responderFromProbe, isLivenessPlane, LIVENESS_PLANES, } from "./liveness.js";
12
13
  import { EndpointPresence } from "./endpoint-presence.js";
@@ -17,45 +18,41 @@ import { armRenewal, renewalDeadlineMs, jitteredBackoffMs } from "./renewal.js";
17
18
  import { requireBrokerFloor } from "./broker-floor.js";
18
19
  import { withDeadline } from "./deadline.js";
19
20
  import { inspectCredHealth } from "./provision.js";
20
- import { parseSecretStoreIdentity, } from "./secret-store.js";
21
+ import {} from "./secret-store.js";
21
22
  import { BIND_SPLIT_REISSUES, resolveService, invokeCommand, goalFollowRefusal, submitAndFollowGoal } from "./endpoint-invoke.js";
22
23
  import { EpEnvelopeError, respondedButUnbound, replyRefusedBeforeEffect, replyTargetUnmapped, EP_BIND_REFUSED } from "./endpoint-envelope.js";
23
24
  import { isRepeatSafeCommand } from "./endpoint-grants.js";
24
25
  import { assertIdToken, assertGeneration, assertRuntimeToken, endpointToken } from "./endpoint-subjects.js";
25
26
  import { servePrincipalRoute, describeContracts } from "./endpoint-serve.js";
26
27
  import { epPrincipalCall } from "./endpoint-call.js";
27
- import { AGENT_CONTRACTS, AGENT_ENDPOINT } from "./agent-endpoint.js";
28
+ import { AGENT_CLUSTER, AGENT_CONTRACTS, AGENT_ENDPOINT } from "./agent-endpoint.js";
28
29
  import { processRuntime } from "./process-runtime.js";
29
30
  import { importNativeSubjectPermissions, issuedPermitsSubject, readAcceptedRow } from "./issued-authority.js";
30
31
  import { REGISTRY_ENDPOINT, REGISTRY_ENDPOINTS_COMMAND } from "./registry-contract.js";
31
32
  import { listRegistryEndpoints, registryListingCaller } from "./registry-client.js";
32
33
  import { lastKvEntries } from "./kv-scan.js";
33
- import { isConsumerAlreadyExists, isConsumerNotFound, isJetStreamMissing, isStreamNotFound } from "./jetstream-missing.js";
34
- import { isRestoredConsumer } from "./backup.js";
35
- import { ARTIFACT_PART_KIND, isArtifactPart } from "./artifact.js";
34
+ import { isConsumerNotFound, isJetStreamMissing, isStreamNotFound } from "./jetstream-missing.js";
36
35
  import { assertValidKind } from "./agent-file.js";
37
36
  import { assertValidName } from "./resolve.js";
38
- import { EVICT_PRINCIPALS_MAX } from "./evict.js";
39
37
  import { nextHeartbeat } from "./presence-heartbeat.js";
40
- import { anycastStoreDefinition, createSpaceStreams, dmDurableConfig, dlvDurableConfig, taskDurableConfig, fanoutDurableConfig, inboxReaderConfig, MANAGER_LEASE_TTL_MS, MANAGER_LEASE_ATTEMPT_MS, PRESENCE_STORAGE } from "./streams.js";
41
- import { DELIVERY_LIMIT_HEADER, HISTORY_NO_COUNT, PLANE3_MAX_MSGS_PER_SUBJECT } from "./backup-config.js";
42
- import { jetstream, jetstreamManager, AckPolicy, DeliverPolicy, JetStreamApiCodes, JetStreamApiError, } from "@nats-io/jetstream";
38
+ import { anycastStoreDefinition, createSpaceStreams, dmDurableConfig, dlvDurableConfig, taskDurableConfig, MANAGER_LEASE_TTL_MS, MANAGER_LEASE_ATTEMPT_MS, PRESENCE_STORAGE } from "./streams.js";
39
+ import { DELIVERY_LIMIT_HEADER, PLANE3_MAX_MSGS_PER_SUBJECT } from "./backup-config.js";
40
+ import { jetstream, jetstreamManager, AckPolicy, DeliverPolicy, JetStreamApiCodes, } from "@nats-io/jetstream";
43
41
  import {} from "@nats-io/jetstream";
44
42
  import { Kvm } from "@nats-io/kv";
45
43
  import { Bucket, KvWatchInclude } from "@nats-io/kv/internal";
46
- import { openMembersRegistry, commitMemberRevision, tombstoneMember, activateMember, readMember, deleteMember, listMembers, listLifecycleMemberChannels, durableEligible, } from "./members.js";
47
- import { openAclRegistry, readAcl, readAclForAlias, aclRetired, AmbiguousAclAlias, commitAcl as writeAclRecord, reissueAcl as writeAclReissue } from "./acls.js";
44
+ import { openMembersRegistry, commitMemberRevision, tombstoneMember, activateMember, readMember, deleteMember, listMembers, } from "./members.js";
45
+ import { openAclRegistry, readAcl, readAclForAlias, AmbiguousAclAlias, commitAcl as writeAclRecord, reissueAcl as writeAclReissue } from "./acls.js";
48
46
  import { EndpointDeliveryLease } from "./endpoint-delivery-lease.js";
47
+ import { EndpointDeliveryPlane } from "./endpoint-delivery-plane.js";
48
+ import { ackWaitOf, authenticatedChannelMessage, durableDelivery, isRecord } from "./endpoint-inbound.js";
49
49
  import { openChannelRegistry, effectiveReplay, effectiveReplayWindowMs, effectiveDeliveryClass, readChannelConfig, readChannelDefaults, decodeChannelEntry, } from "./channels.js";
50
50
  import { isCasLoss } from "./endpoint-records.js";
51
51
  import { ANYCAST_CANCEL_OP, ANYCAST_NOTICE_KEY, ANYCAST_RECORDS_CREDENTIAL_OUTDATED, ANYCAST_STATUS_OP, ANYCAST_TAKES_OP, AnycastRequestAuthority, isAnycastSeq, } from "./anycast-requests.js";
52
- import { anycastSubject, assertValidOwnerToken, assertValidRole, CHANNEL_DEFAULTS_KEY, chatStream, chatHistDurable, chatSubject, historyReaderConsumer, calloutReaderId, HISTORY_READERS, controlServiceSubject, livenessSubject, livenessServeFilter, CONTROL_DELIVERY, CONTROL_DELIVERY_ADMIN, dmStream, dmDurable, dlvStream, dlvDurable, dlvSubject, dinboxSubject, inboxStream, parseDinboxPrincipal, FANOUT_DURABLE, INBOX_READER_DURABLE, managerBucket, MANAGER_LEASE_KEY, MANAGER_RENEWAL_LEASE_KEY, managerLeaseKey, assertValidChannel, channelInAllow, membershipInAllow, isConcreteChannel, isChannelFamily, familiesCovering, normalizeMentions, MENTION_NOTICE_KEY, parseSubject, isPrincipalOwnerToken, assertInboxConnId, presenceBucket, channelBucket, membershipBucket, MEMBERSHIP_FEED_KEY, principalKey, parsePrincipalKey, assertLifecycleToken, mintLifecycleUid, lifecycleNameKey, memberKey, DEV_OWNER, spacePrefix, spaceWildcard, subjectMatches, taskStream, taskDurable, takeServeFilter, takeSubject, parseTakeSubject, TAKE_ROUTE, token, unicastSubject, unicastRecvFilter, } from "./subjects.js";
52
+ import { anycastSubject, assertValidOwnerToken, assertValidRole, CHANNEL_DEFAULTS_KEY, chatStream, chatSubject, controlServiceSubject, livenessSubject, livenessServeFilter, CONTROL_DELIVERY, CONTROL_DELIVERY_ADMIN, dmStream, dmDurable, dlvStream, dinboxSubject, FANOUT_DURABLE, managerBucket, MANAGER_LEASE_KEY, MANAGER_RENEWAL_LEASE_KEY, managerLeaseKey, membershipInAllow, isConcreteChannel, isChannelFamily, normalizeMentions, MENTION_NOTICE_KEY, parseSubject, isPrincipalOwnerToken, assertInboxConnId, presenceBucket, channelBucket, membershipBucket, MEMBERSHIP_FEED_KEY, principalKey, parsePrincipalKey, assertLifecycleToken, mintLifecycleUid, lifecycleNameKey, memberKey, DEV_OWNER, spacePrefix, spaceWildcard, subjectMatches, taskStream, taskDurable, takeServeFilter, takeSubject, parseTakeSubject, TAKE_ROUTE, token, unicastSubject, unicastRecvFilter, } from "./subjects.js";
53
53
  import { isPermissionDenied } from "./permission-denied.js";
54
54
  export { isPermissionDenied };
55
55
  export const DEFAULT_SERVER = "nats://127.0.0.1:4222";
56
- const PLANE3_FRAME_HEADER = "Cotal-Delivery-Frame";
57
- /** The `ctl.delivery` ops a sender calls about one of its own anycast requests (SPEC §8.1). */
58
- const ANYCAST_CONTROL_OPS = new Set([ANYCAST_STATUS_OP, ANYCAST_CANCEL_OP, ANYCAST_TAKES_OP]);
59
56
  /** Takes a daemon answers at once; one more is refused as busy, so the holder keeps its copy and
60
57
  * asks again rather than queueing work the daemon cannot bound. */
61
58
  const MAX_TAKES_IN_FLIGHT = 64;
@@ -63,8 +60,6 @@ const MAX_TAKES_IN_FLIGHT = 64;
63
60
  const ANYCAST_SWEEP_MS = 10 * 60 * 1000;
64
61
  /** How long a holder waits for the daemon to answer a take. */
65
62
  const ANYCAST_TAKE_TIMEOUT_MS = 5_000;
66
- /** How long the trusted writer reuses its read of the DLV count before reading it again. */
67
- const DELIVERY_LIMIT_READ_MS = 5_000;
68
63
  /** The DLV count a stored copy was written under, from its {@link DELIVERY_LIMIT_HEADER} (SPEC §8):
69
64
  * a whole number of at least 1, null for `none`, and 1000 when the header is absent, since only a
70
65
  * daemon from before the count was configurable writes a copy without it and its count was 1000.
@@ -80,31 +75,6 @@ export function deliveryLimitOf(value) {
80
75
  }
81
76
  /** Space joined when none is given on the CLI (the `cotal-<space>` cmux tab, etc.). */
82
77
  export const DEFAULT_SPACE = "main";
83
- /** How many channel filters one multi-filter consumer create may carry.
84
- *
85
- * A create names every requested channel in one request, so the request grows with the channel
86
- * count and the CLIENT's request timeout is what gives way, not the broker: the create is never
87
- * refused, it just does not answer. Measured on an isolated broker, one message per channel,
88
- * `limit=5`: 70 filters answer in 43ms, 1,000 in 246ms, 5,000 in 5,345ms, and 10,000 does not
89
- * answer at all, failing `timeout` after 5,023ms. Note 5,000 SUCCEEDED while taking longer than
90
- * 10,000 took to fail, which is what says the ceiling is on the create request rather than on the
91
- * read: past roughly 5s the create itself is what times out.
92
- *
93
- * 1,000 is chosen from that sweep rather than from the failure point: it is a fifth of the largest
94
- * count that still answered, and it answers in a quarter second, so a batch stays far away from
95
- * both the timeout and the response-deadline budget the dashboard has to share with its DM read.
96
- * A space with the 69 chat channels this was built for is still ONE read, so the round-trip claim
97
- * in #1210 is unchanged at that size. */
98
- export const MULTI_FILTER_BATCH = 1_000;
99
- /** How many filter batches may be in flight at once. Bounded because the point of #1210 was to stop
100
- * issuing one read per channel: a space large enough to need batches must not get the fan-out back
101
- * under another name. */
102
- export const MULTI_FILTER_READ_CONCURRENCY = 4;
103
- /** A history reader's inactive threshold. It bounds only a reader whose delete never ran (its link
104
- * dropped, or its process stopped), because every read deletes its readers when it ends. Five
105
- * minutes is what the nats.js ordered consumers it replaces carried, and it keeps a reader alive
106
- * across the gap between pulls on a slow link. */
107
- const HISTORY_READER_INACTIVE_MS = 5 * 60_000;
108
78
  /** The longest delay `setTimeout` honors, 2^31-1 ms. */
109
79
  const MAX_TIMER_MS = 2_147_483_647;
110
80
  /** This endpoint's own `dm_` durable does not exist. A DM-only endpoint, and any endpoint on an
@@ -133,35 +103,6 @@ export class DmBindRefusedError extends Error {
133
103
  this.name = "DmBindRefusedError";
134
104
  }
135
105
  }
136
- /**
137
- * Events: "message" (CotalMessage), "presence" (PresenceEvent), "roster" (Presence[]), "error" (Error),
138
- * "connection" ({ connected: boolean }) — true on every successful (re)bind (initial start, manual
139
- * reconnect, AND background self-heal), false the moment the connection drops (rebuild null window /
140
- * terminal close). Lets an in-process agent track connectedness off the endpoint's own (re)binds
141
- * instead of an imperative flag the self-heal path can't reach; "transport" ({ connected, server? })
142
- * is the lower-level NATS socket edge, true before the full bind finishes and false during an internal
143
- * nats.js reconnect without changing `connection` readiness. "stored-copies-full" ({ limit: number }): a
144
- * stored copy arrived with the store at the count the copy was written under ({@link deliveryLimitOf}),
145
- * so the store may have discarded older copies before this session read them; emitted for each such
146
- * copy. A lowering of the count or an age set on the store removes copies without this event.
147
- *
148
- * Callers MUST attach an "error" listener before `start()`: async faults (incl. NATS
149
- * permission denials, surfaced via `watchStatus`) are emitted as "error", and Node throws
150
- * synchronously on an unhandled "error" — a missing listener turns any such fault into a
151
- * process crash instead of a logged denial.
152
- */
153
- /** Plane-3 trusted-reader redelivery ceiling: a dinbox entry that keeps failing re-auth-defer
154
- * (unknown owner) or DELIVER transfer is `term()`d + surfaced after this many redeliveries, so one
155
- * stuck/poison entry can't head-of-line the single shared reader forever. */
156
- const READER_MAX_REDELIVERIES = 10;
157
- /** Page size for the mediated history read when the caller names none — matches `channelHistory`'s
158
- * own default, since the mediated read exists to be a drop-in for it. */
159
- const READ_HISTORY_DEFAULT_LIMIT = 100;
160
- /** Hard server-side ceiling on one mediated page. The caller PROPOSES a limit and the mediator
161
- * decides: the reader is pooled and privileged, so an unbounded caller-chosen limit would let one
162
- * request pull a channel's whole retained set through it. Clamped, not refused — a UI asking for
163
- * more than a page should get a page, and `complete: false` already tells it more remains. */
164
- const READ_HISTORY_MAX_LIMIT = 200;
165
106
  /** The NEWEST prefix-from-the-end of `items` whose serialized `readHistory` reply fits `budget`
166
107
  * bytes, order preserved. Returns `[]` when not even the newest single message fits — the caller
167
108
  * must refuse loudly there rather than serve an empty page, which would read as "no history".
@@ -210,6 +151,13 @@ function membershipChange(e, update) {
210
151
  }
211
152
  return { type: "member", id: e.key, operation, revision: e.revision, update, member };
212
153
  }
154
+ /** The highest `prompts` presence `seq` this process used under each runtime id (SPEC §6). A reader
155
+ * orders lists by `seq` within one `runtime.id`, and a runtime id lives as long as the process, so
156
+ * this does too: across a stopped serve, a stopped endpoint and a new endpoint for the lifecycle. */
157
+ const promptsSeqs = new Map();
158
+ /** The principal endpoint each `{space, runtime id, endpoint}` is served by in this process. Two
159
+ * would subscribe the same subject and both answer one request (SPEC 13.21). */
160
+ const principalServers = new Map();
213
161
  export class CotalEndpoint extends EventEmitter {
214
162
  card;
215
163
  space;
@@ -302,10 +250,52 @@ export class CotalEndpoint extends EventEmitter {
302
250
  /** Caller-owned membership watches survive a connection rebuild as INTENT. Their iterators are
303
251
  * connection-scoped and are stopped/re-created around the epoch swap. */
304
252
  membershipFeedWatches = new Set();
305
- /** The live `ctl.delivery` serve subscription (delivery daemon) — re-created on every (re)connect by
306
- * {@link armDeliveryControl}; tracked so the stale one is dropped on reconnect. */
307
- deliveryServeSub;
308
- deliveryAdminServeSub;
253
+ /** The plane-3 delivery daemon side of this endpoint (fan-out writer, trusted reader, control
254
+ * responders) and the agent's DELIVER pump. Every dependency is read when it is used. */
255
+ deliveryPlane = new EndpointDeliveryPlane({
256
+ ackWaitMs: () => this.ackWaitMs,
257
+ actor: () => this.actor,
258
+ owner: () => this.owner,
259
+ allowSubscribe: () => this.allowSubscribe,
260
+ anycastUnavailable: () => this.anycastUnavailable,
261
+ card: () => this.card,
262
+ js: () => this.js,
263
+ jsm: () => this.jsm,
264
+ nc: () => this.nc,
265
+ ownLifecycleUid: () => this.ownLifecycleUid,
266
+ residentSwap: () => this.residentSwap,
267
+ space: () => this.space,
268
+ stopped: () => this.stopped,
269
+ streamMsgs: () => this.streamMsgs,
270
+ subs: () => this.subs,
271
+ userMode: () => this.userMode,
272
+ watchDeletePermissions: () => this.watchDeletePermissions,
273
+ maxPayload: () => this.maxPayload,
274
+ aclForAlias: (...args) => this.aclForAlias(...args),
275
+ channelHistory: (...args) => this.channelHistory(...args),
276
+ deliveryClassFresh: (...args) => this.deliveryClassFresh(...args),
277
+ durableJoinFor: (...args) => this.durableJoinFor(...args),
278
+ durableLeaveFor: (...args) => this.durableLeaveFor(...args),
279
+ ownerMemberships: (...args) => this.ownerMemberships(...args),
280
+ ownsDeliveryLease: (...args) => this.ownsDeliveryLease(...args),
281
+ readDeliveryLease: (...args) => this.readDeliveryLease(...args),
282
+ readDeliveryLeaseEntry: (...args) => this.readDeliveryLeaseEntry(...args),
283
+ reloadCreds: (...args) => this.reloadCreds(...args),
284
+ serveControl: (...args) => this.serveControl(...args),
285
+ aclRegistry: (...args) => this.aclRegistry(...args),
286
+ membersRegistry: (...args) => this.membersRegistry(...args),
287
+ armAnycastRequests: (...args) => this.armAnycastRequests(...args),
288
+ deliveryAnycast: (...args) => this.deliveryAnycast(...args),
289
+ dropAnycastRequests: (...args) => this.dropAnycastRequests(...args),
290
+ manager: (...args) => this.manager(...args),
291
+ notLiveMsg: (...args) => this.notLiveMsg(...args),
292
+ plane3MsgId: (...args) => this.plane3MsgId(...args),
293
+ publishDinbox: (...args) => this.publishDinbox(...args),
294
+ resolveOwnerByName: (...args) => this.resolveOwnerByName(...args),
295
+ scheduleResidentSwap: (...args) => this.scheduleResidentSwap(...args),
296
+ sendMentionNotice: (...args) => this.sendMentionNotice(...args),
297
+ emit: (event, ...args) => this.emit(event, ...args),
298
+ });
309
299
  /** The anycast take responder, the request records it writes, and their maintenance timer
310
300
  * (delivery daemon, SPEC §8.1). Bound with the other Plane-3 responders and dropped with them. */
311
301
  takeServeSub;
@@ -317,24 +307,6 @@ export class CotalEndpoint extends EventEmitter {
317
307
  anycastUnavailable;
318
308
  anycastSweep;
319
309
  takesInFlight = 0;
320
- /** The lifecycles whose membership inventory this daemon has been asked for, as principal to uids
321
- * (#4965). A durable join or leave bound to one of them is refused, so no row of a retiring lifecycle
322
- * is written after its inventory read starts. An entry goes when a later inventory for the same
323
- * principal finds its ACL row retired; an alias that never retires again keeps its one entry until
324
- * the daemon restarts. This is the second layer of the fence: it covers a manager that does not
325
- * narrow the ACL row before the inventory, on this daemon process only. The first is the ACL row
326
- * each join re-reads around its write (`durableJoinFor`'s `aclFence`), which holds across daemons. */
327
- retiringLifecycles = new Map();
328
- /** The durable joins and leaves running now, by `<principal>/<lifecycleUid>`: the writes an
329
- * inventory of that lifecycle waits for before it reads. */
330
- membershipWrites = new Map();
331
- /** True once {@link quiescePlane3} has stopped serving this shard pending an ownership answer.
332
- * Guards {@link armPlane3} so a RECONNECT cannot silently resume serving mid-question. */
333
- plane3Quiesced = false;
334
- /** When set, this endpoint hosts the Plane-3 fan-out writer + trusted reader (the server-side delivery
335
- * daemon). `aclFor` maps an owner id to its current read ACL (`allowSubscribe`) for the reader's
336
- * re-authorization — read FRESH per entry from the durable ACL registry KV, hence async. */
337
- plane3;
338
310
  /** Local cache of the channel registry (key = channel token), kept by a KV watch or, on an
339
311
  * endpoint that reads it on demand, by {@link readChannels}. */
340
312
  channelConfigs = new Map();
@@ -344,9 +316,31 @@ export class CotalEndpoint extends EventEmitter {
344
316
  /** Orders {@link readChannels} results: a read that settles after a newer one was applied is dropped. */
345
317
  channelReadGeneration = 0;
346
318
  channelReadApplied = 0;
347
- /** Serializes history reads ({@link collectHistory}): they share the fixed per-instance
348
- * `chathist_<id>` consumer, so overlapping reads would delete/recreate it under one another. */
349
- histLock = Promise.resolve();
319
+ history = new EndpointHistory({
320
+ connection: () => this.nc,
321
+ jetstream: () => this.js,
322
+ manager: () => this.jsm,
323
+ setManager: (manager) => { this.jsm = manager; },
324
+ space: () => this.space,
325
+ owner: () => this.owner,
326
+ actor: () => this.actor,
327
+ userMode: () => this.userMode,
328
+ connectionId: () => this.connId,
329
+ card: () => this.card,
330
+ notLiveMsg: () => this.notLiveMsg(),
331
+ requireLifecycleUid: (operation) => this.requireLifecycleUid(operation),
332
+ requestDelivery: (op, args) => this.requestDelivery(op, args),
333
+ deleteConsumerIfPresent: (stream, name) => this.deleteConsumerIfPresent(stream, name),
334
+ joinPolicyFresh: (channel) => this.joinPolicyFresh(channel),
335
+ chatMayHaveLost: (subject, sinceSeq) => this.chatMayHaveLost(subject, sinceSeq),
336
+ emit: (event, ...args) => this.emit(event, ...args),
337
+ decodeHistory: historyMessageFromDelivery,
338
+ isRecord,
339
+ isUsableMessageId,
340
+ isFromSubjectSender,
341
+ authenticatedMessage,
342
+ isPublishPermissionDenied,
343
+ });
350
344
  subs = [];
351
345
  /** The observer feeds {@link tap} was asked for. This is INTENT, re-subscribed by {@link bindTap}
352
346
  * on every (re)connect: a rebuild after an outage the library could not ride out closes the old
@@ -401,9 +395,6 @@ export class CotalEndpoint extends EventEmitter {
401
395
  /** `$JS.API.CONSUMER.DELETE.<stream>.<prefix>_` for each of this connection's own KV watches and
402
396
  * running membership scans (see {@link recordOwnWatchConsumer}). */
403
397
  ownWatchDeletePrefixes = new Set();
404
- /** Per stream, the history-reader names of this connection's current reader id and the reads
405
- * waiting for one (see {@link acquireHistoryReader}). */
406
- historyReaderSlots = new Map();
407
398
  /** True until the first successful connect completes its boot backfill — distinguishes first-connect
408
399
  * (backfill the boot channels' history) from a reconnect (reopen the core-subs, no re-backfill).
409
400
  * Persists across reconnect (NOT connection-scoped). Replaces the legacy chat-durable consumed-cursor
@@ -470,6 +461,8 @@ export class CotalEndpoint extends EventEmitter {
470
461
  * now instead of awaiting the full retryMs. */
471
462
  backoffResolve;
472
463
  backoffTimer;
464
+ /** Wakes for the membership retry loops' pauses (see {@link membershipPause}); stop() runs them all. */
465
+ membershipPauses = new Set();
473
466
  retryMs = 3000;
474
467
  /** Consecutive failed rebuilds, driving {@link nextRetryDelayMs}. Reset on every success. */
475
468
  retryAttempt = 0;
@@ -1667,19 +1660,19 @@ export class CotalEndpoint extends EventEmitter {
1667
1660
  // Handles that armDeliveryControl created on a previous connection: null them so a rebind
1668
1661
  // does not carry a dead protocol's subs forward. Best-effort unsubscribe first (the sub is
1669
1662
  // dead with its connection either way; unsubscribing an already-dead sub is a noop).
1670
- if (this.deliveryServeSub) {
1663
+ if (this.deliveryPlane.deliveryServeSub) {
1671
1664
  try {
1672
- this.deliveryServeSub.unsubscribe();
1665
+ this.deliveryPlane.deliveryServeSub.unsubscribe();
1673
1666
  }
1674
1667
  catch { /* dead with the old connection */ }
1675
- this.deliveryServeSub = undefined;
1668
+ this.deliveryPlane.deliveryServeSub = undefined;
1676
1669
  }
1677
- if (this.deliveryAdminServeSub) {
1670
+ if (this.deliveryPlane.deliveryAdminServeSub) {
1678
1671
  try {
1679
- this.deliveryAdminServeSub.unsubscribe();
1672
+ this.deliveryPlane.deliveryAdminServeSub.unsubscribe();
1680
1673
  }
1681
1674
  catch { /* dead with the old connection */ }
1682
- this.deliveryAdminServeSub = undefined;
1675
+ this.deliveryPlane.deliveryAdminServeSub = undefined;
1683
1676
  }
1684
1677
  this.dropAnycastRequests();
1685
1678
  for (const sub of this.subs) {
@@ -1921,6 +1914,21 @@ export class CotalEndpoint extends EventEmitter {
1921
1914
  this.backoffTimer = undefined;
1922
1915
  }
1923
1916
  }
1917
+ /** Capped exponential pause between membership retries. stop() ends it early, so a loop waiting
1918
+ * on an absent delivery daemon never holds a stopped endpoint's process open for the delay. */
1919
+ membershipPause(attempt) {
1920
+ if (this.stopped)
1921
+ return Promise.resolve();
1922
+ return new Promise((resolve) => {
1923
+ const wake = () => {
1924
+ clearTimeout(timer);
1925
+ this.membershipPauses.delete(wake);
1926
+ resolve();
1927
+ };
1928
+ const timer = setTimeout(wake, Math.min(30_000, 1000 * 2 ** attempt));
1929
+ this.membershipPauses.add(wake);
1930
+ });
1931
+ }
1924
1932
  /** Manual reconnect: tear down the current connection and rebuild, WITHOUT the permanent
1925
1933
  * stop (stopped/stopping stay false). Serialized with the self-heal supervisor via
1926
1934
  * {@link rebuild}, and interruptible — if a backoff is in flight, kick it so the attempt
@@ -1939,7 +1947,14 @@ export class CotalEndpoint extends EventEmitter {
1939
1947
  throw e;
1940
1948
  }
1941
1949
  }
1942
- /** The presence epoch moves first: a bind still awaiting the broker must find it moved
1950
+ /** Stop the endpoint and close its connection.
1951
+ *
1952
+ * Every principal serve ends first: its handlers' `signal` aborts, and stop waits for each
1953
+ * handler in flight until its request's deadline at most, so a handler that never settles does
1954
+ * not hold stop open longer than 30 s, the route's cap on a deadline. A reply that comes after
1955
+ * that is dropped.
1956
+ *
1957
+ * The presence epoch moves first: a bind still awaiting the broker must find it moved
1943
1958
  * before any await below gives it a window to install a watch on a stopped endpoint
1944
1959
  * (see {@link startPresenceWatch}). */
1945
1960
  stop() {
@@ -1954,10 +1969,16 @@ export class CotalEndpoint extends EventEmitter {
1954
1969
  }
1955
1970
  this.activeGoalFollowers.clear();
1956
1971
  this.presenceEpoch++;
1972
+ // Every principal serve ends now, so a `ready` still waiting for the broker rejects and no new
1973
+ // request is taken. Its handlers in flight finish below, before the connection they reply on
1974
+ // closes, or their requests' deadlines pass.
1975
+ const principalStops = [...this.principalServes.values()].map((intent) => this.stopPrincipalServe(intent, false));
1957
1976
  this.presence.cancelRebind();
1958
1977
  // Wake a reestablishLoop sitting in backoff so it sees `stopped` and exits instead of
1959
1978
  // sleeping out retryMs; also clears the timer so it can't fire later.
1960
1979
  this.kickBackoff();
1980
+ for (const wake of [...this.membershipPauses])
1981
+ wake();
1961
1982
  if (this.heartbeatTimer)
1962
1983
  clearInterval(this.heartbeatTimer);
1963
1984
  if (this.sweepTimer)
@@ -1999,6 +2020,7 @@ export class CotalEndpoint extends EventEmitter {
1999
2020
  this.retireKvWatch(this.channelWatchIter);
2000
2021
  this.channelWatchIter = undefined;
2001
2022
  this.stopDeliveryLeaseWatch();
2023
+ await Promise.all(principalStops);
2002
2024
  try {
2003
2025
  if (this.doRegister) {
2004
2026
  await this.presence.publishOffline();
@@ -2176,6 +2198,7 @@ export class CotalEndpoint extends EventEmitter {
2176
2198
  *
2177
2199
  * `expectedLastSubjectSeq` is a parameter because it is a header VALUE: sizing at 0 and publishing
2178
2200
  * at 123456 differ by five bytes.
2201
+ * `expectedLastMsgId` is measured when given, so a caller that may send it can size for it.
2179
2202
  *
2180
2203
  * Residual: `ts` is re-stamped at publish, so the two differ in value — not in length until
2181
2204
  * epoch-millis needs a 14th digit.
@@ -2193,6 +2216,10 @@ export class CotalEndpoint extends EventEmitter {
2193
2216
  const mh = headers();
2194
2217
  mh.set("Nats-Msg-Id", opts.id);
2195
2218
  mh.set("Nats-Expected-Last-Subject-Sequence", `${opts.expectedLastSubjectSeq}`);
2219
+ if (opts.expectedLastMsgId !== undefined) {
2220
+ assertIdToken(opts.expectedLastMsgId, "expected last msg id");
2221
+ mh.set("Nats-Expected-Last-Msg-Id", opts.expectedLastMsgId);
2222
+ }
2196
2223
  // `encode()` is on the client's header implementation but not on the published `MsgHdrs` type,
2197
2224
  // so it is reached through a cast rather than copied. If a client version drops it, this throws
2198
2225
  // immediately and loudly — the calibration cell would also fail — instead of returning a number
@@ -2218,6 +2245,10 @@ export class CotalEndpoint extends EventEmitter {
2218
2245
  * means "the subject must be empty". A mismatch throws, and the throw stays classifiable by the
2219
2246
  * already-public {@link isCasLoss} — the error is deliberately **not wrapped**, since wrapping
2220
2247
  * would hide the `err_code` that classification reads.
2248
+ * - `expectedLastMsgId`, when given, becomes `Nats-Expected-Last-Msg-Id`. The broker checks it after
2249
+ * its dedup cache, so a fresh value nobody has published turns a publish into a probe: an id the
2250
+ * cache holds answers `duplicate: true` with the stored sequence, and any other id is refused
2251
+ * with error 10070 before anything is stored.
2221
2252
  *
2222
2253
  * @throws if the endpoint is not live, the channel is not concrete, `id` is malformed, `parts` is
2223
2254
  * empty, or `expectedLastSubjectSeq` is not a non-negative safe integer.
@@ -2235,11 +2266,13 @@ export class CotalEndpoint extends EventEmitter {
2235
2266
  throw new Error(`expectedLastSubjectSeq must be a non-negative safe integer, got ${JSON.stringify(expected)}`);
2236
2267
  if (!Array.isArray(opts.parts) || opts.parts.length === 0)
2237
2268
  throw new Error("multicastExpecting requires at least one part");
2269
+ if (opts.expectedLastMsgId !== undefined)
2270
+ assertIdToken(opts.expectedLastMsgId, "expected last msg id");
2238
2271
  const message = this.casEnvelope(opts);
2239
2272
  assertPartsSerializable(message.parts);
2240
2273
  // Publish DIRECTLY rather than through publishMsg: this path must set the expectation and read
2241
2274
  // the ack, and publishMsg deliberately does neither.
2242
- const ack = await this.js.publish(chatSubject(this.space, this.owner, this.actor, opts.channel), JSON.stringify(message), { msgID: opts.id, expect: { lastSubjectSequence: expected } });
2275
+ const ack = await this.js.publish(chatSubject(this.space, this.owner, this.actor, opts.channel), JSON.stringify(message), { msgID: opts.id, expect: { lastSubjectSequence: expected, ...(opts.expectedLastMsgId !== undefined ? { lastMsgID: opts.expectedLastMsgId } : {}) } });
2243
2276
  return { message, ack: { seq: ack.seq, duplicate: ack.duplicate === true } };
2244
2277
  }
2245
2278
  /** Publish a payload frame on this principal's forge-locked sibling event rail. */
@@ -3767,13 +3800,7 @@ export class CotalEndpoint extends EventEmitter {
3767
3800
  * published by that principal: the same guarantee a live subscription to that subject gives.
3768
3801
  * Without it the read is the channel from any sender. */
3769
3802
  async channelHistory(channel, opts) {
3770
- const from = opts?.publisher;
3771
- // A publisher is one principal. A `*` in either slot would widen the read back to any sender
3772
- // while the caller believes it narrowed it, so both slots are checked as concrete tokens.
3773
- const subject = from === undefined
3774
- ? chatSubject(this.space, "*", "*", channel)
3775
- : chatSubject(this.space, assertValidOwnerToken(from.owner), assertValidOwnerToken(from.actor), channel);
3776
- return (await this.streamHistory(chatStream(this.space), [subject], opts?.limit ?? 100, undefined, opts?.signal)).map((r) => r.msg);
3803
+ return this.history.channelHistory(channel, opts);
3777
3804
  }
3778
3805
  /** Read one concrete channel using this lifecycle's single-filter CHAT grant (SPEC §8/§9).
3779
3806
  * No observer authority, presence or delivery ACL mirror is required. Returns the newest page,
@@ -3781,87 +3808,7 @@ export class CotalEndpoint extends EventEmitter {
3781
3808
  * Reads have a ten-second budget, shortened by `signal`. Concurrent connections sharing this
3782
3809
  * lifecycle wait for native consumer inactivity cleanup; they never delete another reader. */
3783
3810
  async channelHistoryPage(channel, opts) {
3784
- assertValidChannel(channel);
3785
- if (!isConcreteChannel(channel))
3786
- throw new Error("channelHistoryPage requires one concrete channel");
3787
- const limit = opts?.limit ?? 50;
3788
- if (!Number.isSafeInteger(limit) || limit < 1 || limit > 1000)
3789
- throw new Error("channelHistoryPage limit must be a whole number from 1 to 1000");
3790
- if (!this.nc || !this.js)
3791
- throw new Error(this.notLiveMsg());
3792
- const jsm = this.jsm ??= await jetstreamManager(this.nc);
3793
- const signal = AbortSignal.any([AbortSignal.timeout(10_000), ...(opts?.signal ? [opts.signal] : [])]);
3794
- signal.throwIfAborted();
3795
- const stream = chatStream(this.space);
3796
- const name = chatHistDurable(this.owner, this.actor, this.requireLifecycleUid("channelHistoryPage"));
3797
- const subject = chatSubject(this.space, "*", "*", channel);
3798
- const marker = randomUUID();
3799
- let created;
3800
- for (;;) {
3801
- signal.throwIfAborted();
3802
- try {
3803
- // The pinned client's add is CREATE-only. A unique description makes even an otherwise
3804
- // identical occupied config conflict. Never bind/update it or pre-delete by shared name.
3805
- created = await jsm.consumers.add(stream, {
3806
- name, description: marker, filter_subject: subject,
3807
- ack_policy: AckPolicy.None, deliver_policy: DeliverPolicy.All,
3808
- mem_storage: true, inactive_threshold: nanos(250),
3809
- });
3810
- break;
3811
- }
3812
- catch (e) {
3813
- if (!(e instanceof JetStreamApiError) || e.code !== 10148)
3814
- throw e; // consumer already exists
3815
- await delay(50, undefined, { signal });
3816
- }
3817
- }
3818
- signal.throwIfAborted();
3819
- const owns = (info) => info.created === created.created &&
3820
- info.config.description === marker && info.config.filter_subject === subject;
3821
- if (!owns(created))
3822
- throw new Error("channelHistoryPage: consumer ownership was not established");
3823
- const consumer = await this.js.consumers.get(stream, name);
3824
- const pending = created.num_pending;
3825
- const out = [];
3826
- let delivered = 0;
3827
- if (pending > 0) {
3828
- const iter = await consumer.consume({ max_messages: Math.min(pending, 32) });
3829
- const stop = () => iter.stop(abortReason(signal));
3830
- signal.addEventListener("abort", stop, { once: true });
3831
- try {
3832
- signal.throwIfAborted();
3833
- for await (const m of iter) {
3834
- signal.throwIfAborted();
3835
- if (!subjectMatches(subject, m.subject))
3836
- throw new Error("channelHistoryPage: consumer delivered another channel; ownership was lost");
3837
- delivered++;
3838
- const msg = historyMessageFromDelivery(m);
3839
- if (msg) {
3840
- out.push(msg);
3841
- if (out.length > limit + 1)
3842
- out.shift();
3843
- }
3844
- if (delivered >= pending)
3845
- break;
3846
- }
3847
- }
3848
- finally {
3849
- signal.removeEventListener("abort", stop);
3850
- iter.stop();
3851
- }
3852
- signal.throwIfAborted();
3853
- if (delivered !== pending)
3854
- throw new Error(`channelHistoryPage: read ${delivered} of ${pending} messages before the window was cut short`);
3855
- }
3856
- signal.throwIfAborted();
3857
- // A legacy join/recall reader can delete/recreate this fixed name. A final ownership check
3858
- // makes that interference loud. It is detection, not a fence against those legacy readers.
3859
- if (!owns(await consumer.info()))
3860
- throw new Error("channelHistoryPage: consumer ownership was lost");
3861
- signal.throwIfAborted();
3862
- // No DELETE, even after abort: a delayed owner could otherwise delete its successor between
3863
- // INFO and DELETE. Native inactivity reaps this transient reader without that TOCTOU race.
3864
- return { items: out.slice(-limit), complete: out.length <= limit };
3811
+ return this.history.channelHistoryPage(channel, opts);
3865
3812
  }
3866
3813
  /**
3867
3814
  * The newest `limit` chat messages across concrete channels or one channel family, oldest-first within the page,
@@ -3892,68 +3839,7 @@ export class CotalEndpoint extends EventEmitter {
3892
3839
  * so the broker checks the credential's grant for that filter. No additional authority is minted.
3893
3840
  */
3894
3841
  async multiChannelHistory(channels, opts) {
3895
- const subjects = [...new Set(channels.map((channel) => {
3896
- if (!isConcreteChannel(channel) && !(channels.length === 1 && isChannelFamily(channel)))
3897
- throw new Error(`multiChannelHistory: "${channel}" is a wildcard channel - name one channel family alone or concrete channels without wildcards`);
3898
- // `chatSubject` builds the filter through `token()`, which REWRITES rather than refuses: it
3899
- // maps a character a subject may not carry to `_`, trims each segment, and drops empty ones.
3900
- // So "foo/bar" would filter on `foo_bar`, ".lead" on `lead`, and "team..b" on `team.b` - the
3901
- // caller names one channel and the broker returns another, which is the exact promise this
3902
- // method makes ("a channel left out of the list never crosses the link") inverted. This is the
3903
- // same aliasing `assertValidChannel` was written for on the policy path; the read path needs
3904
- // it too, and it fails loud rather than serving a channel nobody asked for.
3905
- assertValidChannel(channel);
3906
- return chatSubject(this.space, "*", "*", channel);
3907
- }))];
3908
- // No channels is not an empty stream, but it IS an empty answer, and asking the broker for a
3909
- // consumer with no filter would read the WHOLE stream instead of none of it.
3910
- if (subjects.length === 0)
3911
- return [];
3912
- const limit = opts?.limit ?? 100;
3913
- // ONE CREATE CANNOT CARRY AN UNBOUNDED FILTER LIST. The create names every subject in one
3914
- // request and the client's request timeout is what gives way, so past roughly 5,000 filters the
3915
- // read does not answer at all and the route loses the whole chat source: measured on an
3916
- // isolated broker, 10,000 channels failed `timeout` after 5,023ms while the fan-out this
3917
- // replaced still returned 2,739 messages on the same corpus. Reading in batches keeps the
3918
- // single-read cost at the sizes this was built for (69 channels is one batch, so #1210's
3919
- // round-trip numbers are unchanged) and degrades to a few reads instead of none above that.
3920
- // `batch` is a knob rather than a constant so the batched path can be compared against the
3921
- // single-create path on ONE corpus: forcing a small batch makes a space that would otherwise be
3922
- // one read take many, which is the only way to assert the two select the same messages.
3923
- const size = Math.max(1, Math.trunc(opts?.batch ?? MULTI_FILTER_BATCH));
3924
- const batches = [];
3925
- for (let i = 0; i < subjects.length; i += size)
3926
- batches.push(subjects.slice(i, i + size));
3927
- const pages = new Array(batches.length);
3928
- let next = 0;
3929
- const readBatch = async () => {
3930
- for (;;) {
3931
- const i = next++;
3932
- if (i >= batches.length)
3933
- return;
3934
- pages[i] = await this.streamHistory(chatStream(this.space), batches[i], limit, undefined, opts?.signal);
3935
- }
3936
- };
3937
- // A BATCH THAT FAILS FAILS THE READ. Returning the batches that answered would be the newest
3938
- // `limit` across SOME of the channels asked for while looking like the newest across all of
3939
- // them, which is a wrong page presented as a right one. Throwing keeps the caller's existing
3940
- // envelope: the dashboard marks `chat` missing and says the page is partial, which is what it
3941
- // already did when this method was one read.
3942
- await Promise.all(Array.from({ length: Math.min(MULTI_FILTER_READ_CONCURRENCY, batches.length) }, readBatch));
3943
- // MERGE ON ARRIVAL, NOT ON `ts`. Each batch returns the newest `limit` within its own subjects,
3944
- // and the newest `limit` overall is a subset of their union, so re-selecting by stream sequence
3945
- // reproduces exactly what one create over the whole list would have selected. Sorting by the
3946
- // payload's `ts` here instead would change which messages the page holds, which is the
3947
- // selection question this pull request already had to answer once.
3948
- const rows = pages.flat().sort((a, b) => a.seq - b.seq).slice(-limit);
3949
- return rows.map(({ seq, subject, msg }) => {
3950
- const p = parseSubject(subject);
3951
- // The filter set is built from chat subjects, so this cannot fire against a healthy broker.
3952
- // It is here because the alternative to raising is tagging a message with a guess.
3953
- if (p?.kind !== "chat")
3954
- throw new Error(`multiChannelHistory: the broker delivered ${subject}, which is not a chat subject on this space`);
3955
- return { channel: p.rest, seq, msg };
3956
- });
3842
+ return this.history.multiChannelHistory(channels, opts);
3957
3843
  }
3958
3844
  /** Read a channel's recent history through the delivery daemon, using a provisioned ACL mirror.
3959
3845
  * SPEC's ban on raw control-surface stream reads does not remove the single-channel CHAT grant.
@@ -3983,16 +3869,7 @@ export class CotalEndpoint extends EventEmitter {
3983
3869
  * different authorization model, and one handler with two authz paths is the wrong shape on the
3984
3870
  * surface where a mistake exposes private messages). */
3985
3871
  async readHistory(channel, opts) {
3986
- const reply = await this.requestDelivery("readHistory", { channel, limit: opts?.limit });
3987
- if (!reply.ok)
3988
- throw new Error(reply.error ?? "readHistory failed");
3989
- const data = reply.data;
3990
- // Validate rather than coerce. A malformed reply must not be massaged into a plausible page:
3991
- // defaulting `complete` would invent the very signal a caller uses to decide whether it is
3992
- // looking at the start of a conversation.
3993
- if (!Array.isArray(data?.items) || typeof data?.complete !== "boolean")
3994
- throw new Error("readHistory: the delivery daemon returned a malformed page (expected { items, complete })");
3995
- return { items: data.items, complete: data.complete };
3872
+ return this.history.readHistory(channel, opts);
3996
3873
  }
3997
3874
  /** Fetch recent DMs (any sender→any recipient) from the space's DM backlog. `signal` cancels the
3998
3875
  * active pull and reclaims its ephemeral consumer. God-view only:
@@ -4002,15 +3879,7 @@ export class CotalEndpoint extends EventEmitter {
4002
3879
  * returned — a stored row missing `ts` / `space` / `parts` / `from.name` is dropped, never
4003
3880
  * returned with that member silently `undefined`. */
4004
3881
  async dmHistory(opts) {
4005
- try {
4006
- // every inst.<recipOwner>.<recipActor>.<sndOwner>.<sndActor> DM — the whole DM subtree (god-view)
4007
- return (await this.streamHistory(dmStream(this.space), [`${spacePrefix(this.space)}.inst.>`], opts?.limit ?? 100, undefined, opts?.signal)).map((r) => r.msg);
4008
- }
4009
- catch (e) {
4010
- if (isPermissionDenied(e))
4011
- return [];
4012
- throw e;
4013
- }
3882
+ return this.history.dmHistory(opts);
4014
3883
  }
4015
3884
  /** The newest `limit` DMs (100 by default) sent to or by any of `principals`, oldest first. Each
4016
3885
  * principal is a concrete `<owner>.<actor>`; anything else throws, because a wildcard would widen
@@ -4069,434 +3938,7 @@ export class CotalEndpoint extends EventEmitter {
4069
3938
  * `before` pages toward the past: pass the `seq` of the oldest message you already have.
4070
3939
  */
4071
3940
  async streamHistory(stream, subjects, limit, before, signal) {
4072
- if (!this.nc)
4073
- throw new Error(this.notLiveMsg());
4074
- signal?.throwIfAborted();
4075
- // A LIMIT THAT IS NOT A FINITE NUMBER HAS NO ANSWER, AND THE SEARCH BELOW CANNOT REFUSE IT.
4076
- // Every comparison against NaN is false, so `limit <= 0` does not fire for one, and neither of
4077
- // the widening loop's exits can ever be true either: `page.length >= NaN` is false forever and
4078
- // `start === 1` compares against a `start` that is itself NaN. The loop does not return
4079
- // everything, it never returns. Measured through the dashboard on a real broker: no answer
4080
- // after 30s, and the abandoned read still consuming half a core fifteen seconds after its
4081
- // caller had gone, while the process kept serving every other route so nothing announced it.
4082
- // `Infinity` reaches the other end of the same hole: it passes the guard, `start` collapses to
4083
- // 1, and `slice(-Infinity)` is the subject's whole retained set.
4084
- // A caller that computed a limit it cannot state is told so, rather than handed a process that
4085
- // quietly spins. `0` and negatives keep their existing meaning, an empty page.
4086
- // A COUNT OF MESSAGES IS A WHOLE NUMBER THIS SERVER CAN COUNT EXACTLY, and the check sits ABOVE
4087
- // the zero check on purpose: `-Infinity <= 0` is true, so a guard placed below it would fold a
4088
- // limit nobody can answer into a silent empty page. `NaN` never returns (both widening exits
4089
- // compare against it and are false forever). `Infinity` and any magnitude past the safe range
4090
- // collapse `start` to 1. A fraction is the quietest of the set: the page is taken with
4091
- // `slice(-limit)` and slice truncates toward zero, so any limit in (0,1) becomes `slice(0)` and
4092
- // hands back the ENTIRE retained history. Measured on 30 retained: 0.5 and 0.9 both returned all
4093
- // 30, and 2.5 returned 2, a page nobody asked for.
4094
- if (!Number.isSafeInteger(limit))
4095
- throw new Error(`history limit must be a whole number of messages this server can count exactly, received ${limit}`);
4096
- if (limit <= 0)
4097
- return [];
4098
- const nc = this.nc;
4099
- let reader;
4100
- try {
4101
- reader = await this.acquireHistoryReader(nc, this.jsm ?? (this.jsm = await jetstreamManager(nc)), stream, signal);
4102
- // THE EXACT CEILING, on the already-granted surface. A one-shot consumer with
4103
- // `DeliverPolicy.Last` plus this subject's filter reports the newest MATCHING sequence, and its
4104
- // `num_pending` of zero means the channel is genuinely empty. Using the STREAM's last sequence
4105
- // instead (which is all `STREAM.INFO` offers) was a loose upper bound, and on a quiet channel
4106
- // in a busy stream the gap between the two is the whole problem: every window near the stream
4107
- // head is empty, so the search widened over and over before finding anything.
4108
- //
4109
- // Deliberately NOT `getMessage({ last_by_subj })`, which would be the obvious way to ask: it
4110
- // needs `$JS.API.STREAM.MSG.GET`, which read credentials do not hold. That grant hole already
4111
- // shipped once from this function and turned every non-admin history read into an empty list.
4112
- const ceiling = before !== undefined ? before - 1 : await this.lastMatchingSeq(reader, subjects, signal);
4113
- if (ceiling < 1)
4114
- return [];
4115
- // Widen from the exact ceiling until a window holds a full page, or until the window IS the
4116
- // whole subject. Draining each attempt is bounded, and that is the point: a window only fails
4117
- // when it holds FEWER than `limit` matches, so every wasted drain moves less than one page.
4118
- // Geometric growth keeps the number of attempts logarithmic, so total wasted transfer is a
4119
- // small multiple of a page.
4120
- //
4121
- // THE FIRST WINDOW IS ONE PAGE WIDE, NOT FOUR. It used to open at `limit * 4` on the reasoning
4122
- // that a filtered subject is sparse inside its stream and a wider first look would land the
4123
- // page in one drain. That is true for one channel of a busy stream and false for a subject
4124
- // that IS most of its stream, and the second case is the one a whole-backlog read takes: the
4125
- // window succeeds on the first attempt and drains four pages to keep one, because
4126
- // `drainWindow` delivers everything in the window and keeps the tail. Measured on `/api/dms`
4127
- // at limit 500 against a 2500-message backlog: 1,995,854 to 1,995,859 bytes moved across
4128
- // four runs to return a 346,001-byte page, and 8161ms to 8753ms on that link, so all four
4129
- // missed the dashboard's 8000ms deadline with nothing else on the connection (Cotal #1210).
4130
- // That cost does not keep growing with the backlog, and saying it did was wrong: the old first
4131
- // span was `max(limit * 4, 64)`, so at limit 500 it is 2000 messages, and ANY backlog of 2000
4132
- // or more drains the same 2000-message window. The point is that the window, not the backlog,
4133
- // sets the cost, and four pages to return one is already past the deadline on this link for
4134
- // every such deployment. One page wide makes the
4135
- // SUCCESSFUL drain
4136
- // obey the same bound the failed ones already promised: it moves at most one page. A sparse
4137
- // subject pays one more widening step for that, which is four round trips against a transfer
4138
- // several times the size of the answer.
4139
- //
4140
- // A SHORT PAGE is either "the channel is exhausted" or "the window is still above the
4141
- // first match". Sequence 1 is the wrong floor for the first of those: three matches at
4142
- // the high end of a busy stream are exhausted as soon as the window's lower edge passes
4143
- // the subject's first matching sequence, and walking on to 1 re-reads everyone else's
4144
- // retained set (#840). Probe that floor only after a short drain so a dense page (one
4145
- // drain, full) never pays for it. The remaining unbounded-looking case is a subject
4146
- // whose FIRST match really is near sequence 1; that span is the channel's own, not the
4147
- // stream's.
4148
- let span = Math.max(limit, 64);
4149
- let floor = 1;
4150
- let floorKnown = false;
4151
- for (;;) {
4152
- signal?.throwIfAborted();
4153
- const start = Math.max(floor, ceiling - span + 1);
4154
- const page = await this.drainWindow(reader, subjects, start, ceiling, limit, signal);
4155
- if (page.length >= limit)
4156
- return page.slice(-limit);
4157
- if (!floorKnown) {
4158
- floor = Math.max(1, await this.firstMatchingSeq(reader, subjects, signal));
4159
- floorKnown = true;
4160
- }
4161
- if (start <= floor)
4162
- return page.slice(-limit);
4163
- span *= 4;
4164
- }
4165
- }
4166
- catch (e) {
4167
- // NARROW. This catch is how the MSG.GET grant bug shipped: it turned a Permissions Violation
4168
- // into an empty history, so every non-admin read looked exactly like a quiet channel, and the
4169
- // smokes stayed green because they run as admin against a local broker. Emitting on the error
4170
- // event is not enough either — callers consume the RETURN VALUE, and the dashboard renders
4171
- // empty regardless of what an operator log says.
4172
- //
4173
- // So only one thing may still produce an empty result here: the stream does not exist (a space
4174
- // with no history yet). Anything else — a denial, a timeout, a 503, a protocol or
4175
- // consumer-create failure — is raised, because "no history" and "I could not read the history"
4176
- // are different answers and only one of them is safe to render as an empty conversation.
4177
- // `dmHistory` alone turns a denial into its documented empty god-view result.
4178
- const msg = String(e?.message ?? "");
4179
- if (/stream not found/i.test(msg) || e?.code === 404)
4180
- return [];
4181
- throw e;
4182
- }
4183
- finally {
4184
- if (reader)
4185
- this.releaseHistoryReader(reader);
4186
- }
4187
- }
4188
- /** This connection's history-reader id, the key of its credential's reader deletes
4189
- * ({@link historyReaderConsumer}). A static credential's is its connId, the nkey it signs with.
4190
- * Under the auth callout the connId is the inbox nonce this client chose, so the callout keys the
4191
- * deletes on {@link calloutReaderId} instead: the server's id and the client id it assigned this
4192
- * connection, which the client reads from the server's INFO and does not choose. */
4193
- historyReaderId(nc) {
4194
- if (!this.userMode)
4195
- return this.connId;
4196
- const info = nc.info;
4197
- if (!info)
4198
- throw new Error("history: the server has not sent its INFO, so this connection has no history reader id yet");
4199
- return calloutReaderId(info.server_id, info.client_id);
4200
- }
4201
- /** Take one of this connection's history-reader names on `stream` ({@link historyReaderConsumer})
4202
- * for one read. The read holds the name for its whole walk and creates and deletes a consumer
4203
- * under it at each step, so this connection keeps at most {@link HISTORY_READERS} reader consumers
4204
- * on a stream (#4123). A read past that many waits for a name, and `signal` ends the wait. A
4205
- * reconnect under the auth callout changes the reader id, and reads from then on take the new
4206
- * id's names. */
4207
- async acquireHistoryReader(nc, jsm, stream, signal) {
4208
- for (;;) {
4209
- signal?.throwIfAborted();
4210
- const readerId = this.historyReaderId(nc);
4211
- let pool = this.historyReaderSlots.get(stream);
4212
- if (pool?.readerId !== readerId) {
4213
- pool = { readerId, free: Array.from({ length: HISTORY_READERS }, (_, i) => HISTORY_READERS - 1 - i), held: 0, waiting: [], refusedUntil: 0 };
4214
- this.historyReaderSlots.set(stream, pool);
4215
- }
4216
- // A read here would leave one more reader it may not delete, so it fails before creating one.
4217
- if (pool.refusedUntil > Date.now())
4218
- throw this.readerDeleteRefused(stream);
4219
- const slot = pool.free.pop();
4220
- if (slot !== undefined) {
4221
- pool.held++;
4222
- return { nc, js: jetstream(nc), jsm, stream, pool, slot, name: historyReaderConsumer(readerId, slot), held: true, left: false };
4223
- }
4224
- // No read here holds a name, so none will come back from one: every name is set aside (see
4225
- // openHistoryReader and closeHistoryReader). Waiting would hang until those expire.
4226
- if (pool.held === 0)
4227
- throw new Error(`history: every history reader name this connection has on ${stream} is set aside, held by a consumer it did not create (another process using the same credential, or one that stopped mid-read) or by a reader it could not confirm deleted; they expire ${HISTORY_READER_INACTIVE_MS / 60_000} minutes after their last use`);
4228
- const waitOn = pool;
4229
- await new Promise((resolve, reject) => {
4230
- const wake = () => { signal?.removeEventListener("abort", stop); resolve(); };
4231
- const stop = () => { waitOn.waiting.splice(waitOn.waiting.indexOf(wake), 1); reject(abortReason(signal)); };
4232
- waitOn.waiting.push(wake);
4233
- signal?.addEventListener("abort", stop, { once: true });
4234
- });
4235
- }
4236
- }
4237
- /** Give a read's history-reader name back and wake one read waiting for a name. A name whose
4238
- * delete failed is set aside instead. */
4239
- releaseHistoryReader(reader) {
4240
- if (!reader.held)
4241
- return;
4242
- reader.held = false;
4243
- const pool = reader.pool;
4244
- pool.held--;
4245
- if (!reader.left) {
4246
- pool.free.push(reader.slot);
4247
- pool.waiting.shift()?.();
4248
- return;
4249
- }
4250
- this.setAsideHistoryReader(pool, reader.slot);
4251
- // Every waiter checks again, so one that no running read can now serve fails instead of hanging.
4252
- if (pool.held === 0)
4253
- for (const wake of pool.waiting.splice(0))
4254
- wake();
4255
- }
4256
- /** Keep `slot` out of use until a consumer under it would have expired, then free it. */
4257
- setAsideHistoryReader(pool, slot) {
4258
- setTimeout(() => { pool.free.push(slot); pool.waiting.shift()?.(); }, HISTORY_READER_INACTIVE_MS).unref();
4259
- }
4260
- /** Create the reader consumer for one step of a read, under the read's name. It is a plain pull
4261
- * consumer: an nats.js ordered consumer renames itself on every rebuild, and the observer and admin
4262
- * credentials may delete only these exact names.
4263
- *
4264
- * Every create carries a fresh description. NATS answers a create whose config equals the existing
4265
- * consumer's with that consumer, delivery state and all (measured on nats-server 2.15), and a read
4266
- * repeats the same config on the same name. Without the description a reader left under the name
4267
- * would come back as if new: a `DeliverPolicy.Last` reader that already delivered its message
4268
- * reports nothing pending, which reads as an empty channel. With it, a create on a name in use is
4269
- * refused, and a consumer that does not carry this step's description is never used.
4270
- *
4271
- * A consumer already under the name is never deleted, whoever left it: another holder of this
4272
- * credential, a process that stopped mid-read, or this endpoint when a delete's answer was lost and
4273
- * the name was taken again since. The read moves to a free name, and the taken one stays out of use
4274
- * until such a consumer would have expired. */
4275
- async openHistoryReader(reader, config) {
4276
- // A single filter must ride the filter-bearing API subject so scoped read grants apply.
4277
- if (config.filter_subjects?.length === 1) {
4278
- const { filter_subjects, ...rest } = config;
4279
- config = { ...rest, filter_subject: filter_subjects[0] };
4280
- }
4281
- const pool = reader.pool;
4282
- for (;;) {
4283
- // A reconnect under the auth callout moved this connection to new reader names, and its
4284
- // credential may no longer delete this one. Creating under it would leave the reader behind.
4285
- if (this.historyReaderId(reader.nc) !== pool.readerId)
4286
- throw new Error(`history: the connection was re-established during the read on ${reader.stream}, so its history reader names changed; the read is not served`);
4287
- const description = `cotal history read ${randomUUID()}`;
4288
- let info;
4289
- try {
4290
- info = await reader.jsm.consumers.add(reader.stream, {
4291
- ...config,
4292
- name: reader.name,
4293
- description,
4294
- ack_policy: AckPolicy.None,
4295
- max_deliver: 1,
4296
- mem_storage: true,
4297
- num_replicas: 1,
4298
- inactive_threshold: nanos(HISTORY_READER_INACTIVE_MS),
4299
- });
4300
- }
4301
- catch (e) {
4302
- if (!isConsumerAlreadyExists(e))
4303
- throw e;
4304
- }
4305
- if (info?.config.description === description)
4306
- return reader.js.consumers.getConsumerFromInfo(info);
4307
- this.setAsideHistoryReader(pool, reader.slot);
4308
- const next = pool.free.pop();
4309
- if (next === undefined) {
4310
- // The read ends holding no name. Every waiter checks again, so one that no running read can
4311
- // now serve fails instead of hanging.
4312
- reader.held = false;
4313
- pool.held--;
4314
- for (const wake of pool.waiting.splice(0))
4315
- wake();
4316
- throw new Error(`history: reader ${reader.name} on ${reader.stream} is held by a consumer this read did not create, and no other reader name is free`);
4317
- }
4318
- reader.slot = next;
4319
- reader.name = historyReaderConsumer(pool.readerId, next);
4320
- }
4321
- }
4322
- /** Delete the read's reader consumer; already gone is the same outcome. A failed delete may still
4323
- * have run with its answer lost, and the name may be taken again before this endpoint could tell,
4324
- * so the name is set aside when the read gives it back rather than deleted again later. A refused
4325
- * delete means the credential was minted before the observer and admin profiles held their reader
4326
- * deletes. Each read would then leave a reader for the inactive threshold, which is how a polling
4327
- * dashboard filled the stream's consumer limit (#4123), so the read fails and says to remint
4328
- * rather than leak quietly, and reads on the stream fail at once until that reader expires. */
4329
- async closeHistoryReader(reader) {
4330
- try {
4331
- await reader.jsm.consumers.delete(reader.stream, reader.name);
4332
- }
4333
- catch (e) {
4334
- if (isConsumerNotFound(e))
4335
- return;
4336
- reader.left = true;
4337
- // No `cause`: a permission-shaped cause would let streamHistory read this as the admin-only
4338
- // DM denial and return an empty page.
4339
- if (isPublishPermissionDenied(e)) {
4340
- if (this.historyReaderId(reader.nc) !== reader.pool.readerId)
4341
- throw new Error(`history: the connection was re-established during the read on ${reader.stream}, so it may no longer delete its reader ${reader.name}, which expires on its own; the read is not served`);
4342
- reader.pool.refusedUntil = Date.now() + HISTORY_READER_INACTIVE_MS;
4343
- throw this.readerDeleteRefused(reader.stream);
4344
- }
4345
- throw e;
4346
- }
4347
- }
4348
- readerDeleteRefused(stream) {
4349
- return new Error(`history: this credential may not delete its own history readers on ${stream}; it predates the observer and admin reader-delete grant, so remint it`);
4350
- }
4351
- /** The oldest stream sequence matching `subject`, or 0 when the subject has no messages.
4352
- * Mirror of {@link lastMatchingSeq}: same CREATE/INFO/NEXT/DELETE surface, `DeliverPolicy.All`
4353
- * instead of `Last`, first delivered seq instead of last. Read credentials already hold this. */
4354
- async firstMatchingSeq(reader, subjects, signal) {
4355
- signal?.throwIfAborted();
4356
- const consumer = await this.openHistoryReader(reader, {
4357
- filter_subjects: subjects,
4358
- deliver_policy: DeliverPolicy.All,
4359
- });
4360
- try {
4361
- if ((await consumer.info(true)).num_pending === 0)
4362
- return 0;
4363
- signal?.throwIfAborted();
4364
- const iter = await consumer.fetch({ max_messages: 1 });
4365
- const stop = () => iter.stop(abortReason(signal));
4366
- signal?.addEventListener("abort", stop, { once: true });
4367
- try {
4368
- signal?.throwIfAborted();
4369
- for await (const m of iter)
4370
- return m.seq;
4371
- }
4372
- finally {
4373
- signal?.removeEventListener("abort", stop);
4374
- iter.stop();
4375
- }
4376
- throw new Error(`history: the broker reported messages on ${subjectLabel(subjects)} but delivered none - the read was cut short, not empty`);
4377
- }
4378
- finally {
4379
- await this.closeHistoryReader(reader);
4380
- }
4381
- }
4382
- /** The newest stream sequence matching `subject`, or 0 when the subject has no messages.
4383
- *
4384
- * One reader consumer at `DeliverPolicy.Last` with this subject's filter: its `num_pending`
4385
- * (available from the create, before anything is delivered) is 0 for an empty subject, and
4386
- * otherwise one message carries the sequence. Same CREATE/INFO/NEXT/DELETE surface `drainWindow`
4387
- * already uses, so no broker authority is added. */
4388
- async lastMatchingSeq(reader, subjects, signal) {
4389
- signal?.throwIfAborted();
4390
- const consumer = await this.openHistoryReader(reader, {
4391
- filter_subjects: subjects,
4392
- deliver_policy: DeliverPolicy.Last,
4393
- });
4394
- try {
4395
- // Bind-time zero is the ONLY thing that means "this subject has no messages".
4396
- if ((await consumer.info(true)).num_pending === 0)
4397
- return 0;
4398
- signal?.throwIfAborted();
4399
- const iter = await consumer.fetch({ max_messages: 1 });
4400
- const stop = () => iter.stop(abortReason(signal));
4401
- signal?.addEventListener("abort", stop, { once: true });
4402
- try {
4403
- signal?.throwIfAborted();
4404
- for await (const m of iter)
4405
- return m.seq;
4406
- }
4407
- finally {
4408
- signal?.removeEventListener("abort", stop);
4409
- iter.stop();
4410
- }
4411
- // Bind said a message was pending and none arrived. The pinned client's pull iterator ends
4412
- // CLEANLY when the connection closes ("we don't propagate the error here"), so this is what a
4413
- // dropped link looks like from here. Returning 0 would make the caller report an empty
4414
- // channel, which is the same "could not read means no history" lie the narrowed catch above
4415
- // exists to stop.
4416
- throw new Error(`history: the broker reported messages on ${subjectLabel(subjects)} but delivered none - the read was cut short, not empty`);
4417
- }
4418
- finally {
4419
- await this.closeHistoryReader(reader);
4420
- }
4421
- }
4422
- /** Drain every message matching `subject` with sequence in `[start, ceiling]`, oldest-first.
4423
- * One reader consumer, one batched pull — `AckPolicy.None`, so no per-message ack
4424
- * round trip. Fetches exactly the pending count so it returns as soon as the window is
4425
- * delivered rather than blocking for the pull's full expiry. */
4426
- async drainWindow(reader, subjects, start, ceiling, limit, signal) {
4427
- signal?.throwIfAborted();
4428
- const out = [];
4429
- const consumer = await this.openHistoryReader(reader, {
4430
- filter_subjects: subjects,
4431
- deliver_policy: DeliverPolicy.StartSequence,
4432
- opt_start_seq: start,
4433
- });
4434
- try {
4435
- // A freshly created consumer already carries its ConsumerInfo, so read the CACHED copy: the
4436
- // explicit uncached `info()` this used to call was a round trip for data we already had.
4437
- const pending = (await consumer.info(true)).num_pending;
4438
- if (pending === 0)
4439
- return out;
4440
- signal?.throwIfAborted();
4441
- // Keep only a small rolling buffer in flight. Stopping a client iterator cannot unsend bytes the
4442
- // broker already committed to that pull request, so fetching the whole page let an aborted history
4443
- // read keep filling a constrained shared connection and starve the next dashboard poll. `consume`
4444
- // replenishes this bounded buffer as it is read, so large pages still complete without committing
4445
- // all of their bytes to the connection up front.
4446
- const iter = await consumer.consume({ max_messages: Math.min(pending, 32) });
4447
- const stop = () => iter.stop(abortReason(signal));
4448
- signal?.addEventListener("abort", stop, { once: true });
4449
- // PROVE THE WINDOW COMPLETED. The pull iterator ends cleanly on a dropped connection, so a
4450
- // close after three of ten deliveries would otherwise return a convincing three-message page.
4451
- // The window is done when we have reached its upper bound or consumed everything bind said
4452
- // was pending; anything else is a cut-short read and must say so.
4453
- let delivered = 0;
4454
- let complete = false;
4455
- try {
4456
- signal?.throwIfAborted();
4457
- for await (const m of iter) {
4458
- delivered++;
4459
- // A plain consumer does not rebuild itself the way an ordered one did, so a gap in its
4460
- // delivery sequence is a message this page would silently lack.
4461
- if (m.info.deliverySequence !== delivered)
4462
- throw new Error(`history: delivery ${m.info.deliverySequence} arrived where ${delivered} was due on ${subjectLabel(subjects)} - the window has a gap, so the page is not served`);
4463
- if (m.seq >= ceiling) { // reached the page's upper bound
4464
- if (m.seq === ceiling) {
4465
- const msg = historyMessageFromDelivery(m);
4466
- if (msg)
4467
- out.push({ seq: m.seq, subject: m.subject, msg });
4468
- }
4469
- complete = true;
4470
- break;
4471
- }
4472
- const msg = historyMessageFromDelivery(m);
4473
- if (msg) {
4474
- out.push({ seq: m.seq, subject: m.subject, msg });
4475
- if (out.length > limit)
4476
- out.shift();
4477
- }
4478
- if (delivered >= pending) {
4479
- complete = true;
4480
- break;
4481
- }
4482
- }
4483
- }
4484
- finally {
4485
- signal?.removeEventListener("abort", stop);
4486
- iter.stop();
4487
- }
4488
- if (!complete)
4489
- throw new Error(`history: read ${delivered} of ${pending} messages on ${subjectLabel(subjects)} before the stream ended early - the window was cut short, not empty`);
4490
- return out;
4491
- }
4492
- finally {
4493
- // DELETE THE READER. Left alone it would stay for its inactive threshold, and the widening
4494
- // search can run several steps per call while a polling dashboard repeats every call on each
4495
- // poll. The observer and admin profiles used to hold no delete at all (#691), so a dashboard
4496
- // polling every ten seconds filled the stream's consumer limit and every later read failed
4497
- // (#4123). Each reader now goes when its step ends.
4498
- await this.closeHistoryReader(reader);
4499
- }
3941
+ return this.history.streamHistory(stream, subjects, limit, before, signal);
4500
3942
  }
4501
3943
  // ---- internals -----------------------------------------------------------
4502
3944
  /**
@@ -4545,9 +3987,10 @@ export class CotalEndpoint extends EventEmitter {
4545
3987
  continue;
4546
3988
  if (s.error instanceof PermissionViolationError && this.chatSubRefusals.has(s.error))
4547
3989
  continue;
4548
- // A principal-route refusal belongs to the call that throws it or the serve that ends on it
4549
- // (SPEC 13.21); nothing else on this private connection publishes or subscribes there.
4550
- if (s.error instanceof PermissionViolationError && this.isPrincipalRouteSubject(s.error.subject))
3990
+ // A refused principal-route request publish or subscription belongs to the call that throws
3991
+ // it or the serve that ends on it (SPEC 13.21). A refused reply publish has no owner, so it
3992
+ // is reported below.
3993
+ if (s.error instanceof PermissionViolationError && this.ownedPrincipalRouteRefusal(s.error))
4551
3994
  continue;
4552
3995
  if (s.error instanceof PermissionViolationError && handedToRequest.has(s.error))
4553
3996
  continue;
@@ -5396,7 +4839,7 @@ export class CotalEndpoint extends EventEmitter {
5396
4839
  /** Privileged durable-LEAVE write: tombstone the membership at `leaveCursor = frontier` so the
5397
4840
  * backstop denies `seq > leaveCursor` while a pre-leave entry stays deliverable (SPEC §7 interval). */
5398
4841
  async durableLeaveFor(owner, channel, lifecycleUid, expectedGeneration) {
5399
- if (!this.plane3)
4842
+ if (!this.deliveryPlane.plane3)
5400
4843
  return; // not a Plane-3 host — no membership to tombstone
5401
4844
  const kv = await this.membersRegistry();
5402
4845
  // expectedGeneration (captured by the agent at durableJoin) refuses a stale leave from tombstoning
@@ -5488,302 +4931,7 @@ export class CotalEndpoint extends EventEmitter {
5488
4931
  * {@link armPlane3} on EVERY (re)connect — a reconnect drains the old connection, so re-binding both
5489
4932
  * is required, not optional (the responder would otherwise be lost on a broker blip). */
5490
4933
  async startPlane3(aclFor, opts = {}) {
5491
- if (!this.js)
5492
- throw new Error(this.notLiveMsg());
5493
- this.plane3 = { aclFor, reloadMembershipCreds: opts.reloadMembershipCreds, evictPrincipal: opts.evictPrincipal, evictPrincipals: opts.evictPrincipals, planeConnLiveness: opts.planeConnLiveness, principalLiveness: opts.principalLiveness, reloadStoreIdentity: opts.reloadStoreIdentity, onDeliveryCredsAdopted: opts.onDeliveryCredsAdopted, startupComplete: opts.startupComplete };
5494
- await this.armPlane3();
5495
- }
5496
- /** Whether an ALREADY-DISPATCHED unit of Plane-3 work may still take effect.
5497
- *
5498
- * Unsubscribing stops NEW work; it cannot recall work already in flight. A handler that entered
5499
- * before {@link quiescePlane3} and awaited broker I/O inside it resumes AFTER the freeze, and a
5500
- * reviewer traced the ordering that makes that a split rather than a latency blip: the loser
5501
- * accepts a unit and awaits, the loser is descheduled, the successor acquires the shard and flips
5502
- * its lease READY, then the loser resumes and answers or acks for a shard it no longer holds.
5503
- * Subscription counts and parked-pull readings cannot see it, because the effect is the reply and
5504
- * the ack rather than the binding.
5505
- *
5506
- * So each work path re-asks HERE, at the point of effect, after its awaits and before it acts.
5507
- * The lease keeps one ROW, not one SERVER; this is the half that keeps one server. An unacked
5508
- * message is not lost by refusing: a stopped consumer redelivers it to whoever holds the shard
5509
- * next, which is the same property quiescing already relies on. */
5510
- plane3MayAct() {
5511
- return !this.plane3Quiesced;
5512
- }
5513
- /** Serve one runtime durable-membership control request (the server-side delivery daemon). The caller
5514
- * id is the authenticated subject sender ({@link serveControl} fail-closes on a mismatch). Validation
5515
- * is against the durable ACL registry — the SAME KV the reader re-auths against (single source of
5516
- * truth, no in-memory ledger to drift). */
5517
- async handleDeliveryControl(req) {
5518
- // FENCE: entered before a quiesce, resuming after it. Answering now would put a second server on
5519
- // this shard's control rail while the winner is already READY.
5520
- if (!this.plane3MayAct())
5521
- return { ok: false, error: "delivery: this daemon is not serving this shard (it is re-checking ownership); retry" };
5522
- const caller = req.from.id;
5523
- const args = req.args ?? {};
5524
- if (req.op === "durableJoin")
5525
- return this.deliveryJoin(caller, args);
5526
- if (req.op === "durableLeave")
5527
- return this.deliveryLeave(caller, args);
5528
- if (req.op === "readHistory")
5529
- return this.deliveryReadHistory(caller, args);
5530
- if (req.op === ANYCAST_TAKES_OP)
5531
- return this.anycastUnavailable ? { ok: false, error: `delivery: ${this.anycastUnavailable}` } : { ok: true, data: { takes: true } };
5532
- if (req.op === ANYCAST_STATUS_OP || req.op === ANYCAST_CANCEL_OP)
5533
- return this.deliveryAnycast(req.op, caller, args);
5534
- if (req.op === "listMemberships") {
5535
- if (typeof args.lifecycleUid !== "string")
5536
- return { ok: false, error: "listMemberships: the caller's lifecycleUid is required (membership rows are lifecycle-keyed, SPEC 13.1)" };
5537
- let uid;
5538
- try {
5539
- uid = assertLifecycleToken(args.lifecycleUid);
5540
- }
5541
- catch (e) {
5542
- return { ok: false, error: e.message };
5543
- }
5544
- let memberships;
5545
- if (args.channel !== undefined) {
5546
- const channel = this.checkDurableChannelArg(args, "listMemberships", { family: true });
5547
- if (typeof channel !== "string")
5548
- return channel;
5549
- memberships = (await listMembers(await this.membersRegistry(), { owner: caller }))
5550
- .filter((r) => r.channel === channel && r.leaveCursor === undefined)
5551
- .map((r) => ({ channel: r.channel, generation: r.generation, activated: r.activated === true, lifecycleUid: r.lifecycleUid }));
5552
- }
5553
- else {
5554
- memberships = await this.ownerMemberships(caller, uid);
5555
- }
5556
- // FENCE AFTER THE AWAIT, which is the one that matters. The entry check above cannot catch the
5557
- // case this exists for: a request admitted WHILE SERVING, then descheduled inside its broker
5558
- // read, resuming after a successor has taken the shard and reached READY. Re-ask at the point
5559
- // of effect, so the answer is never served by a daemon that has stood down.
5560
- if (!this.plane3MayAct())
5561
- return { ok: false, error: "delivery: this daemon stopped serving this shard while the request was in flight; retry" };
5562
- return { ok: true, data: { memberships } };
5563
- }
5564
- return { ok: false, error: `op "${req.op}" not supported on the delivery control service` };
5565
- }
5566
- /** Validate the channel ARG shape only: non-blank, valid, concrete (NO ACL check, that is op-specific).
5567
- * `family` also admits a channel family such as `review.>` (a membership join or leave, #5404).
5568
- * Returns the channel on success or a ControlReply error to short-circuit. */
5569
- checkDurableChannelArg(args, op, opts = {}) {
5570
- const channel = typeof args.channel === "string" ? args.channel.trim() : "";
5571
- if (!channel)
5572
- return { ok: false, error: `${op}: channel must be a non-blank string` };
5573
- try {
5574
- assertValidChannel(channel);
5575
- }
5576
- catch (e) {
5577
- return { ok: false, error: e.message };
5578
- }
5579
- if (opts.family && isChannelFamily(channel))
5580
- return channel;
5581
- if (!isConcreteChannel(channel))
5582
- return {
5583
- ok: false,
5584
- error: opts.family
5585
- ? `${op}: "${channel}" stays live only: a durable membership takes a concrete channel or a family such as review.>, never the root > or a pattern with *`
5586
- : `${op}: "${channel}" must be a concrete channel (durable membership is per-concrete-channel, not wildcard)`,
5587
- };
5588
- return channel;
5589
- }
5590
- /** JOIN requires the channel be within the caller's CURRENT read ACL (you can't durable-subscribe a
5591
- * channel you may not read). A family must lie inside one ACL entry, so every channel under it is
5592
- * readable. */
5593
- async deliveryJoin(caller, args) {
5594
- const channel = this.checkDurableChannelArg(args, "durableJoin", { family: true });
5595
- if (typeof channel !== "string")
5596
- return channel; // a ControlReply error
5597
- // The caller arrives as an ALIAS (the control subject carries owner+actor, never a uid): resolve
5598
- // its single live lifecycle-keyed ACL row SERVER-SIDE — the trusted registry, never a caller
5599
- // assertion, decides which lifecycle joins. Two live rows (a reservation breach / unfinished
5600
- // teardown) refuse loudly rather than binding a membership to a guessed lifecycle.
5601
- let acl;
5602
- try {
5603
- acl = await this.aclForAlias(caller);
5604
- }
5605
- catch (e) {
5606
- return { ok: false, error: e.message };
5607
- }
5608
- if (acl === undefined)
5609
- return { ok: false, error: `durableJoin: no read ACL on record for ${caller} (not provisioned for durable delivery)` };
5610
- if (!membershipInAllow(acl.allowSubscribe, channel))
5611
- return { ok: false, error: `channel "${channel}" is not within your read ACL [${acl.allowSubscribe.join(", ")}]` };
5612
- const uid = acl.lifecycleUid;
5613
- // The lookup above may be stale by the time the row is written (a retirement can fence and purge
5614
- // this lifecycle meanwhile), so the write re-checks the exact ACL row itself (`aclFence`).
5615
- try {
5616
- return { ok: true, data: await this.membershipWrite(caller, uid, () => this.durableJoinFor(caller, channel, uid, { aclFence: true })) };
5617
- }
5618
- catch (e) {
5619
- return { ok: false, error: e.message };
5620
- }
5621
- }
5622
- /** Whether `lifecycleUid`'s exact ACL row still allows reads, so an inventory of it came from a
5623
- * caller that did not fence. A row narrowed to no reads, or retired or absent, is fenced. A read
5624
- * that fails answers true: the caller keeps its in-memory mark, which only ever refuses. */
5625
- async lifecycleAclUnfenced(principal, lifecycleUid) {
5626
- try {
5627
- const row = await readAcl(await this.aclRegistry(), principal, lifecycleUid);
5628
- return row !== undefined && row.record.allowSubscribe.length > 0;
5629
- }
5630
- catch {
5631
- return true;
5632
- }
5633
- }
5634
- /** Run one durable join or leave that writes `principal`'s rows of `lifecycleUid`, refused once that
5635
- * lifecycle's inventory has been asked for ({@link retiringLifecycles}). The check and the
5636
- * registration have no await between them, so an inventory either finds this write running and
5637
- * waits for it, or this write finds the inventory's mark and is refused. */
5638
- async membershipWrite(principal, lifecycleUid, write) {
5639
- if (this.retiringLifecycles.get(principal)?.has(lifecycleUid))
5640
- throw new Error(`lifecycle ${lifecycleUid} of ${principal} is retiring: its membership inventory was requested, so its durable memberships no longer change`);
5641
- const key = `${principal}/${lifecycleUid}`;
5642
- const running = write();
5643
- let writes = this.membershipWrites.get(key);
5644
- if (writes === undefined)
5645
- this.membershipWrites.set(key, (writes = new Set()));
5646
- writes.add(running);
5647
- try {
5648
- return await running;
5649
- }
5650
- finally {
5651
- writes.delete(running);
5652
- if (writes.size === 0)
5653
- this.membershipWrites.delete(key);
5654
- }
5655
- }
5656
- /** LEAVE must NOT require current-ACL coverage. Leave fires precisely when the ACL was narrowed/revoked
5657
- * (a refused live sub → {@link closeRefusedMembership}); gating the tombstone on the current ACL would
5658
- * loop forever and leave the SPEC §7 boundary open (the membership could resume if the ACL is later
5659
- * restored). The guards are: authenticated caller (serveControl), concrete channel, a finite generation
5660
- * (the join epoch — without it a stale/replayed leave could tombstone a newer rejoin), and an EXISTING
5661
- * own membership; `durableLeaveFor` → `tombstoneMember` then enforces the generation match. */
5662
- async deliveryLeave(caller, args) {
5663
- const channel = this.checkDurableChannelArg(args, "durableLeave", { family: true });
5664
- if (typeof channel !== "string")
5665
- return channel; // a ControlReply error
5666
- if (typeof args.generation !== "number" || !Number.isFinite(args.generation))
5667
- return { ok: false, error: "durableLeave: a finite generation is required (fail-closed stale-leave guard)" };
5668
- // The LEAVE carries the leaver's own lifecycleUid: leave must work precisely when the ACL row was
5669
- // narrowed or already purged (see the method doc), so the alias→row resolution join uses is not
5670
- // available here. The uid is caller-asserted but harmless to lie about: the member key it selects
5671
- // is confined to the AUTHENTICATED caller's own alias (`<channel>/<caller>.<uid>`), so the worst a
5672
- // false uid reaches is the caller's own retired incarnation's row — legitimate cleanup — and the
5673
- // generation guard still applies.
5674
- if (typeof args.lifecycleUid !== "string")
5675
- return { ok: false, error: "durableLeave: the leaving incarnation's lifecycleUid is required (membership rows are lifecycle-keyed, SPEC 13.1)" };
5676
- let uid;
5677
- try {
5678
- uid = assertLifecycleToken(args.lifecycleUid);
5679
- }
5680
- catch (e) {
5681
- return { ok: false, error: e.message };
5682
- }
5683
- const existing = await readMember(await this.membersRegistry(), channel, caller, uid);
5684
- if (!existing)
5685
- return { ok: true, data: { channel, alreadyLeft: true } }; // nothing to tombstone — idempotent
5686
- const generation = args.generation;
5687
- try {
5688
- await this.membershipWrite(caller, uid, () => this.durableLeaveFor(caller, channel, uid, generation));
5689
- }
5690
- catch (e) {
5691
- return { ok: false, error: e.message };
5692
- }
5693
- return { ok: true, data: { channel } };
5694
- }
5695
- /** Serve one MEDIATED HISTORY READ (`readHistory`, the client side is {@link readHistory}). The daemon
5696
- * holds the consumer so the caller does not have to: this is a privilege reduction, not a new
5697
- * capability, and it is the shape SPEC's "Mediated reads (normative)" rule asks for.
5698
- *
5699
- * THE ONE INVARIANT THIS METHOD EXISTS FOR: authorization is read FRESH on every call and never
5700
- * cached across calls. A consumer pins its authorization when it is created, so a revoked caller
5701
- * keeps being served by a consumer it already holds; re-reading here is what makes a revocation
5702
- * stop the very NEXT read. Cache this and the verb loses its only advantage over the raw consumer
5703
- * path.
5704
- *
5705
- * AUTHORITY (SPEC §9.6): "current read ACL" is the effective broker-accepted credential. The
5706
- * durable registry is a live mirror of that credential, not a second grant source. History
5707
- * authorizes against `allowSubscribe ∩ issuedAllowSubscribe` — the live row (so revocation via
5708
- * plain commitAcl still stops the next read) intersected with the mint-time ceiling (so a
5709
- * registry widen without a remint cannot grant what the JWT does not). Raising the ceiling is
5710
- * {@link reissueAcl}, which is what provision does when it bakes the list into the JWT.
5711
- *
5712
- * The caller arrives as an alias (the control subject carries owner+actor, never a uid) and is the
5713
- * AUTHENTICATED subject sender — `serveControl` has already fail-closed on any payload that names a
5714
- * different principal, so `caller` cannot be self-asserted. */
5715
- async deliveryReadHistory(caller, args) {
5716
- const channel = this.checkDurableChannelArg(args, "readHistory");
5717
- if (typeof channel !== "string")
5718
- return channel; // a ControlReply error
5719
- // The caller PROPOSES a limit; the mediator decides. Reject a nonsense limit loudly rather than
5720
- // silently substituting a default — a caller asking for -1 has a bug and should hear about it.
5721
- const asked = args.limit === undefined ? READ_HISTORY_DEFAULT_LIMIT : args.limit;
5722
- if (typeof asked !== "number" || !Number.isInteger(asked) || asked < 1)
5723
- return { ok: false, error: `readHistory: limit must be a positive integer (got ${JSON.stringify(args.limit)})` };
5724
- const limit = Math.min(asked, READ_HISTORY_MAX_LIMIT);
5725
- // FRESH per call — see the method doc. Same registry, same alias resolution, and the same loud
5726
- // refusal on two live rows that `durableJoin` uses: a stale lifecycle must never authorize its
5727
- // successor's read.
5728
- //
5729
- // SPEC §9.6: "current read ACL" is the effective broker-accepted credential. The registry row is
5730
- // a live mirror and can be rewritten by plain commitAcl without a remint. Authorizing history
5731
- // from the row ALONE let a widen grant reads channelHistory still broker-denies — a new
5732
- // capability wearing a privilege-reduction label (panel BLOCKING at 914fd7b0). History therefore
5733
- // requires the channel in BOTH the live row AND the mint-time ceiling (`issuedAllowSubscribe`).
5734
- // Revocation still works: narrowing allowSubscribe stops the next read. Raising the ceiling
5735
- // requires reissueAcl (provision/remint), which is the same act that bakes the list into the JWT.
5736
- let acl;
5737
- try {
5738
- acl = await this.aclForAlias(caller);
5739
- }
5740
- catch (e) {
5741
- return { ok: false, error: e.message };
5742
- }
5743
- if (acl === undefined)
5744
- return { ok: false, error: `readHistory: no read ACL on record for ${caller} - not permitted` };
5745
- const inLive = channelInAllow(acl.allowSubscribe, channel);
5746
- const inIssued = channelInAllow(acl.issuedAllowSubscribe, channel);
5747
- if (!inLive || !inIssued) {
5748
- // Name the live list in the error — that is what operators edit and what revocation clears.
5749
- // A ceiling miss (registry widened past the credential) is the same refusal shape as a live
5750
- // miss so a caller cannot probe which half failed.
5751
- return {
5752
- ok: false,
5753
- error: `readHistory: channel "${channel}" is not within your read ACL [${acl.allowSubscribe.join(", ")}] - refused`,
5754
- };
5755
- }
5756
- // One message MORE than the page is the completeness probe: if the backlog hands back limit+1,
5757
- // something older exists behind the page and `complete` is false. Cheaper and more honest than a
5758
- // separate count, which could race the page it describes.
5759
- let page;
5760
- try {
5761
- page = await this.channelHistory(channel, { limit: limit + 1 });
5762
- }
5763
- catch (e) {
5764
- // Surface the read failure. `streamHistory` deliberately raises rather than returning empty on a
5765
- // denial or a cut-short window, and that distinction must survive the rail: an error here would
5766
- // otherwise reach the caller as an empty page and render as "no history".
5767
- return { ok: false, error: `readHistory: ${e.message}` };
5768
- }
5769
- const reachedStart = page.length <= limit;
5770
- const wanted = reachedStart ? page : page.slice(-limit);
5771
- // A page that cannot be SENT is not a page. The reply rides one NATS message, so `limit` alone is
5772
- // the wrong bound: 200 large messages serialize past `max_payload`, and `serveControl` can then only
5773
- // answer with a refusal in place of the page.
5774
- //
5775
- // So bound by BYTES too, keeping the NEWEST that fit — which needs no new vocabulary, because
5776
- // `complete: false` already means "older history remains behind this page". Trimming here is the
5777
- // documented truncation signal doing its job, not a silent degradation.
5778
- const fitted = fitHistoryPage(wanted, this.maxPayload);
5779
- // `wanted.length > 0` is load-bearing, not defensive: an EMPTY channel also fits nothing, and
5780
- // without this guard a channel nobody has posted to was refused with "the newest message exceeds
5781
- // the payload budget" — a confident, entirely wrong explanation for a legitimately empty result.
5782
- // Genuine emptiness is `{ items: [], complete: true }`; only a message too large to ever send is
5783
- // the error.
5784
- if (wanted.length > 0 && fitted.length === 0)
5785
- return { ok: false, error: `readHistory: the newest message on "${channel}" alone exceeds the broker payload budget (${this.maxPayload} bytes) - refused loudly rather than served as an empty page` };
5786
- return { ok: true, data: { items: fitted, complete: reachedStart && fitted.length === wanted.length } };
4934
+ return this.deliveryPlane.startPlane3(aclFor, opts);
5787
4935
  }
5788
4936
  /** Stop serving Plane-3 WITHOUT tearing down the connection, so a daemon that has just learned its
5789
4937
  * lease may no longer be its own can stop acting on the shard while it finds out for certain.
@@ -5805,37 +4953,13 @@ export class CotalEndpoint extends EventEmitter {
5805
4953
  * Deliberately not `stop()`: the connection, the lease KV handles and the control rails must stay
5806
4954
  * up, because the daemon still has to ask the broker who owns the shard. */
5807
4955
  async quiescePlane3() {
5808
- this.plane3Quiesced = true;
5809
- if (this.deliveryServeSub) {
5810
- try {
5811
- this.deliveryServeSub.unsubscribe();
5812
- }
5813
- catch { /* already dead */ }
5814
- this.deliveryServeSub = undefined;
5815
- }
5816
- if (this.deliveryAdminServeSub) {
5817
- try {
5818
- this.deliveryAdminServeSub.unsubscribe();
5819
- }
5820
- catch { /* already dead */ }
5821
- this.deliveryAdminServeSub = undefined;
5822
- }
5823
- this.dropAnycastRequests();
5824
- // Stopping the consumers ends the `for await` loops that drive fan-out and the reader. In-flight
5825
- // messages are NOT acked by a stopped consumer, so they redeliver to whoever holds the shard
5826
- // next: quiescing loses no message, it only stops this process from claiming them.
5827
- for (const msgs of this.streamMsgs.splice(0)) {
5828
- try {
5829
- msgs.stop();
5830
- }
5831
- catch { /* already draining */ }
5832
- }
4956
+ return this.deliveryPlane.quiescePlane3();
5833
4957
  }
5834
4958
  /** Whether Plane-3 is currently quiesced: unbound, and refusing to re-bind until ownership is proven.
5835
4959
  * Read-only. The flag is the daemon's own answer to "am I serving this shard", so a cell that asserts
5836
4960
  * a recovery worked can check the endpoint's state rather than inferring it from a log line. */
5837
4961
  plane3IsQuiesced() {
5838
- return this.plane3Quiesced;
4962
+ return this.deliveryPlane.plane3IsQuiesced();
5839
4963
  }
5840
4964
  /** Resume serving Plane-3 after {@link quiescePlane3}, once ownership has been re-established.
5841
4965
  * Idempotent, and a no-op when the daemon was never quiesced.
@@ -5848,80 +4972,19 @@ export class CotalEndpoint extends EventEmitter {
5848
4972
  * That is a readiness lie surviving a transient broker error, the daemon claims a responder it does
5849
4973
  * not have, which is the #1318 outage wearing the readiness flag instead of the exit path. */
5850
4974
  async rearmPlane3() {
5851
- if (!this.plane3Quiesced)
5852
- return;
5853
- // `armPlane3` refuses while quiesced (a reconnect must not resume mid-question), so the flag is
5854
- // lowered for the attempt and RAISED AGAIN on failure. A partial bind leaves the endpoint quiesced,
5855
- // which is the honest state: not serving, and a later retry will attempt the whole sequence again.
5856
- this.plane3Quiesced = false;
5857
- try {
5858
- await this.armPlane3();
5859
- }
5860
- catch (e) {
5861
- this.plane3Quiesced = true;
5862
- // Drop whatever DID bind. Leaving half-bound responders up would answer control requests for a
5863
- // shard this process has just declared itself not to be serving.
5864
- this.quiescePlane3();
5865
- throw e;
5866
- }
4975
+ return this.deliveryPlane.rearmPlane3();
5867
4976
  }
5868
- /** (Re)bind the Plane-3 fan-out writer + trusted reader. Idempotent — the durables resume from their
5869
- * cursor. Called by {@link startPlane3} once AND by {@link connectAndBind} on every (re)connect, so
5870
- * the delivery daemon's reconnect RE-ARMS the backstop + the ctl.delivery responder. Without this, a broker blip would silently kill
5871
- * the loops while `durableJoinFor` kept reporting `durable:true` (the impl-review's BLOCKER-1). No-op
5872
- * unless this endpoint hosts Plane-3 (`this.plane3` set). */
4977
+ /** Forwards to the delivery plane's {@link EndpointDeliveryPlane.plane3MayAct}. */
4978
+ plane3MayAct() {
4979
+ return this.deliveryPlane.plane3MayAct();
4980
+ }
4981
+ /** Forwards to the delivery plane's {@link EndpointDeliveryPlane.armPlane3}. */
5873
4982
  async armPlane3() {
5874
- if (!this.plane3 || !this.js)
5875
- return;
5876
- // A quiesced endpoint must not be re-armed by a RECONNECT: the reconnect path calls this too,
5877
- // and silently resuming there would restore exactly the double-serving this guards against
5878
- // while the ownership question is still open.
5879
- if (this.plane3Quiesced)
5880
- return;
5881
- await this.manager(); // the manager runs consume:false, so this.jsm is lazy — ensure it
5882
- this.armDeliveryControl();
5883
- await this.armAnycastRequests();
5884
- await this.runFanout();
5885
- await this.runReader();
5886
- }
5887
- /** (Re)register the `ctl.delivery` control responder on the CURRENT connection. A reconnect drains the
5888
- * old connection (the old sub is dead and `clearConnectionScoped` leaves caller-owned subs alone), so
5889
- * this MUST run on every arm — otherwise durable join/leave/list silently lose their responder after a
5890
- * broker blip. The stale sub is dropped (unsubscribed + removed from `this.subs`) before re-creating.
5891
- * `boundReply` is essential here: the daemon holds a wildcard reply-publish grant, so the serve path
5892
- * must reject any reply target outside the authenticated sender's own subtree (confused-deputy fix). */
5893
- armDeliveryControl() {
5894
- if (this.deliveryServeSub) {
5895
- try {
5896
- this.deliveryServeSub.unsubscribe();
5897
- }
5898
- catch { /* dead with the old connection */ }
5899
- const i = this.subs.indexOf(this.deliveryServeSub);
5900
- if (i >= 0)
5901
- this.subs.splice(i, 1);
5902
- }
5903
- // A request's status and cancel touch only the TASK queue and the anycast records, so they do
5904
- // not wait behind a channel join on the in-order chain.
5905
- this.deliveryServeSub = this.serveControl(CONTROL_DELIVERY, (req) => this.handleDeliveryControl(req), {
5906
- boundReply: true,
5907
- concurrent: (req) => ANYCAST_CONTROL_OPS.has(req.op),
5908
- });
5909
- if (this.deliveryAdminServeSub) {
5910
- try {
5911
- this.deliveryAdminServeSub.unsubscribe();
5912
- }
5913
- catch { /* dead with the old connection */ }
5914
- const i = this.subs.indexOf(this.deliveryAdminServeSub);
5915
- if (i >= 0)
5916
- this.subs.splice(i, 1);
5917
- }
5918
- // The membership inventory is read-only and shares nothing with the other admin commands, so it
5919
- // does not queue behind them. In a mass stop each retirement's inventory followed every other
5920
- // seat's `evictPrincipal` on this rail and missed its 5 s deadline (#4965).
5921
- this.deliveryAdminServeSub = this.serveControl(CONTROL_DELIVERY_ADMIN, (req) => this.handleDeliveryAdmin(req), {
5922
- boundReply: true,
5923
- concurrent: (req) => req.op === "lifecycleMemberships",
5924
- });
4983
+ return this.deliveryPlane.armPlane3();
4984
+ }
4985
+ /** Forwards to the delivery plane's {@link EndpointDeliveryPlane.pumpDlv}. */
4986
+ async pumpDlv() {
4987
+ return this.deliveryPlane.pumpDlv();
5925
4988
  }
5926
4989
  /** (Re)bind the anycast take responder and the request records maintenance on the current
5927
4990
  * connection (SPEC §8.1). The records bucket is created here as well as at space setup, so a mesh
@@ -6309,23 +5372,28 @@ export class CotalEndpoint extends EventEmitter {
6309
5372
  return "refused";
6310
5373
  }
6311
5374
  /**
6312
- * Serve this process's own endpoint on the principal route (SPEC 13.21): a core-defined command
6313
- * set about this principal, addressed by its own principal, lifecycle UID and runtime id, with no
6314
- * registration. Revision 1 defines one endpoint, `agent` (SPEC 13.22).
5375
+ * Serve this process's `agent` endpoint on the principal route (SPEC 13.21, 13.22): the
5376
+ * `AGENT_CLUSTER` command set about this principal, addressed by its own principal, lifecycle UID
5377
+ * and runtime id, with no registration. `handlers` serves the cluster's commands with
5378
+ * `AGENT_CONTRACTS`; `describe` is built in.
6315
5379
  *
6316
5380
  * Throws on an open mesh and on a static credential, where no row names who may call, without a
6317
- * lifecycle UID, before the endpoint has started, for an endpoint it already serves, and for a
6318
- * cluster with a command that declares no `authorities` (see {@link servePrincipalRoute}). It
6319
- * subscribes this process's own subject with no queue group.
5381
+ * lifecycle UID, before the endpoint has started, while it already serves `agent`, for an
5382
+ * `endpoint` or `cluster` field, which this entry point does not take, and for a set of handlers
5383
+ * that is not the cluster's commands with its contracts. It subscribes this process's own subject
5384
+ * with no queue group.
6320
5385
  *
6321
5386
  * `ready` is a positive check: it resolves only after the broker accepted the subscription
6322
5387
  * (subscribe, flush, and no permission violation for that subject), and rejects with the
6323
5388
  * refusal, which a credential minted before the rows existed gets. The serve is INTENT: every
6324
5389
  * replaced connection subscribes again and runs the check again, and a refusal there ends the
6325
- * serve, resolves `failed` and clears the presence field. `stop` unsubscribes and waits for the
6326
- * handlers in flight.
5390
+ * serve, resolves `failed` and clears the presence field. `stop` unsubscribes, aborts the
5391
+ * handlers' `signal`, and waits for each handler in flight until its request's deadline at most.
6327
5392
  */
6328
- servePrincipal(serve) {
5393
+ servePrincipal(spec) {
5394
+ if ("endpoint" in spec || "cluster" in spec)
5395
+ throw new Error(`servePrincipal serves only the "${AGENT_ENDPOINT}" endpoint with AGENT_CLUSTER and takes handlers alone; an endpoint or cluster is refused, never ignored (SPEC 13.22)`);
5396
+ const serve = { endpoint: AGENT_ENDPOINT, cluster: AGENT_CLUSTER, handlers: spec.handlers };
6329
5397
  if (!this.authed)
6330
5398
  throw new Error("the principal route is served only on an authenticated mesh: on an open mesh no row says who may call (SPEC 13.21)");
6331
5399
  if (!this.userMode)
@@ -6340,16 +5408,24 @@ export class CotalEndpoint extends EventEmitter {
6340
5408
  if (this.principalServes.has(serve.endpoint))
6341
5409
  throw new Error(`this endpoint already serves "${serve.endpoint}" on the principal route`);
6342
5410
  const self = { owner: this.owner, actor: this.actor, uid, runtimeId: assertRuntimeToken(processRuntime(uid).id) };
5411
+ const serverKey = `${this.space}\n${self.runtimeId}\n${serve.endpoint}`;
5412
+ if (principalServers.has(serverKey))
5413
+ throw new Error(`another endpoint in this process already serves "${serve.endpoint}" for lifecycle ${uid}: both would subscribe the same principal-route subject and answer one request twice (SPEC 13.21); stop that one first`);
6343
5414
  let resolveReady, rejectReady, resolveFailed;
6344
5415
  const ready = new Promise((resolve, reject) => { resolveReady = resolve; rejectReady = reject; });
6345
5416
  ready.catch(() => { });
6346
5417
  const failed = new Promise((resolve) => { resolveFailed = resolve; });
6347
5418
  const intent = {
6348
- serve, self, handles: [], ended: false, readySettled: false, resolveReady, rejectReady, resolveFailed,
5419
+ serve, self, serverKey, handles: [], ended: false, readySettled: false, resolveReady, rejectReady, resolveFailed,
6349
5420
  };
6350
5421
  // Construction refuses an invalid cluster before anything is kept or subscribed.
6351
5422
  const handle = this.bindPrincipalHandle(intent, nc);
6352
5423
  this.principalServes.set(serve.endpoint, intent);
5424
+ principalServers.set(serverKey, intent);
5425
+ // A fresh `seq` per serve: the last serve may have ended with prompts open, and the pair
5426
+ // `(runtime.id, seq)` it showed then must not name this serve's empty list (SPEC §6).
5427
+ if (serve.endpoint === AGENT_ENDPOINT)
5428
+ this.promptsPresence = { open: 0, seq: this.nextPromptsSeq() };
6353
5429
  void this.armPrincipalServe(intent, nc, handle);
6354
5430
  return { ready, failed, stop: () => this.stopPrincipalServe(intent) };
6355
5431
  }
@@ -6357,15 +5433,25 @@ export class CotalEndpoint extends EventEmitter {
6357
5433
  bindPrincipalHandle(intent, nc) {
6358
5434
  return servePrincipalRoute(nc, this.space, intent.self, intent.serve);
6359
5435
  }
6360
- /** A request or reply subject (or filter) of the principal route in this space. */
6361
- isPrincipalRouteSubject(subject) {
5436
+ /** A principal-route refusal a call or a serve reports itself: a request publish, a serve
5437
+ * subscription or a reply read. Only a reply publish is left, which no call or serve awaits. */
5438
+ ownedPrincipalRouteRefusal(err) {
6362
5439
  const p = `${spacePrefix(this.space)}.ep.`;
6363
- return subject.startsWith(`${p}prin.`) || subject.startsWith(`${p}prin-reply.`);
5440
+ if (err.operation === "publish")
5441
+ return err.subject.startsWith(`${p}prin.`);
5442
+ return err.subject.startsWith(`${p}prin.`) || err.subject.startsWith(`${p}prin-reply.`);
6364
5443
  }
6365
5444
  /** Track one binding: a refusal ends the serve, and an accepted subscription on the current
6366
5445
  * connection shows the presence field. */
6367
5446
  async armPrincipalServe(intent, nc, handle) {
6368
5447
  intent.binding = { nc, handle };
5448
+ // Every earlier binding is on a replaced connection: retire it once its handlers finish.
5449
+ for (const old of intent.handles)
5450
+ void old.stop().finally(() => {
5451
+ const i = intent.handles.indexOf(old);
5452
+ if (i >= 0)
5453
+ intent.handles.splice(i, 1);
5454
+ });
6369
5455
  intent.handles.push(handle);
6370
5456
  intent.readyOn = undefined;
6371
5457
  void handle.refused.then((err) => {
@@ -6427,10 +5513,7 @@ export class CotalEndpoint extends EventEmitter {
6427
5513
  endPrincipalServe(intent, err) {
6428
5514
  if (intent.ended)
6429
5515
  return;
6430
- intent.ended = true;
6431
- intent.readyOn = undefined;
6432
- if (this.principalServes.get(intent.serve.endpoint) === intent)
6433
- this.principalServes.delete(intent.serve.endpoint);
5516
+ this.forgetPrincipalServe(intent);
6434
5517
  const wasReady = intent.readySettled;
6435
5518
  if (!intent.readySettled) {
6436
5519
  intent.readySettled = true;
@@ -6447,19 +5530,30 @@ export class CotalEndpoint extends EventEmitter {
6447
5530
  if (wasReady)
6448
5531
  this.emitRecoverable(new Error(`stopped serving "${intent.serve.endpoint}" on the principal route: ${err.message}`, { cause: err }));
6449
5532
  }
6450
- async stopPrincipalServe(intent) {
5533
+ /** Mark `intent` ended and release what it holds in this endpoint and in this process. */
5534
+ forgetPrincipalServe(intent) {
5535
+ intent.ended = true;
5536
+ intent.readyOn = undefined;
5537
+ if (this.principalServes.get(intent.serve.endpoint) === intent)
5538
+ this.principalServes.delete(intent.serve.endpoint);
5539
+ if (principalServers.get(intent.serverKey) === intent)
5540
+ principalServers.delete(intent.serverKey);
5541
+ }
5542
+ /** Stop serving `intent` and wait for its handlers in flight, each until its request's deadline
5543
+ * at most. The ending is synchronous, so a
5544
+ * `ready` still waiting for the broker rejects before the caller's next step. `republish` writes
5545
+ * presence without the `prompts` field; the endpoint's own stop writes it offline instead. */
5546
+ async stopPrincipalServe(intent, republish = true) {
6451
5547
  if (!intent.ended) {
6452
- intent.ended = true;
6453
- intent.readyOn = undefined;
6454
- if (this.principalServes.get(intent.serve.endpoint) === intent)
6455
- this.principalServes.delete(intent.serve.endpoint);
5548
+ this.forgetPrincipalServe(intent);
6456
5549
  if (!intent.readySettled) {
6457
5550
  intent.readySettled = true;
6458
5551
  intent.rejectReady(new Error(`stopped serving "${intent.serve.endpoint}" before the broker confirmed the subscription`));
6459
5552
  }
6460
5553
  if (intent.serve.endpoint === AGENT_ENDPOINT) {
6461
5554
  this.promptsPresence = undefined;
6462
- this.publishPresence().catch((e) => this.emitRecoverable(e));
5555
+ if (republish)
5556
+ this.publishPresence().catch((e) => this.emitRecoverable(e));
6463
5557
  }
6464
5558
  }
6465
5559
  await Promise.all(intent.handles.map((handle) => handle.stop()));
@@ -6468,24 +5562,42 @@ export class CotalEndpoint extends EventEmitter {
6468
5562
  * Set this process's `prompts` presence field and write presence at once (SPEC §6): how many
6469
5563
  * prompts it holds `open` or `sent`, and a counter that goes up by one each time a prompt opens or
6470
5564
  * changes state. The field shows only while the `agent` endpoint is served and the broker accepted
6471
- * its subscription on the current connection, and it shows `{ open: 0, seq: 0 }` until this is
6472
- * called. Throws when the `agent` endpoint is not served.
5565
+ * its subscription on the current connection. Until this is called it shows `open: 0` with the
5566
+ * fresh `seq` the serve took from {@link nextPromptsSeq} when it started. Throws when the `agent`
5567
+ * endpoint is not served, and a RangeError for a `seq` below one this runtime already used.
6473
5568
  */
6474
5569
  async setPromptsPresence(prompts) {
6475
5570
  for (const k of ["open", "seq"])
6476
5571
  if (!Number.isSafeInteger(prompts?.[k]) || prompts[k] < 0)
6477
5572
  throw new TypeError(`prompts.${k} must be a non-negative safe integer`);
6478
- if (!this.principalServes.has(AGENT_ENDPOINT))
5573
+ const intent = this.principalServes.get(AGENT_ENDPOINT);
5574
+ if (!intent)
6479
5575
  throw new Error(`the prompts presence field belongs to a served "${AGENT_ENDPOINT}" endpoint: call servePrincipal first (SPEC §6)`);
5576
+ const used = promptsSeqs.get(intent.self.runtimeId) ?? 0;
5577
+ if (prompts.seq < used)
5578
+ throw new RangeError(`prompts.seq ${prompts.seq} is below ${used}, which this process already used under runtime ${intent.self.runtimeId}: seq never goes back under one runtime id (SPEC §6)`);
5579
+ promptsSeqs.set(intent.self.runtimeId, prompts.seq);
6480
5580
  this.promptsPresence = { open: prompts.open, seq: prompts.seq };
6481
5581
  await this.publishPresence();
6482
5582
  }
5583
+ /**
5584
+ * Take the next `prompts` `seq` for this process's runtime (SPEC §6): 0 the first time, then one
5585
+ * above every `seq` this process used under its runtime id, whichever endpoint object used it. A
5586
+ * prompt desk takes every `seq` it shows from here. Throws without a lifecycle UID.
5587
+ */
5588
+ nextPromptsSeq() {
5589
+ const runtimeId = processRuntime(this.requireLifecycleUid("nextPromptsSeq")).id;
5590
+ const used = promptsSeqs.get(runtimeId);
5591
+ const next = used === undefined ? 0 : used + 1;
5592
+ promptsSeqs.set(runtimeId, next);
5593
+ return next;
5594
+ }
6483
5595
  /** The `prompts` field this process's presence carries now, or none. */
6484
5596
  servedPrompts() {
6485
5597
  const intent = this.principalServes.get(AGENT_ENDPOINT);
6486
5598
  if (!intent || intent.ended || !this.nc || intent.readyOn !== this.nc)
6487
5599
  return undefined;
6488
- return this.promptsPresence ?? { open: 0, seq: 0 };
5600
+ return this.promptsPresence;
6489
5601
  }
6490
5602
  /**
6491
5603
  * Call one command of a principal's own endpoint on the principal route (SPEC 13.21), as this
@@ -6514,671 +5626,6 @@ export class CotalEndpoint extends EventEmitter {
6514
5626
  ...(call.args !== undefined ? { args: call.args } : {}),
6515
5627
  }, { deadlineMs: call.deadlineMs, ...(call.signal ? { signal: call.signal } : {}) });
6516
5628
  }
6517
- /** Serve one PRIVILEGED delivery-admin request (the D5 rail-split). The cred layer is the caller
6518
- * boundary — only the supervisor profile can publish here — and `serveControl`'s sender check +
6519
- * bounded reply still apply on top. `reloadCreds` is the class-2 renewal ADOPTION step: re-read
6520
- * the renewal-owner-re-signed creds file, pin, swap the live connection, reconnect the membership
6521
- * feed's rw connection, and reply with proof (identities + the adopted JWT windows) — or a
6522
- * structured failure (e.g. the file was never re-signed), never a silent partial. */
6523
- async handleDeliveryAdmin(req) {
6524
- // A reload's resident swap reconnects the connection every reply here rides. A caller's next
6525
- // request, such as a manager's eviction right after its reload, is answered once that
6526
- // connection is back, not while it is down.
6527
- await this.residentSwap;
6528
- // FENCE: same reason as the runtime rail above.
6529
- if (!this.plane3MayAct())
6530
- return { ok: false, error: "delivery: this daemon is not serving this shard (it is re-checking ownership); retry" };
6531
- if (req.op === "reloadCreds") {
6532
- // NOT BEFORE START-UP IS DONE (#2304). Both halves below would run: the delivery half commits
6533
- // and schedules `nc.reconnect()`, which lands underneath start-up awaits still in flight on
6534
- // this connection (the lease watch's consumer create times out and the daemon exits). Refuse
6535
- // before either half acts, so nothing is adopted and the renewal owner records why.
6536
- if (this.plane3?.startupComplete?.() === false)
6537
- return { ok: false, error: "delivery: this daemon has not finished starting; nothing adopted - the next renewal pass or the daemon's 75% re-read adopts the re-signed creds" };
6538
- // The renewal owner's EXPECTED-generation tokens (SHA-256 of each JWT it re-signed), per
6539
- // component. A missing entry means "no expectation" (the passive backstop still adopts).
6540
- const expected = (req.args?.expected ?? {});
6541
- // Prove BOTH components INDEPENDENTLY (non-short-circuit): one failing must neither hide the
6542
- // other's outcome nor be masked by it. Top-level `ok` is false if EITHER proof failed, so the
6543
- // manager can never record a green adoption while a component was refused.
6544
- // Each component's outcome claims only what is PROVEN: `brokerAccepted` (the preflight) carries
6545
- // the accepted generation's window; `residentSwap` records that the wire swap is best-effort and
6546
- // self-healing, NOT witnessed. Neither claims verified resident reauth (that would need a
6547
- // generation-tied reconnect witness on both connections).
6548
- const [delivery, membership] = await Promise.all([
6549
- this.reloadCreds(expected.delivery).then((brokerAccepted) => ({ ok: true, brokerAccepted, residentSwap: "scheduled" }), (e) => ({ ok: false, error: e.message })),
6550
- this.plane3?.reloadMembershipCreds
6551
- ? this.plane3.reloadMembershipCreds(expected.membership).then((result) => ({ ok: true, ...result }), (e) => ({ ok: false, error: e.message }))
6552
- : Promise.resolve({ ok: true, skipped: "no-membership-hook" }),
6553
- ]);
6554
- const failures = [
6555
- delivery.ok ? undefined : `delivery: ${delivery.error}`,
6556
- membership.ok ? undefined : `membership: ${membership.error}`,
6557
- ].filter((m) => m !== undefined);
6558
- // Arm the resident wire swap ONLY now — after BOTH proofs settled, right before the reply is
6559
- // returned+responded — so a slow membership proof can never let the delivery reconnect strand
6560
- // this reply. Only when delivery actually adopted a new candidate (currentCreds was updated).
6561
- if (delivery.ok) {
6562
- this.plane3?.onDeliveryCredsAdopted?.();
6563
- this.scheduleResidentSwap();
6564
- }
6565
- return failures.length
6566
- ? { ok: false, error: failures.join("; "), data: { delivery, membership } }
6567
- : { ok: true, data: { delivery, membership } };
6568
- }
6569
- if (req.op === "evictPrincipal") {
6570
- // The LIVE-EVICTION executor: force-drop a denied principal's connections.
6571
- // Composition-root hook because the $SYS observer/evictor creds live outside this endpoint's
6572
- // trust boundary; absent hook = a daemon build without the executor, refused loudly.
6573
- if (!this.plane3?.evictPrincipal)
6574
- return { ok: false, error: "evictPrincipal: no eviction executor wired on this daemon" };
6575
- const principal = typeof req.args?.principal === "string" ? req.args.principal.trim() : "";
6576
- if (!principal)
6577
- return { ok: false, error: "evictPrincipal: a principal (owner.actor dot-form) is required" };
6578
- try {
6579
- return { ok: true, data: await this.plane3.evictPrincipal(principal) };
6580
- }
6581
- catch (e) {
6582
- return { ok: false, error: e.message };
6583
- }
6584
- }
6585
- if (req.op === "evictPrincipals") {
6586
- // The same executor over a SET: one shared sweep instead of one request and one scan per
6587
- // principal. The set is bounded so one request stays inside one sweep's work.
6588
- if (!this.plane3?.evictPrincipals)
6589
- return { ok: false, error: "evictPrincipals: no batch eviction executor wired on this daemon" };
6590
- const raw = req.args?.principals;
6591
- const principals = Array.isArray(raw) ? raw.map((p) => (typeof p === "string" ? p.trim() : "")) : [];
6592
- if (principals.length === 0 || principals.some((p) => !p))
6593
- return { ok: false, error: "evictPrincipals: principals must be a non-empty array of owner.actor dot-form principals" };
6594
- if (principals.length > EVICT_PRINCIPALS_MAX || new Set(principals).size !== principals.length)
6595
- return { ok: false, error: `evictPrincipals: principals must be distinct and at most ${EVICT_PRINCIPALS_MAX}` };
6596
- try {
6597
- return { ok: true, data: await this.plane3.evictPrincipals(principals) };
6598
- }
6599
- catch (e) {
6600
- return { ok: false, error: e.message };
6601
- }
6602
- }
6603
- if (req.op === "planeConnLiveness") {
6604
- // The plane-claim liveness oracle (#29 HIGH 3): a CLOSED read-only verb — two claimed
6605
- // scanner tuples in, two bound verdicts + sweep completeness out. The executor hook owns
6606
- // the closed query validation (it holds the $SYS observer cred outside this trust boundary);
6607
- // absent hook = a daemon build without the oracle, refused loudly (the auth plane treats
6608
- // that refusal as UNKNOWN and never reclaims over it).
6609
- if (!this.plane3?.planeConnLiveness)
6610
- return { ok: false, error: "planeConnLiveness: no plane-liveness oracle wired on this daemon" };
6611
- try {
6612
- return { ok: true, data: await this.plane3.planeConnLiveness(req.args?.query) };
6613
- }
6614
- catch (e) {
6615
- return { ok: false, error: e.message };
6616
- }
6617
- }
6618
- if (req.op === "lifecycleMemberships") {
6619
- // The terminal teardown's membership INVENTORY: a complete, read-only, lifecycle-exact listing
6620
- // of one principal's durable membership rows (tombstones included) from the trusted daemon that
6621
- // owns the members bucket. The caller deletes only the exact keys it gets back, through its
6622
- // target-pinned deprovisioner grant; this verb deletes nothing and returns no other lifecycle.
6623
- const principal = typeof req.args?.principal === "string" ? req.args.principal.trim() : "";
6624
- const uid = typeof req.args?.lifecycleUid === "string" ? req.args.lifecycleUid : "";
6625
- if (!parsePrincipalKey(principal))
6626
- return { ok: false, error: "lifecycleMemberships: a principal (owner.actor dot-form) is required" };
6627
- try {
6628
- assertLifecycleToken(uid);
6629
- }
6630
- catch (e) {
6631
- return { ok: false, error: `lifecycleMemberships: ${e.message}` };
6632
- }
6633
- // A retirement fences the lifecycle's ACL row before it asks (fenceLifecycleAcl), and every join
6634
- // re-reads that row around the row it creates, so a row written after this read begins removes
6635
- // itself, on whichever daemon wrote it. For a manager that does not fence, this daemon's own
6636
- // memberships of the lifecycle stop changing here (#4965): a join or leave bound to it is refused
6637
- // from now on, and the read starts after the ones already running have finished.
6638
- let marked = this.retiringLifecycles.get(principal);
6639
- if (marked === undefined)
6640
- this.retiringLifecycles.set(principal, (marked = new Set()));
6641
- // An earlier lifecycle of this principal whose ACL row is retired can bind no join or leave any
6642
- // more, so its mark goes. A failed read keeps the mark, which only ever refuses.
6643
- for (const earlier of [...marked])
6644
- if (earlier !== uid && await aclRetired(await this.aclRegistry(), principal, earlier).catch(() => false))
6645
- marked.delete(earlier);
6646
- // A mark is kept only for an inventory whose caller did not fence (#4996): a lifecycle whose ACL
6647
- // row is narrowed to no reads, or already retired, needs none, since every join re-reads that
6648
- // row around its write. A read that fails proves nothing, so the mark is kept, which only refuses.
6649
- if (await this.lifecycleAclUnfenced(principal, uid))
6650
- marked.add(uid);
6651
- else if (marked.size === 0)
6652
- this.retiringLifecycles.delete(principal);
6653
- await Promise.allSettled(this.membershipWrites.get(`${principal}/${uid}`) ?? []);
6654
- try {
6655
- const channels = await listLifecycleMemberChannels(await this.membersRegistry(), principal, uid);
6656
- return { ok: true, data: { complete: true, channels } };
6657
- }
6658
- catch (e) {
6659
- return { ok: false, error: `lifecycleMemberships: the inventory read did not complete (${e.message})` };
6660
- }
6661
- }
6662
- if (req.op === "principalLiveness") {
6663
- // The freeze-holder liveness probe (#391): the READ-ONLY half of `evictPrincipal`. A repair
6664
- // that must REFUSE while the holder is alive cannot use eviction as its own precheck — that
6665
- // kills the holder before anything can refuse on its behalf. Same executor-hook shape as the
6666
- // verbs above (the $SYS observer cred lives outside this trust boundary); absent hook =
6667
- // refused loudly, and the caller maps a refusal to UNKNOWN and never repairs over it.
6668
- if (!this.plane3?.principalLiveness)
6669
- return { ok: false, error: "principalLiveness: no liveness oracle wired on this daemon" };
6670
- const principal = typeof req.args?.principal === "string" ? req.args.principal.trim() : "";
6671
- if (!principal)
6672
- return { ok: false, error: "principalLiveness: a principal (owner.actor dot-form) is required" };
6673
- try {
6674
- return { ok: true, data: await this.plane3.principalLiveness(principal) };
6675
- }
6676
- catch (e) {
6677
- return { ok: false, error: e.message };
6678
- }
6679
- }
6680
- if (req.op === "reloadStoreIdentity") {
6681
- // Proof that this daemon reloads standing creds from ONE named store. The manager
6682
- // compares it to its remint store before every renewal pass. Absent hook
6683
- // is a daemon that cannot name its store, which is itself a divergent composition.
6684
- if (!this.plane3?.reloadStoreIdentity)
6685
- return { ok: false, error: "reloadStoreIdentity: this daemon did not name the SecretStore it reloads from" };
6686
- try {
6687
- const identity = this.plane3.reloadStoreIdentity();
6688
- // WHO IS ANSWERING is part of the answer. This rail is queue-grouped, so the broker hands
6689
- // the request to any bound responder, and every process with a `delivery` credential for
6690
- // the space can bind it, while only the DELIVERY LEASE HOLDER actually reloads the standing
6691
- // credentials. A bare store identity therefore lets a second responder's store stand in for
6692
- // the reloading process's, and the manager's remint decision is then made about a process
6693
- // that reloads nothing. So the reply carries this responder's wire identity and whether it
6694
- // holds the lease, and the caller requires the binding (#1694).
6695
- //
6696
- // The lease read is the LIVE row, not a cached claim: a holder that lost the shard must not
6697
- // keep asserting it. A read that fails is reported as `false` rather than thrown, because
6698
- // "I cannot prove I hold the shard" is what a non-holder's answer says, and it sends the
6699
- // caller down the same fail-closed path rather than inventing an authority.
6700
- let holdsDeliveryLease = false;
6701
- try {
6702
- const own = await this.readDeliveryLeaseEntry(0);
6703
- holdsDeliveryLease = own !== undefined && this.ownsDeliveryLease(own.info);
6704
- }
6705
- catch {
6706
- holdsDeliveryLease = false;
6707
- }
6708
- // Round-trip through the closed parser so a hook cannot smuggle extra fields onto the rail.
6709
- return { ok: true, data: { identity: parseSecretStoreIdentity(identity), responder: this.card.id, holdsDeliveryLease } };
6710
- }
6711
- catch (e) {
6712
- return { ok: false, error: e.message };
6713
- }
6714
- }
6715
- return { ok: false, error: `op "${req.op}" not supported on the delivery admin service` };
6716
- }
6717
- /** Create one of the daemon's Plane-3 durables. A named create is idempotent for the same config,
6718
- * and a denial is never taken for existence. The one conflict accepted is this durable as a full
6719
- * restore recreated it, starting after its recorded ACK floor ({@link isRestoredConsumer}); every
6720
- * other failure propagates. */
6721
- async addPlane3Durable(stream, config) {
6722
- try {
6723
- await this.jsm.consumers.add(stream, config);
6724
- }
6725
- catch (e) {
6726
- if (!isConsumerAlreadyExists(e) || !isRestoredConsumer(this.space, stream, await this.jsm.consumers.info(stream, config.durable_name), config))
6727
- throw e;
6728
- }
6729
- }
6730
- /** Fan-out loop: bind the privileged `fanout` durable on CHAT and route each message (routing only —
6731
- * the trusted reader is the auth gate). */
6732
- async runFanout() {
6733
- if (!this.js || !this.jsm)
6734
- return;
6735
- await this.addPlane3Durable(chatStream(this.space), fanoutDurableConfig(this.space, { ackWaitMs: this.ackWaitMs }));
6736
- const consumer = await this.js.consumers.get(chatStream(this.space), FANOUT_DURABLE);
6737
- const msgs = await consumer.consume();
6738
- this.streamMsgs.push(msgs);
6739
- void (async () => {
6740
- for await (const m of msgs) {
6741
- try {
6742
- await this.fanOutMessage(m);
6743
- }
6744
- catch (e) {
6745
- if (!this.stopped)
6746
- this.emit("error", e);
6747
- try {
6748
- m.nak();
6749
- }
6750
- catch { /* draining */ }
6751
- }
6752
- }
6753
- })().catch((e) => { if (!this.stopped)
6754
- this.emit("error", e); });
6755
- }
6756
- /** Route ONE chat message to eligible owners' mixed inboxes. `durable` channel → its `durable-active`
6757
- * members within interval; `live` channel → `@mention` targets authorized to read it (ACL only).
6758
- * On either class, a mention of an agent whose read ACL excludes the channel is reported to the
6759
- * sender with a mention notice. Members KV is scanned FRESH per message (no cache — red-team
6760
- * BLOCKER-1 catch-up correctness). */
6761
- async fanOutMessage(m) {
6762
- // FENCE, and NOT followed by an ack: this unit was dispatched before the quiesce and is resuming
6763
- // after it, so writing the fan-out now would put entries in member inboxes on behalf of a shard
6764
- // this daemon no longer serves. Returning WITHOUT acking is the whole point - a stopped consumer
6765
- // redelivers to whoever holds the shard next, so refusing here loses no message, while acking it
6766
- // would consume the successor's work.
6767
- if (!this.plane3MayAct())
6768
- return;
6769
- const parsed = parseSubject(m.subject);
6770
- if (!parsed || parsed.kind !== "chat") {
6771
- m.ack();
6772
- return;
6773
- }
6774
- const channel = parsed.rest;
6775
- let msg;
6776
- try {
6777
- msg = m.json();
6778
- }
6779
- catch {
6780
- m.ack();
6781
- return;
6782
- }
6783
- if (!isFromSubjectSender(msg, parsed)) {
6784
- m.ack();
6785
- return;
6786
- } // authenticity (SPEC §5)
6787
- const seq = m.seq;
6788
- const normalizedMsg = authenticatedChannelMessage(msg, channel);
6789
- // SECOND FENCE, AT THE COMMIT POINT. The entry check above cannot cover this unit's own awaits:
6790
- // the class read and the member scan below both hit the broker, and a daemon descheduled inside
6791
- // them can resume after a successor owns the shard. Re-ask immediately before the first WRITE,
6792
- // and again return without acking so the entry redelivers rather than being consumed here.
6793
- if (!this.plane3MayAct())
6794
- return;
6795
- const durable = (await this.deliveryClassFresh(channel)) === "durable";
6796
- // Each mention is resolved once. A live channel delivers to the agents among them that may read
6797
- // it, and on either class a mention of an agent that may not read the channel is reported to the
6798
- // sender below, so a post never claims a reader it cannot have.
6799
- const mentioned = [];
6800
- for (const name of msg.mentions ?? []) {
6801
- const owner = this.resolveOwnerByName(name);
6802
- if (!owner || owner === msg.from.id)
6803
- continue;
6804
- // A mention arrives as an ALIAS (the roster names no lifecycle): resolve its single live ACL
6805
- // row for BOTH the read-authorization and the lifecycle to address. Ambiguity (two live rows)
6806
- // refuses THIS copy loudly rather than guessing a lifecycle.
6807
- let row;
6808
- try {
6809
- row = await this.aclForAlias(owner);
6810
- }
6811
- catch (e) {
6812
- this.emit("error", e);
6813
- continue;
6814
- }
6815
- if (row)
6816
- mentioned.push({ name, owner, row });
6817
- }
6818
- if (durable) {
6819
- // The channel's own rows and every family row above it (`review.>`). A lifecycle holding two of
6820
- // them gets two publishes and keeps one copy: the dedupe id is per message and lifecycle (#5404).
6821
- for (const rec of await listMembers(await this.membersRegistry(), { covering: channel })) {
6822
- if (rec.owner === msg.from.id)
6823
- continue; // never backstop the sender's own post
6824
- if (!durableEligible(rec, seq))
6825
- continue; // routing fast-filter (reader re-checks)
6826
- // Address the member row's RECORDED lifecycle: a retired row (tombstone pending) routes to the
6827
- // retired lifecycle's inbox, never the alias's new occupant (SPEC 13.1 cross-plane scoping).
6828
- // Store the AUTHENTICATED-channel copy (main's normalization): the durable frame validates
6829
- // msg.channel === frame.channel, and payload to/toService are stripped.
6830
- // Per-copy, because the scan above and every publish below is a broker round trip: a fan-out
6831
- // to fifty members can straddle a quiesce in its middle. Stopping part-way is safe where
6832
- // stopping late is not - the entry is never acked here, so the successor redelivers it and
6833
- // re-sends the whole set; publishDinbox is idempotent per (owner, lifecycle, msgID).
6834
- if (!this.plane3MayAct())
6835
- return;
6836
- await this.publishDinbox(rec.owner, rec.lifecycleUid, {
6837
- msg: normalizedMsg, channel, seq, reason: "durable-channel", generation: rec.generation,
6838
- ...(rec.channel !== channel ? { row: rec.channel } : {}),
6839
- });
6840
- }
6841
- }
6842
- else {
6843
- for (const { owner, row } of mentioned) {
6844
- if (!channelInAllow(row.allowSubscribe, channel))
6845
- continue; // @mention can't bypass the read ACL
6846
- if (!this.plane3MayAct())
6847
- return; // same rule on the live-mention path
6848
- await this.publishDinbox(owner, row.lifecycleUid, { msg: normalizedMsg, channel, seq, reason: "live-mention", generation: 0 });
6849
- }
6850
- }
6851
- const unreadable = mentioned.filter(({ row }) => !channelInAllow(row.allowSubscribe, channel)).map(({ name }) => name);
6852
- if (unreadable.length) {
6853
- if (!this.plane3MayAct())
6854
- return;
6855
- await this.sendMentionNotice(msg, channel, seq, unreadable);
6856
- }
6857
- // FINAL FENCE, BEFORE THE ACK ITSELF, which is the real commit: acking tells the stream this
6858
- // shard's work is done. A daemon that stopped serving mid-fan-out must not make that claim, or
6859
- // the successor never sees the entry and the message is lost outright rather than duplicated.
6860
- if (!this.plane3MayAct())
6861
- return;
6862
- m.ack();
6863
- }
6864
- deliveryLimitRead;
6865
- /** The DLV count the trusted writer stamps on a copy (SPEC §8): the stream's `max_msgs_per_subject`,
6866
- * read at most {@link DELIVERY_LIMIT_READ_MS} before the write, as a whole number or `none`. */
6867
- async deliveryLimitStamp() {
6868
- const at = Date.now();
6869
- if (this.deliveryLimitRead && at - this.deliveryLimitRead.at < DELIVERY_LIMIT_READ_MS)
6870
- return this.deliveryLimitRead.stamp;
6871
- const count = (await this.jsm.streams.info(dlvStream(this.space))).config.max_msgs_per_subject;
6872
- const stamp = count === HISTORY_NO_COUNT || count < 1 ? "none" : String(count);
6873
- this.deliveryLimitRead = { at, stamp };
6874
- return stamp;
6875
- }
6876
- /** Trusted-reader loop: bind the single privileged `reader` durable over `dinbox.>` and re-authorize
6877
- * + transfer each entry. */
6878
- async runReader() {
6879
- if (!this.js || !this.jsm)
6880
- return;
6881
- await this.addPlane3Durable(inboxStream(this.space), inboxReaderConfig(this.space, { ackWaitMs: this.ackWaitMs }));
6882
- const consumer = await this.js.consumers.get(inboxStream(this.space), INBOX_READER_DURABLE);
6883
- const msgs = await consumer.consume();
6884
- this.streamMsgs.push(msgs);
6885
- void (async () => {
6886
- for await (const m of msgs) {
6887
- try {
6888
- await this.readerHandle(m);
6889
- }
6890
- catch (e) {
6891
- if (!this.stopped)
6892
- this.emit("error", e);
6893
- try {
6894
- m.nak();
6895
- }
6896
- catch { /* draining */ }
6897
- }
6898
- }
6899
- })().catch((e) => { if (!this.stopped)
6900
- this.emit("error", e); });
6901
- }
6902
- /** Whether one of the lifecycle's rows covering concrete `channel`, the channel's own or a family
6903
- * above it, is eligible for the post at `seq`. One exact read per row, the channel's own first. */
6904
- async lifecycleCoversPost(owner, lifecycleUid, channel, seq) {
6905
- const kv = await this.membersRegistry();
6906
- for (const row of [channel, ...familiesCovering(channel).reverse()]) {
6907
- const rec = await readMember(kv, row, owner, lifecycleUid);
6908
- if (rec && durableEligible(rec.record, seq))
6909
- return true;
6910
- }
6911
- return false;
6912
- }
6913
- /** Remove a retired lifecycle's member row for `channel`, when one exists. Best effort: the row
6914
- * authorizes nothing (its ACL row is retired), and the next entry fan-out writes against it
6915
- * brings the reader back here. */
6916
- async removeRetiredMemberRow(channel, owner, lifecycleUid) {
6917
- try {
6918
- const kv = await this.membersRegistry();
6919
- if (await readMember(kv, channel, owner, lifecycleUid) === undefined)
6920
- return;
6921
- await deleteMember(kv, channel, owner, lifecycleUid);
6922
- }
6923
- catch (e) {
6924
- this.emit("warning", new Error(`plane-3 reader: could not remove retired lifecycle ${owner}.${lifecycleUid}'s "${channel}" member row: ${e.message}`));
6925
- }
6926
- }
6927
- /** Re-authorize ONE mixed-inbox entry and transfer it to the owner's DELIVER store. Deny (drop) on a
6928
- * revoked/narrowed ACL or out-of-interval seq; on transfer success, ack the mixed entry (durability
6929
- * has moved to DLV — an §8 equivalent per-member at-least-once mechanism). The agent acks DLV. */
6930
- async readerHandle(m) {
6931
- // FENCE, same rule as the fan-out writer: no effect and no ack once this daemon has stopped
6932
- // serving the shard. The entry redelivers to the holder rather than being consumed here.
6933
- if (!this.plane3MayAct())
6934
- return;
6935
- const pr = parseDinboxPrincipal(m.subject);
6936
- if (!pr) {
6937
- m.ack();
6938
- return;
6939
- } // unparseable subject (incl. a pre-cut 5-segment form) — not a real entry
6940
- const owner = `${pr.owner}.${pr.actor}`; // the member principal dot-form (acl/member keys, msgID)
6941
- let entry;
6942
- try {
6943
- entry = m.json();
6944
- }
6945
- catch {
6946
- m.ack();
6947
- return;
6948
- } // undecodable — drop
6949
- if (!isRecord(entry)) {
6950
- m.ack();
6951
- return;
6952
- } // non-object envelope — permanently invalid
6953
- const redeliveries = m.info?.deliveryCount ?? 1; // JsMsg delivery attempts (1 on first delivery)
6954
- // Lifecycle-exact ACL re-auth (SPEC 13.1): the entry was addressed to pr.lifecycleUid's inbox, so
6955
- // the row read is that lifecycle's exact key — a retired lifecycle's purged row reads as unknown
6956
- // and its residual entries are removed below, never authorized against the successor's row.
6957
- const acl = await this.plane3?.aclFor(owner, pr.lifecycleUid);
6958
- if (acl === undefined) {
6959
- // RETIRED lifecycle: its exact ACL key carries retirement's tombstone, so no attempt can ever
6960
- // deliver this entry. Drop it AND remove it from the store. Deferring it to the ceiling and
6961
- // keeping it would make every reader that starts from a fresh cursor pay the same sweep again.
6962
- // An ACL row that is merely absent proves nothing and still defers below.
6963
- if (await aclRetired(await this.aclRegistry(), owner, pr.lifecycleUid)) {
6964
- if (!this.plane3MayAct())
6965
- return;
6966
- // Remove BEFORE the ack. The ack ends the reader's work on this entry, so acking first and
6967
- // then failing the delete would keep the entry with nothing left to remove it. A failed delete
6968
- // stays pending and retries, bounded by the same ceiling; an entry the stream no longer holds
6969
- // is already removed (the broker answers a delete of a missing sequence with 10043).
6970
- let failure;
6971
- try {
6972
- await this.jsm.streams.deleteMessage(inboxStream(this.space), m.seq, false);
6973
- }
6974
- catch (e) {
6975
- if (!isJetStreamMissing(e, 10043))
6976
- failure = e instanceof Error ? e : new Error(String(e));
6977
- }
6978
- // FENCE AFTER THE DELETE, which is broker I/O. A daemon that stopped serving while it was in
6979
- // flight leaves the entry to the holder: a term here would move the shared durable past an entry
6980
- // the failed delete left stored, so the holder would never retry it.
6981
- if (!this.plane3MayAct())
6982
- return;
6983
- if (failure) {
6984
- if (redeliveries >= READER_MAX_REDELIVERIES) {
6985
- m.term();
6986
- this.emit("error", new Error(`plane-3 reader: gave up removing entry ${m.seq} for retired lifecycle ${owner}.${pr.lifecycleUid} after ${redeliveries} redeliveries: ${failure.message}`));
6987
- return;
6988
- }
6989
- m.nak(2000);
6990
- return;
6991
- }
6992
- // The member row that addressed this entry is one a daemon left behind when it died between
6993
- // its create and its post-write ACL read (#4994). It authorizes nothing, but fan-out copies
6994
- // every later message on the channel into this inbox until it goes, and this reader then
6995
- // removes each copy. Remove the row where that cost is paid, by its exact key, only when it
6996
- // exists: a purge of an absent key would write a marker per dropped entry.
6997
- if (entry.reason === "durable-channel")
6998
- await this.removeRetiredMemberRow(entry.row ?? entry.channel, owner, pr.lifecycleUid);
6999
- if (!this.plane3MayAct())
7000
- return;
7001
- m.ack();
7002
- return;
7003
- }
7004
- // UNKNOWN owner — the manager has not (re)hydrated this owner's ACL yet (e.g. right after a
7005
- // manager PROCESS restart). This is NOT a revocation: DEFER (redeliver), never drop — an ack here
7006
- // would lose at-least-once on restart (impl-review BLOCKER-2). A delayed nak + a redelivery
7007
- // ceiling stops one perma-unknown owner from head-of-lining the shared reader.
7008
- // (Follow-up: the manager does not yet rehydrate its managed set across a process restart — until
7009
- // it does, a long-unknown owner's entries term after the ceiling; tracked, not a silent ack-drop.)
7010
- if (redeliveries >= READER_MAX_REDELIVERIES) {
7011
- m.term();
7012
- this.emit("error", new Error(`plane-3 reader: gave up on entry for unknown owner ${owner} after ${redeliveries} redeliveries`));
7013
- return;
7014
- }
7015
- m.nak(2000);
7016
- return;
7017
- }
7018
- // KNOWN owner whose CURRENT ACL no longer covers the channel — a revocation/narrowing. Drop: the
7019
- // entry is no longer authorized (SPEC §7 current-ACL gate before surfacing).
7020
- if (!channelInAllow(acl, entry.channel)) {
7021
- m.ack();
7022
- return;
7023
- }
7024
- // INTERVAL re-auth (not a current-member boolean): a pre-leave entry (seq ≤ leaveCursor) stays
7025
- // deliverable; seq > leaveCursor (or after a rejoin's newer joinCursor) is the hard cut. Any of the
7026
- // lifecycle's rows that cover the channel may hold the post, whichever row addressed the copy.
7027
- if (entry.reason === "durable-channel" && !(await this.lifecycleCoversPost(owner, pr.lifecycleUid, entry.channel, entry.seq))) {
7028
- m.ack();
7029
- return;
7030
- }
7031
- // COMMIT-POINT FENCE. Everything above this line is broker I/O - the ACL read and the member
7032
- // re-read - so the entry check at the top of the handler is stale by now if this unit was
7033
- // descheduled inside either. The transfer below is the effect that must not happen on behalf of
7034
- // a shard we no longer serve; returning without acking leaves it for the holder.
7035
- if (!this.plane3MayAct())
7036
- return;
7037
- try {
7038
- // DLV has no original chat subject, so preserve the channel the trusted fan-out reader derived
7039
- // from CHAT. Never let the publisher-controlled payload label choose connector attention.
7040
- const frame = {
7041
- version: 1,
7042
- channel: entry.channel,
7043
- msg: authenticatedChannelMessage(entry.msg, entry.channel),
7044
- };
7045
- const frameHeaders = headers();
7046
- frameHeaders.set(PLANE3_FRAME_HEADER, "1");
7047
- // The count read may be broker I/O, so the fence is asked again before the write.
7048
- frameHeaders.set(DELIVERY_LIMIT_HEADER, await this.deliveryLimitStamp());
7049
- if (!this.plane3MayAct())
7050
- return;
7051
- // Lifecycle-keyed subject + msgID (ours): the DLV consumer filters on the lifecycle-scoped
7052
- // subject (streams.ts filter_subject), and a predecessor's transferred copy must never suppress
7053
- // a successor's under the same alias (disjoint lifecycles, stream-wide dedupe).
7054
- await this.js.publish(dlvSubject(this.space, pr.owner, pr.actor, pr.lifecycleUid), JSON.stringify(frame), {
7055
- ...this.plane3MsgId(entry, `${owner}:${pr.lifecycleUid}`),
7056
- headers: frameHeaders,
7057
- });
7058
- }
7059
- catch {
7060
- // EVERY transfer failure is deliberately a retry, never a successful result. Keep the entry
7061
- // pending (redeliver), bounded by the same ceiling so a poison
7062
- // entry can't head-of-line the shared reader forever.
7063
- if (redeliveries >= READER_MAX_REDELIVERIES) {
7064
- m.term();
7065
- this.emit("error", new Error(`plane-3 reader: gave up transferring ${entry.msg.id} for ${owner} after ${redeliveries} redeliveries`));
7066
- return;
7067
- }
7068
- m.nak(2000);
7069
- return;
7070
- }
7071
- // The ack is the last commit: the transfer above already happened, so the only question left is
7072
- // whether THIS daemon may retire the source entry. If it stopped serving mid-transfer, leave the
7073
- // entry pending - the successor re-transfers and stream-wide dedupe on the msgID collapses the
7074
- // duplicate, which is the recoverable direction. Acking here is the unrecoverable one.
7075
- if (!this.plane3MayAct())
7076
- return;
7077
- m.ack();
7078
- }
7079
- /** Agent-side: bind + pump our pre-created Plane-3 DELIVER durable (`dlv_<id>`). Every message here is
7080
- * delivery-daemon-written (DLV is delivery-write-only, broker-enforced) and is a CHANNEL message by contract
7081
- * (the backstop never carries DMs), so `kind=channel` is path-derived (SPEC §4) and the body is
7082
- * trusted (no spoof-guard). `durable:true` — real JetStream ack, coalesced with the core-sub live
7083
- * copy by `MeshAgent.ingest`. No-op when the durable isn't present (open mode / not provisioned). A read
7084
- * list declared on a credential that cannot fetch the durable refuses. */
7085
- async pumpDlv() {
7086
- if (!this.js)
7087
- return;
7088
- if (!this.ownLifecycleUid)
7089
- return; // no lifecycle uid — never provisioned for Plane-3 (its durable is lifecycle-keyed)
7090
- if (this.allowSubscribe?.length === 0)
7091
- return; // known channel-less grant: no DLV durable or bind rows exist
7092
- const durable = dlvDurable(this.owner, this.actor, this.ownLifecycleUid);
7093
- if (this.allowSubscribe === undefined && !this.userMode && this.nc) {
7094
- // Native credential-only clients have no separate ACL option. Use the publish permissions
7095
- // captured when THIS connection authenticated; a callout sentinel does not carry the grant.
7096
- const permission = this.watchDeletePermissions.get(this.nc)?.();
7097
- // Only an absent allow selects no binding. A declared bind with a deny still reaches the
7098
- // broker and fails, so a refusal cannot masquerade as a channel-less credential.
7099
- if (permission && !issuedPermitsSubject({ ...permission, deny: [] }, `$JS.API.CONSUMER.INFO.${dlvStream(this.space)}.${durable}`))
7100
- return;
7101
- }
7102
- let consumer;
7103
- try {
7104
- consumer = await this.js.consumers.get(dlvStream(this.space), durable);
7105
- }
7106
- catch (e) {
7107
- if (isJetStreamMissing(e, JetStreamApiCodes.ConsumerNotFound)) {
7108
- // A channel-less credential may ask about its durable but holds no fetch row for it. A read
7109
- // list declared on such a credential disagrees with the grant, so the start refuses.
7110
- const issued = this.nc && this.watchDeletePermissions.get(this.nc)?.();
7111
- if ((this.allowSubscribe?.length ?? 0) > 0 && issued
7112
- && !issuedPermitsSubject({ ...issued, deny: [] }, `$JS.API.CONSUMER.MSG.NEXT.${dlvStream(this.space)}.${durable}`))
7113
- throw new Error(`this connection declared a read list, but its credential cannot fetch delivery durable "${durable}" for space "${this.space}"; the read list and the credential disagree`);
7114
- return;
7115
- }
7116
- throw e; // a denied bind is not evidence that the read ACL is empty or the durable is absent
7117
- }
7118
- const lease = await this.readDeliveryLease(0);
7119
- const liveDeliveryPlane = lease?.ready === true;
7120
- if (!liveDeliveryPlane)
7121
- this.emit("warning", new Error(`delivery durable "${durable}" for space "${this.space}" bound while the plane this connection reaches has no ready delivery lease, so the durable delivers nothing until a delivery daemon serves this plane. If a daemon serves this space on another plane, reconnect against it, which re-binds the durable there.`));
7122
- const ackWaitMs = await ackWaitOf(consumer);
7123
- const msgs = await consumer.consume();
7124
- this.streamMsgs.push(msgs);
7125
- void (async () => {
7126
- for await (const m of msgs) {
7127
- // A copy with a full store behind it means the store held its count, so older copies may have
7128
- // been discarded unread (#5404); NATS gives no notice of the discard itself. The count is the
7129
- // one the trusted writer stamped on this copy. Checked on every delivery, ahead of the checks
7130
- // below and the connector's id dedupe, so a copy the agent also got live cannot hide it. The
7131
- // connector folds repeats into one notice.
7132
- let limit;
7133
- try {
7134
- limit = deliveryLimitOf(m.headers?.get(DELIVERY_LIMIT_HEADER));
7135
- }
7136
- catch (e) {
7137
- this.emit("error", e);
7138
- limit = null;
7139
- }
7140
- if (limit !== null && m.info.pending >= limit - 1)
7141
- this.emit("stored-copies-full", { limit });
7142
- // The header is minted only by the trusted DLV writer. A body-only discriminator is
7143
- // insufficient because old Cotal envelopes permit unknown fields and could imitate it.
7144
- if (m.headers?.get(PLANE3_FRAME_HEADER) !== "1") {
7145
- this.emit("error", new Error("plane-3 delivery: unauthenticated or unversioned DLV entry terminated"));
7146
- try {
7147
- m.term();
7148
- }
7149
- catch { /* draining */ }
7150
- continue;
7151
- }
7152
- let raw;
7153
- try {
7154
- raw = m.json();
7155
- }
7156
- catch (e) {
7157
- this.emit("error", e);
7158
- try {
7159
- m.term();
7160
- }
7161
- catch { /* draining */ }
7162
- continue;
7163
- }
7164
- if (!isPlane3DeliveryFrame(raw)) {
7165
- this.emit("error", new Error("plane-3 delivery: malformed versioned DLV entry terminated"));
7166
- try {
7167
- m.term();
7168
- }
7169
- catch { /* draining */ }
7170
- continue;
7171
- }
7172
- const msg = authenticatedChannelMessage(raw.msg, raw.channel);
7173
- if (msg.from?.id === this.card.id) {
7174
- m.ack();
7175
- continue;
7176
- } // own echo (defensive)
7177
- this.emit("message", msg, durableDelivery(m, ackWaitMs), { historical: false, kind: "channel" });
7178
- }
7179
- })().catch((e) => { if (!this.stopped)
7180
- this.emit("error", e); });
7181
- }
7182
5629
  /** Agent-side: request a Plane-3 durable backstop for a channel via the server-side delivery daemon (ctl.delivery). Throws
7183
5630
  * when no privileged writer is present (open / no delivery daemon). 30s timeout — activation catch-up may
7184
5631
  * run before the reply (the window is small, but a busy channel can take more than the 5s default). */
@@ -7218,7 +5665,7 @@ export class CotalEndpoint extends EventEmitter {
7218
5665
  catch (e) {
7219
5666
  if (attempt === 0)
7220
5667
  this.emitRecoverable(new Error(`channel "${channel}": Plane-3 durable membership (generation ${generation}) not yet tombstoned after a refused live sub - retrying; §7 boundary may be open until it succeeds (${e.message})`));
7221
- await new Promise((r) => setTimeout(r, Math.min(30_000, 1000 * 2 ** attempt)));
5668
+ await this.membershipPause(attempt);
7222
5669
  }
7223
5670
  }
7224
5671
  }
@@ -7387,7 +5834,7 @@ export class CotalEndpoint extends EventEmitter {
7387
5834
  return; // already reconciling
7388
5835
  this.pendingBootJoins.add(channel);
7389
5836
  for (let attempt = 0;; attempt++) {
7390
- await new Promise((r) => setTimeout(r, Math.min(30_000, 1000 * 2 ** attempt)));
5837
+ await this.membershipPause(attempt);
7391
5838
  if (this.stopped || !this.channels.includes(channel) || this.plane3Channels.has(channel)) {
7392
5839
  this.pendingBootJoins.delete(channel);
7393
5840
  return; // stopped, left, or another path established it
@@ -7878,60 +6325,7 @@ export class CotalEndpoint extends EventEmitter {
7878
6325
  * calls must be serial — every reader here awaits to completion, so they are.
7879
6326
  */
7880
6327
  async collectHistory(subject, start, opts = {}) {
7881
- // Serialize on the per-instance lock: the fixed `chathist_<id>` name means two concurrent reads
7882
- // (recall + join-backfill + drop-marker can race in-process) would delete/recreate the consumer
7883
- // under each other and cross-feed results. The chain makes the "serial callers" assumption true.
7884
- const run = this.histLock.then(() => this.collectHistoryInner(subject, start, opts));
7885
- this.histLock = run.catch(() => { }); // keep the chain alive on error
7886
- return run;
7887
- }
7888
- async collectHistoryInner(subject, start, opts = {}) {
7889
- if (!this.jsm || !this.js)
7890
- throw new Error(this.notLiveMsg());
7891
- const stream = chatStream(this.space);
7892
- const name = chatHistDurable(this.owner, this.actor, this.requireLifecycleUid("chat history reads"));
7893
- const out = [];
7894
- // Clear any consumer leaked by a crashed prior read before re-creating it with THIS read's
7895
- // single filter (the read ACL is enforced at create — see the doc above).
7896
- await this.deleteConsumerIfPresent(stream, name);
7897
- await this.jsm.consumers.add(stream, {
7898
- name,
7899
- filter_subject: subject,
7900
- ack_policy: AckPolicy.None,
7901
- mem_storage: true,
7902
- inactive_threshold: nanos(30_000),
7903
- ...("time" in start
7904
- ? { deliver_policy: DeliverPolicy.StartTime, opt_start_time: start.time.toISOString() }
7905
- : { deliver_policy: DeliverPolicy.StartSequence, opt_start_seq: start.seq }),
7906
- });
7907
- try {
7908
- const consumer = await this.js.consumers.get(stream, name);
7909
- let pending = (await consumer.info()).num_pending;
7910
- while (pending > 0) {
7911
- const want = Math.min(pending, 256);
7912
- const iter = await consumer.fetch({ max_messages: want, expires: 5_000 });
7913
- let got = 0;
7914
- for await (const m of iter) {
7915
- got++;
7916
- if (opts.untilSeq !== undefined && m.seq > opts.untilSeq)
7917
- return out; // crossed the frontier
7918
- // Belt-and-suspenders over the lock: only keep messages on the requested channel subject
7919
- // (the consumer's filter already bounds this; guards against any stale-consumer edge).
7920
- if (!subjectMatches(subject, m.subject))
7921
- continue;
7922
- out.push(m);
7923
- if (opts.limit !== undefined && out.length >= opts.limit)
7924
- return out;
7925
- }
7926
- if (got < want)
7927
- break; // drained early
7928
- pending -= got;
7929
- }
7930
- }
7931
- finally {
7932
- await this.deleteConsumerIfPresent(stream, name);
7933
- }
7934
- return out;
6328
+ return this.history.collectHistory(subject, start, opts);
7935
6329
  }
7936
6330
  /** Read a channel's retained history up to `upToSeq` (the join frontier) and emit each message
7937
6331
  * as a `historical` "message" event. `sinceMs` bounds how far back via a native consumer
@@ -7945,44 +6339,7 @@ export class CotalEndpoint extends EventEmitter {
7945
6339
  * after it, which is exactly the catch-up. New messages (`seq > upToSeq`) are skipped — the live
7946
6340
  * tail owns them. Reads through the contained {@link collectHistory}. */
7947
6341
  async backfillChannel(channel, upToSeq, sinceMs, floorSeq) {
7948
- if (floorSeq !== undefined && floorSeq >= upToSeq)
7949
- return 0;
7950
- const subject = chatSubject(this.space, "*", "*", channel);
7951
- // A window that reaches back before 1970 covers the whole retained window (#7323). Its start time
7952
- // would be a date JetStream does not read, or not a valid date at all, and the read replayed nothing.
7953
- const since = sinceMs === undefined ? undefined : Date.now() - sinceMs;
7954
- const start = floorSeq !== undefined ? { seq: floorSeq + 1 } : since === undefined || !(since > 0) ? { seq: 1 } : { time: new Date(since) };
7955
- let msgs;
7956
- try {
7957
- msgs = await this.collectHistory(subject, start, { untilSeq: upToSeq });
7958
- }
7959
- catch (e) {
7960
- this.emit("error", e);
7961
- if (isPermissionDenied(e))
7962
- throw e;
7963
- return 0;
7964
- }
7965
- const noop = { ack: () => { }, nak: () => { }, durable: false };
7966
- let n = 0;
7967
- for (const sm of msgs) {
7968
- let msg;
7969
- try {
7970
- msg = sm.json();
7971
- }
7972
- catch {
7973
- continue; // skip undecodable
7974
- }
7975
- // Same authenticity guard as the tail; skip our own echoes in history.
7976
- if (!isRecord(msg) || !isUsableMessageId(msg.id))
7977
- continue; // malformed envelope (SPEC sec 5) — history skips
7978
- const parsed = parseSubject(sm.subject);
7979
- if (!parsed || !isFromSubjectSender(msg, parsed) || msg.from.id === this.card.id)
7980
- continue;
7981
- // Backfill only ever reads the chat stream, so the authenticated class is always "channel".
7982
- this.emit("message", authenticatedMessage(msg, parsed), noop, { historical: true, kind: "channel" });
7983
- n++;
7984
- }
7985
- return n;
6342
+ return this.history.backfillChannel(channel, upToSeq, sinceMs, floorSeq);
7986
6343
  }
7987
6344
  /**
7988
6345
  * Replay-gated pull of a channel's retained ambient from `sinceSeq` (exclusive) forward — the
@@ -8005,55 +6362,7 @@ export class CotalEndpoint extends EventEmitter {
8005
6362
  * the window was empty and complete.
8006
6363
  */
8007
6364
  async recallChannel(channel, sinceSeq) {
8008
- if (!this.jsm)
8009
- throw new Error(this.notLiveMsg());
8010
- if (!isConcreteChannel(channel))
8011
- return { messages: [], seqs: [], dropped: false, unanswered: true };
8012
- const policy = await this.joinPolicyFresh(channel);
8013
- if (!policy.replay)
8014
- return { messages: [], seqs: [], dropped: true, unanswered: true };
8015
- const subject = chatSubject(this.space, "*", "*", channel);
8016
- let raw;
8017
- try {
8018
- raw = await this.collectHistory(subject, { seq: sinceSeq + 1 });
8019
- }
8020
- catch (e) {
8021
- this.emit("error", e);
8022
- if (isPermissionDenied(e))
8023
- throw e;
8024
- return { messages: [], seqs: [], dropped: true, unanswered: true };
8025
- }
8026
- const collected = [];
8027
- const seqs = [];
8028
- for (const sm of raw) {
8029
- let msg;
8030
- try {
8031
- msg = sm.json();
8032
- }
8033
- catch {
8034
- continue; // skip undecodable
8035
- }
8036
- // Same authenticity guard as the tail/backfill; skip our own echoes.
8037
- if (!isRecord(msg) || !isUsableMessageId(msg.id))
8038
- continue; // malformed envelope (SPEC sec 5) — recall skips
8039
- const parsed = parseSubject(sm.subject);
8040
- if (!parsed || !isFromSubjectSender(msg, parsed) || msg.from.id === this.card.id)
8041
- continue;
8042
- collected.push(authenticatedMessage(msg, parsed));
8043
- seqs.push(sm.seq);
8044
- }
8045
- let dropped;
8046
- try {
8047
- dropped = await this.chatMayHaveLost(subject, sinceSeq);
8048
- }
8049
- catch (e) {
8050
- // Completeness could not be read, so the read is reported as unanswered: nothing settles on it.
8051
- this.emit("error", e);
8052
- if (isPermissionDenied(e))
8053
- throw e;
8054
- return { messages: [], seqs: [], dropped: true, unanswered: true };
8055
- }
8056
- return { messages: collected, seqs, dropped, unanswered: false };
6365
+ return this.history.recallChannel(channel, sinceSeq);
8057
6366
  }
8058
6367
  /** Whether the chat stream's limits may have removed posts on `subject` after `sinceSeq`, read from
8059
6368
  * the stream's own config rather than a compiled default. Throws when it cannot read, so no caller
@@ -8168,12 +6477,6 @@ export class CotalEndpoint extends EventEmitter {
8168
6477
  return this.presence.sweep();
8169
6478
  }
8170
6479
  }
8171
- /** What a history read failure NAMES when it could not finish. One filter subject is the useful
8172
- * thing to print; a set of sixty-nine of them is a wall of text in a message a human has to read,
8173
- * so a set says its size and the stream it was read from instead. */
8174
- function subjectLabel(subjects) {
8175
- return subjects.length === 1 ? subjects[0] : `${subjects.length} filtered subjects`;
8176
- }
8177
6480
  /** A usable delivery-message id (#624): a string, possibly empty (the never-a-key case), but
8178
6481
  * never absent and never a non-string. An absent or non-string id is a malformed envelope under
8179
6482
  * SPEC sec 5; each delivery pump handles it per its own class (durable term, live drop, history
@@ -8220,12 +6523,6 @@ function authenticatedMessage(msg, parsed) {
8220
6523
  return authenticatedAnycastMessage(msg, parsed.rest);
8221
6524
  return msg;
8222
6525
  }
8223
- function authenticatedChannelMessage(msg, channel) {
8224
- if (msg.channel === channel && msg.to === undefined && msg.toService === undefined)
8225
- return msg;
8226
- const { to: _to, toService: _toService, ...base } = msg;
8227
- return { ...base, channel };
8228
- }
8229
6526
  function authenticatedDmMessage(msg, to) {
8230
6527
  if (msg.to === to && msg.channel === undefined && msg.toService === undefined)
8231
6528
  return msg;
@@ -8238,23 +6535,6 @@ function authenticatedAnycastMessage(msg, toService) {
8238
6535
  const { to: _to, channel: _channel, ...base } = msg;
8239
6536
  return { ...base, toService };
8240
6537
  }
8241
- /** The ack wait the broker reports for a bound consumer: how long a copy it delivers stays with
8242
- * its reader without an ack. */
8243
- async function ackWaitOf(consumer) {
8244
- const { config } = await consumer.info(true);
8245
- if (config.ack_wait === undefined)
8246
- throw new Error(`consumer ${config.durable_name} reports no ack wait`);
8247
- return millis(config.ack_wait);
8248
- }
8249
- /** A JetStream copy's {@link Delivery}, carrying the broker's own redelivery state for it. */
8250
- function durableDelivery(m, ackWaitMs) {
8251
- return {
8252
- ack: () => m.ack(),
8253
- nak: () => m.nak(),
8254
- durable: true,
8255
- redelivery: { deliveryCount: m.info.deliveryCount, ackWaitMs, working: () => m.working() },
8256
- };
8257
- }
8258
6538
  /** A DM-only copy's delivery. An ack, nak or in-progress signal on a closed connection is refused
8259
6539
  * with a throw instead of being dropped silently: the copy stays unacked and the broker redelivers it. */
8260
6540
  function dmOnlyDelivery(m, ackWaitMs, nc) {
@@ -8347,59 +6627,6 @@ function isReadableMessagePart(value) {
8347
6627
  function isHistoryDrainEnvelope(value) {
8348
6628
  return isRecord(value) && isUsableMessageId(value.id) && isRecord(value.from);
8349
6629
  }
8350
- function isPlane3DeliveryFrame(value) {
8351
- if (!isRecord(value))
8352
- return false;
8353
- const keys = Object.keys(value);
8354
- if (keys.length !== 3 || !keys.includes("version") || !keys.includes("channel") || !keys.includes("msg"))
8355
- return false;
8356
- if (value.version !== 1 || typeof value.channel !== "string" || !isConcreteChannel(value.channel))
8357
- return false;
8358
- return isCotalMessage(value.msg) && value.msg.channel === value.channel;
8359
- }
8360
- function isCotalMessage(value) {
8361
- if (!isRecord(value))
8362
- return false;
8363
- const routes = ["channel", "to", "toService"].filter((key) => key in value);
8364
- if (routes.length !== 1 || typeof value[routes[0]] !== "string")
8365
- return false;
8366
- if (typeof value.id !== "string" || typeof value.ts !== "number" || !Number.isFinite(value.ts) ||
8367
- typeof value.space !== "string" || !isEndpointRef(value.from) || !Array.isArray(value.parts) ||
8368
- !value.parts.every(isMessagePart))
8369
- return false;
8370
- if (value.mentions !== undefined && (!Array.isArray(value.mentions) || !value.mentions.every((name) => typeof name === "string")))
8371
- return false;
8372
- if (value.replyTo !== undefined && typeof value.replyTo !== "string")
8373
- return false;
8374
- if (value.contextId !== undefined && typeof value.contextId !== "string")
8375
- return false;
8376
- return true;
8377
- }
8378
- function isEndpointRef(value) {
8379
- return isRecord(value) && typeof value.id === "string" && typeof value.name === "string" &&
8380
- (value.role === undefined || typeof value.role === "string");
8381
- }
8382
- /** A conformant message part: the three CORE kinds, or a reverse-DNS extension kind (SPEC §5).
8383
- *
8384
- * Reached from {@link isCotalMessage}, which gates the Plane-3 delivery frame — so a core kind
8385
- * missing an arm here is not a schema nicety: the durable backstop DROPS every message carrying
8386
- * it, silently, and the drop surfaces nowhere near the feature that added the part. `artifact`
8387
- * was exactly that case before it was added (a bare kind has no dot, so it fell through to the
8388
- * extension regex and failed). */
8389
- function isMessagePart(value) {
8390
- if (!isRecord(value) || typeof value.kind !== "string")
8391
- return false;
8392
- if (value.kind === "text")
8393
- return typeof value.text === "string";
8394
- if (value.kind === "data")
8395
- return Object.prototype.hasOwnProperty.call(value, "data");
8396
- if (value.kind === ARTIFACT_PART_KIND)
8397
- return isArtifactPart(value);
8398
- return /^[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)+$/.test(value.kind);
8399
- }
8400
- function isRecord(value) {
8401
- return !!value && typeof value === "object" && !Array.isArray(value);
8402
- }
8403
6630
  /** A `data` part value the wire can carry faithfully: a JSON value at EVERY depth (SPEC §5's
8404
6631
  * `<any JSON value>`). `JSON.stringify` does not reject what it cannot represent — it silently
8405
6632
  * rewrites: `undefined` (and a function-valued member) drops the key, `NaN`/`Infinity` store as
@@ -8588,12 +6815,6 @@ function membershipConsumerReleased(err) {
8588
6815
  return (isJetStreamMissing(err, JetStreamApiCodes.ConsumerNotFound, JetStreamApiCodes.StreamNotFound)
8589
6816
  || isPublishPermissionDenied(err));
8590
6817
  }
8591
- /** The signal's exact abort reason, or the platform-standard AbortError when none was supplied. */
8592
- function abortReason(signal) {
8593
- return signal?.reason instanceof Error
8594
- ? signal.reason
8595
- : new DOMException("This operation was aborted", "AbortError");
8596
- }
8597
6818
  /** True ONLY for a denial on a **publish** — the single case that proves the message was never
8598
6819
  * ACCEPTED or stored. (Not "never reached the server": the server necessarily received enough of
8599
6820
  * it to reject it. The distinction matters precisely here, because this helper exists to separate