@cotal-ai/core 0.169.0 → 0.170.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-file.d.ts +9 -5
- package/dist/agent-file.d.ts.map +1 -1
- package/dist/agent-file.js +18 -8
- package/dist/agent-file.js.map +1 -1
- package/dist/anycast-requests.d.ts +8 -1
- package/dist/anycast-requests.d.ts.map +1 -1
- package/dist/anycast-requests.js +47 -0
- package/dist/anycast-requests.js.map +1 -1
- package/dist/backup-config.d.ts +5 -2
- package/dist/backup-config.d.ts.map +1 -1
- package/dist/backup-config.js +7 -2
- package/dist/backup-config.js.map +1 -1
- package/dist/backup.d.ts +6 -0
- package/dist/backup.d.ts.map +1 -1
- package/dist/backup.js +11 -4
- package/dist/backup.js.map +1 -1
- package/dist/endpoint-binding.d.ts.map +1 -1
- package/dist/endpoint-binding.js +2 -0
- package/dist/endpoint-binding.js.map +1 -1
- package/dist/endpoint-channel-discovery.d.ts +102 -0
- package/dist/endpoint-channel-discovery.d.ts.map +1 -0
- package/dist/endpoint-channel-discovery.js +170 -0
- package/dist/endpoint-channel-discovery.js.map +1 -0
- package/dist/endpoint-control-plane.d.ts +204 -0
- package/dist/endpoint-control-plane.d.ts.map +1 -0
- package/dist/endpoint-control-plane.js +627 -0
- package/dist/endpoint-control-plane.js.map +1 -0
- package/dist/endpoint-durable-backstop.d.ts +249 -0
- package/dist/endpoint-durable-backstop.d.ts.map +1 -0
- package/dist/endpoint-durable-backstop.js +485 -0
- package/dist/endpoint-durable-backstop.js.map +1 -0
- package/dist/endpoint-liveness.d.ts +105 -0
- package/dist/endpoint-liveness.d.ts.map +1 -0
- package/dist/endpoint-liveness.js +223 -0
- package/dist/endpoint-liveness.js.map +1 -0
- package/dist/endpoint-messaging.d.ts +229 -0
- package/dist/endpoint-messaging.d.ts.map +1 -0
- package/dist/endpoint-messaging.js +420 -0
- package/dist/endpoint-messaging.js.map +1 -0
- package/dist/endpoint-presence.d.ts +15 -0
- package/dist/endpoint-presence.d.ts.map +1 -1
- package/dist/endpoint-presence.js +35 -0
- package/dist/endpoint-presence.js.map +1 -1
- package/dist/endpoint-subscriptions.d.ts +199 -0
- package/dist/endpoint-subscriptions.d.ts.map +1 -0
- package/dist/endpoint-subscriptions.js +543 -0
- package/dist/endpoint-subscriptions.js.map +1 -0
- package/dist/endpoint.d.ts +32 -91
- package/dist/endpoint.d.ts.map +1 -1
- package/dist/endpoint.js +246 -1744
- package/dist/endpoint.js.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/provision-jwt.d.ts.map +1 -1
- package/dist/provision-jwt.js +1 -0
- package/dist/provision-jwt.js.map +1 -1
- package/dist/provision-permissions.d.ts +1 -1
- package/dist/provision-permissions.d.ts.map +1 -1
- package/dist/provision-permissions.js +3 -0
- package/dist/provision-permissions.js.map +1 -1
- package/dist/run-host.d.ts +4 -1
- package/dist/run-host.d.ts.map +1 -1
- package/dist/run-record.d.ts +4 -0
- package/dist/run-record.d.ts.map +1 -1
- package/dist/run-record.js +36 -0
- package/dist/run-record.js.map +1 -1
- package/dist/space-quiescence-resources.d.ts +30 -0
- package/dist/space-quiescence-resources.d.ts.map +1 -0
- package/dist/space-quiescence-resources.js +45 -0
- package/dist/space-quiescence-resources.js.map +1 -0
- package/package.json +1 -1
package/dist/endpoint.js
CHANGED
|
@@ -1,15 +1,14 @@
|
|
|
1
1
|
import { EndpointHistory } from "./endpoint-history.js";
|
|
2
2
|
export { MULTI_FILTER_BATCH, MULTI_FILTER_READ_CONCURRENCY } from "./endpoint-history.js";
|
|
3
|
-
import { eventPayloadSubject, eventPayloadStream } from "./event-payloads.js";
|
|
4
3
|
import { requestSafeReconnect } from "./request-safe-reconnect.js";
|
|
5
4
|
import { nativeBindingAfter, nativeConnectionBinding } from "./native-connection.js";
|
|
6
5
|
import { EventEmitter } from "node:events";
|
|
7
6
|
import { drainAndClose } from "./connection-cleanup.js";
|
|
8
7
|
import { randomUUID } from "node:crypto";
|
|
9
8
|
import { createConnection, isIP } from "node:net";
|
|
10
|
-
import { connect,
|
|
9
|
+
import { connect, tokenAuthenticator, nanos, AuthorizationError, PermissionViolationError, UserAuthenticationExpiredError, NoRespondersError, RequestError, TimeoutError, ConnectionError, } from "@nats-io/transport-node";
|
|
11
10
|
import { wsconnect } from "@nats-io/nats-core";
|
|
12
|
-
import {
|
|
11
|
+
import { responderFromLease, responderFromProbe, } from "./liveness.js";
|
|
13
12
|
import { EndpointPresence } from "./endpoint-presence.js";
|
|
14
13
|
export { PresenceWriteStuckError } from "./endpoint-presence.js";
|
|
15
14
|
import { credsClaims, credsFingerprint, credsRenewalDelayMs, idFromCreds } from "./identity.js";
|
|
@@ -19,37 +18,40 @@ import { requireBrokerFloor } from "./broker-floor.js";
|
|
|
19
18
|
import { withDeadline } from "./deadline.js";
|
|
20
19
|
import { inspectCredHealth } from "./provision.js";
|
|
21
20
|
import {} from "./secret-store.js";
|
|
22
|
-
import { BIND_SPLIT_REISSUES, resolveService
|
|
23
|
-
import { EpEnvelopeError, respondedButUnbound, replyRefusedBeforeEffect, replyTargetUnmapped
|
|
21
|
+
import { BIND_SPLIT_REISSUES, resolveService } from "./endpoint-invoke.js";
|
|
22
|
+
import { EpEnvelopeError, respondedButUnbound, replyRefusedBeforeEffect, replyTargetUnmapped } from "./endpoint-envelope.js";
|
|
24
23
|
import { isRepeatSafeCommand } from "./endpoint-grants.js";
|
|
25
|
-
import {
|
|
24
|
+
import { assertGeneration, assertRuntimeToken, endpointToken } from "./endpoint-subjects.js";
|
|
26
25
|
import { servePrincipalRoute, describeContracts } from "./endpoint-serve.js";
|
|
27
26
|
import { epPrincipalCall } from "./endpoint-call.js";
|
|
28
27
|
import { AGENT_CLUSTER, AGENT_CONTRACTS, AGENT_ENDPOINT } from "./agent-endpoint.js";
|
|
29
28
|
import { processRuntime } from "./process-runtime.js";
|
|
30
29
|
import { importNativeSubjectPermissions, issuedPermitsSubject, readAcceptedRow } from "./issued-authority.js";
|
|
31
|
-
import {
|
|
32
|
-
import { listRegistryEndpoints, registryListingCaller } from "./registry-client.js";
|
|
33
|
-
import { lastKvEntries } from "./kv-scan.js";
|
|
30
|
+
import { listRegistryEndpoints } from "./registry-client.js";
|
|
34
31
|
import { isConsumerNotFound, isJetStreamMissing, isStreamNotFound } from "./jetstream-missing.js";
|
|
35
32
|
import { assertValidKind } from "./agent-file.js";
|
|
36
33
|
import { assertValidName } from "./resolve.js";
|
|
37
34
|
import { nextHeartbeat } from "./presence-heartbeat.js";
|
|
38
|
-
import { anycastStoreDefinition, createSpaceStreams, dmDurableConfig, dlvDurableConfig, taskDurableConfig,
|
|
35
|
+
import { anycastStoreDefinition, createSpaceStreams, dmDurableConfig, dlvDurableConfig, taskDurableConfig, PRESENCE_STORAGE } from "./streams.js";
|
|
39
36
|
import { DELIVERY_LIMIT_HEADER, PLANE3_MAX_MSGS_PER_SUBJECT } from "./backup-config.js";
|
|
40
37
|
import { jetstream, jetstreamManager, AckPolicy, DeliverPolicy, JetStreamApiCodes, } from "@nats-io/jetstream";
|
|
41
38
|
import {} from "@nats-io/jetstream";
|
|
42
39
|
import { Kvm } from "@nats-io/kv";
|
|
43
|
-
import {
|
|
44
|
-
import {
|
|
45
|
-
import { openAclRegistry, readAcl, readAclForAlias, AmbiguousAclAlias, commitAcl as writeAclRecord, reissueAcl as writeAclReissue } from "./acls.js";
|
|
40
|
+
import { listMembers } from "./members.js";
|
|
41
|
+
import { readAcl } from "./acls.js";
|
|
46
42
|
import { EndpointDeliveryLease } from "./endpoint-delivery-lease.js";
|
|
47
43
|
import { EndpointDeliveryPlane } from "./endpoint-delivery-plane.js";
|
|
44
|
+
import { EndpointControlPlane } from "./endpoint-control-plane.js";
|
|
45
|
+
import { EndpointMessaging, assertPartsSerializable } from "./endpoint-messaging.js";
|
|
46
|
+
import { EndpointDurableBackstop } from "./endpoint-durable-backstop.js";
|
|
47
|
+
import { EndpointSubscriptions, membershipConsumerReleased } from "./endpoint-subscriptions.js";
|
|
48
|
+
import { EndpointChannelDiscovery } from "./endpoint-channel-discovery.js";
|
|
49
|
+
import { EndpointLiveness } from "./endpoint-liveness.js";
|
|
48
50
|
import { ackWaitOf, authenticatedChannelMessage, durableDelivery, isRecord } from "./endpoint-inbound.js";
|
|
49
|
-
import { openChannelRegistry, effectiveReplay, effectiveReplayWindowMs,
|
|
51
|
+
import { openChannelRegistry, effectiveReplay, effectiveReplayWindowMs, readChannelConfig, readChannelDefaults, } from "./channels.js";
|
|
50
52
|
import { isCasLoss } from "./endpoint-records.js";
|
|
51
53
|
import { ANYCAST_CANCEL_OP, ANYCAST_NOTICE_KEY, ANYCAST_RECORDS_CREDENTIAL_OUTDATED, ANYCAST_STATUS_OP, ANYCAST_TAKES_OP, AnycastRequestAuthority, isAnycastSeq, } from "./anycast-requests.js";
|
|
52
|
-
import {
|
|
54
|
+
import { assertValidRole, chatStream, chatSubject, controlServiceSubject, CONTROL_DELIVERY, CONTROL_DELIVERY_ADMIN, dmStream, dmDurable, dlvStream, membershipInAllow, isConcreteChannel, isChannelFamily, MENTION_NOTICE_KEY, parseSubject, isPrincipalOwnerToken, assertInboxConnId, presenceBucket, MEMBERSHIP_FEED_KEY, principalKey, parsePrincipalKey, assertLifecycleToken, mintLifecycleUid, lifecycleNameKey, DEV_OWNER, spacePrefix, spaceWildcard, subjectMatches, taskStream, taskDurable, takeServeFilter, takeSubject, parseTakeSubject, TAKE_ROUTE, token, unicastSubject, unicastRecvFilter, } from "./subjects.js";
|
|
53
55
|
import { isPermissionDenied } from "./permission-denied.js";
|
|
54
56
|
export { isPermissionDenied };
|
|
55
57
|
export const DEFAULT_SERVER = "nats://127.0.0.1:4222";
|
|
@@ -126,31 +128,6 @@ export function fitHistoryPage(items, budget) {
|
|
|
126
128
|
}
|
|
127
129
|
return first === items.length ? [] : items.slice(first);
|
|
128
130
|
}
|
|
129
|
-
/** One feed entry, decoded the way {@link CotalEndpoint.readMembership} decodes it: the reserved
|
|
130
|
-
* heartbeat key carries `asOf`, every other key is one agent's record, and an undecodable record
|
|
131
|
-
* is a member change with no record (never a crash in the consumer callback). */
|
|
132
|
-
function membershipChange(e, update) {
|
|
133
|
-
const operation = e.operation;
|
|
134
|
-
if (e.key === MEMBERSHIP_FEED_KEY) {
|
|
135
|
-
let asOf;
|
|
136
|
-
if (operation === "PUT") {
|
|
137
|
-
try {
|
|
138
|
-
asOf = e.json().observedAt;
|
|
139
|
-
}
|
|
140
|
-
catch { /* heartbeat garbled; leave undefined */ }
|
|
141
|
-
}
|
|
142
|
-
return { type: "heartbeat", revision: e.revision, update, asOf };
|
|
143
|
-
}
|
|
144
|
-
let member;
|
|
145
|
-
if (operation === "PUT") {
|
|
146
|
-
try {
|
|
147
|
-
const rec = e.json();
|
|
148
|
-
member = { id: e.key, live: rec.live ?? [], durable: rec.durable ?? [], observedAt: rec.observedAt };
|
|
149
|
-
}
|
|
150
|
-
catch { /* undecodable: delivered as a change with no record */ }
|
|
151
|
-
}
|
|
152
|
-
return { type: "member", id: e.key, operation, revision: e.revision, update, member };
|
|
153
|
-
}
|
|
154
131
|
/** The highest `prompts` presence `seq` this process used under each runtime id (SPEC §6). A reader
|
|
155
132
|
* orders lists by `seq` within one `runtime.id`, and a runtime id lives as long as the process, so
|
|
156
133
|
* this does too: across a stopped serve, a stopped endpoint and a new endpoint for the lifecycle. */
|
|
@@ -2077,33 +2054,25 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
2077
2054
|
return open.map((nc) => nativeConnectionBinding(nc, "endpoint connection"));
|
|
2078
2055
|
}
|
|
2079
2056
|
// ---- messaging -----------------------------------------------------------
|
|
2057
|
+
/** The messaging verbs of this endpoint: multicast, unicast, anycast and the event-payload
|
|
2058
|
+
* channel. Every dependency is read when it is used, so a field assigned after construction is seen. */
|
|
2059
|
+
messaging = new EndpointMessaging({
|
|
2060
|
+
nc: () => this.nc,
|
|
2061
|
+
js: () => this.js,
|
|
2062
|
+
jsm: () => this.jsm,
|
|
2063
|
+
space: () => this.space,
|
|
2064
|
+
owner: () => this.owner,
|
|
2065
|
+
actor: () => this.actor,
|
|
2066
|
+
channels: () => this.channels,
|
|
2067
|
+
notLiveMsg: () => this.notLiveMsg(),
|
|
2068
|
+
ref: () => this.ref(),
|
|
2069
|
+
publishMsg: (subject, msg) => this.publishMsg(subject, msg),
|
|
2070
|
+
streamHistory: (...args) => this.streamHistory(...args),
|
|
2071
|
+
emitRecoverable: (err) => this.emitRecoverable(err),
|
|
2072
|
+
});
|
|
2080
2073
|
/** Multicast: broadcast to everyone on a channel. */
|
|
2081
2074
|
async multicast(text, opts) {
|
|
2082
|
-
|
|
2083
|
-
// wildcard. Default to the first concrete channel we're on (channels[0] may
|
|
2084
|
-
// itself be a wildcard subscription like `team.>`). On no channels there is NO default: refuse
|
|
2085
|
-
// rather than pick `general`, which is a destination the caller never chose and, for an agent
|
|
2086
|
-
// whose cred carries no channel row, one the broker would deny anyway.
|
|
2087
|
-
const channel = opts?.channel ?? this.channels.find(isConcreteChannel);
|
|
2088
|
-
if (channel === undefined)
|
|
2089
|
-
throw new Error("no default channel: this endpoint is not on any concrete channel, so there is nothing to broadcast to - name a channel explicitly, or join one first");
|
|
2090
|
-
if (!isConcreteChannel(channel))
|
|
2091
|
-
throw new Error(`cannot publish to wildcard channel "${channel}" - pick a concrete sub-channel`);
|
|
2092
|
-
const msg = {
|
|
2093
|
-
id: randomUUID(),
|
|
2094
|
-
ts: Date.now(),
|
|
2095
|
-
space: this.space,
|
|
2096
|
-
from: this.ref(),
|
|
2097
|
-
channel,
|
|
2098
|
-
// Priority/wake hint, not routing — validation (against the roster) is the connector's
|
|
2099
|
-
// job; core just canonicalizes and omits the field when empty.
|
|
2100
|
-
mentions: normalizeMentions(opts?.mentions),
|
|
2101
|
-
parts: opts?.parts ?? [{ kind: "text", text }],
|
|
2102
|
-
replyTo: opts?.replyTo,
|
|
2103
|
-
contextId: opts?.contextId,
|
|
2104
|
-
};
|
|
2105
|
-
await this.publishMsg(chatSubject(this.space, this.owner, this.actor, channel), msg);
|
|
2106
|
-
return msg;
|
|
2075
|
+
return this.messaging.multicast(text, opts);
|
|
2107
2076
|
}
|
|
2108
2077
|
/** The broker's live `max_payload` — the CEILING a frame is measured against, not a budget for a
|
|
2109
2078
|
* caller's own payload.
|
|
@@ -2117,10 +2086,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
2117
2086
|
* the client's own headers, are charged against the same ceiling and the caller never sees them.
|
|
2118
2087
|
* Use {@link encodedSize}, which measures what will actually be sent. */
|
|
2119
2088
|
get maxPayload() {
|
|
2120
|
-
|
|
2121
|
-
if (typeof max !== "number" || !Number.isFinite(max) || max <= 0)
|
|
2122
|
-
throw new Error(`${this.notLiveMsg()} - max_payload is only known while connected`);
|
|
2123
|
-
return max;
|
|
2089
|
+
return this.messaging.maxPayload;
|
|
2124
2090
|
}
|
|
2125
2091
|
/**
|
|
2126
2092
|
* Verify the PRECONDITION {@link multicastExpecting} depends on: that the chat stream evaluates
|
|
@@ -2146,42 +2112,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
2146
2112
|
* one replica. Never degrades to a warning: the failure it prevents is silent.
|
|
2147
2113
|
*/
|
|
2148
2114
|
async assertExpectationSemantics() {
|
|
2149
|
-
|
|
2150
|
-
throw new Error(this.notLiveMsg());
|
|
2151
|
-
const stream = chatStream(this.space);
|
|
2152
|
-
let replicas;
|
|
2153
|
-
try {
|
|
2154
|
-
replicas = (await this.jsm.streams.info(stream)).config.num_replicas;
|
|
2155
|
-
}
|
|
2156
|
-
catch (e) {
|
|
2157
|
-
throw new Error(`cannot verify expectation semantics: stream "${stream}" info unavailable (${e.message}). ` +
|
|
2158
|
-
`Serialized appends are refused rather than run on an unverified stream.`);
|
|
2159
|
-
}
|
|
2160
|
-
// `undefined` is NOT treated as 1. A server that does not report the field is a server whose
|
|
2161
|
-
// ordering we have not established, which is the case this check exists for.
|
|
2162
|
-
if (replicas !== 1)
|
|
2163
|
-
throw new Error(`stream "${stream}" reports num_replicas=${String(replicas)}; serialized appends on THIS ` +
|
|
2164
|
-
`stream require 1 — a property of this one stream, not of the broker, so a clustered ` +
|
|
2165
|
-
`deployment is fine so long as this stream is R1, which a cluster can host. ` +
|
|
2166
|
-
`On a replicated stream the dedup cache is consulted before the subject expectation, so a ` +
|
|
2167
|
-
`retry returns a duplicate ack instead of a conflict and a lost message reads as success.`);
|
|
2168
|
-
}
|
|
2169
|
-
/** The envelope {@link multicastExpecting} publishes, built in ONE place so that a frame and any
|
|
2170
|
-
* measurement of that frame cannot describe different messages. The fields this adds — `ts`,
|
|
2171
|
-
* `space`, `from`, `channel` and the normalized `mentions` — are exactly the ones a caller
|
|
2172
|
-
* holding only its parts cannot account for. */
|
|
2173
|
-
casEnvelope(opts) {
|
|
2174
|
-
return {
|
|
2175
|
-
id: opts.id,
|
|
2176
|
-
ts: Date.now(),
|
|
2177
|
-
space: this.space,
|
|
2178
|
-
from: this.ref(),
|
|
2179
|
-
channel: opts.channel,
|
|
2180
|
-
mentions: normalizeMentions(opts.mentions),
|
|
2181
|
-
parts: opts.parts,
|
|
2182
|
-
replyTo: opts.replyTo,
|
|
2183
|
-
contextId: opts.contextId,
|
|
2184
|
-
};
|
|
2115
|
+
return this.messaging.assertExpectationSemantics();
|
|
2185
2116
|
}
|
|
2186
2117
|
/**
|
|
2187
2118
|
* The bytes this frame will ACTUALLY put on the wire, to compare against {@link maxPayload}.
|
|
@@ -2204,28 +2135,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
2204
2135
|
* epoch-millis needs a 14th digit.
|
|
2205
2136
|
*/
|
|
2206
2137
|
encodedSize(opts) {
|
|
2207
|
-
|
|
2208
|
-
// would have been refused before it ever reached the wire.
|
|
2209
|
-
if (!isConcreteChannel(opts.channel))
|
|
2210
|
-
throw new Error(`cannot publish to wildcard channel "${opts.channel}" - pick a concrete sub-channel`);
|
|
2211
|
-
assertIdToken(opts.id, "publish id");
|
|
2212
|
-
if (!Number.isSafeInteger(opts.expectedLastSubjectSeq) || opts.expectedLastSubjectSeq < 0)
|
|
2213
|
-
throw new Error(`expectedLastSubjectSeq must be a non-negative safe integer, got ${JSON.stringify(opts.expectedLastSubjectSeq)}`);
|
|
2214
|
-
if (!Array.isArray(opts.parts) || opts.parts.length === 0)
|
|
2215
|
-
throw new Error("encodedSize requires at least one part");
|
|
2216
|
-
const mh = headers();
|
|
2217
|
-
mh.set("Nats-Msg-Id", opts.id);
|
|
2218
|
-
mh.set("Nats-Expected-Last-Subject-Sequence", `${opts.expectedLastSubjectSeq}`);
|
|
2219
|
-
if (opts.expectedLastMsgId !== undefined) {
|
|
2220
|
-
assertIdToken(opts.expectedLastMsgId, "expected last msg id");
|
|
2221
|
-
mh.set("Nats-Expected-Last-Msg-Id", opts.expectedLastMsgId);
|
|
2222
|
-
}
|
|
2223
|
-
// `encode()` is on the client's header implementation but not on the published `MsgHdrs` type,
|
|
2224
|
-
// so it is reached through a cast rather than copied. If a client version drops it, this throws
|
|
2225
|
-
// immediately and loudly — the calibration cell would also fail — instead of returning a number
|
|
2226
|
-
// that is quietly wrong near the ceiling.
|
|
2227
|
-
const headerBytes = mh.encode().length;
|
|
2228
|
-
return headerBytes + Buffer.byteLength(JSON.stringify(this.casEnvelope(opts)), "utf8");
|
|
2138
|
+
return this.messaging.encodedSize(opts);
|
|
2229
2139
|
}
|
|
2230
2140
|
/**
|
|
2231
2141
|
* Multicast with an OPTIMISTIC-CONCURRENCY expectation and a caller-chosen dedup id, returning
|
|
@@ -2254,138 +2164,39 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
2254
2164
|
* empty, or `expectedLastSubjectSeq` is not a non-negative safe integer.
|
|
2255
2165
|
*/
|
|
2256
2166
|
async multicastExpecting(opts) {
|
|
2257
|
-
|
|
2258
|
-
throw new Error(this.notLiveMsg());
|
|
2259
|
-
if (!isConcreteChannel(opts.channel))
|
|
2260
|
-
throw new Error(`cannot publish to wildcard channel "${opts.channel}" - pick a concrete sub-channel`);
|
|
2261
|
-
// Reuse the existing id grammar rather than mint a second one: [A-Za-z0-9_-]{1,64} admits a
|
|
2262
|
-
// UUID and rejects every character that could break a wire header (CR, LF, space, colon).
|
|
2263
|
-
assertIdToken(opts.id, "publish id");
|
|
2264
|
-
const expected = opts.expectedLastSubjectSeq;
|
|
2265
|
-
if (!Number.isSafeInteger(expected) || expected < 0)
|
|
2266
|
-
throw new Error(`expectedLastSubjectSeq must be a non-negative safe integer, got ${JSON.stringify(expected)}`);
|
|
2267
|
-
if (!Array.isArray(opts.parts) || opts.parts.length === 0)
|
|
2268
|
-
throw new Error("multicastExpecting requires at least one part");
|
|
2269
|
-
if (opts.expectedLastMsgId !== undefined)
|
|
2270
|
-
assertIdToken(opts.expectedLastMsgId, "expected last msg id");
|
|
2271
|
-
const message = this.casEnvelope(opts);
|
|
2272
|
-
assertPartsSerializable(message.parts);
|
|
2273
|
-
// Publish DIRECTLY rather than through publishMsg: this path must set the expectation and read
|
|
2274
|
-
// the ack, and publishMsg deliberately does neither.
|
|
2275
|
-
const ack = await this.js.publish(chatSubject(this.space, this.owner, this.actor, opts.channel), JSON.stringify(message), { msgID: opts.id, expect: { lastSubjectSequence: expected, ...(opts.expectedLastMsgId !== undefined ? { lastMsgID: opts.expectedLastMsgId } : {}) } });
|
|
2276
|
-
return { message, ack: { seq: ack.seq, duplicate: ack.duplicate === true } };
|
|
2167
|
+
return this.messaging.multicastExpecting(opts);
|
|
2277
2168
|
}
|
|
2278
2169
|
/** Publish a payload frame on this principal's forge-locked sibling event rail. */
|
|
2279
2170
|
async publishEventPayload(opts) {
|
|
2280
|
-
|
|
2281
|
-
throw new Error(this.notLiveMsg());
|
|
2282
|
-
assertIdToken(opts.id, "publish id");
|
|
2283
|
-
if (!Number.isSafeInteger(opts.expectedLastSubjectSeq) || opts.expectedLastSubjectSeq < 0)
|
|
2284
|
-
throw new Error("payload expectation must be a non-negative safe integer");
|
|
2285
|
-
if (!opts.parts.length)
|
|
2286
|
-
throw new Error("payload frame requires parts");
|
|
2287
|
-
const message = this.casEnvelope({ ...opts, channel: `events.${this.owner}.${this.actor}` });
|
|
2288
|
-
assertPartsSerializable(message.parts);
|
|
2289
|
-
const ack = await this.js.publish(eventPayloadSubject(this.space, this.owner, this.actor), JSON.stringify(message), { msgID: opts.id, expect: { lastSubjectSequence: opts.expectedLastSubjectSeq } });
|
|
2290
|
-
return { ack: { seq: ack.seq, duplicate: ack.duplicate === true } };
|
|
2171
|
+
return this.messaging.publishEventPayload(opts);
|
|
2291
2172
|
}
|
|
2292
2173
|
/** Read payload history through a single native-filtered consumer, never through CHAT. */
|
|
2293
2174
|
async eventPayloadHistory(owner, actor, opts) {
|
|
2294
|
-
return
|
|
2295
|
-
.map(row => row.msg);
|
|
2175
|
+
return this.messaging.eventPayloadHistory(owner, actor, opts);
|
|
2296
2176
|
}
|
|
2297
2177
|
/** Verify the sibling stream and learn its actual message ceiling. A pre-upgrade space without
|
|
2298
2178
|
* the stream or its info-read permission returns null and emits a warning. Other authorization,
|
|
2299
2179
|
* transport and config errors throw. */
|
|
2300
2180
|
async eventPayloadLimit() {
|
|
2301
|
-
|
|
2302
|
-
throw new Error(this.notLiveMsg());
|
|
2303
|
-
const stream = eventPayloadStream(this.space);
|
|
2304
|
-
const info = await this.jsm.streams.info(stream).catch((err) => {
|
|
2305
|
-
const denial = err instanceof PermissionViolationError ? err
|
|
2306
|
-
: err?.cause;
|
|
2307
|
-
const infoDenied = denial instanceof PermissionViolationError && denial.operation === "publish"
|
|
2308
|
-
&& denial.subject === `$JS.API.STREAM.INFO.${stream}`;
|
|
2309
|
-
if (!infoDenied && !isJetStreamMissing(err, JetStreamApiCodes.StreamNotFound))
|
|
2310
|
-
throw err;
|
|
2311
|
-
const reason = infoDenied ? `missing publish permission on $JS.API.STREAM.INFO.${stream}` : `stream ${stream} is absent`;
|
|
2312
|
-
this.emitRecoverable(new Error(`event payloads unavailable: ${reason}; publishing tool names and times only. Upgrade the space's broker setup and credentials to enable payloads`));
|
|
2313
|
-
return null;
|
|
2314
|
-
});
|
|
2315
|
-
if (info === null)
|
|
2316
|
-
return null;
|
|
2317
|
-
const { config } = info;
|
|
2318
|
-
if (config.num_replicas !== 1)
|
|
2319
|
-
throw new Error("event payload stream requires one replica for serialized appends");
|
|
2320
|
-
const limits = [this.maxPayload, config.max_msg_size, config.max_bytes].filter((n) => typeof n === "number" && n > 0);
|
|
2321
|
-
return Math.min(...limits);
|
|
2181
|
+
return this.messaging.eventPayloadLimit();
|
|
2322
2182
|
}
|
|
2323
2183
|
/** Unicast: direct message to one specific instance. */
|
|
2324
2184
|
async unicast(instanceId, text, opts) {
|
|
2325
|
-
|
|
2326
|
-
return msg;
|
|
2185
|
+
return this.messaging.unicast(instanceId, text, opts);
|
|
2327
2186
|
}
|
|
2328
2187
|
/** Unicast, returning the JetStream publish ack alongside the message: the only truthful proof
|
|
2329
2188
|
* a sender has is that the broker stored the message at a sequence, never that anyone read it. */
|
|
2330
2189
|
async unicastAttributed(instanceId, text, opts) {
|
|
2331
|
-
|
|
2332
|
-
throw new Error(this.notLiveMsg());
|
|
2333
|
-
const msg = {
|
|
2334
|
-
id: randomUUID(),
|
|
2335
|
-
ts: Date.now(),
|
|
2336
|
-
space: this.space,
|
|
2337
|
-
from: this.ref(),
|
|
2338
|
-
to: instanceId,
|
|
2339
|
-
parts: opts?.parts ?? [{ kind: "text", text }],
|
|
2340
|
-
replyTo: opts?.replyTo,
|
|
2341
|
-
contextId: opts?.contextId,
|
|
2342
|
-
};
|
|
2343
|
-
// The recipient id is a principal dot-form `<owner>.<actor>` (owner+actor grammar); the 4-token DM
|
|
2344
|
-
// subject forge-locks recipient AND sender, so both are split into their tokens here.
|
|
2345
|
-
const recip = parsePrincipalKey(instanceId);
|
|
2346
|
-
if (!recip)
|
|
2347
|
-
throw new Error(`unicast: "${instanceId}" is not a valid recipient principal <owner>.<actor>`);
|
|
2348
|
-
assertPartsSerializable(msg.parts);
|
|
2349
|
-
// Publish DIRECTLY rather than through publishMsg, the way multicastExpecting does: this path
|
|
2350
|
-
// must read the ack, and publishMsg deliberately discards it.
|
|
2351
|
-
const subject = unicastSubject(this.space, recip.owner, recip.actor, this.owner, this.actor);
|
|
2352
|
-
try {
|
|
2353
|
-
const ack = await this.js.publish(subject, JSON.stringify(msg), { msgID: msg.id });
|
|
2354
|
-
return { msg, ack: { seq: ack.seq, duplicate: ack.duplicate === true } };
|
|
2355
|
-
}
|
|
2356
|
-
catch (error) {
|
|
2357
|
-
if (error instanceof TimeoutError)
|
|
2358
|
-
throw new Error(`${error.message} publishing to ${JSON.stringify(subject)}. The send outcome is unknown: the broker may have stored the message. Check before resending.`, { cause: error });
|
|
2359
|
-
throw error;
|
|
2360
|
-
}
|
|
2190
|
+
return this.messaging.unicastAttributed(instanceId, text, opts);
|
|
2361
2191
|
}
|
|
2362
2192
|
/** Anycast: deliver to ANY one instance of a service (role) — queue-group load balancing. */
|
|
2363
2193
|
async anycast(service, text, opts) {
|
|
2364
|
-
|
|
2365
|
-
return msg;
|
|
2194
|
+
return this.messaging.anycast(service, text, opts);
|
|
2366
2195
|
}
|
|
2367
2196
|
/** Anycast, returning the JetStream publish ack alongside the message: the broker stored the
|
|
2368
2197
|
* request on the role's work queue at a sequence, which proves neither a holder nor a read. */
|
|
2369
2198
|
async anycastAttributed(service, text, opts) {
|
|
2370
|
-
|
|
2371
|
-
throw new Error(this.notLiveMsg());
|
|
2372
|
-
// The subject builder admits `*` for allow rules; a publish needs one concrete role.
|
|
2373
|
-
assertValidRole(service);
|
|
2374
|
-
const msg = {
|
|
2375
|
-
id: randomUUID(),
|
|
2376
|
-
ts: Date.now(),
|
|
2377
|
-
space: this.space,
|
|
2378
|
-
from: this.ref(),
|
|
2379
|
-
toService: service,
|
|
2380
|
-
parts: opts?.parts ?? [{ kind: "text", text }],
|
|
2381
|
-
replyTo: opts?.replyTo,
|
|
2382
|
-
contextId: opts?.contextId,
|
|
2383
|
-
};
|
|
2384
|
-
assertPartsSerializable(msg.parts);
|
|
2385
|
-
// Publish DIRECTLY rather than through publishMsg, the way unicastAttributed does: this path
|
|
2386
|
-
// must read the ack, and publishMsg deliberately discards it.
|
|
2387
|
-
const ack = await this.js.publish(anycastSubject(this.space, service, this.owner, this.actor), JSON.stringify(msg), { msgID: msg.id });
|
|
2388
|
-
return { msg, ack: { seq: ack.seq, duplicate: ack.duplicate === true } };
|
|
2199
|
+
return this.messaging.anycastAttributed(service, text, opts);
|
|
2389
2200
|
}
|
|
2390
2201
|
/** Subscribe to a read-only observer feed. Defaults to the whole space; an observer under
|
|
2391
2202
|
* auth must pass `chatWildcard(space)` since its `sub.allow` only covers chat (DM/anycast
|
|
@@ -2440,6 +2251,35 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
2440
2251
|
})().catch((e) => this.emit("error", e));
|
|
2441
2252
|
}
|
|
2442
2253
|
// ---- control plane (request/reply) --------------------------------------
|
|
2254
|
+
/** The control plane and the service-invocation client of this endpoint. Every dependency is
|
|
2255
|
+
* read when it is used, so a field assigned after construction is seen. */
|
|
2256
|
+
controlPlane = new EndpointControlPlane({
|
|
2257
|
+
nc: () => this.nc,
|
|
2258
|
+
space: () => this.space,
|
|
2259
|
+
owner: () => this.owner,
|
|
2260
|
+
actor: () => this.actor,
|
|
2261
|
+
stopped: () => this.stopped,
|
|
2262
|
+
authed: () => this.authed,
|
|
2263
|
+
connId: () => this.connId,
|
|
2264
|
+
presentedExp: () => this.presentedExp,
|
|
2265
|
+
issuedGeneration: () => this.issuedGeneration,
|
|
2266
|
+
acceptedToken: () => this.acceptedToken,
|
|
2267
|
+
rebuildPromise: () => this.rebuildPromise,
|
|
2268
|
+
subs: () => this.subs,
|
|
2269
|
+
resolvedServices: () => this.resolvedServices,
|
|
2270
|
+
resolvedCommandServices: () => this.resolvedCommandServices,
|
|
2271
|
+
serviceInvocations: () => this.serviceInvocations,
|
|
2272
|
+
requestsAwaitingReply: () => this.requestsAwaitingReply,
|
|
2273
|
+
activeGoalFollowers: () => this.activeGoalFollowers,
|
|
2274
|
+
splitsRecovered: () => this.splitsRecovered,
|
|
2275
|
+
setSplitsRecovered: (count) => { this.splitsRecovered = count; },
|
|
2276
|
+
notLiveMsg: () => this.notLiveMsg(),
|
|
2277
|
+
requireLifecycleUid: (what) => this.requireLifecycleUid(what),
|
|
2278
|
+
reportSubscriptionDenial: (error) => this.reportSubscriptionDenial(error),
|
|
2279
|
+
ref: () => this.ref(),
|
|
2280
|
+
reconnect: () => this.reconnect(),
|
|
2281
|
+
emit: (event, ...args) => this.emit(event, ...args),
|
|
2282
|
+
});
|
|
2443
2283
|
/** Serve control requests for a service. Returns the subscription so a caller that re-registers on
|
|
2444
2284
|
* reconnect (the delivery daemon) can drop the stale one. `boundReply` is REQUIRED for any service
|
|
2445
2285
|
* whose responder holds a wildcard publish grant over the service subtree (the delivery daemon's
|
|
@@ -2454,117 +2294,11 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
2454
2294
|
* running, and holds up none behind it. Only for a request that shares no state with the service's
|
|
2455
2295
|
* other requests. */
|
|
2456
2296
|
serveControl(service, handler, opts = {}) {
|
|
2457
|
-
|
|
2458
|
-
throw new Error(this.notLiveMsg());
|
|
2459
|
-
const sub = this.nc.subscribe(controlServiceSubject(this.space, service, "*", "*"), {
|
|
2460
|
-
queue: service,
|
|
2461
|
-
});
|
|
2462
|
-
this.subs.push(sub);
|
|
2463
|
-
const inOwnTime = opts.concurrent;
|
|
2464
|
-
// With `concurrent`, the in-order requests are answered one after another on this chain, and the
|
|
2465
|
-
// reading below never waits on it, so a request answered in its own time is read and answered
|
|
2466
|
-
// while an in-order one is still running.
|
|
2467
|
-
let inOrder = Promise.resolve();
|
|
2468
|
-
void (async () => {
|
|
2469
|
-
for await (const m of sub) {
|
|
2470
|
-
// Sender-bound reply guard (confused-deputy fix): never respond to a reply target outside the
|
|
2471
|
-
// authenticated request subject's own `.reply.` subtree. Drop silently (don't inject elsewhere).
|
|
2472
|
-
if (opts.boundReply && (!m.reply || !m.reply.startsWith(`${m.subject}.reply.`))) {
|
|
2473
|
-
this.emit("error", new Error(`rejected ${service} request on ${m.subject}: reply target "${m.reply ?? "(none)"}" is not under the sender's own reply subtree`));
|
|
2474
|
-
continue;
|
|
2475
|
-
}
|
|
2476
|
-
const answer = () => this.answerControl(service, handler, m);
|
|
2477
|
-
if (inOwnTime === undefined) {
|
|
2478
|
-
await answer();
|
|
2479
|
-
continue;
|
|
2480
|
-
}
|
|
2481
|
-
let own = false;
|
|
2482
|
-
try {
|
|
2483
|
-
own = inOwnTime(m.json());
|
|
2484
|
-
}
|
|
2485
|
-
catch { /* unreadable: answered in order, as a refusal */ }
|
|
2486
|
-
if (own)
|
|
2487
|
-
void answer().catch((e) => this.emit("error", e));
|
|
2488
|
-
else
|
|
2489
|
-
inOrder = inOrder.then(answer).catch((e) => { this.emit("error", e); });
|
|
2490
|
-
}
|
|
2491
|
-
})().catch((e) => {
|
|
2492
|
-
if (!this.reportSubscriptionDenial(e))
|
|
2493
|
-
this.emit("error", e);
|
|
2494
|
-
});
|
|
2495
|
-
return sub;
|
|
2297
|
+
return this.controlPlane.serveControl(service, handler, opts);
|
|
2496
2298
|
}
|
|
2497
|
-
/**
|
|
2498
|
-
async answerControl(service, handler, m) {
|
|
2499
|
-
let body;
|
|
2500
|
-
try {
|
|
2501
|
-
let reply;
|
|
2502
|
-
const req = m.json();
|
|
2503
|
-
// Authenticity guard (fail closed): control is the most privileged surface
|
|
2504
|
-
// (start/stop). The sender is encoded in the subject (ctl.<svc>.<sender>), which
|
|
2505
|
-
// the server policed who could publish; the payload `from` is advisory and must
|
|
2506
|
-
// match. Reject before the handler acts on a request claiming a forged sender.
|
|
2507
|
-
const parsed = parseSubject(m.subject);
|
|
2508
|
-
if (!parsed || !isFromSubjectSender(req, parsed)) {
|
|
2509
|
-
this.emit("error", new Error(`rejected control request on ${m.subject}: from ${req.from?.id ?? "(none)"} ` +
|
|
2510
|
-
`does not match subject sender ${parsed?.sender ?? "(unparseable)"}`));
|
|
2511
|
-
reply = { ok: false, error: "sender mismatch - request rejected" };
|
|
2512
|
-
}
|
|
2513
|
-
else {
|
|
2514
|
-
reply = await handler(req);
|
|
2515
|
-
}
|
|
2516
|
-
body = JSON.stringify(reply);
|
|
2517
|
-
}
|
|
2518
|
-
catch (e) {
|
|
2519
|
-
body = JSON.stringify({ ok: false, error: e.message });
|
|
2520
|
-
}
|
|
2521
|
-
try {
|
|
2522
|
-
m.respond(body);
|
|
2523
|
-
}
|
|
2524
|
-
catch (e) {
|
|
2525
|
-
// `respond` signals a missing reply inbox by returning false; it throws when the reply exceeds the
|
|
2526
|
-
// broker's `max_payload`. Silence would leave the caller a bare timeout it cannot tell apart from
|
|
2527
|
-
// a dead service.
|
|
2528
|
-
const refused = { ok: false, error: `the ${service} reply is ${Buffer.byteLength(body)} bytes and was not published: ${e.message}` };
|
|
2529
|
-
try {
|
|
2530
|
-
m.respond(JSON.stringify(refused));
|
|
2531
|
-
}
|
|
2532
|
-
catch (err) {
|
|
2533
|
-
this.emit("error", new Error(`could not answer ${service} request on ${m.subject}: ${refused.error}, and the refusal failed too: ${err.message}`));
|
|
2534
|
-
}
|
|
2535
|
-
}
|
|
2536
|
-
}
|
|
2537
|
-
/** Keep native named-reply and NoResponders handling, but settle when its connection closes:
|
|
2538
|
-
* nats.js can close a no-mux subscription without settling its request. One close observer per
|
|
2539
|
-
* connection avoids retaining a closed-promise callback for every completed request. */
|
|
2299
|
+
/** Forwards to the control plane's {@link EndpointControlPlane.requestBoundReply}. */
|
|
2540
2300
|
async requestBoundReply(nc, subject, body, reply, timeoutMs) {
|
|
2541
|
-
|
|
2542
|
-
if (!pending) {
|
|
2543
|
-
const waiting = new Set();
|
|
2544
|
-
this.requestsAwaitingReply.set(nc, waiting);
|
|
2545
|
-
void nc.closed().then((error) => {
|
|
2546
|
-
this.requestsAwaitingReply.delete(nc);
|
|
2547
|
-
const failure = error ?? new ClosedConnectionError();
|
|
2548
|
-
for (const reject of waiting)
|
|
2549
|
-
reject(failure);
|
|
2550
|
-
waiting.clear();
|
|
2551
|
-
});
|
|
2552
|
-
pending = waiting;
|
|
2553
|
-
}
|
|
2554
|
-
let rejectClosed;
|
|
2555
|
-
const closed = new Promise((_resolve, reject) => {
|
|
2556
|
-
rejectClosed = reject;
|
|
2557
|
-
pending.add(reject);
|
|
2558
|
-
});
|
|
2559
|
-
try {
|
|
2560
|
-
return await Promise.race([
|
|
2561
|
-
nc.request(subject, body, { timeout: timeoutMs, noMux: true, reply }),
|
|
2562
|
-
closed,
|
|
2563
|
-
]);
|
|
2564
|
-
}
|
|
2565
|
-
finally {
|
|
2566
|
-
pending.delete(rejectClosed);
|
|
2567
|
-
}
|
|
2301
|
+
return this.controlPlane.requestBoundReply(nc, subject, body, reply, timeoutMs);
|
|
2568
2302
|
}
|
|
2569
2303
|
/** Send a control request to a service and await its reply (client side). Like {@link requestDelivery},
|
|
2570
2304
|
* the reply rides a BOUNDED subject UNDER the request subject (`ctl.<service>.<id>.reply.<uuid>`), not
|
|
@@ -2573,120 +2307,20 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
2573
2307
|
* defense-in-depth (a predictable suffix would let a peer target an in-flight named reply sub). The
|
|
2574
2308
|
* reply sits under the sender's OWN request subject, so the responder's `boundReply` guard accepts it. */
|
|
2575
2309
|
async requestControl(service, req, timeoutMs = 5000) {
|
|
2576
|
-
|
|
2577
|
-
throw new Error(this.notLiveMsg());
|
|
2578
|
-
const reqSubject = controlServiceSubject(this.space, service, this.owner, this.actor);
|
|
2579
|
-
const reply = `${reqSubject}.reply.${randomUUID()}`;
|
|
2580
|
-
const body = { ...req, from: req.from ?? this.ref() };
|
|
2581
|
-
const m = await this.requestBoundReply(this.nc, reqSubject, JSON.stringify(body), reply, timeoutMs);
|
|
2582
|
-
return m.json();
|
|
2310
|
+
return this.controlPlane.requestControl(service, req, timeoutMs);
|
|
2583
2311
|
}
|
|
2584
2312
|
/** The caller this connection's endpoint requests ride as: the legacy triple, or the issued
|
|
2585
2313
|
* caller with its discovered generation (SPEC 13.15). It names the reply rail this connection's
|
|
2586
2314
|
* credential reads, which is where a manager's turn notice for it lands (SPEC 14.3). */
|
|
2587
2315
|
serviceRailCaller() {
|
|
2588
|
-
return this.
|
|
2589
|
-
}
|
|
2590
|
-
/** This endpoint's v0.4 caller triple (§13.2) — the identity its minted ep-rail rows pin. The
|
|
2591
|
-
* owner/actor principal is mode-correct by construction (static: DEV_OWNER + the connection
|
|
2592
|
-
* identity; user mode: the bearer's callout-derived pair), and the lifecycle UID is the
|
|
2593
|
-
* launcher-supplied incarnation the rows are keyed on (ledger-consistent: the §13.1 presence
|
|
2594
|
-
* lifecycle-proof refuses a divergent uid before any publish). */
|
|
2595
|
-
serviceCaller() {
|
|
2596
|
-
const triple = { owner: this.owner, actor: this.actor, uid: this.requireLifecycleUid("invokeService") };
|
|
2597
|
-
if (this.acceptedToken === undefined)
|
|
2598
|
-
return triple;
|
|
2599
|
-
if (this.issuedGeneration === undefined)
|
|
2600
|
-
throw new Error("invokeService: the issued generation is not yet discovered on this connection");
|
|
2601
|
-
return { ...triple, generation: this.issuedGeneration };
|
|
2316
|
+
return this.controlPlane.serviceRailCaller();
|
|
2602
2317
|
}
|
|
2603
2318
|
/** Follow this caller's goal on its standing connection, subscribing before submission.
|
|
2604
2319
|
* A submitter may use a separate short-lived control credential for the same caller triple;
|
|
2605
2320
|
* the accepted action's readiness budget can outlive that credential. This endpoint retains
|
|
2606
2321
|
* its existing credential renewal and caller-scoped progress grants. */
|
|
2607
2322
|
async followServiceGoal(endpoint, submit, deadlineMs = 10_000, opts = {}) {
|
|
2608
|
-
|
|
2609
|
-
throw new Error(this.notLiveMsg());
|
|
2610
|
-
if (this.stopped)
|
|
2611
|
-
throw new Error("endpoint stopped - cannot follow goal");
|
|
2612
|
-
const abortController = new AbortController();
|
|
2613
|
-
const onAbort = () => abortController.abort();
|
|
2614
|
-
if (opts.signal?.aborted)
|
|
2615
|
-
onAbort();
|
|
2616
|
-
else
|
|
2617
|
-
opts.signal?.addEventListener("abort", onAbort, { once: true });
|
|
2618
|
-
let reconnectHandler;
|
|
2619
|
-
const entry = {
|
|
2620
|
-
cancel: () => abortController.abort(),
|
|
2621
|
-
reconnected: (nc) => reconnectHandler?.(nc),
|
|
2622
|
-
};
|
|
2623
|
-
this.activeGoalFollowers.add(entry);
|
|
2624
|
-
try {
|
|
2625
|
-
const reconcile = opts.reconcile ?? (async (goalId, _attributed, context) => {
|
|
2626
|
-
const current = this.serviceCaller();
|
|
2627
|
-
if (current.owner !== context.caller.owner || current.actor !== context.caller.actor || current.uid !== context.caller.uid)
|
|
2628
|
-
throw new Error("the renewed goal reader no longer has the accepting caller's full identity");
|
|
2629
|
-
// Static callers retain their authorized class route; knowing an instance id grants no rights.
|
|
2630
|
-
const q = await this.invokeService(endpoint, "goal-result", { goalId }, {
|
|
2631
|
-
deadlineMs: Math.min(deadlineMs, context.deadlineMs), signal: context.signal,
|
|
2632
|
-
});
|
|
2633
|
-
if (q.reply.ok !== true)
|
|
2634
|
-
throw new Error(q.reply.error.message);
|
|
2635
|
-
return q.reply.data;
|
|
2636
|
-
});
|
|
2637
|
-
return await submitAndFollowGoal(this.nc, this.space, endpoint, this.serviceCaller(), deadlineMs, submit, {
|
|
2638
|
-
reconcile,
|
|
2639
|
-
currentNc: () => this.nc,
|
|
2640
|
-
onReconnect: (cb) => {
|
|
2641
|
-
reconnectHandler = cb;
|
|
2642
|
-
return () => { reconnectHandler = undefined; };
|
|
2643
|
-
},
|
|
2644
|
-
signal: abortController.signal,
|
|
2645
|
-
prepare: opts.prepare,
|
|
2646
|
-
});
|
|
2647
|
-
}
|
|
2648
|
-
finally {
|
|
2649
|
-
opts.signal?.removeEventListener("abort", onAbort);
|
|
2650
|
-
this.activeGoalFollowers.delete(entry);
|
|
2651
|
-
}
|
|
2652
|
-
}
|
|
2653
|
-
/** Drop every cached handle for one endpoint after refresh or a currency split. */
|
|
2654
|
-
clearResolvedService(endpoint) {
|
|
2655
|
-
this.resolvedServices.delete(endpoint);
|
|
2656
|
-
this.resolvedCommandServices.delete(endpoint);
|
|
2657
|
-
}
|
|
2658
|
-
/** Hold connection recovery through the whole pre-acceptance phase, including the progress
|
|
2659
|
-
* subscription flush. Release on setup failure or the command's answer, never on goal terminal. */
|
|
2660
|
-
beginServiceInvocation(nc) {
|
|
2661
|
-
let pending = this.serviceInvocations.get(nc);
|
|
2662
|
-
if (!pending) {
|
|
2663
|
-
pending = new Set();
|
|
2664
|
-
this.serviceInvocations.set(nc, pending);
|
|
2665
|
-
}
|
|
2666
|
-
let settle;
|
|
2667
|
-
const request = new Promise((resolve) => { settle = resolve; });
|
|
2668
|
-
pending.add(request);
|
|
2669
|
-
return () => { pending.delete(request); settle(); };
|
|
2670
|
-
}
|
|
2671
|
-
async waitServiceRecovery(recovery, deadline, signal, timeout) {
|
|
2672
|
-
let onAbort;
|
|
2673
|
-
const cancelled = new Promise((_, reject) => {
|
|
2674
|
-
onAbort = () => reject(signal?.reason);
|
|
2675
|
-
if (signal?.aborted)
|
|
2676
|
-
onAbort();
|
|
2677
|
-
else
|
|
2678
|
-
signal?.addEventListener("abort", onAbort, { once: true });
|
|
2679
|
-
});
|
|
2680
|
-
try {
|
|
2681
|
-
await withDeadline(Promise.race([recovery, cancelled]), deadline - Date.now(), timeout);
|
|
2682
|
-
}
|
|
2683
|
-
finally {
|
|
2684
|
-
if (onAbort)
|
|
2685
|
-
signal?.removeEventListener("abort", onAbort);
|
|
2686
|
-
}
|
|
2687
|
-
signal?.throwIfAborted();
|
|
2688
|
-
if (deadline <= Date.now())
|
|
2689
|
-
throw timeout();
|
|
2323
|
+
return this.controlPlane.followServiceGoal(endpoint, submit, deadlineMs, opts);
|
|
2690
2324
|
}
|
|
2691
2325
|
/** The resolved, digest-verified command surface of `endpoint` as this connection's caller sees
|
|
2692
2326
|
* it. Unpinned calls cache the complete discovery surface, and `refresh` drops both the full
|
|
@@ -2694,39 +2328,21 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
2694
2328
|
* reading or populating either class cache, as a pinned invoke does. The caller's existing
|
|
2695
2329
|
* credential must permit that route. */
|
|
2696
2330
|
async describeService(endpoint, opts = {}) {
|
|
2697
|
-
|
|
2698
|
-
throw new Error(this.notLiveMsg());
|
|
2699
|
-
if (opts.refresh)
|
|
2700
|
-
this.clearResolvedService(endpoint);
|
|
2701
|
-
const cached = opts.instanceId === undefined ? this.resolvedServices.get(endpoint) : undefined;
|
|
2702
|
-
if (cached)
|
|
2703
|
-
return cached;
|
|
2704
|
-
const svc = await resolveService(this.nc, this.space, endpoint, this.serviceCaller(), { deadlineMs: opts.deadlineMs ?? 10_000, signal: opts.signal, ...(opts.instanceId !== undefined ? { instanceId: opts.instanceId } : {}) });
|
|
2705
|
-
if (opts.instanceId === undefined)
|
|
2706
|
-
this.resolvedServices.set(endpoint, svc);
|
|
2707
|
-
return svc;
|
|
2331
|
+
return this.controlPlane.describeService(endpoint, opts);
|
|
2708
2332
|
}
|
|
2709
2333
|
/** The caller a registry listing rides (SPEC 13.15). An issued connection lists as itself. Any
|
|
2710
2334
|
* other connection reads the listing generation the issuer bound to the credential it presented,
|
|
2711
2335
|
* under the token its connection id and that credential's expiry derive. `undefined`: that
|
|
2712
2336
|
* credential carries no listing binding, because it predates listing issuance or has no expiry. */
|
|
2713
2337
|
async listingCaller() {
|
|
2714
|
-
|
|
2715
|
-
throw new Error(this.notLiveMsg());
|
|
2716
|
-
return registryListingCaller(this.nc, this.space, this.serviceCaller(), { connId: this.connId, ...(this.presentedExp !== undefined ? { exp: this.presentedExp } : {}) });
|
|
2338
|
+
return this.controlPlane.listingCaller();
|
|
2717
2339
|
}
|
|
2718
2340
|
/** One page of the registry's `endpoints` answer (SPEC 13.20) over this connection
|
|
2719
2341
|
* ({@link listRegistryEndpoints}). The describe rides this connection's own caller; the question
|
|
2720
2342
|
* rides the listing caller on an authed mesh. `unbound`: this credential carries no listing
|
|
2721
2343
|
* binding. */
|
|
2722
2344
|
async listEndpoints(input, opts = {}) {
|
|
2723
|
-
|
|
2724
|
-
throw new Error(this.notLiveMsg());
|
|
2725
|
-
const nc = this.nc;
|
|
2726
|
-
const listing = this.authed ? await this.listingCaller() : undefined;
|
|
2727
|
-
if (this.authed && listing === undefined)
|
|
2728
|
-
return { unbound: true };
|
|
2729
|
-
return this.listEndpointsOver(nc, listing, input, opts);
|
|
2345
|
+
return this.controlPlane.listEndpoints(input, opts);
|
|
2730
2346
|
}
|
|
2731
2347
|
/** Ask the registry over `nc` under `listing`: this connection, or another of this caller triple
|
|
2732
2348
|
* such as a manager-call connection. The describe rides this connection, whose view holds the
|
|
@@ -2734,17 +2350,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
2734
2350
|
* describe of it reads the surface afresh. `unbound`: the broker refused the question under
|
|
2735
2351
|
* `listing`, whose credential holds no listing row. */
|
|
2736
2352
|
async listEndpointsOver(nc, listing, input, opts = {}) {
|
|
2737
|
-
|
|
2738
|
-
throw new Error(this.notLiveMsg());
|
|
2739
|
-
const own = this.nc;
|
|
2740
|
-
const result = await listRegistryEndpoints(this.space, (deadlineMs, signal) => resolveService(own, this.space, REGISTRY_ENDPOINT, this.serviceCaller(), {
|
|
2741
|
-
deadlineMs, signal, commands: [REGISTRY_ENDPOINTS_COMMAND],
|
|
2742
|
-
}), { nc, ...(listing ? { caller: listing } : {}) }, input, opts);
|
|
2743
|
-
if (result.unbound)
|
|
2744
|
-
return result;
|
|
2745
|
-
for (const row of result.page?.rows ?? [])
|
|
2746
|
-
this.clearResolvedService(row.endpoint);
|
|
2747
|
-
return result;
|
|
2353
|
+
return this.controlPlane.listEndpointsOver(nc, listing, input, opts);
|
|
2748
2354
|
}
|
|
2749
2355
|
/** GENERIC v0.4 service invoke over this endpoint's own connection: resolve
|
|
2750
2356
|
* the named endpoint's registered surface — describe, §13.7 store fetch, digest-verified
|
|
@@ -2772,252 +2378,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
2772
2378
|
* Errors from the responder come back structurally on the attributed reply
|
|
2773
2379
|
* (`reply.ok === false`); transport/validation refusals throw {@link EpEnvelopeError}. */
|
|
2774
2380
|
async invokeService(endpoint, command, args, opts = {}) {
|
|
2775
|
-
|
|
2776
|
-
const rebuilding = this.rebuildPromise;
|
|
2777
|
-
if (rebuilding) {
|
|
2778
|
-
await this.waitServiceRecovery(rebuilding, deadline, opts.signal, () => new EpEnvelopeError("deadline-exceeded", "the invocation deadline elapsed waiting for connection recovery", undefined, "not-executed"));
|
|
2779
|
-
opts = { ...opts, deadlineMs: deadline - Date.now() };
|
|
2780
|
-
}
|
|
2781
|
-
const nc = this.nc;
|
|
2782
|
-
if (!nc)
|
|
2783
|
-
throw new Error(this.notLiveMsg());
|
|
2784
|
-
try {
|
|
2785
|
-
return await this.invokeServiceOnce(endpoint, command, args, opts, rebuilding ? deadline : undefined);
|
|
2786
|
-
}
|
|
2787
|
-
catch (e) {
|
|
2788
|
-
if (!(e instanceof EpEnvelopeError) || e.code !== "permission-denied" || e.outcome !== "not-executed"
|
|
2789
|
-
|| deadline <= Date.now())
|
|
2790
|
-
throw e;
|
|
2791
|
-
opts.signal?.throwIfAborted();
|
|
2792
|
-
const recovery = (async () => {
|
|
2793
|
-
// Re-read the live set: calls admitted while earlier ones settle must keep their replies.
|
|
2794
|
-
// The empty check and rebuild admission have no await between them. Later calls wait on
|
|
2795
|
-
// rebuildPromise instead of publishing on the connection being replaced.
|
|
2796
|
-
for (;;) {
|
|
2797
|
-
opts.signal?.throwIfAborted();
|
|
2798
|
-
if (deadline <= Date.now())
|
|
2799
|
-
throw e;
|
|
2800
|
-
const pending = this.serviceInvocations.get(nc);
|
|
2801
|
-
if (!pending?.size)
|
|
2802
|
-
break;
|
|
2803
|
-
await Promise.allSettled([...pending]);
|
|
2804
|
-
}
|
|
2805
|
-
if (this.nc === nc)
|
|
2806
|
-
await this.reconnect();
|
|
2807
|
-
else
|
|
2808
|
-
await this.rebuildPromise;
|
|
2809
|
-
})();
|
|
2810
|
-
await this.waitServiceRecovery(recovery, deadline, opts.signal, () => e);
|
|
2811
|
-
this.clearResolvedService(endpoint);
|
|
2812
|
-
return this.invokeServiceOnce(endpoint, command, args, { ...opts, deadlineMs: deadline - Date.now() }, deadline);
|
|
2813
|
-
}
|
|
2814
|
-
}
|
|
2815
|
-
async invokeServiceOnce(endpoint, command, args, opts = {}, retryDeadline) {
|
|
2816
|
-
if (!this.nc)
|
|
2817
|
-
throw new Error(this.notLiveMsg());
|
|
2818
|
-
const nc = this.nc;
|
|
2819
|
-
const caller = this.serviceCaller();
|
|
2820
|
-
const resolve = async (signal = opts.signal, deadlineMs = opts.deadlineMs ?? 10_000) => {
|
|
2821
|
-
signal?.throwIfAborted();
|
|
2822
|
-
if (retryDeadline !== undefined)
|
|
2823
|
-
deadlineMs = Math.min(deadlineMs, retryDeadline - Date.now());
|
|
2824
|
-
if (deadlineMs <= 0)
|
|
2825
|
-
throw new EpEnvelopeError("deadline-exceeded", "the invocation deadline elapsed before resolve", undefined, "not-executed");
|
|
2826
|
-
const needed = opts.follow ? [command, "goal-result"] : [command];
|
|
2827
|
-
const classSelected = opts.instanceId === undefined && opts.classInstanceId === undefined;
|
|
2828
|
-
const full = classSelected ? this.resolvedServices.get(endpoint) : undefined;
|
|
2829
|
-
if (full && needed.every((name) => full.commands.has(name)))
|
|
2830
|
-
return full;
|
|
2831
|
-
const cacheKey = [...needed].sort().join("\u0000");
|
|
2832
|
-
const commandCache = classSelected ? this.resolvedCommandServices.get(endpoint) : undefined;
|
|
2833
|
-
const cached = commandCache?.get(cacheKey);
|
|
2834
|
-
if (cached)
|
|
2835
|
-
return cached;
|
|
2836
|
-
const svc = await resolveService(nc, this.space, endpoint, caller, {
|
|
2837
|
-
deadlineMs,
|
|
2838
|
-
signal,
|
|
2839
|
-
commands: needed,
|
|
2840
|
-
...(opts.follow && command !== "goal-result" ? { optionalCommands: ["goal-result"] } : {}),
|
|
2841
|
-
...(opts.instanceId !== undefined ? { instanceId: opts.instanceId } : {}),
|
|
2842
|
-
...(opts.classInstanceId !== undefined ? { classInstanceId: opts.classInstanceId } : {}),
|
|
2843
|
-
});
|
|
2844
|
-
// Pinned resolves never enter either class cache. Narrow unpinned handles stay in their own
|
|
2845
|
-
// command-set cache, so they cannot truncate discovery or evict another command's handle.
|
|
2846
|
-
if (classSelected) {
|
|
2847
|
-
let byCommand = this.resolvedCommandServices.get(endpoint);
|
|
2848
|
-
if (!byCommand) {
|
|
2849
|
-
byCommand = new Map();
|
|
2850
|
-
this.resolvedCommandServices.set(endpoint, byCommand);
|
|
2851
|
-
}
|
|
2852
|
-
byCommand.set(cacheKey, svc);
|
|
2853
|
-
}
|
|
2854
|
-
return svc;
|
|
2855
|
-
};
|
|
2856
|
-
const invokeOpts = { ...(opts.target ? { target: opts.target } : {}), ...(opts.deadlineMs !== undefined ? { deadlineMs: opts.deadlineMs } : {}) };
|
|
2857
|
-
const invokeResolved = (service, signal) => {
|
|
2858
|
-
signal?.throwIfAborted();
|
|
2859
|
-
const unfollowable = opts.follow ? goalFollowRefusal(service) : undefined;
|
|
2860
|
-
if (unfollowable)
|
|
2861
|
-
return Promise.resolve(unfollowable);
|
|
2862
|
-
const deadlineMs = retryDeadline === undefined ? invokeOpts.deadlineMs : retryDeadline - Date.now();
|
|
2863
|
-
if (deadlineMs !== undefined && deadlineMs <= 0)
|
|
2864
|
-
throw new EpEnvelopeError("deadline-exceeded", "the invocation deadline elapsed before publish", undefined, "not-executed");
|
|
2865
|
-
return invokeCommand(nc, this.space, service, command, args, { ...invokeOpts, deadlineMs, signal });
|
|
2866
|
-
};
|
|
2867
|
-
// A `failed-precondition` from the resolve is its own refusal, raised before any command was
|
|
2868
|
-
// published, so resolving once more is a repair. The repair spends what is left of the same
|
|
2869
|
-
// deadline, and with nothing left the refusal stands.
|
|
2870
|
-
const resolveFirst = async (signal = opts.signal, deadline = Date.now() + (opts.deadlineMs ?? 10_000)) => {
|
|
2871
|
-
try {
|
|
2872
|
-
return await resolve(signal, deadline - Date.now());
|
|
2873
|
-
}
|
|
2874
|
-
catch (e) {
|
|
2875
|
-
if (!(e instanceof EpEnvelopeError) || e.code !== "failed-precondition")
|
|
2876
|
-
throw e;
|
|
2877
|
-
const left = deadline - Date.now();
|
|
2878
|
-
if (left <= 0)
|
|
2879
|
-
throw e;
|
|
2880
|
-
return await resolve(signal, left);
|
|
2881
|
-
}
|
|
2882
|
-
};
|
|
2883
|
-
const doInvoke = async (service, signal = opts.signal) => {
|
|
2884
|
-
signal?.throwIfAborted();
|
|
2885
|
-
try {
|
|
2886
|
-
let r = await invokeResolved(service, signal);
|
|
2887
|
-
// THE RESPONDER FENCED IT (§13.2 `ai.cotal.ep.bind-refused`): a class member saw the call
|
|
2888
|
-
// was bound to a different incarnation and refused BEFORE running the command.
|
|
2889
|
-
//
|
|
2890
|
-
// Handled here rather than below because it arrives as an ordinary `ok:false` REPLY, not a
|
|
2891
|
-
// throw, so the `respondedButUnbound` recovery never sees it — a long-lived client would
|
|
2892
|
-
// otherwise keep its stale bind and meet the same refusal forever.
|
|
2893
|
-
//
|
|
2894
|
-
// The re-issue is NOT gated on {@link isRepeatSafeCommand}: the responder states the command
|
|
2895
|
-
// did not run, so each re-issue is a FIRST attempt, not a second. It repeats up to the bound
|
|
2896
|
-
// the CLI uses, because the re-issue rides the same class queue and splits again at the same
|
|
2897
|
-
// rate; repairing once left a quarter of all calls in a two-manager space failing (#443).
|
|
2898
|
-
// A member that holds no mapping for the target also ran nothing, and on the class rail a
|
|
2899
|
-
// sibling may host it. A pinned call named its instance, so that refusal is its answer.
|
|
2900
|
-
// That member looked the target up, so when the re-issues run out its refusal is surfaced
|
|
2901
|
-
// in place of a trailing bind refusal, which says nothing about the target.
|
|
2902
|
-
const unmappedHere = (x) => opts.instanceId === undefined && replyTargetUnmapped(x.reply.error);
|
|
2903
|
-
let unmapped = unmappedHere(r) ? r : undefined;
|
|
2904
|
-
for (let reissues = 0; reissues < BIND_SPLIT_REISSUES && r.reply.ok === false
|
|
2905
|
-
&& (replyRefusedBeforeEffect(r.reply.error) || unmappedHere(r)); reissues++) {
|
|
2906
|
-
// Counted before it is repaired: a recovery that leaves no trace takes the split rate with it.
|
|
2907
|
-
this.splitsRecovered++;
|
|
2908
|
-
// `boundTo` is the other half of `servedBy`: who the handle THOUGHT it was talking to,
|
|
2909
|
-
// against who actually answered. Without it a listener sees that a split was recovered
|
|
2910
|
-
// but not which bind went stale, so it cannot tell one handle's repeated staleness from
|
|
2911
|
-
// splits spread across many — and that is the difference between a handle to drop and a
|
|
2912
|
-
// class that is churning.
|
|
2913
|
-
this.emit("split-recovered", {
|
|
2914
|
-
endpoint, command, servedBy: r.responder, splitsRecovered: this.splitsRecovered,
|
|
2915
|
-
boundTo: (r.reply.error.details ?? []).find((d) => d.kind === EP_BIND_REFUSED)?.boundTo,
|
|
2916
|
-
});
|
|
2917
|
-
this.clearResolvedService(endpoint);
|
|
2918
|
-
// A FAILED RE-ISSUE RETHROWS THE ORIGINAL REFUSAL, not the resolve error: if the endpoint
|
|
2919
|
-
// has since retired, the resolve times out and the caller would otherwise get `no describe
|
|
2920
|
-
// reply within 10000ms` — about a describe it never asked for — losing the one fact that
|
|
2921
|
-
// says its handle is stale AND that nothing ran. The resolve failure is named as the
|
|
2922
|
-
// reason the repair could not be attempted.
|
|
2923
|
-
//
|
|
2924
|
-
// The code must be the refusal's own (`failed-precondition` for a different instance,
|
|
2925
|
-
// `expired` for the same instance at ANY other epoch — the fence does not compare
|
|
2926
|
-
// direction, so a superseded incarnation still answering produces it too); the marker on
|
|
2927
|
-
// any other code is incoherent, so it is rejected rather than trusted (§13.3).
|
|
2928
|
-
const raw = r.reply.error.code;
|
|
2929
|
-
const refusalCode = raw === "expired" ? "expired"
|
|
2930
|
-
: raw === "failed-precondition" ? "failed-precondition" : undefined;
|
|
2931
|
-
if (refusalCode === undefined)
|
|
2932
|
-
throw new EpEnvelopeError("internal", `${endpoint}.${command} came back marked as refused before it ran, but with code ${String(raw)}; the fence produces only failed-precondition or expired (SPEC 13.2)`, r.reply.error.details);
|
|
2933
|
-
// ONLY THE RESOLVE IS WRAPPED — do not widen this `try` to cover the re-issue. A re-issue
|
|
2934
|
-
// that PUBLISHED AND RAN can still throw (an unfenced responder answers and the post-reply
|
|
2935
|
-
// currency check raises `respondedButUnbound`); wrapping that as the first hop's refusal
|
|
2936
|
-
// would hand the caller `WAS NOT RUN` for a command that executed, and its next attempt
|
|
2937
|
-
// would be a second one believing it was the first. A re-issue that fails on its own terms
|
|
2938
|
-
// must propagate its OWN error, because `respondedButUnbound` asserts the opposite of what
|
|
2939
|
-
// the fence's marker does: a responder answered and the effect may have landed.
|
|
2940
|
-
let reissueTarget;
|
|
2941
|
-
try {
|
|
2942
|
-
reissueTarget = await resolve(signal);
|
|
2943
|
-
const unfollowable = opts.follow ? goalFollowRefusal(reissueTarget) : undefined;
|
|
2944
|
-
if (unfollowable)
|
|
2945
|
-
throw new Error(unfollowable.reply.error.message);
|
|
2946
|
-
}
|
|
2947
|
-
catch (reissue) {
|
|
2948
|
-
throw new EpEnvelopeError(refusalCode, `${endpoint}.${command} WAS NOT RUN - the incarnation that received it refused it before any effect, and the re-issue could not be resolved: ${reissue instanceof Error ? reissue.message : String(reissue)}. Re-resolve and re-issue when the endpoint is reachable (SPEC 13.2)`, r.reply.error.details,
|
|
2949
|
-
// §13.3: the message asserts the command did not run, so the FIELD a caller keys on
|
|
2950
|
-
// must assert it too. Omitted, it MUST be read as `unknown`, which is the opposite of
|
|
2951
|
-
// what this path knows: the responder fenced the call before any effect and the
|
|
2952
|
-
// re-issue never went out. Prose is for the reader; this is for the machine.
|
|
2953
|
-
"not-executed");
|
|
2954
|
-
}
|
|
2955
|
-
r = await invokeResolved(reissueTarget, signal);
|
|
2956
|
-
if (unmappedHere(r))
|
|
2957
|
-
unmapped = r;
|
|
2958
|
-
}
|
|
2959
|
-
return replyRefusedBeforeEffect(r.reply.error) ? unmapped ?? r : r;
|
|
2960
|
-
}
|
|
2961
|
-
catch (e) {
|
|
2962
|
-
// DO NOT auto-retry a command a responder already ANSWERED. This path covers the responders
|
|
2963
|
-
// WITHOUT the fence above: they ignore `bind`, run the command, and the error is raised
|
|
2964
|
-
// afterwards, so core cannot tell a repair from a duplicate and the allowlist is the only
|
|
2965
|
-
// guard left. (`failed-precondition` here is not only supersession — it also fires when a
|
|
2966
|
-
// DIFFERENT live instance wins the class queue and replies, an ordinary split.)
|
|
2967
|
-
//
|
|
2968
|
-
// Dropping the retry outright is not an option: in a two-manager space roughly half of all
|
|
2969
|
-
// class-queue calls split, so every other `ps` would surface an error this absorbs, and
|
|
2970
|
-
// re-running a read costs nothing.
|
|
2971
|
-
//
|
|
2972
|
-
// So it is gated on an ALLOWLIST (`REPEAT_SAFE_COMMANDS`), keyed by ENDPOINT because this
|
|
2973
|
-
// method is endpoint-agnostic. Polarity matters more than membership: an allowlist fails
|
|
2974
|
-
// CLOSED. Not a `GOAL_BEARING_COMMANDS`-only gate — `purge` is neither goal-bearing nor
|
|
2975
|
-
// convergent, and its second run deletes messages published after the first completed, so
|
|
2976
|
-
// the boundary is "changes anything", which core cannot see: commands carry no idempotency
|
|
2977
|
-
// declaration and every manager command shares `class: "ephemeral"`.
|
|
2978
|
-
//
|
|
2979
|
-
// Keyed on the MARKER, not the error code, because the same fact has two producers: a
|
|
2980
|
-
// DIFFERENT instance answering (`failed-precondition`) and the SAME instance at ANY OTHER
|
|
2981
|
-
// EPOCH (`expired`) — epoch inequality, not "a successor", so a superseded incarnation
|
|
2982
|
-
// still answering produces it too. Same rule for both, or a long-lived client stays bound
|
|
2983
|
-
// to a dead epoch while every call reaches whoever is actually there.
|
|
2984
|
-
if (respondedButUnbound(e)) {
|
|
2985
|
-
// Drop the stale bind FIRST, whichever way this goes: it names an incarnation that is not
|
|
2986
|
-
// the one answering, so every later call would reuse it and meet the same refusal. This is
|
|
2987
|
-
// not a retry — nothing is re-issued by it, and for a command that is not repeat-safe the
|
|
2988
|
-
// next call is the caller's, made after it has verified.
|
|
2989
|
-
this.clearResolvedService(endpoint);
|
|
2990
|
-
if (!isRepeatSafeCommand(endpoint, command))
|
|
2991
|
-
throw e;
|
|
2992
|
-
return await invokeResolved(await resolve(signal), signal);
|
|
2993
|
-
}
|
|
2994
|
-
throw e;
|
|
2995
|
-
}
|
|
2996
|
-
};
|
|
2997
|
-
// A goal-bearing command (spawn/launch) follows its acceptance to the terminal so
|
|
2998
|
-
// the caller still returns on the real outcome (UX unchanged); every other command replies directly.
|
|
2999
|
-
const release = this.beginServiceInvocation(nc);
|
|
3000
|
-
try {
|
|
3001
|
-
if (!opts.follow)
|
|
3002
|
-
return await doInvoke(await resolveFirst());
|
|
3003
|
-
// Resolve and confirm observation before submission. Hold the barrier continuously across
|
|
3004
|
-
// both phases, then release when the command answers so accepted goals can outlive recovery.
|
|
3005
|
-
let service;
|
|
3006
|
-
return await this.followServiceGoal(endpoint, async (signal) => {
|
|
3007
|
-
try {
|
|
3008
|
-
return await doInvoke(service, signal);
|
|
3009
|
-
}
|
|
3010
|
-
finally {
|
|
3011
|
-
release();
|
|
3012
|
-
}
|
|
3013
|
-
}, opts.deadlineMs ?? 10_000, {
|
|
3014
|
-
signal: opts.signal,
|
|
3015
|
-
prepare: async (signal, deadline) => { service = await resolveFirst(signal, deadline); },
|
|
3016
|
-
});
|
|
3017
|
-
}
|
|
3018
|
-
finally {
|
|
3019
|
-
release();
|
|
3020
|
-
}
|
|
2381
|
+
return this.controlPlane.invokeService(endpoint, command, args, opts);
|
|
3021
2382
|
}
|
|
3022
2383
|
/** Send a durable-membership request to the SERVER-SIDE delivery daemon (`ctl.delivery`) and await its
|
|
3023
2384
|
* reply. The reply rides a subject UNDER `ctl.delivery.<id>.>` (not the per-id `_INBOX`), so the
|
|
@@ -3123,20 +2484,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
3123
2484
|
* untouched (unlike `variant`, it is never deleted, since a caller that does not know the
|
|
3124
2485
|
* provider says nothing about whether one still applies). */
|
|
3125
2486
|
async setCardModel(model, variant, provider) {
|
|
3126
|
-
|
|
3127
|
-
const v = variant?.trim();
|
|
3128
|
-
const p = provider?.trim();
|
|
3129
|
-
if (!m || (this.card.meta?.model === m && this.card.meta.variant === v && (p === undefined || this.card.meta?.provider === p)))
|
|
3130
|
-
return;
|
|
3131
|
-
const meta = { ...(this.card.meta ?? {}), model: m };
|
|
3132
|
-
if (v)
|
|
3133
|
-
meta.variant = v;
|
|
3134
|
-
else
|
|
3135
|
-
delete meta.variant;
|
|
3136
|
-
if (p)
|
|
3137
|
-
meta.provider = p;
|
|
3138
|
-
this.card.meta = meta;
|
|
3139
|
-
await this.publishPresence();
|
|
2487
|
+
return this.presence.setCardModel(model, variant, provider);
|
|
3140
2488
|
}
|
|
3141
2489
|
/** Overlay the session's working directory onto the card's display-only `meta.cwd`, then
|
|
3142
2490
|
* republish presence. For connectors that learn the directory from the harness after launch (e.g.
|
|
@@ -3145,16 +2493,37 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
3145
2493
|
* or end in whitespace, so it is never trimmed or normalized. An empty string names no directory
|
|
3146
2494
|
* and leaves the card unchanged. */
|
|
3147
2495
|
async setCardCwd(cwd) {
|
|
3148
|
-
|
|
3149
|
-
return;
|
|
3150
|
-
this.card.meta = { ...(this.card.meta ?? {}), cwd };
|
|
3151
|
-
await this.publishPresence();
|
|
2496
|
+
return this.presence.setCardCwd(cwd);
|
|
3152
2497
|
}
|
|
3153
2498
|
// ---- channel discovery ---------------------------------------------------
|
|
3154
|
-
/**
|
|
3155
|
-
*
|
|
2499
|
+
/** The channel registry cache, its policy lookups and the registry watch that fills it, of this
|
|
2500
|
+
* endpoint. Every dependency is read when it is used, so a field assigned after construction is seen. */
|
|
2501
|
+
channelDiscovery = new EndpointChannelDiscovery({
|
|
2502
|
+
nc: () => this.nc,
|
|
2503
|
+
space: () => this.space,
|
|
2504
|
+
card: () => this.card,
|
|
2505
|
+
doConsume: () => this.doConsume,
|
|
2506
|
+
doWatch: () => this.doWatch,
|
|
2507
|
+
doWatchChannels: () => this.doWatchChannels,
|
|
2508
|
+
channelKv: () => this.channelKv,
|
|
2509
|
+
setChannelWatchIter: (iter) => { this.channelWatchIter = iter; },
|
|
2510
|
+
channelConfigs: () => this.channelConfigs,
|
|
2511
|
+
channelDefaults: () => this.channelDefaults,
|
|
2512
|
+
setChannelDefaults: (defaults) => { this.channelDefaults = defaults; },
|
|
2513
|
+
channelRead: () => this.channelRead,
|
|
2514
|
+
setChannelRead: (read) => { this.channelRead = read; },
|
|
2515
|
+
channelReadGeneration: () => this.channelReadGeneration,
|
|
2516
|
+
setChannelReadGeneration: (n) => { this.channelReadGeneration = n; },
|
|
2517
|
+
channelReadApplied: () => this.channelReadApplied,
|
|
2518
|
+
setChannelReadApplied: (n) => { this.channelReadApplied = n; },
|
|
2519
|
+
notLiveMsg: () => this.notLiveMsg(),
|
|
2520
|
+
bindKvWatch: (kv) => this.bindKvWatch(kv),
|
|
2521
|
+
emitRecoverable: (err) => this.emitRecoverable(err),
|
|
2522
|
+
emit: (event, ...args) => this.emit(event, ...args),
|
|
2523
|
+
});
|
|
2524
|
+
/** Forwards to the channel discovery component's {@link EndpointChannelDiscovery.readsChannelsOnDemand}. */
|
|
3156
2525
|
get readsChannelsOnDemand() {
|
|
3157
|
-
return this.
|
|
2526
|
+
return this.channelDiscovery.readsChannelsOnDemand;
|
|
3158
2527
|
}
|
|
3159
2528
|
/**
|
|
3160
2529
|
* Read the channel registry from the broker now, for a consumer that reads presence on demand. One
|
|
@@ -3166,85 +2535,73 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
3166
2535
|
* throws and changes nothing. A credential that may not make the read is refused with the grant it lacks.
|
|
3167
2536
|
*/
|
|
3168
2537
|
async readChannels() {
|
|
3169
|
-
|
|
3170
|
-
throw new Error("readChannels is for a consumer that reads presence on demand; this endpoint watches the channel registry or keeps no cache of it");
|
|
3171
|
-
const kv = this.channelKv;
|
|
3172
|
-
if (!kv)
|
|
3173
|
-
throw new Error(this.notLiveMsg());
|
|
3174
|
-
if (this.channelRead)
|
|
3175
|
-
return this.channelRead;
|
|
3176
|
-
const generation = ++this.channelReadGeneration;
|
|
3177
|
-
const read = (async () => {
|
|
3178
|
-
let entries;
|
|
3179
|
-
try {
|
|
3180
|
-
entries = await lastKvEntries(kv);
|
|
3181
|
-
}
|
|
3182
|
-
catch (e) {
|
|
3183
|
-
if (!isPublishPermissionDenied(e))
|
|
3184
|
-
throw e;
|
|
3185
|
-
throw new Error(`channel registry read refused for ${this.card.id}: this credential may not publish $JS.API.DIRECT.GET.KV_${channelBucket(this.space)}, the on-demand registry read (SPEC Appendix B). A credential minted by an older issuer can lack this grant even when newly minted; upgrade the manager or auth service that issues credentials, then mint a new credential. Reusing credentials or reconnecting does not add the grant.`, { cause: e });
|
|
3186
|
-
}
|
|
3187
|
-
if (generation <= this.channelReadApplied)
|
|
3188
|
-
return;
|
|
3189
|
-
this.channelReadApplied = generation;
|
|
3190
|
-
this.channelConfigs.clear();
|
|
3191
|
-
this.channelDefaults = {};
|
|
3192
|
-
for (const e of entries)
|
|
3193
|
-
this.handleChannelEntry(e);
|
|
3194
|
-
})();
|
|
3195
|
-
this.channelRead = read;
|
|
3196
|
-
const settled = () => { if (this.channelRead === read)
|
|
3197
|
-
this.channelRead = undefined; };
|
|
3198
|
-
read.then(settled, settled);
|
|
3199
|
-
return read;
|
|
2538
|
+
return this.channelDiscovery.readChannels();
|
|
3200
2539
|
}
|
|
3201
2540
|
/** This channel's registry config from the local cache (undefined if unset). */
|
|
3202
2541
|
getChannelConfig(channel) {
|
|
3203
|
-
return this.
|
|
2542
|
+
return this.channelDiscovery.getChannelConfig(channel);
|
|
3204
2543
|
}
|
|
3205
2544
|
/** Effective replay-on-join policy for a channel: per-channel override ?? space default ??
|
|
3206
2545
|
* true. Reads the live cache, so it reflects runtime registry edits. */
|
|
3207
2546
|
channelReplay(channel) {
|
|
3208
|
-
return
|
|
2547
|
+
return this.channelDiscovery.channelReplay(channel);
|
|
3209
2548
|
}
|
|
3210
2549
|
/** Effective replay window for a channel (per-channel override ?? space default), or undefined
|
|
3211
2550
|
* for the full retained window. Only meaningful when {@link channelReplay} is true. */
|
|
3212
2551
|
channelReplayWindow(channel) {
|
|
3213
|
-
return this.
|
|
2552
|
+
return this.channelDiscovery.channelReplayWindow(channel);
|
|
3214
2553
|
}
|
|
3215
2554
|
/** A channel's effective replay policy read straight from the registry bucket, for a caller that
|
|
3216
2555
|
* reads it right after start, before any watch cache has caught up, or that watches nothing. A
|
|
3217
2556
|
* wildcard reads the space defaults only. A space with no registry bucket has the built-in policy.
|
|
3218
2557
|
* `registered` says whether the channel has its own registry entry. */
|
|
3219
2558
|
async channelReplayPolicy(channel) {
|
|
3220
|
-
|
|
3221
|
-
throw new Error(this.notLiveMsg());
|
|
3222
|
-
const kv = this.channelKv ?? (await openChannelRegistry(this.nc, this.space, { create: false }));
|
|
3223
|
-
let cfg;
|
|
3224
|
-
let defaults;
|
|
3225
|
-
try {
|
|
3226
|
-
[cfg, defaults] = await Promise.all([
|
|
3227
|
-
isConcreteChannel(channel) ? readChannelConfig(kv, channel) : Promise.resolve(undefined),
|
|
3228
|
-
readChannelDefaults(kv),
|
|
3229
|
-
]);
|
|
3230
|
-
}
|
|
3231
|
-
catch (e) {
|
|
3232
|
-
// `open` binds lazily, so a space with no registry bucket surfaces on the first read.
|
|
3233
|
-
if (!isJetStreamMissing(e, JetStreamApiCodes.StreamNotFound))
|
|
3234
|
-
throw e;
|
|
3235
|
-
}
|
|
3236
|
-
return { replay: effectiveReplay(cfg, defaults), window: cfg?.replayWindow ?? defaults?.replayWindow ?? null, registered: cfg !== undefined };
|
|
2559
|
+
return this.channelDiscovery.channelReplayPolicy(channel);
|
|
3237
2560
|
}
|
|
3238
2561
|
/** Effective delivery class for a channel (per-channel override ?? space default ?? "durable"),
|
|
3239
2562
|
* from the live watch cache — drives the non-gating delivery-health surface (only durable-class
|
|
3240
2563
|
* channels have a Plane-3 backstop to report on). */
|
|
3241
2564
|
channelDeliveryClass(channel) {
|
|
3242
|
-
return
|
|
2565
|
+
return this.channelDiscovery.channelDeliveryClass(channel);
|
|
3243
2566
|
}
|
|
3244
2567
|
// ---- dynamic subscription (join / leave mid-session) ---------------------
|
|
2568
|
+
/** Join, leave and channel listing, and the membership feed watches, of this endpoint. Every
|
|
2569
|
+
* dependency is read when it is used, so a field assigned after construction is seen. */
|
|
2570
|
+
subscriptions = new EndpointSubscriptions({
|
|
2571
|
+
nc: () => this.nc,
|
|
2572
|
+
jsm: () => this.jsm,
|
|
2573
|
+
space: () => this.space,
|
|
2574
|
+
card: () => this.card,
|
|
2575
|
+
stopped: () => this.stopped,
|
|
2576
|
+
authed: () => this.authed,
|
|
2577
|
+
channels: () => this.channels,
|
|
2578
|
+
channelConfigs: () => this.channelConfigs,
|
|
2579
|
+
channelDefaults: () => this.channelDefaults,
|
|
2580
|
+
chatSubDenied: () => this.chatSubDenied,
|
|
2581
|
+
plane3Channels: () => this.plane3Channels,
|
|
2582
|
+
readsChannelsOnDemand: () => this.readsChannelsOnDemand,
|
|
2583
|
+
membershipFeedKv: () => this.membershipFeedKv,
|
|
2584
|
+
setMembershipFeedKv: (kv) => { this.membershipFeedKv = kv; },
|
|
2585
|
+
membershipFeedWatches: () => this.membershipFeedWatches,
|
|
2586
|
+
watchDeletePermissions: () => this.watchDeletePermissions,
|
|
2587
|
+
notLiveMsg: () => this.notLiveMsg(),
|
|
2588
|
+
readChannels: () => this.readChannels(),
|
|
2589
|
+
armJoin: (channels) => this.armJoin(channels),
|
|
2590
|
+
backfillArmed: (frontiers, floorSeq) => this.backfillArmed(frontiers, floorSeq),
|
|
2591
|
+
subscribeChat: (channel) => this.subscribeChat(channel),
|
|
2592
|
+
unsubscribeChat: (channel) => this.unsubscribeChat(channel),
|
|
2593
|
+
confirmChatSub: () => this.confirmChatSub(),
|
|
2594
|
+
durableJoinChannel: (channel) => this.durableJoinChannel(channel),
|
|
2595
|
+
durableLeaveChannel: (...args) => this.durableLeaveChannel(...args),
|
|
2596
|
+
fetchMemberships: (channel) => this.fetchMemberships(channel),
|
|
2597
|
+
recordOwnWatchConsumer: (info) => this.recordOwnWatchConsumer(info),
|
|
2598
|
+
watchConsumerDeleteDenied: (stream, name, owner) => this.watchConsumerDeleteDenied(stream, name, owner),
|
|
2599
|
+
deleteMembershipConsumer: (jsm, stream, name) => this.deleteMembershipConsumer(jsm, stream, name),
|
|
2600
|
+
emit: (event, ...args) => this.emit(event, ...args),
|
|
2601
|
+
});
|
|
3245
2602
|
/** The channels this endpoint is currently subscribed to (live — reflects join/leave). */
|
|
3246
2603
|
joinedChannels() {
|
|
3247
|
-
return
|
|
2604
|
+
return this.subscriptions.joinedChannels();
|
|
3248
2605
|
}
|
|
3249
2606
|
/**
|
|
3250
2607
|
* Join a channel mid-session: open a native core subscription (manager-free live read, broker-
|
|
@@ -3256,68 +2613,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
3256
2613
|
* when a durable channel couldn't get one).
|
|
3257
2614
|
*/
|
|
3258
2615
|
async joinChannel(channel) {
|
|
3259
|
-
|
|
3260
|
-
throw new Error(this.notLiveMsg());
|
|
3261
|
-
if (this.channels.includes(channel))
|
|
3262
|
-
return { joined: false, backfilled: 0, durable: this.plane3Channels.has(channel) };
|
|
3263
|
-
// The delivery class below comes from the cache, so a seat reads it current before joining.
|
|
3264
|
-
if (this.readsChannelsOnDemand)
|
|
3265
|
-
await this.readChannels();
|
|
3266
|
-
// Arm the watermark BEFORE going live: the backfill reads ≤ frontier and the core-sub only ever
|
|
3267
|
-
// delivers post-subscribe live messages (> frontier), so the two never overlap.
|
|
3268
|
-
const armed = await this.armJoin([channel]);
|
|
3269
|
-
// Live read (SPEC v0.3): open the native core subscription — MANAGER-FREE, broker-enforced by
|
|
3270
|
-
// sub.allow. This is what lets an agent join a channel's live feed on its own. The sub.allow
|
|
3271
|
-
// refusal is async — broker-confirm before committing local join state; the subscribe handler
|
|
3272
|
-
// ALSO drops a channel on ANY refusal (incl. a late one), so this is not a timing gamble (#13).
|
|
3273
|
-
this.subscribeChat(channel);
|
|
3274
|
-
try {
|
|
3275
|
-
await this.confirmChatSub();
|
|
3276
|
-
}
|
|
3277
|
-
catch (e) {
|
|
3278
|
-
// The confirm boundary (flush) failed — the connection drained/closed mid-join, so we have NO
|
|
3279
|
-
// confirmation the subscribe was accepted. Fail closed: undo the half-open join rather than
|
|
3280
|
-
// returning as if it were confirmed (a reconnect re-confirms from this.channels, which we never
|
|
3281
|
-
// pushed to). unsubscribeChat clears chatSubs.
|
|
3282
|
-
this.unsubscribeChat(channel);
|
|
3283
|
-
throw new Error(`cannot join "${channel}": live subscription could not be confirmed (${e.message})`);
|
|
3284
|
-
}
|
|
3285
|
-
if (this.chatSubDenied.has(channel)) {
|
|
3286
|
-
this.unsubscribeChat(channel);
|
|
3287
|
-
throw new Error(`cannot join "${channel}": not within this agent's read ACL (allowSubscribe)`);
|
|
3288
|
-
}
|
|
3289
|
-
this.channels.push(channel);
|
|
3290
|
-
// Durable backstop. The live core-sub above already delivers (manager-free). For a `durable`-class
|
|
3291
|
-
// channel, request a Plane-3 per-member backstop from the server-side delivery daemon (durableJoin via ctl.delivery) so a post reaches a
|
|
3292
|
-
// busy/offline turn — the core-sub stays as the live wake-hint, dedup-coalesced with the Plane-3
|
|
3293
|
-
// copy by id-dedup. No manager (open dev / manager-less) ⇒ joined LIVE only, surfaced via `reason`
|
|
3294
|
-
// (never silent). A `live`-class channel takes no backstop (joined live is the contract). A family
|
|
3295
|
-
// (`review.>`) always asks for its row: each channel under it decides its own class at fan-out
|
|
3296
|
-
// (#5404). Any other pattern has no backstop, and says so.
|
|
3297
|
-
let durable = false;
|
|
3298
|
-
let reason;
|
|
3299
|
-
const family = isChannelFamily(channel);
|
|
3300
|
-
if (!family && !isConcreteChannel(channel))
|
|
3301
|
-
reason = "a pattern with * or the root > has no stored copies; follow a concrete channel or a family such as review.>";
|
|
3302
|
-
else if (family || effectiveDeliveryClass(this.channelConfigs.get(channel), this.channelDefaults) === "durable") {
|
|
3303
|
-
try {
|
|
3304
|
-
const r = await this.durableJoinChannel(channel);
|
|
3305
|
-
if (r.durable) {
|
|
3306
|
-
this.plane3Channels.set(channel, r.generation ?? 0);
|
|
3307
|
-
durable = true;
|
|
3308
|
-
}
|
|
3309
|
-
else {
|
|
3310
|
-
reason = r.reason ?? "durable backstop unavailable";
|
|
3311
|
-
}
|
|
3312
|
-
}
|
|
3313
|
-
catch (e) {
|
|
3314
|
-
// No privileged writer (no delivery daemon) or the write was rejected — joined live, backstop
|
|
3315
|
-
// unavailable. NOT a join failure: the live subscription is up and authorized.
|
|
3316
|
-
reason = `durable backstop unavailable (${e.message})`;
|
|
3317
|
-
}
|
|
3318
|
-
}
|
|
3319
|
-
const backfilled = await this.backfillArmed(armed);
|
|
3320
|
-
return { joined: true, backfilled, durable, ...(reason !== undefined ? { reason } : {}) };
|
|
2616
|
+
return this.subscriptions.joinChannel(channel);
|
|
3321
2617
|
}
|
|
3322
2618
|
/** Leave a channel mid-session — MANAGER-FREE for the live read: close the core subscription. For a
|
|
3323
2619
|
* Plane-3 durable channel, the membership is tombstoned FIRST at the leave cursor (SPEC §7: leave is
|
|
@@ -3327,38 +2623,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
3327
2623
|
* read while the backstop keeps delivering. `legacyDiscoveryMessage` lets the caller supply the
|
|
3328
2624
|
* remedy when membership discovery predates lifecycle support, without changing the refusal. */
|
|
3329
2625
|
async leaveChannel(channel, legacyDiscoveryMessage) {
|
|
3330
|
-
|
|
3331
|
-
throw new Error(this.notLiveMsg());
|
|
3332
|
-
let left = this.channels.includes(channel);
|
|
3333
|
-
if (this.readsChannelsOnDemand)
|
|
3334
|
-
await this.readChannels();
|
|
3335
|
-
// Auth + durable-class ⇒ a Plane-3 membership may exist; tombstone it BEFORE touching local state.
|
|
3336
|
-
// Discover the channel's rows across this principal's lifecycles, including memberships absent
|
|
3337
|
-
// from the local mirror. Keep each row's generation guard. fetchMemberships throws on a
|
|
3338
|
-
// responder-present error, so a leave whose
|
|
3339
|
-
// tombstone can't be confirmed propagates (live sub stays up, mirror intact) for the caller to retry
|
|
3340
|
-
// — reporting `left` while the trusted reader keeps transferring to DLV is the fail-open leak. A
|
|
3341
|
-
// genuine no-responder (open / no delivery daemon, no Plane-3) means there is no membership to tombstone.
|
|
3342
|
-
if (this.authed && (isChannelFamily(channel) || effectiveDeliveryClass(this.channelConfigs.get(channel), this.channelDefaults) === "durable")) {
|
|
3343
|
-
const memberships = await this.fetchMemberships(channel);
|
|
3344
|
-
if (memberships === undefined && this.plane3Channels.has(channel))
|
|
3345
|
-
await this.durableLeaveChannel(channel, this.plane3Channels.get(channel));
|
|
3346
|
-
for (const member of memberships ?? []) {
|
|
3347
|
-
// A daemon before 0.166.0 answers with this lifecycle's rows and no lifecycleUid, so it cannot
|
|
3348
|
-
// show an earlier lifecycle's row. Refuse with the remedy rather than leave that row open.
|
|
3349
|
-
if (member.lifecycleUid === undefined)
|
|
3350
|
-
throw new Error(legacyDiscoveryMessage ?? "the delivery daemon predates lifecycle-aware membership discovery, so this durable leave was not applied and the channel stays joined; restart the daemon at this version (`cotal up` runs it)");
|
|
3351
|
-
await this.durableLeaveChannel(channel, member.generation, member.lifecycleUid);
|
|
3352
|
-
left = true;
|
|
3353
|
-
}
|
|
3354
|
-
if (memberships !== undefined)
|
|
3355
|
-
this.plane3Channels.delete(channel);
|
|
3356
|
-
}
|
|
3357
|
-
this.unsubscribeChat(channel);
|
|
3358
|
-
const i = this.channels.indexOf(channel);
|
|
3359
|
-
if (i >= 0)
|
|
3360
|
-
this.channels.splice(i, 1);
|
|
3361
|
-
return { left };
|
|
2626
|
+
return this.subscriptions.leaveChannel(channel, legacyDiscoveryMessage);
|
|
3362
2627
|
}
|
|
3363
2628
|
/**
|
|
3364
2629
|
* What the chat stream still holds and what removes the rest. `perSender` is the stream's
|
|
@@ -3370,93 +2635,18 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
3370
2635
|
* not exist, which holds nothing. `lastSeq` is the stream's newest sequence, 0 when it has none.
|
|
3371
2636
|
*/
|
|
3372
2637
|
async chatRetention() {
|
|
3373
|
-
|
|
3374
|
-
throw new Error(this.notLiveMsg());
|
|
3375
|
-
const mgr = await jetstreamManager(this.nc);
|
|
3376
|
-
let info;
|
|
3377
|
-
try {
|
|
3378
|
-
info = await mgr.streams.info(chatStream(this.space), { subjects_filter: ">" });
|
|
3379
|
-
}
|
|
3380
|
-
catch (e) {
|
|
3381
|
-
// As in listChannels: an absent CHAT stream holds nothing yet; any other failure stays loud.
|
|
3382
|
-
if (!isJetStreamMissing(e, JetStreamApiCodes.StreamNotFound))
|
|
3383
|
-
throw e;
|
|
3384
|
-
return { perSender: null, maxAgeMs: null, oldestAt: null, senders: [], stream: false, lastSeq: 0 };
|
|
3385
|
-
}
|
|
3386
|
-
const senders = [];
|
|
3387
|
-
for (const [subject, messages] of Object.entries(info.state.subjects ?? {})) {
|
|
3388
|
-
const p = parseSubject(subject);
|
|
3389
|
-
if (p?.kind === "chat" && isPrincipalOwnerToken(p.owner))
|
|
3390
|
-
senders.push({ channel: p.rest, sender: p.sender, messages });
|
|
3391
|
-
}
|
|
3392
|
-
const cap = info.config.max_msgs_per_subject;
|
|
3393
|
-
const age = info.config.max_age;
|
|
3394
|
-
return {
|
|
3395
|
-
perSender: typeof cap === "number" && cap > 0 ? cap : null,
|
|
3396
|
-
maxAgeMs: typeof age === "number" && age > 0 ? Math.round(age / 1_000_000) : null,
|
|
3397
|
-
// The stream's oldest post, of any channel; every post sent before it is gone.
|
|
3398
|
-
oldestAt: info.state.messages > 0 ? info.state.first_ts : null,
|
|
3399
|
-
senders: senders.sort((a, b) => a.channel.localeCompare(b.channel) || a.sender.localeCompare(b.sender)),
|
|
3400
|
-
stream: true,
|
|
3401
|
-
lastSeq: info.state.last_seq,
|
|
3402
|
-
};
|
|
2638
|
+
return this.subscriptions.chatRetention();
|
|
3403
2639
|
}
|
|
3404
2640
|
/** Whether this space's direct-message stream is on its broker. A missing stream is false.
|
|
3405
2641
|
* Any other failure, including a permission denial, is raised. */
|
|
3406
2642
|
async dmStreamPresent() {
|
|
3407
|
-
|
|
3408
|
-
throw new Error(this.notLiveMsg());
|
|
3409
|
-
try {
|
|
3410
|
-
await (await jetstreamManager(this.nc)).streams.info(dmStream(this.space));
|
|
3411
|
-
return true;
|
|
3412
|
-
}
|
|
3413
|
-
catch (e) {
|
|
3414
|
-
if (!isJetStreamMissing(e, JetStreamApiCodes.StreamNotFound))
|
|
3415
|
-
throw e;
|
|
3416
|
-
return false;
|
|
3417
|
-
}
|
|
2643
|
+
return this.subscriptions.dmStreamPresent();
|
|
3418
2644
|
}
|
|
3419
2645
|
/** One coherent channel model for dashboards: every channel that has messages OR a registry
|
|
3420
2646
|
* entry (configured-but-empty), each tagged with its {@link ChannelConfig}. Works even on
|
|
3421
2647
|
* observer endpoints (no consumers needed). */
|
|
3422
2648
|
async listChannels() {
|
|
3423
|
-
|
|
3424
|
-
throw new Error(this.notLiveMsg());
|
|
3425
|
-
if (this.readsChannelsOnDemand)
|
|
3426
|
-
await this.readChannels();
|
|
3427
|
-
const mgr = await jetstreamManager(this.nc);
|
|
3428
|
-
// Subjects carry the sender (chat.<sender>.<channel>), so collapse across senders: sum
|
|
3429
|
-
// each channel's counts regardless of who published.
|
|
3430
|
-
const counts = new Map();
|
|
3431
|
-
try {
|
|
3432
|
-
const info = await mgr.streams.info(chatStream(this.space), { subjects_filter: ">" });
|
|
3433
|
-
if (info.state.subjects) {
|
|
3434
|
-
for (const [subject, count] of Object.entries(info.state.subjects)) {
|
|
3435
|
-
const p = parseSubject(subject);
|
|
3436
|
-
// Same surfacing-boundary defense as the message guards: parseSubject splits only, so an
|
|
3437
|
-
// old-shape alias (`chat.<nkey>.team.backend`) structurally parses with a raw-nkey owner and a
|
|
3438
|
-
// misattributed channel. Reject a non-principal owner token here too, or retained pre-flip
|
|
3439
|
-
// subjects would inflate this channel-count surface with phantom channels.
|
|
3440
|
-
if (p?.kind === "chat" && isPrincipalOwnerToken(p.owner))
|
|
3441
|
-
counts.set(p.rest, (counts.get(p.rest) ?? 0) + count);
|
|
3442
|
-
}
|
|
3443
|
-
}
|
|
3444
|
-
}
|
|
3445
|
-
catch (e) {
|
|
3446
|
-
// A genuinely absent CHAT stream means there are no retained message counts yet. Every other
|
|
3447
|
-
// failure, especially a STREAM.INFO permission denial, means the count view could not be read
|
|
3448
|
-
// and must stay loud rather than returning a valid-looking registry-only subset.
|
|
3449
|
-
if (!isJetStreamMissing(e, JetStreamApiCodes.StreamNotFound))
|
|
3450
|
-
throw e;
|
|
3451
|
-
}
|
|
3452
|
-
const channels = new Set([...counts.keys(), ...this.channelConfigs.keys()]);
|
|
3453
|
-
return [...channels]
|
|
3454
|
-
.map((channel) => ({
|
|
3455
|
-
channel,
|
|
3456
|
-
messages: counts.get(channel) ?? 0,
|
|
3457
|
-
config: this.channelConfigs.get(channel),
|
|
3458
|
-
}))
|
|
3459
|
-
.sort((a, b) => a.channel.localeCompare(b.channel));
|
|
2649
|
+
return this.subscriptions.listChannels();
|
|
3460
2650
|
}
|
|
3461
2651
|
async channelMembers(channel) {
|
|
3462
2652
|
const members = (await listMembers(await this.membersRegistry())).filter((r) => r.leaveCursor === undefined && r.activated === true);
|
|
@@ -3493,16 +2683,6 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
3493
2683
|
arr.sort(byName);
|
|
3494
2684
|
return map;
|
|
3495
2685
|
}
|
|
3496
|
-
/** Lazily open the DERIVED membership FEED KV (`cotal_membership_<space>`; admin/observer read, the
|
|
3497
|
-
* delivery daemon writes it) — the display-only who-is-subscribed view. Distinct from the authoritative
|
|
3498
|
-
* {@link membersRegistry} (`cotal_members_<space>`, the Plane-3 durable-membership source of truth): the
|
|
3499
|
-
* two names look alike, so this one is explicitly "feed". Read-only here; agents hold no grant. */
|
|
3500
|
-
async membershipFeedRegistry() {
|
|
3501
|
-
if (!this.nc)
|
|
3502
|
-
throw new Error(this.notLiveMsg());
|
|
3503
|
-
this.membershipFeedKv ??= await new Kvm(this.nc).open(membershipBucket(this.space));
|
|
3504
|
-
return this.membershipFeedKv;
|
|
3505
|
-
}
|
|
3506
2686
|
/**
|
|
3507
2687
|
* Snapshot the broker-sourced channel-membership feed (admin/observer read): every agent's
|
|
3508
2688
|
* `{live, durable}` record plus `asOf` — the feed's freshness heartbeat (epoch ms of the daemon's last
|
|
@@ -3511,38 +2691,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
3511
2691
|
* when the feed has never been written (no daemon → the dashboard degrades to traffic-only).
|
|
3512
2692
|
*/
|
|
3513
2693
|
async readMembership() {
|
|
3514
|
-
|
|
3515
|
-
const members = [];
|
|
3516
|
-
let asOf;
|
|
3517
|
-
// Paged batched Direct Gets, as the on-demand presence read: a few round trips regardless of N,
|
|
3518
|
-
// a short page refused rather than reported as the whole feed, and no consumer bound. The scan
|
|
3519
|
-
// this replaced bound an ordered consumer per read whose delete the observer and admin profiles
|
|
3520
|
-
// may not send (#691), so every dashboard poll left a consumer to the broker's inactive threshold
|
|
3521
|
-
// and a refused `CONSUMER.DELETE` line in its log (#4123).
|
|
3522
|
-
let entries;
|
|
3523
|
-
try {
|
|
3524
|
-
entries = await lastKvEntries(kv);
|
|
3525
|
-
}
|
|
3526
|
-
catch (e) {
|
|
3527
|
-
if (!isPublishPermissionDenied(e))
|
|
3528
|
-
throw e;
|
|
3529
|
-
throw new Error(`membership read refused for ${this.card.id}: this credential may not publish $JS.API.DIRECT.GET.KV_${membershipBucket(this.space)}, the membership feed read (SPEC Appendix B). A credential minted by an older issuer can lack this grant even when newly minted; upgrade the manager or auth service that issues credentials, then mint a new credential. Reusing credentials or reconnecting does not add the grant.`, { cause: e });
|
|
3530
|
-
}
|
|
3531
|
-
for (const e of entries) {
|
|
3532
|
-
if (e.key === MEMBERSHIP_FEED_KEY) {
|
|
3533
|
-
try {
|
|
3534
|
-
asOf = e.json().observedAt;
|
|
3535
|
-
}
|
|
3536
|
-
catch { /* heartbeat garbled; leave undefined */ }
|
|
3537
|
-
continue;
|
|
3538
|
-
}
|
|
3539
|
-
try {
|
|
3540
|
-
const rec = e.json();
|
|
3541
|
-
members.push({ id: e.key, live: rec.live ?? [], durable: rec.durable ?? [], observedAt: rec.observedAt });
|
|
3542
|
-
}
|
|
3543
|
-
catch { /* skip undecodable */ }
|
|
3544
|
-
}
|
|
3545
|
-
return { asOf, members };
|
|
2694
|
+
return this.subscriptions.readMembership();
|
|
3546
2695
|
}
|
|
3547
2696
|
/** Watch the membership feed for changes (admin/observer): `onChange` receives every KV entry as a
|
|
3548
2697
|
* {@link MembershipChange} — the decoded member record (or its deletion), or the feed heartbeat's
|
|
@@ -3552,122 +2701,11 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
3552
2701
|
* a known-ungranted delete leaves it to native inactivity reclamation. A feed the cred can't read
|
|
3553
2702
|
* (or absent) surfaces as an `error` event and the dashboard keeps its last snapshot. */
|
|
3554
2703
|
async watchMembership(onChange, onClosed) {
|
|
3555
|
-
|
|
3556
|
-
throw new Error("endpoint stopped - cannot watch membership");
|
|
3557
|
-
const watch = { onChange, onClosed, stopped: false, arm: Promise.resolve() };
|
|
3558
|
-
this.membershipFeedWatches.add(watch);
|
|
3559
|
-
watch.arm = watch.arm.catch(() => { }).then(() => this.armMembershipWatch(watch));
|
|
3560
|
-
try {
|
|
3561
|
-
await watch.arm;
|
|
3562
|
-
}
|
|
3563
|
-
catch (err) {
|
|
3564
|
-
watch.stopped = true;
|
|
3565
|
-
this.membershipFeedWatches.delete(watch);
|
|
3566
|
-
await this.disarmMembershipWatch(watch);
|
|
3567
|
-
throw err;
|
|
3568
|
-
}
|
|
3569
|
-
return { stop: async () => {
|
|
3570
|
-
if (watch.stopPromise)
|
|
3571
|
-
return watch.stopPromise;
|
|
3572
|
-
watch.stopped = true;
|
|
3573
|
-
watch.stopPromise = new Promise((resolve, reject) => {
|
|
3574
|
-
watch.resolveStop = resolve;
|
|
3575
|
-
watch.rejectStop = reject;
|
|
3576
|
-
});
|
|
3577
|
-
watch.arm = watch.arm.catch(() => { }).then(async () => {
|
|
3578
|
-
await this.disarmMembershipWatch(watch);
|
|
3579
|
-
await this.deleteRetainedMembershipConsumer(watch);
|
|
3580
|
-
this.finishMembershipWatchStop(watch);
|
|
3581
|
-
}).catch((err) => {
|
|
3582
|
-
// A real authorization/server failure remains loud. A terminal close is retained rather than
|
|
3583
|
-
// rejected by the cleanup helpers, so its promise stays pending for fresh-epoch cleanup.
|
|
3584
|
-
watch.rejectStop?.(err);
|
|
3585
|
-
});
|
|
3586
|
-
return watch.stopPromise;
|
|
3587
|
-
} };
|
|
2704
|
+
return this.subscriptions.watchMembership(onChange, onClosed);
|
|
3588
2705
|
}
|
|
3589
|
-
/**
|
|
3590
|
-
* per intent, so a public watch call cannot race a reconnect rearm into two consumers. */
|
|
3591
|
-
async armMembershipWatch(watch) {
|
|
3592
|
-
if (watch.stopped)
|
|
3593
|
-
return;
|
|
3594
|
-
const kv = await this.membershipFeedRegistry();
|
|
3595
|
-
await this.deleteRetainedMembershipConsumer(watch);
|
|
3596
|
-
if (!(kv instanceof Bucket))
|
|
3597
|
-
throw new Error("membership watch needs the @nats-io/kv Bucket implementation");
|
|
3598
|
-
const cc = kv._buildCC(">", KvWatchInclude.LastValue, { headers_only: false });
|
|
3599
|
-
const consumer = await kv.js.consumers.getPushConsumer(kv.stream, cc);
|
|
3600
|
-
const info = await consumer.info(true);
|
|
3601
|
-
this.recordOwnWatchConsumer(info);
|
|
3602
|
-
// The broker resource exists now. Record its identity BEFORE consume() so a concurrent stop or
|
|
3603
|
-
// connection close always leaves enough state for strict cleanup or fresh-epoch retry.
|
|
3604
|
-
watch.consumer = consumer;
|
|
3605
|
-
const permission = this.nc && this.watchDeletePermissions.get(this.nc);
|
|
3606
|
-
if (permission)
|
|
3607
|
-
this.watchDeletePermissions.set(consumer, permission);
|
|
3608
|
-
watch.consumerStream = info.stream_name;
|
|
3609
|
-
watch.consumerName = info.name;
|
|
3610
|
-
// The replay is the bucket's last value per key, bound at `info`; it ends with the entry that
|
|
3611
|
-
// leaves nothing pending (the same rule as nats.js `watch()`'s `initializedFn`).
|
|
3612
|
-
let replaying = info.num_pending > 0;
|
|
3613
|
-
let iter;
|
|
3614
|
-
try {
|
|
3615
|
-
iter = await consumer.consume({ callback: (jm) => {
|
|
3616
|
-
if (watch.stopped || watch.consumer !== consumer)
|
|
3617
|
-
return;
|
|
3618
|
-
const entry = kv.jmToWatchEntry(jm, !replaying);
|
|
3619
|
-
watch.onChange(membershipChange(entry, !replaying));
|
|
3620
|
-
if (replaying && jm.info.pending === 0) {
|
|
3621
|
-
replaying = false;
|
|
3622
|
-
watch.onChange({ type: "replayed", revision: entry.revision });
|
|
3623
|
-
}
|
|
3624
|
-
} });
|
|
3625
|
-
}
|
|
3626
|
-
catch (err) {
|
|
3627
|
-
await this.disarmMembershipWatch(watch);
|
|
3628
|
-
throw err;
|
|
3629
|
-
}
|
|
3630
|
-
watch.iter = iter;
|
|
3631
|
-
if (watch.stopped) {
|
|
3632
|
-
await this.disarmMembershipWatch(watch);
|
|
3633
|
-
return;
|
|
3634
|
-
}
|
|
3635
|
-
if (!replaying)
|
|
3636
|
-
watch.onChange({ type: "replayed", revision: 0 }); // an empty feed replays nothing
|
|
3637
|
-
iter.closed().then(() => {
|
|
3638
|
-
if (watch.stopped || watch.consumer !== consumer)
|
|
3639
|
-
return;
|
|
3640
|
-
const err = new Error("membership watch closed");
|
|
3641
|
-
watch.onClosed?.(err);
|
|
3642
|
-
this.emit("error", err);
|
|
3643
|
-
}).catch(() => { });
|
|
3644
|
-
}
|
|
3645
|
-
/** Delete identity retained across a failed/closed-epoch consumer object using the CURRENT connection. */
|
|
2706
|
+
/** Forwards to the subscriptions component's {@link EndpointSubscriptions.deleteRetainedMembershipConsumer}. */
|
|
3646
2707
|
async deleteRetainedMembershipConsumer(watch) {
|
|
3647
|
-
|
|
3648
|
-
return;
|
|
3649
|
-
// A terminal close is not deletion success and not a public-stop failure. Keep the identity
|
|
3650
|
-
// endpoint-owned; rearmMembershipWatches retries it through the next live JetStream manager.
|
|
3651
|
-
if (!this.nc || this.nc.isClosed())
|
|
3652
|
-
return;
|
|
3653
|
-
if (this.watchConsumerDeleteDenied(watch.consumerStream, watch.consumerName, this.nc)) {
|
|
3654
|
-
watch.consumerStream = undefined;
|
|
3655
|
-
watch.consumerName = undefined;
|
|
3656
|
-
return;
|
|
3657
|
-
}
|
|
3658
|
-
const jsm = await jetstreamManager(this.nc);
|
|
3659
|
-
if (await this.deleteMembershipConsumer(jsm, watch.consumerStream, watch.consumerName)) {
|
|
3660
|
-
watch.consumerStream = undefined;
|
|
3661
|
-
watch.consumerName = undefined;
|
|
3662
|
-
}
|
|
3663
|
-
}
|
|
3664
|
-
finishMembershipWatchStop(watch) {
|
|
3665
|
-
if (!watch.stopped || watch.consumer || watch.iter || watch.consumerStream || watch.consumerName)
|
|
3666
|
-
return;
|
|
3667
|
-
this.membershipFeedWatches.delete(watch);
|
|
3668
|
-
watch.resolveStop?.();
|
|
3669
|
-
watch.resolveStop = undefined;
|
|
3670
|
-
watch.rejectStop = undefined;
|
|
2708
|
+
return this.subscriptions.deleteRetainedMembershipConsumer(watch);
|
|
3671
2709
|
}
|
|
3672
2710
|
/** Record the consumer behind one of this connection's own KV watches or membership scans, and
|
|
3673
2711
|
* return the delete-subject prefix it recorded. nats.js names it `<prefix>_<serial>` and,
|
|
@@ -3721,72 +2759,13 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
3721
2759
|
throw err;
|
|
3722
2760
|
}
|
|
3723
2761
|
}
|
|
3724
|
-
/**
|
|
3725
|
-
* cleanup custody without a request; the broker reclaims the consumer at its inactive threshold. */
|
|
2762
|
+
/** Forwards to the subscriptions component's {@link EndpointSubscriptions.disarmMembershipWatch}. */
|
|
3726
2763
|
async disarmMembershipWatch(watch) {
|
|
3727
|
-
|
|
3728
|
-
const consumer = watch.consumer;
|
|
3729
|
-
watch.iter = undefined;
|
|
3730
|
-
watch.consumer = undefined;
|
|
3731
|
-
try {
|
|
3732
|
-
iter?.stop();
|
|
3733
|
-
}
|
|
3734
|
-
catch { /* already closed */ }
|
|
3735
|
-
if (consumer) {
|
|
3736
|
-
if (watch.consumerStream && watch.consumerName && this.watchConsumerDeleteDenied(watch.consumerStream, watch.consumerName, consumer)) {
|
|
3737
|
-
watch.consumerStream = undefined;
|
|
3738
|
-
watch.consumerName = undefined;
|
|
3739
|
-
return;
|
|
3740
|
-
}
|
|
3741
|
-
try {
|
|
3742
|
-
const deleted = await consumer.delete();
|
|
3743
|
-
if (deleted) {
|
|
3744
|
-
watch.consumerStream = undefined;
|
|
3745
|
-
watch.consumerName = undefined;
|
|
3746
|
-
}
|
|
3747
|
-
}
|
|
3748
|
-
catch (err) {
|
|
3749
|
-
if (membershipConsumerReleased(err)) {
|
|
3750
|
-
watch.consumerStream = undefined;
|
|
3751
|
-
watch.consumerName = undefined;
|
|
3752
|
-
}
|
|
3753
|
-
else {
|
|
3754
|
-
const closedEpoch = err instanceof ClosedConnectionError;
|
|
3755
|
-
const timeout = err instanceof TimeoutError;
|
|
3756
|
-
// Cleanup of an ordered consumer: a delete timeout means the broker did not answer in time,
|
|
3757
|
-
// not that the endpoint is unusable. The broker reaps an idle/ephemeral consumer anyway.
|
|
3758
|
-
// Throwing here killed a live observer over a slow VPN (#1047). Catch, surface, continue.
|
|
3759
|
-
if (timeout || closedEpoch) {
|
|
3760
|
-
this.emit("error", err);
|
|
3761
|
-
}
|
|
3762
|
-
else {
|
|
3763
|
-
throw err;
|
|
3764
|
-
}
|
|
3765
|
-
}
|
|
3766
|
-
// A terminal close leaves stream/name intact. The endpoint-owned stopped intent is retried
|
|
3767
|
-
// through the fresh JetStream manager before its public stop promise may resolve.
|
|
3768
|
-
}
|
|
3769
|
-
}
|
|
2764
|
+
return this.subscriptions.disarmMembershipWatch(watch);
|
|
3770
2765
|
}
|
|
3771
|
-
/**
|
|
2766
|
+
/** Forwards to the subscriptions component's {@link EndpointSubscriptions.rearmMembershipWatches}. */
|
|
3772
2767
|
async rearmMembershipWatches() {
|
|
3773
|
-
|
|
3774
|
-
// clearConnectionScoped already resets a rejected prior arm before scheduling disarm. At this
|
|
3775
|
-
// point the queue is the completing cleanup promise; append fresh-epoch cleanup first.
|
|
3776
|
-
watch.arm = watch.arm.catch(() => { }).then(async () => {
|
|
3777
|
-
await this.disarmMembershipWatch(watch);
|
|
3778
|
-
await this.deleteRetainedMembershipConsumer(watch);
|
|
3779
|
-
if (!watch.stopped)
|
|
3780
|
-
await this.armMembershipWatch(watch);
|
|
3781
|
-
});
|
|
3782
|
-
try {
|
|
3783
|
-
await watch.arm;
|
|
3784
|
-
this.finishMembershipWatchStop(watch);
|
|
3785
|
-
}
|
|
3786
|
-
catch (err) {
|
|
3787
|
-
this.emit("error", err);
|
|
3788
|
-
}
|
|
3789
|
-
}));
|
|
2768
|
+
return this.subscriptions.rearmMembershipWatches();
|
|
3790
2769
|
}
|
|
3791
2770
|
/** Fetch recent messages from a channel's JetStream backlog. `signal` cancels the active pull and
|
|
3792
2771
|
* reclaims its ephemeral consumer before the promise rejects. Returns `HistoryMessage[]`
|
|
@@ -4209,20 +3188,41 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
4209
3188
|
// DELIVER store (`dlv.<owner>`), which the agent binds + acks via native JetStream. The agent holds no
|
|
4210
3189
|
// read on the mixed store. (v3: this all moved off the manager — the manager is lifecycle-only; it
|
|
4211
3190
|
// records the read-ACL at mint via commitAcl.) See `.internal/research/stage4-impl-design.md`.
|
|
4212
|
-
/**
|
|
3191
|
+
/** The Plane-3 registries, manager and daemon-renewal leases, and durable membership writes of this
|
|
3192
|
+
* endpoint. Every dependency is read when it is used, so a field assigned after construction is seen. */
|
|
3193
|
+
durableBackstop = new EndpointDurableBackstop({
|
|
3194
|
+
nc: () => this.nc,
|
|
3195
|
+
js: () => this.js,
|
|
3196
|
+
jsm: () => this.jsm,
|
|
3197
|
+
setJsm: (manager) => { this.jsm = manager; },
|
|
3198
|
+
space: () => this.space,
|
|
3199
|
+
card: () => this.card,
|
|
3200
|
+
authed: () => this.authed,
|
|
3201
|
+
channelKv: () => this.channelKv,
|
|
3202
|
+
presence: () => this.presence,
|
|
3203
|
+
deliveryPlane: () => this.deliveryPlane,
|
|
3204
|
+
membersKv: () => this.membersKv,
|
|
3205
|
+
setMembersKv: (kv) => { this.membersKv = kv; },
|
|
3206
|
+
aclKv: () => this.aclKv,
|
|
3207
|
+
setAclKv: (kv) => { this.aclKv = kv; },
|
|
3208
|
+
managerLeaseKv: () => this.managerLeaseKv,
|
|
3209
|
+
setManagerLeaseKv: (kv) => { this.managerLeaseKv = kv; },
|
|
3210
|
+
daemonRenewalLeaseRevision: () => this.daemonRenewalLeaseRevision,
|
|
3211
|
+
setDaemonRenewalLeaseRevision: (revision) => { this.daemonRenewalLeaseRevision = revision; },
|
|
3212
|
+
notLiveMsg: () => this.notLiveMsg(),
|
|
3213
|
+
consumerInfo: (stream, durable) => this.consumerInfo(stream, durable),
|
|
3214
|
+
manager: () => this.manager(),
|
|
3215
|
+
chatFrontier: () => this.chatFrontier(),
|
|
3216
|
+
aclAllows: (owner, lifecycleUid, channel) => this.aclAllows(owner, lifecycleUid, channel),
|
|
3217
|
+
catchupCopy: (...args) => this.catchupCopy(...args),
|
|
3218
|
+
});
|
|
3219
|
+
/** Forwards to the durable backstop's {@link EndpointDurableBackstop.membersRegistry}. */
|
|
4213
3220
|
async membersRegistry() {
|
|
4214
|
-
|
|
4215
|
-
throw new Error(this.notLiveMsg());
|
|
4216
|
-
this.membersKv ??= await openMembersRegistry(this.nc, this.space);
|
|
4217
|
-
return this.membersKv;
|
|
3221
|
+
return this.durableBackstop.membersRegistry();
|
|
4218
3222
|
}
|
|
4219
|
-
/**
|
|
4220
|
-
* ACL at mint); the delivery daemon reads it fresh per durable entry to re-authorize. */
|
|
3223
|
+
/** Forwards to the durable backstop's {@link EndpointDurableBackstop.aclRegistry}. */
|
|
4221
3224
|
async aclRegistry() {
|
|
4222
|
-
|
|
4223
|
-
throw new Error(this.notLiveMsg());
|
|
4224
|
-
this.aclKv ??= await openAclRegistry(this.nc, this.space);
|
|
4225
|
-
return this.aclKv;
|
|
3225
|
+
return this.durableBackstop.aclRegistry();
|
|
4226
3226
|
}
|
|
4227
3227
|
/** Privileged ({@link DurableProvisioner}): record an agent's read ACL in the durable registry at
|
|
4228
3228
|
* provision/mint time — the same act as baking it into the JWT, persisted so the server-side
|
|
@@ -4230,7 +3230,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
4230
3230
|
* durable-joins without holding any in-memory ledger. Written ATOMICALLY ({@link writeAclRecord}),
|
|
4231
3231
|
* so a present record is always complete (`[]` = known no-read, never a half-write). */
|
|
4232
3232
|
async commitAcl(targetId, lifecycleUid, allowSubscribe) {
|
|
4233
|
-
|
|
3233
|
+
return this.durableBackstop.commitAcl(targetId, lifecycleUid, allowSubscribe);
|
|
4234
3234
|
}
|
|
4235
3235
|
/**
|
|
4236
3236
|
* Raise the mint-time ACL ceiling. Provision/remint only — see {@link reissueAcl}.
|
|
@@ -4238,28 +3238,21 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
4238
3238
|
* write is not crypto-bound to credential bytes.
|
|
4239
3239
|
*/
|
|
4240
3240
|
async reissueAcl(targetId, lifecycleUid, allowSubscribe) {
|
|
4241
|
-
|
|
3241
|
+
return this.durableBackstop.reissueAcl(targetId, lifecycleUid, allowSubscribe);
|
|
4242
3242
|
}
|
|
4243
3243
|
/** The server-side delivery daemon's fresh-per-entry ACL read: one LIFECYCLE's current read ACL
|
|
4244
3244
|
* (`allowSubscribe`) from the durable registry (exact key `<owner>.<actor>.<uid>`), or `undefined`
|
|
4245
3245
|
* if no record (an unknown lifecycle — the reader DEFERS, never drops). A present `[]` (known
|
|
4246
3246
|
* no-read) returns `[]` (the reader DROPS). */
|
|
4247
3247
|
async aclForOwner(owner, lifecycleUid) {
|
|
4248
|
-
return
|
|
3248
|
+
return this.durableBackstop.aclForOwner(owner, lifecycleUid);
|
|
4249
3249
|
}
|
|
4250
3250
|
/** Resolve an ALIAS to its single live lifecycle-keyed ACL row (`readAclForAlias`): the daemon's
|
|
4251
3251
|
* authz seam for callers that arrive with alias identity only (a `ctl.delivery` durable-join).
|
|
4252
3252
|
* THROWS {@link AmbiguousAclAlias} on two live rows — first-match would let a stale lifecycle
|
|
4253
3253
|
* authorize the successor (SPEC 13.1: at most one live lifecycle per alias). */
|
|
4254
3254
|
async aclForAlias(principal) {
|
|
4255
|
-
|
|
4256
|
-
if (row === undefined)
|
|
4257
|
-
return undefined;
|
|
4258
|
-
const allow = row.record.allowSubscribe;
|
|
4259
|
-
// Legacy rows predate the ceiling field: treat missing as equal to allowSubscribe so behaviour
|
|
4260
|
-
// matches what that row already exposed. New rows always carry an explicit ceiling.
|
|
4261
|
-
const issued = row.record.issuedAllowSubscribe ?? allow;
|
|
4262
|
-
return { allowSubscribe: allow, issuedAllowSubscribe: issued, lifecycleUid: row.lifecycleUid };
|
|
3255
|
+
return this.durableBackstop.aclForAlias(principal);
|
|
4263
3256
|
}
|
|
4264
3257
|
/** Is this shard's lease row one THIS ENDPOINT INSTANCE wrote? The question a daemon whose renew
|
|
4265
3258
|
* just failed has to answer before it decides whether it still owns the shard.
|
|
@@ -4454,41 +3447,6 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
4454
3447
|
async readDeliveryLeaseEntry(shardIndex) {
|
|
4455
3448
|
return this.deliveryLease.readDeliveryLeaseEntry(shardIndex);
|
|
4456
3449
|
}
|
|
4457
|
-
/** Ensure + bind the manager singleton-lease bucket. Mirrors the presence-bucket pattern (connectAndBind):
|
|
4458
|
-
* AUTH mode OPENs the bucket pre-created at `cotal up` (the scoped `supervisor` cred holds no
|
|
4459
|
-
* STREAM.CREATE — and `open` binds direct=false, so the CAS-conflict `kv.get` inside `acquire` rides
|
|
4460
|
-
* STREAM.MSG.GET, the verb the supervisor grants, never DIRECT.GET). OPEN mode (no creds) create-firsts:
|
|
4461
|
-
* `Kvm.open` binds LAZILY — it does NOT verify the stream exists or throw when it's missing (a fresh
|
|
4462
|
-
* bucket then fails 'stream not found' on the first write), so `create` is the ensure-exists call (it
|
|
4463
|
-
* makes the bucket or, when another endpoint already did, throws and we bind the existing one). Either
|
|
4464
|
-
* way the per-KEY CAS create stays the only single-flight gate, so a lost bucket-create race never reads
|
|
4465
|
-
* as "lease held". */
|
|
4466
|
-
async managerLeaseRegistry() {
|
|
4467
|
-
if (!this.nc)
|
|
4468
|
-
throw new Error(this.notLiveMsg());
|
|
4469
|
-
if (this.managerLeaseKv)
|
|
4470
|
-
return this.managerLeaseKv;
|
|
4471
|
-
// A JetStream client of its own, so every operation on this bucket carries the lease budget's
|
|
4472
|
-
// attempt deadline rather than the library default. The default is TTL/2, which would let one
|
|
4473
|
-
// attempt spend the whole renew window (see MANAGER_LEASE_ATTEMPT_MS). Scoped to this bucket:
|
|
4474
|
-
// every op on it is a single small keyed request, and nothing else shares this client.
|
|
4475
|
-
const kvm = new Kvm(jetstream(this.nc, { timeout: MANAGER_LEASE_ATTEMPT_MS }));
|
|
4476
|
-
if (this.authed) {
|
|
4477
|
-
this.managerLeaseKv = await kvm.open(managerBucket(this.space));
|
|
4478
|
-
}
|
|
4479
|
-
else {
|
|
4480
|
-
try {
|
|
4481
|
-
this.managerLeaseKv = await kvm.create(managerBucket(this.space), { ttl: MANAGER_LEASE_TTL_MS });
|
|
4482
|
-
}
|
|
4483
|
-
catch {
|
|
4484
|
-
// OPEN mode has no broker ACLs, so a permission denial is impossible here. `create` is only
|
|
4485
|
-
// the ensure-exists attempt; `open` below is the authority after either a pre-existing bucket
|
|
4486
|
-
// or a create race, and it still throws if the bucket cannot actually be used.
|
|
4487
|
-
this.managerLeaseKv = await kvm.open(managerBucket(this.space));
|
|
4488
|
-
}
|
|
4489
|
-
}
|
|
4490
|
-
return this.managerLeaseKv;
|
|
4491
|
-
}
|
|
4492
3450
|
/** Take or keep the per-SPACE daemon-credential renewal lease (#1634), returning whether THIS
|
|
4493
3451
|
* instance now holds it. One atomic CAS `create` per pass: it succeeds for whoever arrives first
|
|
4494
3452
|
* and throws for everyone else, so exactly one manager remints even when several share the
|
|
@@ -4497,57 +3455,19 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
4497
3455
|
* never fails a start. A crashed holder's key TTL-expires with the bucket, so the next pass hands
|
|
4498
3456
|
* the lease to a survivor with no operator step. */
|
|
4499
3457
|
async holdDaemonRenewalLease(instanceId) {
|
|
4500
|
-
|
|
4501
|
-
const held = this.daemonRenewalLeaseRevision;
|
|
4502
|
-
if (held !== undefined) {
|
|
4503
|
-
try {
|
|
4504
|
-
this.daemonRenewalLeaseRevision = await kv.update(MANAGER_RENEWAL_LEASE_KEY, this.encodeDaemonRenewalLease(instanceId), held);
|
|
4505
|
-
return true;
|
|
4506
|
-
}
|
|
4507
|
-
catch {
|
|
4508
|
-
// The revision moved (our key TTL-expired and a peer took it). Re-contend below rather than
|
|
4509
|
-
// keep reminting on a lease we no longer hold.
|
|
4510
|
-
this.daemonRenewalLeaseRevision = undefined;
|
|
4511
|
-
}
|
|
4512
|
-
}
|
|
4513
|
-
try {
|
|
4514
|
-
this.daemonRenewalLeaseRevision = await kv.create(MANAGER_RENEWAL_LEASE_KEY, this.encodeDaemonRenewalLease(instanceId));
|
|
4515
|
-
return true;
|
|
4516
|
-
}
|
|
4517
|
-
catch {
|
|
4518
|
-
return false; // another manager holds it: exactly one owner is the point
|
|
4519
|
-
}
|
|
3458
|
+
return this.durableBackstop.holdDaemonRenewalLease(instanceId);
|
|
4520
3459
|
}
|
|
4521
3460
|
/** Release the renewal lease on a clean stop so a peer takes over at once rather than at the TTL.
|
|
4522
3461
|
* CAS-guarded, so a lease we already lost is never deleted out from under its new holder. */
|
|
4523
3462
|
async releaseDaemonRenewalLease() {
|
|
4524
|
-
|
|
4525
|
-
this.daemonRenewalLeaseRevision = undefined;
|
|
4526
|
-
if (held === undefined)
|
|
4527
|
-
return;
|
|
4528
|
-
try {
|
|
4529
|
-
await (await this.managerLeaseRegistry()).delete(MANAGER_RENEWAL_LEASE_KEY, { previousSeq: held });
|
|
4530
|
-
}
|
|
4531
|
-
catch {
|
|
4532
|
-
// Best-effort: a moved revision means it is not ours, and a broker failure is recovered by the
|
|
4533
|
-
// bucket TTL. Shutdown must not claim deletion.
|
|
4534
|
-
}
|
|
4535
|
-
}
|
|
4536
|
-
encodeDaemonRenewalLease(instanceId) {
|
|
4537
|
-
return new TextEncoder().encode(JSON.stringify({ instanceId, since: Date.now() }));
|
|
3463
|
+
return this.durableBackstop.releaseDaemonRenewalLease();
|
|
4538
3464
|
}
|
|
4539
3465
|
/** Read the per-space daemon-credential renewal-lease row (the one {@link holdDaemonRenewalLease}
|
|
4540
3466
|
* CAS-writes), or `undefined` when nothing holds it. A `doctor auth --fix` that lost `hold` calls
|
|
4541
3467
|
* this through the SAME bucket the hold attempt opened, to name the live holder in its refusal —
|
|
4542
3468
|
* `managerLeaseKv` is private, so this is the exported read beside {@link readManagerLease}. */
|
|
4543
3469
|
async readDaemonRenewalLease() {
|
|
4544
|
-
|
|
4545
|
-
if (!e || e.operation !== "PUT")
|
|
4546
|
-
return undefined;
|
|
4547
|
-
return JSON.parse(new TextDecoder().decode(e.value));
|
|
4548
|
-
}
|
|
4549
|
-
encodeManagerLease(info) {
|
|
4550
|
-
return new TextEncoder().encode(JSON.stringify(info));
|
|
3470
|
+
return this.durableBackstop.readDaemonRenewalLease();
|
|
4551
3471
|
}
|
|
4552
3472
|
/** Acquire THIS logical instance's liveness lease via ATOMIC CAS create on its own per-instance key
|
|
4553
3473
|
* ({@link managerLeaseKey}). THROWS only if that SAME instance id already holds a live key (a same-root
|
|
@@ -4556,12 +3476,12 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
4556
3476
|
* key and coexists. A crashed holder's key auto-expires (bucket TTL). Returns the
|
|
4557
3477
|
* lease revision (for renew). */
|
|
4558
3478
|
async acquireManagerLease(info) {
|
|
4559
|
-
return
|
|
3479
|
+
return this.durableBackstop.acquireManagerLease(info);
|
|
4560
3480
|
}
|
|
4561
3481
|
/** Renew THIS instance's held key (CAS update against `revision`) before the bucket TTL expires it.
|
|
4562
3482
|
* Throws if the revision moved (lost the lease). Returns the new revision. */
|
|
4563
3483
|
async renewManagerLease(info, revision) {
|
|
4564
|
-
return
|
|
3484
|
+
return this.durableBackstop.renewManagerLease(info, revision);
|
|
4565
3485
|
}
|
|
4566
3486
|
/** Read THIS instance's OWN lease key, keyed (not the `lease.*` sweep {@link readManagerLease} does).
|
|
4567
3487
|
*
|
|
@@ -4573,21 +3493,14 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
4573
3493
|
* `created` is the broker's write time of the row (epoch ms), which is what the bucket TTL counts
|
|
4574
3494
|
* from. */
|
|
4575
3495
|
async readOwnManagerLease(instanceId) {
|
|
4576
|
-
|
|
4577
|
-
if (!e || e.operation !== "PUT")
|
|
4578
|
-
return undefined;
|
|
4579
|
-
return { info: JSON.parse(new TextDecoder().decode(e.value)), revision: e.revision, created: e.created.getTime() };
|
|
3496
|
+
return this.durableBackstop.readOwnManagerLease(instanceId);
|
|
4580
3497
|
}
|
|
4581
3498
|
/** Release THIS instance's key on clean shutdown so a same-id restart re-acquires immediately. CAS-guarded
|
|
4582
3499
|
* by `revision`: if we already LOST it (renew gap) the stored revision has moved, and the conditional
|
|
4583
3500
|
* delete is refused and rejects, like any broker failure; a shutdown that must finish anyway catches
|
|
4584
3501
|
* it. Keyed per instance, so a release NEVER touches a sibling manager's key (security pin 6). */
|
|
4585
3502
|
async releaseManagerLease(instanceId, revision) {
|
|
4586
|
-
|
|
4587
|
-
if (revision === undefined)
|
|
4588
|
-
await kv.delete(managerLeaseKey(instanceId));
|
|
4589
|
-
else
|
|
4590
|
-
await kv.delete(managerLeaseKey(instanceId), { previousSeq: revision });
|
|
3503
|
+
return this.durableBackstop.releaseManagerLease(instanceId, revision);
|
|
4591
3504
|
}
|
|
4592
3505
|
/** Read a live manager liveness lease, or undefined if NONE (no manager instance holds the space). A
|
|
4593
3506
|
* presence/existence check for the CLI's `spawn -f` reuse and `waitLeaseGone`, which only need "is any
|
|
@@ -4605,65 +3518,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
4605
3518
|
* revision PER KEY and drops keys whose final state is a marker, so a peer's DEL can only retire that
|
|
4606
3519
|
* peer's own key and can never mask a live one. */
|
|
4607
3520
|
async readManagerLease() {
|
|
4608
|
-
|
|
4609
|
-
return undefined;
|
|
4610
|
-
try {
|
|
4611
|
-
const jsm = this.jsm ?? (this.jsm = await jetstreamManager(this.nc));
|
|
4612
|
-
const stream = `KV_${managerBucket(this.space)}`;
|
|
4613
|
-
const prefix = `$KV.${managerBucket(this.space)}.${MANAGER_LEASE_KEY}.`;
|
|
4614
|
-
// STREAM.INFO to learn WHICH instance keys exist, then one point-get per key. NOT a bucket
|
|
4615
|
-
// scan: `liveKvEntries` binds a push consumer, and this principal's grant on this bucket is
|
|
4616
|
-
// STREAM.INFO + STREAM.MSG.GET + the `lease.*` publish only (`provision.ts`, supervisor). A
|
|
4617
|
-
// consumer here is a permissions violation for the very callers this probe serves, and no
|
|
4618
|
-
// open-mesh test can see that, because an open mesh has no permissions to violate.
|
|
4619
|
-
const info = await jsm.streams.info(stream, { subjects_filter: `${prefix}*` });
|
|
4620
|
-
let newest;
|
|
4621
|
-
for (const subject of Object.keys(info.state.subjects ?? {})) {
|
|
4622
|
-
// `last_by_subj` is CORRECT PER KEY and wrong across keys. Scoped to one instance's subject
|
|
4623
|
-
// it returns that instance's latest state, so a DEL here retires only its own key. The
|
|
4624
|
-
// defect was asking one wildcard for the newest message in the whole subtree, where a
|
|
4625
|
-
// stopping peer's tombstone outranks a live sibling's older PUT.
|
|
4626
|
-
let m;
|
|
4627
|
-
try {
|
|
4628
|
-
m = await jsm.streams.getMessage(stream, { last_by_subj: subject });
|
|
4629
|
-
}
|
|
4630
|
-
catch (e) {
|
|
4631
|
-
if (e.code === JetStreamApiCodes.NoMessageFound)
|
|
4632
|
-
continue;
|
|
4633
|
-
throw e;
|
|
4634
|
-
}
|
|
4635
|
-
if (m === null)
|
|
4636
|
-
continue; // key vanished between INFO and GET: it is not a live holder
|
|
4637
|
-
// A 10037 above means the key vanished between INFO and GET: it is not a live holder. A
|
|
4638
|
-
// denial or failed read is not absence and is rethrown.
|
|
4639
|
-
const op = m.header?.get("KV-Operation");
|
|
4640
|
-
if (op === "DEL" || op === "PURGE" || m.data.length === 0)
|
|
4641
|
-
continue;
|
|
4642
|
-
if (newest === undefined || m.seq > newest.seq)
|
|
4643
|
-
newest = { data: m.data, seq: m.seq };
|
|
4644
|
-
}
|
|
4645
|
-
if (newest === undefined)
|
|
4646
|
-
return undefined;
|
|
4647
|
-
return JSON.parse(new TextDecoder().decode(newest.data));
|
|
4648
|
-
}
|
|
4649
|
-
catch (e) {
|
|
4650
|
-
// ABSENCE MUST BE PROVEN, NOT INFERRED FROM A FAILED READ. Exactly two outcomes mean "genuinely
|
|
4651
|
-
// no manager": 10037, no message on the subtree, and a missing bucket — this probe is open-only
|
|
4652
|
-
// and on an authed mesh nothing creates it until a manager first takes a lease, so a space that
|
|
4653
|
-
// has never run one has no stream to read. Every OTHER failure (a JetStream hiccup, a request
|
|
4654
|
-
// timeout, a permissions refusal, or `liveKvEntries` refusing a pass that died mid-delivery) used
|
|
4655
|
-
// to return undefined here as well, and every caller reads undefined as "the space is empty": the
|
|
4656
|
-
// CLI's `spawn -f` reuse stands a SECOND manager up against a live one and `waitLeaseGone` reports
|
|
4657
|
-
// the space free. A read that failed is not evidence of absence, so refuse loudly rather than
|
|
4658
|
-
// degrade (AGENTS.md: no fallbacks). This is the wider of the two doors the point-get fix closes:
|
|
4659
|
-
// it needs only a transient error, where the tombstone path needed a peer to stop.
|
|
4660
|
-
const code = e.code;
|
|
4661
|
-
if (code === 10037)
|
|
4662
|
-
return undefined;
|
|
4663
|
-
if (code === 404 || /stream not found/i.test(e?.message ?? ""))
|
|
4664
|
-
return undefined;
|
|
4665
|
-
throw e;
|
|
4666
|
-
}
|
|
3521
|
+
return this.durableBackstop.readManagerLease();
|
|
4667
3522
|
}
|
|
4668
3523
|
/** Privileged: one owner's NON-TOMBSTONED durable memberships as `{channel, generation, activated}` —
|
|
4669
3524
|
* the server-side delivery daemon serves this to a connecting agent (the `listMemberships` op on
|
|
@@ -4672,69 +3527,25 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
4672
3527
|
* still routes under the pure-interval predicate (a crash-stuck pending activation) — without reading
|
|
4673
3528
|
* the privileged KV itself. */
|
|
4674
3529
|
async ownerMemberships(owner, lifecycleUid) {
|
|
4675
|
-
|
|
4676
|
-
// PREDECESSOR's rows/generations (both incarnations share the alias), and a first-match consumer
|
|
4677
|
-
// like leaveChannel could then act on the wrong incarnation's state. Rows are filtered to the
|
|
4678
|
-
// caller's own uid; the uid is caller-asserted but confined to its own authenticated alias, the
|
|
4679
|
-
// same trust shape as durableLeave.
|
|
4680
|
-
const recs = await listMembers(await this.membersRegistry(), { owner });
|
|
4681
|
-
return recs
|
|
4682
|
-
.filter((r) => r.lifecycleUid === lifecycleUid && r.leaveCursor === undefined)
|
|
4683
|
-
.map((r) => ({ channel: r.channel, generation: r.generation, activated: r.activated === true }));
|
|
3530
|
+
return this.durableBackstop.ownerMemberships(owner, lifecycleUid);
|
|
4684
3531
|
}
|
|
4685
3532
|
/** Effective delivery class read AUTHORITATIVELY from the registry KV (not the watch cache) — so a
|
|
4686
3533
|
* `live`→`durable` flip is seen by fan-out without a cache-propagation gap (red-team MED-3), and a
|
|
4687
3534
|
* reader just started need not wait for the cache to fill. */
|
|
4688
3535
|
async deliveryClassFresh(channel) {
|
|
4689
|
-
|
|
4690
|
-
return effectiveDeliveryClass(undefined, undefined);
|
|
4691
|
-
const [cfg, defaults] = await Promise.all([
|
|
4692
|
-
isConcreteChannel(channel) ? readChannelConfig(this.channelKv, channel) : Promise.resolve(undefined),
|
|
4693
|
-
readChannelDefaults(this.channelKv),
|
|
4694
|
-
]);
|
|
4695
|
-
return effectiveDeliveryClass(cfg, defaults);
|
|
3536
|
+
return this.durableBackstop.deliveryClassFresh(channel);
|
|
4696
3537
|
}
|
|
4697
|
-
/**
|
|
4698
|
-
* peer wins; 0 or >1 matches drop (never fan a directed durable copy to an unrelated same-named
|
|
4699
|
-
* bystander — red-team LOW; SPEC §4 unique instance id). */
|
|
3538
|
+
/** Forwards to the durable backstop's {@link EndpointDurableBackstop.resolveOwnerByName}. */
|
|
4700
3539
|
resolveOwnerByName(name) {
|
|
4701
|
-
|
|
4702
|
-
|
|
4703
|
-
}
|
|
4704
|
-
/** Plane-3 publish-dedupe key for a fan-out/transfer entry: the message id plus the member
|
|
4705
|
-
* lifecycle (`keySuffix`). One message reaches one lifecycle once, whichever membership row and
|
|
4706
|
-
* whichever writer (fan-out or any row's catch-up) produced the copy (#5404), within the stream's
|
|
4707
|
-
* duplicate window. An empty `msg.id` contributes NO msgID (#673, SPEC §4: two `id: ""` posts MUST
|
|
4708
|
-
* NOT collapse to one delivery) — a plain publish, never an empty or salted one — so an id-less
|
|
4709
|
-
* message is at-least-once on the durable plane (a redelivery may reach the member twice; the
|
|
4710
|
-
* receiver already treats each as its own delivery). */
|
|
3540
|
+
return this.durableBackstop.resolveOwnerByName(name);
|
|
3541
|
+
}
|
|
3542
|
+
/** Forwards to the durable backstop's {@link EndpointDurableBackstop.plane3MsgId}. */
|
|
4711
3543
|
plane3MsgId(entry, keySuffix) {
|
|
4712
|
-
return
|
|
3544
|
+
return this.durableBackstop.plane3MsgId(entry, keySuffix);
|
|
4713
3545
|
}
|
|
4714
|
-
/**
|
|
4715
|
-
* §13.1: fan-out addresses the member row's RECORDED lifecycle, never the alias's current
|
|
4716
|
-
* occupant), idempotent via `Nats-Msg-Id` (`<msgId>:<principal>:<lifecycleUid>`) so a catch-up copy
|
|
4717
|
-
* and a racing fan-out copy collapse, from the same row or from two rows that both cover the post.
|
|
4718
|
-
* The `principal` is the member's owner+actor dot-form. */
|
|
3546
|
+
/** Forwards to the durable backstop's {@link EndpointDurableBackstop.publishDinbox}. */
|
|
4719
3547
|
async publishDinbox(principal, lifecycleUid, entry) {
|
|
4720
|
-
|
|
4721
|
-
return;
|
|
4722
|
-
const p = parsePrincipalKey(principal);
|
|
4723
|
-
if (!p)
|
|
4724
|
-
throw new Error(`publishDinbox: "${principal}" is not a valid member principal <owner>.<actor>`);
|
|
4725
|
-
await this.js.publish(dinboxSubject(this.space, p.owner, p.actor, lifecycleUid), JSON.stringify(entry), {
|
|
4726
|
-
// JetStream dedupe is STREAM-WIDE, so the id must carry the LIFECYCLE too: with an alias-keyed
|
|
4727
|
-
// id, lifecycle A's copy would suppress a same-alias successor B's copy of the same message —
|
|
4728
|
-
// cross-lifecycle suppression, not dedup. It carries no row generation: two rows of one
|
|
4729
|
-
// lifecycle covering the post must collapse to one copy.
|
|
4730
|
-
...this.plane3MsgId(entry, `${principal}:${lifecycleUid}`),
|
|
4731
|
-
});
|
|
4732
|
-
}
|
|
4733
|
-
/** The fan-out consumer's delivered stream-seq — the activation-fence upper bound (red-team
|
|
4734
|
-
* BLOCKER-1: the shared fan-out cursor advances independently of the stream frontier). */
|
|
4735
|
-
async fanoutDeliveredSeq() {
|
|
4736
|
-
const info = await this.consumerInfo(chatStream(this.space), FANOUT_DURABLE);
|
|
4737
|
-
return info?.delivered?.stream_seq ?? 0;
|
|
3548
|
+
return this.durableBackstop.publishDinbox(principal, lifecycleUid, entry);
|
|
4738
3549
|
}
|
|
4739
3550
|
/**
|
|
4740
3551
|
* Privileged durable-JOIN write (v3: the delivery daemon calls this from its `ctl.delivery` handler
|
|
@@ -4758,44 +3569,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
4758
3569
|
* Without the option the write is a privileged assertion that consults no ACL.
|
|
4759
3570
|
*/
|
|
4760
3571
|
async durableJoinFor(owner, channel, lifecycleUid, opts = {}) {
|
|
4761
|
-
|
|
4762
|
-
throw new Error(this.notLiveMsg());
|
|
4763
|
-
await this.manager(); // ensure jsm — a non-consuming provisioner inits it lazily; catch-up + fence need it
|
|
4764
|
-
const kv = await this.membersRegistry();
|
|
4765
|
-
const existing = await readMember(kv, channel, owner, lifecycleUid);
|
|
4766
|
-
const open = existing?.record.state === "durable-active" && existing.record.leaveCursor === undefined;
|
|
4767
|
-
if (open && existing.record.activated)
|
|
4768
|
-
return { durable: true, generation: existing.record.generation }; // fully activated — idempotent
|
|
4769
|
-
// Either a NEW join (no record / a tombstone to supersede) → fresh joinCursor + bumped generation,
|
|
4770
|
-
// OR a retry of an INCOMPLETE activation (durable-active but not yet activated, from an earlier
|
|
4771
|
-
// crash or failed catch-up) → re-run catch-up over the SAME join window, no bump. The record is committed
|
|
4772
|
-
// `activated:false` first and routes IN-INTERVAL immediately (fan-out + reader deliver via the
|
|
4773
|
-
// pure-interval durableEligible) so no live message published during catch-up is lost. `activated`
|
|
4774
|
-
// gates only the REPORT — durableJoin returns true / channelMembers lists the owner only after the
|
|
4775
|
-
// catch-up confirms. A join that never completes catch-up still routes live (harmless: the agent is
|
|
4776
|
-
// live-subscribed and DLV is id-deduped) but honestly reports durable:false and stays hidden.
|
|
4777
|
-
const joinCursor = open ? existing.record.joinCursor : await this.chatFrontier();
|
|
4778
|
-
const generation = open ? existing.record.generation : (existing?.record.generation ?? 0) + 1;
|
|
4779
|
-
const base = {
|
|
4780
|
-
channel, owner, lifecycleUid, state: "durable-active", joinCursor, generation,
|
|
4781
|
-
activated: false, writerIdentity: this.card.id, updatedAt: Date.now(),
|
|
4782
|
-
};
|
|
4783
|
-
if (!open) {
|
|
4784
|
-
if (opts.aclFence && !(await this.aclAllows(owner, lifecycleUid, channel)))
|
|
4785
|
-
throw new Error(`durableJoin: "${channel}" is no longer within the read ACL of lifecycle ${lifecycleUid} (narrowed or retired); no membership row was written`);
|
|
4786
|
-
const written = await commitMemberRevision(kv, base);
|
|
4787
|
-
if (opts.aclFence)
|
|
4788
|
-
await this.keepRowIfAclAllows(kv, owner, lifecycleUid, channel, written.revision);
|
|
4789
|
-
}
|
|
4790
|
-
const fence = Math.max(await this.chatFrontier(), await this.fanoutDeliveredSeq());
|
|
4791
|
-
await this.catchupCopy(owner, lifecycleUid, channel, joinCursor, fence, generation);
|
|
4792
|
-
// Flip → reported durable, ATOMICALLY: refuse if a concurrent SAME-generation leave (tombstone) or a
|
|
4793
|
-
// rejoin superseded this pending join while catch-up ran. A blind same-gen commit would clobber the
|
|
4794
|
-
// tombstone (clear leaveCursor) and resurrect the membership, reopening §7 (review-general-2 BLOCKER).
|
|
4795
|
-
const activated = await activateMember(kv, channel, owner, lifecycleUid, generation, joinCursor);
|
|
4796
|
-
if (!activated)
|
|
4797
|
-
return { durable: false, reason: "activation superseded by a concurrent leave or rejoin", generation };
|
|
4798
|
-
return { durable: true, generation };
|
|
3572
|
+
return this.durableBackstop.durableJoinFor(owner, channel, lifecycleUid, opts);
|
|
4799
3573
|
}
|
|
4800
3574
|
/** Whether one lifecycle's exact ACL row allows a membership on `channel` now (a family must lie
|
|
4801
3575
|
* inside one entry, {@link membershipInAllow}). A leader-served read: the handle is opened without
|
|
@@ -4804,47 +3578,10 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
4804
3578
|
const row = await readAcl(await this.aclRegistry(), owner, lifecycleUid);
|
|
4805
3579
|
return row !== undefined && membershipInAllow(row.record.allowSubscribe, channel);
|
|
4806
3580
|
}
|
|
4807
|
-
/** The post-write half of {@link durableJoinFor}'s `aclFence`: keep the row just written (at
|
|
4808
|
-
* `revision`) only on proof that the lifecycle's ACL row still allows the channel, and refuse the
|
|
4809
|
-
* join otherwise. When the ACL row proves the channel is no longer allowed, no row of the lifecycle
|
|
4810
|
-
* on that channel is wanted, so the row goes by its exact key whoever wrote its latest revision.
|
|
4811
|
-
* When the read itself fails, nothing is proven either way, so only this write is taken back: a
|
|
4812
|
-
* later revision belongs to another join of the same lifecycle, which may have been authorized. */
|
|
4813
|
-
async keepRowIfAclAllows(kv, owner, lifecycleUid, channel, revision) {
|
|
4814
|
-
let disallowed = false;
|
|
4815
|
-
let why;
|
|
4816
|
-
try {
|
|
4817
|
-
if (await this.aclAllows(owner, lifecycleUid, channel))
|
|
4818
|
-
return;
|
|
4819
|
-
disallowed = true;
|
|
4820
|
-
why = "was narrowed or retired while the row was written";
|
|
4821
|
-
}
|
|
4822
|
-
catch (e) {
|
|
4823
|
-
why = `could not be re-read after the row was written (${e.message})`;
|
|
4824
|
-
}
|
|
4825
|
-
const fail = (outcome, cause) => new Error(`durableJoin: the read ACL of lifecycle ${lifecycleUid} ${why}; ${outcome}`, cause === undefined ? undefined : { cause });
|
|
4826
|
-
try {
|
|
4827
|
-
if (disallowed)
|
|
4828
|
-
await deleteMember(kv, channel, owner, lifecycleUid);
|
|
4829
|
-
else
|
|
4830
|
-
await kv.purge(memberKey(channel, owner, lifecycleUid), { previousSeq: revision });
|
|
4831
|
-
}
|
|
4832
|
-
catch (e) {
|
|
4833
|
-
if (!disallowed && isCasLoss(e))
|
|
4834
|
-
throw fail(`a later write replaced its "${channel}" row, which was left to that write`);
|
|
4835
|
-
throw fail(`removing its "${channel}" row failed (${e.message})`, e);
|
|
4836
|
-
}
|
|
4837
|
-
throw fail(`its "${channel}" row was removed`);
|
|
4838
|
-
}
|
|
4839
3581
|
/** Privileged durable-LEAVE write: tombstone the membership at `leaveCursor = frontier` so the
|
|
4840
3582
|
* backstop denies `seq > leaveCursor` while a pre-leave entry stays deliverable (SPEC §7 interval). */
|
|
4841
3583
|
async durableLeaveFor(owner, channel, lifecycleUid, expectedGeneration) {
|
|
4842
|
-
|
|
4843
|
-
return; // not a Plane-3 host — no membership to tombstone
|
|
4844
|
-
const kv = await this.membersRegistry();
|
|
4845
|
-
// expectedGeneration (captured by the agent at durableJoin) refuses a stale leave from tombstoning
|
|
4846
|
-
// a newer rejoin (StaleMembershipWrite) — a durable-disable primitive otherwise.
|
|
4847
|
-
await tombstoneMember(kv, channel, owner, lifecycleUid, await this.chatFrontier(), this.card.id, expectedGeneration);
|
|
3584
|
+
return this.durableBackstop.durableLeaveFor(owner, channel, lifecycleUid, expectedGeneration);
|
|
4848
3585
|
}
|
|
4849
3586
|
/** Idempotently copy the eligible chat messages in `(fromSeqExcl, toSeqIncl]` for `channel` into the
|
|
4850
3587
|
* owner inbox, via a DEDICATED per-(owner,join) ephemeral consumer (NOT the agent-scoped
|
|
@@ -5164,6 +3901,21 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
5164
3901
|
}
|
|
5165
3902
|
}
|
|
5166
3903
|
// ---- the peer-readable liveness surface (#1577) --------------------------
|
|
3904
|
+
/** The peer-readable liveness responders and the probe, of this endpoint. Every dependency is read
|
|
3905
|
+
* when it is used, so a field assigned after construction is seen. */
|
|
3906
|
+
liveness = new EndpointLiveness({
|
|
3907
|
+
nc: () => this.nc,
|
|
3908
|
+
space: () => this.space,
|
|
3909
|
+
owner: () => this.owner,
|
|
3910
|
+
actor: () => this.actor,
|
|
3911
|
+
subs: () => this.subs,
|
|
3912
|
+
livenessResponders: () => this.livenessResponders,
|
|
3913
|
+
notLiveMsg: () => this.notLiveMsg(),
|
|
3914
|
+
requestBoundReply: (nc, subject, body, reply, timeoutMs) => this.requestBoundReply(nc, subject, body, reply, timeoutMs),
|
|
3915
|
+
isNoResponders: (e) => this.isNoResponders(e),
|
|
3916
|
+
emitRecoverable: (err) => this.emitRecoverable(err),
|
|
3917
|
+
emit: (event, ...args) => this.emit(event, ...args),
|
|
3918
|
+
});
|
|
5167
3919
|
/** Bind this endpoint as the liveness RESPONDER for one plane, answering presence and nothing
|
|
5168
3920
|
* else to any credentialed peer that asks (`live.<plane>.<owner>.<actor>`).
|
|
5169
3921
|
*
|
|
@@ -5206,91 +3958,11 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
5206
3958
|
* the plane answered by the broker's no-responders status while this endpoint is serving. A second
|
|
5207
3959
|
* call for the same plane replaces the first. */
|
|
5208
3960
|
serveLiveness(plane, readState) {
|
|
5209
|
-
|
|
5210
|
-
throw new Error(this.notLiveMsg());
|
|
5211
|
-
const held = this.livenessResponders.get(plane);
|
|
5212
|
-
if (held)
|
|
5213
|
-
held.readState = readState;
|
|
5214
|
-
else
|
|
5215
|
-
this.livenessResponders.set(plane, { readState });
|
|
5216
|
-
this.bindLiveness(plane);
|
|
3961
|
+
return this.liveness.serveLiveness(plane, readState);
|
|
5217
3962
|
}
|
|
5218
|
-
/**
|
|
5219
|
-
* as {@link armDeliveryControl} does. */
|
|
3963
|
+
/** Forwards to the liveness component's {@link EndpointLiveness.bindLiveness}. */
|
|
5220
3964
|
bindLiveness(plane) {
|
|
5221
|
-
|
|
5222
|
-
if (!held || !this.nc)
|
|
5223
|
-
return;
|
|
5224
|
-
if (held.sub) {
|
|
5225
|
-
try {
|
|
5226
|
-
held.sub.unsubscribe();
|
|
5227
|
-
}
|
|
5228
|
-
catch { /* dead with the old connection */ }
|
|
5229
|
-
const i = this.subs.indexOf(held.sub);
|
|
5230
|
-
if (i >= 0)
|
|
5231
|
-
this.subs.splice(i, 1);
|
|
5232
|
-
}
|
|
5233
|
-
const { readState } = held;
|
|
5234
|
-
const sub = this.nc.subscribe(livenessServeFilter(this.space, plane), { queue: `live.${plane}` });
|
|
5235
|
-
held.sub = sub;
|
|
5236
|
-
this.subs.push(sub);
|
|
5237
|
-
// The responder token this bind answers under: AN OPAQUE VALUE MINTED HERE, and deliberately
|
|
5238
|
-
// not any name this process already has. It is what makes "which responder answered" askable
|
|
5239
|
-
// without making "who is this responder" answerable — the two properties the surface has to
|
|
5240
|
-
// hold at once. The endpoint's own `card.id` would be its principal, the lease row's
|
|
5241
|
-
// `instanceId` is a field of a bucket an agent holds no grant on, and either would turn a
|
|
5242
|
-
// presence probe into an identity read for every credentialed peer in the space. A fresh
|
|
5243
|
-
// random token correlates with nothing but this plane's other answers, which is the whole job:
|
|
5244
|
-
// two answers carrying two different tokens came from two different responders.
|
|
5245
|
-
//
|
|
5246
|
-
// Per BIND rather than per process, so a responder that goes away and comes back answers under
|
|
5247
|
-
// a new token. That is the honest reading: a caller comparing answers across a restart is
|
|
5248
|
-
// comparing two different incarnations, and a token that survived the restart would say
|
|
5249
|
-
// otherwise. A re-bind after a reconnect is such a return: the plane had no responder here for
|
|
5250
|
-
// the gap.
|
|
5251
|
-
const instance = randomUUID();
|
|
5252
|
-
void (async () => {
|
|
5253
|
-
for await (const m of sub) {
|
|
5254
|
-
// Sender-bound reply guard. Drop silently rather than publishing somewhere else: a reply
|
|
5255
|
-
// aimed outside the sender's own subtree is not a request we can answer safely.
|
|
5256
|
-
//
|
|
5257
|
-
// REPORTED ON `warning`, NOT ON `error`, AND THAT IS A SAFETY PROPERTY RATHER THAN A CHOICE
|
|
5258
|
-
// OF CHANNEL. A rejected probe is a condition this responder is already surviving: it drops
|
|
5259
|
-
// the frame and keeps serving. `error` cannot carry such a notice on this class, because
|
|
5260
|
-
// Node's `EventEmitter` RETHROWS an `error` emit that has no listener attached, so the
|
|
5261
|
-
// notice would end the process of any embedder that had not attached one — and the plane
|
|
5262
|
-
// would then genuinely be unbound, which a peer's next probe reports as `unbound`. The
|
|
5263
|
-
// condition would have manufactured the state it described. `warning` is observable and
|
|
5264
|
-
// never fatal without a listener (see {@link emitRecoverable} and #891, where retry notices
|
|
5265
|
-
// on `error` killed hosts the endpoint intended to keep running).
|
|
5266
|
-
//
|
|
5267
|
-
// Since ANY credentialed peer may publish a probe, and the reply target inside it is chosen
|
|
5268
|
-
// by that peer and not permission-checked by the broker, reaching this branch is a peer's
|
|
5269
|
-
// decision, not an operator's. Safety here must therefore hold with no listener attached,
|
|
5270
|
-
// which is what CELL F1 runs.
|
|
5271
|
-
if (!m.reply || !m.reply.startsWith(`${m.subject}.reply.`)) {
|
|
5272
|
-
this.emitRecoverable(new Error(`rejected liveness probe on ${m.subject}: reply target "${m.reply ?? "(none)"}" is not under the sender's own reply subtree`));
|
|
5273
|
-
continue;
|
|
5274
|
-
}
|
|
5275
|
-
let responder;
|
|
5276
|
-
try {
|
|
5277
|
-
responder = await readState();
|
|
5278
|
-
}
|
|
5279
|
-
catch {
|
|
5280
|
-
responder = "unknown";
|
|
5281
|
-
}
|
|
5282
|
-
const answer = { plane, responder, instance };
|
|
5283
|
-
try {
|
|
5284
|
-
m.respond(JSON.stringify(answer));
|
|
5285
|
-
}
|
|
5286
|
-
catch { /* the requester is gone */ }
|
|
5287
|
-
}
|
|
5288
|
-
// The loop itself ends only on a SUBSCRIPTION-level fault (the connection went away), which
|
|
5289
|
-
// is not something a peer's probe can cause: the guard above `continue`s, `readState` is
|
|
5290
|
-
// caught, and the respond is caught. So this arm stays on `error` — it reports that this
|
|
5291
|
-
// responder has stopped answering at all, which is a fault an embedder should not be able to
|
|
5292
|
-
// miss, and no credentialed peer can reach it.
|
|
5293
|
-
})().catch((e) => this.emit("error", e));
|
|
3965
|
+
return this.liveness.bindLiveness(plane);
|
|
5294
3966
|
}
|
|
5295
3967
|
/** Bind the liveness responder for the DELIVERY plane, grading itself from its own shard-0 lease
|
|
5296
3968
|
* through the shared classifier. The daemon is the one process that can answer this honestly: it
|
|
@@ -5320,56 +3992,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
5320
3992
|
* The reply rides `<request>.reply.<uuid>`, the sender's own subtree, so the responder's bound
|
|
5321
3993
|
* reply guard accepts it and the responder needs no broad inbox-publish grant to answer. */
|
|
5322
3994
|
async probeLiveness(plane, timeoutMs = 2_000) {
|
|
5323
|
-
|
|
5324
|
-
// A closed set, refused loudly. Answering an unknown plane at all would make this a probe
|
|
5325
|
-
// that returns something for every input, which is indistinguishable from one that is not
|
|
5326
|
-
// measuring anything.
|
|
5327
|
-
throw new Error(`liveness: "${plane}" is not a plane this surface answers for (${[...LIVENESS_PLANES].join(", ")})`);
|
|
5328
|
-
if (!this.nc)
|
|
5329
|
-
throw new Error(this.notLiveMsg());
|
|
5330
|
-
const reqSubject = livenessSubject(this.space, plane, this.owner, this.actor);
|
|
5331
|
-
const reply = `${reqSubject}.reply.${randomUUID()}`;
|
|
5332
|
-
let outcome;
|
|
5333
|
-
let answered;
|
|
5334
|
-
let instance;
|
|
5335
|
-
try {
|
|
5336
|
-
const m = await this.requestBoundReply(this.nc, reqSubject, "", reply, timeoutMs);
|
|
5337
|
-
let body;
|
|
5338
|
-
try {
|
|
5339
|
-
body = m.json();
|
|
5340
|
-
}
|
|
5341
|
-
catch {
|
|
5342
|
-
body = undefined;
|
|
5343
|
-
}
|
|
5344
|
-
const parsed = parseLivenessAnswer(body, plane);
|
|
5345
|
-
// A reply we cannot read is `malformed`, which grades `unknown`. It is NOT promoted to
|
|
5346
|
-
// `bound` on the strength of having replied at all: that promotion is the `pgrep` error,
|
|
5347
|
-
// where evidence a process exists was read as evidence it works.
|
|
5348
|
-
outcome = parsed ? "replied" : "malformed";
|
|
5349
|
-
answered = parsed?.responder;
|
|
5350
|
-
instance = parsed?.instance;
|
|
5351
|
-
}
|
|
5352
|
-
catch (e) {
|
|
5353
|
-
outcome = this.isNoResponders(e) ? "noResponders" : this.probeFailureOutcome(e);
|
|
5354
|
-
}
|
|
5355
|
-
// `instance` rides ONLY a reply that was read. Every other outcome means no responder answered,
|
|
5356
|
-
// so there is no responder to name, and a token attached to an `unbound` or an `unknown` would
|
|
5357
|
-
// claim one had. It stays absent on those arms by construction: nothing assigns it.
|
|
5358
|
-
const verdict = { plane, responder: responderFromProbe(outcome, answered) };
|
|
5359
|
-
return instance === undefined ? verdict : { ...verdict, instance };
|
|
5360
|
-
}
|
|
5361
|
-
/** Grade a failed probe that was NOT a no-responders answer. Both arms return an outcome that
|
|
5362
|
-
* maps to `unknown`; they are told apart so the distinction stays legible at the call site and so
|
|
5363
|
-
* a future caller can render the two differently (a refusal is about YOUR credential, a timeout
|
|
5364
|
-
* is about something being wedged). Neither may ever mean health. */
|
|
5365
|
-
probeFailureOutcome(e) {
|
|
5366
|
-
if (e instanceof AuthorizationError || e instanceof PermissionViolationError)
|
|
5367
|
-
return "refused";
|
|
5368
|
-
if (e instanceof TimeoutError)
|
|
5369
|
-
return "timeout";
|
|
5370
|
-
// Anything else is a transport or client failure: it says something about our link, not about
|
|
5371
|
-
// the plane, so it is graded like a refusal rather than guessed at.
|
|
5372
|
-
return "refused";
|
|
3995
|
+
return this.liveness.probeLiveness(plane, timeoutMs);
|
|
5373
3996
|
}
|
|
5374
3997
|
/**
|
|
5375
3998
|
* Serve this process's `agent` endpoint on the principal route (SPEC 13.21, 13.22): the
|
|
@@ -6430,48 +5053,13 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
6430
5053
|
async readPresenceOnce() {
|
|
6431
5054
|
return this.presence.readPresenceOnce();
|
|
6432
5055
|
}
|
|
6433
|
-
/**
|
|
6434
|
-
* cache. A failed bind throws out of connect; an entry that does not decode or validate is
|
|
6435
|
-
* reported as a `warning` and dropped, so its channel resolves to the default policy. */
|
|
5056
|
+
/** Forwards to the channel discovery component's {@link EndpointChannelDiscovery.startChannelWatch}. */
|
|
6436
5057
|
async startChannelWatch() {
|
|
6437
|
-
|
|
6438
|
-
return;
|
|
6439
|
-
const bound = await this.bindKvWatch(this.channelKv);
|
|
6440
|
-
if (!bound)
|
|
6441
|
-
return;
|
|
6442
|
-
const { iter } = bound;
|
|
6443
|
-
this.channelWatchIter = iter;
|
|
6444
|
-
void (async () => {
|
|
6445
|
-
for await (const e of iter)
|
|
6446
|
-
this.handleChannelEntry(e);
|
|
6447
|
-
})().catch((e) => this.emit("error", e));
|
|
5058
|
+
return this.channelDiscovery.startChannelWatch();
|
|
6448
5059
|
}
|
|
6449
|
-
/**
|
|
5060
|
+
/** Forwards to the channel discovery component's {@link EndpointChannelDiscovery.readChannelsOnce}. */
|
|
6450
5061
|
async readChannelsOnce() {
|
|
6451
|
-
|
|
6452
|
-
return;
|
|
6453
|
-
for (const e of await lastKvEntries(this.channelKv))
|
|
6454
|
-
this.handleChannelEntry(e);
|
|
6455
|
-
}
|
|
6456
|
-
handleChannelEntry(e) {
|
|
6457
|
-
// Cleared before the decode, so an entry that does not decode resolves to the default as the
|
|
6458
|
-
// fresh registry reads do.
|
|
6459
|
-
const isDefaults = e.key === CHANNEL_DEFAULTS_KEY;
|
|
6460
|
-
if (isDefaults)
|
|
6461
|
-
this.channelDefaults = {};
|
|
6462
|
-
else
|
|
6463
|
-
this.channelConfigs.delete(e.key);
|
|
6464
|
-
if (e.operation === "DEL" || e.operation === "PURGE")
|
|
6465
|
-
return;
|
|
6466
|
-
try {
|
|
6467
|
-
if (isDefaults)
|
|
6468
|
-
this.channelDefaults = decodeChannelEntry(e);
|
|
6469
|
-
else
|
|
6470
|
-
this.channelConfigs.set(e.key, decodeChannelEntry(e));
|
|
6471
|
-
}
|
|
6472
|
-
catch (err) {
|
|
6473
|
-
this.emitRecoverable(new Error(`dropped channel registry entry for key ${JSON.stringify(e.key)}: ${err.message}`));
|
|
6474
|
-
}
|
|
5062
|
+
return this.channelDiscovery.readChannelsOnce();
|
|
6475
5063
|
}
|
|
6476
5064
|
sweep() {
|
|
6477
5065
|
return this.presence.sweep();
|
|
@@ -6627,85 +5215,6 @@ function isReadableMessagePart(value) {
|
|
|
6627
5215
|
function isHistoryDrainEnvelope(value) {
|
|
6628
5216
|
return isRecord(value) && isUsableMessageId(value.id) && isRecord(value.from);
|
|
6629
5217
|
}
|
|
6630
|
-
/** A `data` part value the wire can carry faithfully: a JSON value at EVERY depth (SPEC §5's
|
|
6631
|
-
* `<any JSON value>`). `JSON.stringify` does not reject what it cannot represent — it silently
|
|
6632
|
-
* rewrites: `undefined` (and a function-valued member) drops the key, `NaN`/`Infinity` store as
|
|
6633
|
-
* `null`, a `Date` stores as a string, a sparse array's holes store as `null`, a `Map` stores as
|
|
6634
|
-
* `{}`, a `Buffer` stores as `{"type":"Buffer",...}`. A reader then cannot tell a stored `null`
|
|
6635
|
-
* from a real one (#1404 fix round). So the producer checks structurally, matching the exported
|
|
6636
|
-
* `JsonValue`: null, boolean, finite number, string, an array whose every slot (holes included)
|
|
6637
|
-
* passes, or a plain object (prototype `null` or `Object.prototype`) whose every defined member
|
|
6638
|
-
* passes. A cycle is refused here too, never left for stringify's TypeError.
|
|
6639
|
-
*
|
|
6640
|
-
* THE OBJECT/ARRAY ASYMMETRY ON `undefined`: an `undefined`-valued MEMBER of a plain object is
|
|
6641
|
-
* allowed — stringify drops just that key, which is faithful (and the exported type says so) —
|
|
6642
|
-
* while `undefined` at the top level or in an ARRAY SLOT is refused, because there stringify
|
|
6643
|
-
* cannot drop a position: it stores `null`, which is a rewrite a reader cannot distinguish from
|
|
6644
|
-
* a real `null`.
|
|
6645
|
-
*
|
|
6646
|
-
* `seen` is the set of ANCESTORS on the current path, not every object visited: it is deleted
|
|
6647
|
-
* from on the way out, so a SHARED subtree (`{a: x, b: x}`) passes — stringify carries it twice,
|
|
6648
|
-
* which is faithful — while an object that contains itself at any depth still reports its path. */
|
|
6649
|
-
function jsonPathProblem(path, value, seen) {
|
|
6650
|
-
if (value === null)
|
|
6651
|
-
return undefined;
|
|
6652
|
-
const t = typeof value;
|
|
6653
|
-
if (t === "boolean" || t === "string")
|
|
6654
|
-
return undefined;
|
|
6655
|
-
if (t === "number")
|
|
6656
|
-
return Number.isFinite(value) ? undefined : `${path} is not a finite number`;
|
|
6657
|
-
if (t === "undefined" || t === "function" || t === "symbol" || t === "bigint")
|
|
6658
|
-
return `${path} is not a JSON value`;
|
|
6659
|
-
if (typeof value === "object") {
|
|
6660
|
-
if (seen.has(value))
|
|
6661
|
-
return `${path} is cyclic`;
|
|
6662
|
-
seen.add(value);
|
|
6663
|
-
let problem;
|
|
6664
|
-
if (Array.isArray(value)) {
|
|
6665
|
-
for (let i = 0; i < value.length; i++) {
|
|
6666
|
-
problem = jsonPathProblem(`${path}[${i}]`, value[i], seen);
|
|
6667
|
-
if (problem)
|
|
6668
|
-
return problem;
|
|
6669
|
-
}
|
|
6670
|
-
}
|
|
6671
|
-
else {
|
|
6672
|
-
const proto = Object.getPrototypeOf(value);
|
|
6673
|
-
if (proto !== null && proto !== Object.prototype) {
|
|
6674
|
-
seen.delete(value);
|
|
6675
|
-
return `${path} is not a plain object`;
|
|
6676
|
-
}
|
|
6677
|
-
for (const key of Object.keys(value)) {
|
|
6678
|
-
// A member whose value is undefined is the allowed key-drop, not a defect: stringify omits
|
|
6679
|
-
// the key, the object stays a JSON object, so skip it rather than walk it.
|
|
6680
|
-
if (value[key] === undefined)
|
|
6681
|
-
continue;
|
|
6682
|
-
problem = jsonPathProblem(`${path}.${key}`, value[key], seen);
|
|
6683
|
-
if (problem)
|
|
6684
|
-
return problem;
|
|
6685
|
-
}
|
|
6686
|
-
}
|
|
6687
|
-
seen.delete(value);
|
|
6688
|
-
return undefined;
|
|
6689
|
-
}
|
|
6690
|
-
return `${path} is not a JSON value`;
|
|
6691
|
-
}
|
|
6692
|
-
/** Refuse, at publish, a `data` part `JSON.stringify` would silently rewrite. The guard is the
|
|
6693
|
-
* runtime half of `Part`'s `data: JsonValue` arm: with it, a non-JSON value at any depth never
|
|
6694
|
-
* reaches the wire, so every reader (live core-sub, Plane-3 durable, history) gives one answer —
|
|
6695
|
-
* the message was never sent — instead of the old split where history returned a rewritten row
|
|
6696
|
-
* while Plane-3 terminated a keyless one as malformed. The error names the path to the offending
|
|
6697
|
-
* value (e.g. `data[2].at`), so a caller learns which member to fix rather than that stringify
|
|
6698
|
-
* would have mangled something. Throws rather than coercing: no fallback. */
|
|
6699
|
-
function assertPartsSerializable(parts) {
|
|
6700
|
-
for (const p of parts) {
|
|
6701
|
-
if (p.kind !== "data")
|
|
6702
|
-
continue;
|
|
6703
|
-
const problem = jsonPathProblem("data", p.data, new Set());
|
|
6704
|
-
if (problem !== undefined) {
|
|
6705
|
-
throw new Error(`cannot publish a data part carrying a non-JSON value (SPEC §5) - ${problem}`);
|
|
6706
|
-
}
|
|
6707
|
-
}
|
|
6708
|
-
}
|
|
6709
5218
|
function authOpts(a) {
|
|
6710
5219
|
const tls = a.tls ? {} : undefined;
|
|
6711
5220
|
// USER MODE: present the shared auth-account sentinel creds AND the user bearer as `auth_token` (an
|
|
@@ -6808,13 +5317,6 @@ function kvWatchConsumer(iter) {
|
|
|
6808
5317
|
throw new Error("KV watch carries no consumer - the pinned nats.js changed shape");
|
|
6809
5318
|
return consumer;
|
|
6810
5319
|
}
|
|
6811
|
-
/** A membership-watch delete that leaves nothing to retry: the consumer or its stream is already
|
|
6812
|
-
* gone, or the broker refused the delete because the reading profile holds no stream-wide
|
|
6813
|
-
* CONSUMER.DELETE (#691) and reaps the consumer at its inactive threshold. */
|
|
6814
|
-
function membershipConsumerReleased(err) {
|
|
6815
|
-
return (isJetStreamMissing(err, JetStreamApiCodes.ConsumerNotFound, JetStreamApiCodes.StreamNotFound)
|
|
6816
|
-
|| isPublishPermissionDenied(err));
|
|
6817
|
-
}
|
|
6818
5320
|
/** True ONLY for a denial on a **publish** — the single case that proves the message was never
|
|
6819
5321
|
* ACCEPTED or stored. (Not "never reached the server": the server necessarily received enough of
|
|
6820
5322
|
* it to reject it. The distinction matters precisely here, because this helper exists to separate
|