@cotal-ai/core 0.133.13 → 0.135.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-file.d.ts +1 -1
- package/dist/agent-file.d.ts.map +1 -1
- package/dist/agent-file.js +11 -2
- package/dist/agent-file.js.map +1 -1
- package/dist/backup-config.d.ts +46 -5
- package/dist/backup-config.d.ts.map +1 -1
- package/dist/backup-config.js +68 -10
- package/dist/backup-config.js.map +1 -1
- package/dist/connector.d.ts +48 -0
- package/dist/connector.d.ts.map +1 -1
- package/dist/connector.js.map +1 -1
- package/dist/endpoint-caller.d.ts +4 -1
- package/dist/endpoint-caller.d.ts.map +1 -1
- package/dist/endpoint-caller.js +3 -1
- package/dist/endpoint-caller.js.map +1 -1
- package/dist/endpoint-checkpoint.d.ts +2 -0
- package/dist/endpoint-checkpoint.d.ts.map +1 -1
- package/dist/endpoint-checkpoint.js +1 -1
- package/dist/endpoint-checkpoint.js.map +1 -1
- package/dist/endpoint-contract-store.d.ts +14 -2
- package/dist/endpoint-contract-store.d.ts.map +1 -1
- package/dist/endpoint-contract-store.js +16 -3
- package/dist/endpoint-contract-store.js.map +1 -1
- package/dist/endpoint-coverage.d.ts +55 -0
- package/dist/endpoint-coverage.d.ts.map +1 -0
- package/dist/endpoint-coverage.js +206 -0
- package/dist/endpoint-coverage.js.map +1 -0
- package/dist/endpoint-grants.d.ts +3 -2
- package/dist/endpoint-grants.d.ts.map +1 -1
- package/dist/endpoint-grants.js +3 -2
- package/dist/endpoint-grants.js.map +1 -1
- package/dist/endpoint-service.d.ts.map +1 -1
- package/dist/endpoint-service.js +14 -11
- package/dist/endpoint-service.js.map +1 -1
- package/dist/endpoint.d.ts +111 -38
- package/dist/endpoint.d.ts.map +1 -1
- package/dist/endpoint.js +275 -132
- package/dist/endpoint.js.map +1 -1
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -0
- package/dist/index.js.map +1 -1
- package/dist/issued-authority.d.ts +48 -0
- package/dist/issued-authority.d.ts.map +1 -1
- package/dist/issued-authority.js +78 -0
- package/dist/issued-authority.js.map +1 -1
- package/dist/kv-scan.d.ts +7 -2
- package/dist/kv-scan.d.ts.map +1 -1
- package/dist/kv-scan.js +47 -28
- package/dist/kv-scan.js.map +1 -1
- package/dist/members.d.ts +8 -6
- package/dist/members.d.ts.map +1 -1
- package/dist/members.js +12 -8
- package/dist/members.js.map +1 -1
- package/dist/membership-feed.d.ts +5 -0
- package/dist/membership-feed.d.ts.map +1 -1
- package/dist/membership-feed.js +8 -1
- package/dist/membership-feed.js.map +1 -1
- package/dist/provision.d.ts +27 -6
- package/dist/provision.d.ts.map +1 -1
- package/dist/provision.js +92 -11
- package/dist/provision.js.map +1 -1
- package/dist/registry-client.d.ts +39 -0
- package/dist/registry-client.d.ts.map +1 -0
- package/dist/registry-client.js +58 -0
- package/dist/registry-client.js.map +1 -0
- package/dist/registry-contract.d.ts +79 -0
- package/dist/registry-contract.d.ts.map +1 -0
- package/dist/registry-contract.js +105 -0
- package/dist/registry-contract.js.map +1 -0
- package/dist/runtime.d.ts +8 -0
- package/dist/runtime.d.ts.map +1 -1
- package/dist/runtime.js.map +1 -1
- package/dist/schema-profile.d.ts +6 -5
- package/dist/schema-profile.d.ts.map +1 -1
- package/dist/schema-profile.js +18 -22
- package/dist/schema-profile.js.map +1 -1
- package/dist/space-schema-diff.d.ts +1 -1
- package/dist/space-schema-diff.d.ts.map +1 -1
- package/dist/space-schema-diff.js +23 -1
- package/dist/space-schema-diff.js.map +1 -1
- package/dist/startup-confirm.d.ts +18 -4
- package/dist/startup-confirm.d.ts.map +1 -1
- package/dist/startup-confirm.js +61 -12
- package/dist/startup-confirm.js.map +1 -1
- package/dist/streams.d.ts +38 -5
- package/dist/streams.d.ts.map +1 -1
- package/dist/streams.js +103 -8
- package/dist/streams.js.map +1 -1
- package/dist/subjects.d.ts +28 -12
- package/dist/subjects.d.ts.map +1 -1
- package/dist/subjects.js +49 -15
- package/dist/subjects.js.map +1 -1
- package/dist/turn-contract.d.ts +10 -0
- package/dist/turn-contract.d.ts.map +1 -1
- package/dist/turn-contract.js +18 -0
- package/dist/turn-contract.js.map +1 -1
- package/dist/types.d.ts +14 -6
- package/dist/types.d.ts.map +1 -1
- package/package.json +3 -2
package/dist/endpoint.js
CHANGED
|
@@ -18,6 +18,8 @@ import { EpEnvelopeError, respondedButUnbound, replyRefusedBeforeEffect, replyTa
|
|
|
18
18
|
import { isRepeatSafeCommand } from "./endpoint-grants.js";
|
|
19
19
|
import { assertIdToken, assertGeneration } from "./endpoint-subjects.js";
|
|
20
20
|
import { importNativeSubjectPermissions, issuedPermitsSubject, readAcceptedRow } from "./issued-authority.js";
|
|
21
|
+
import { REGISTRY_ENDPOINT, REGISTRY_ENDPOINTS_COMMAND } from "./registry-contract.js";
|
|
22
|
+
import { listRegistryEndpoints, registryListingCaller } from "./registry-client.js";
|
|
21
23
|
import { lastKvEntries, liveKvEntries } from "./kv-scan.js";
|
|
22
24
|
import { isConsumerAlreadyExists, isConsumerNotFound, isJetStreamMissing } from "./jetstream-missing.js";
|
|
23
25
|
import { isRestoredConsumer } from "./backup.js";
|
|
@@ -25,7 +27,8 @@ import { ARTIFACT_PART_KIND, isArtifactPart } from "./artifact.js";
|
|
|
25
27
|
import { assertValidKind } from "./agent-file.js";
|
|
26
28
|
import { assertValidName } from "./resolve.js";
|
|
27
29
|
import { EVICT_PRINCIPALS_MAX } from "./evict.js";
|
|
28
|
-
import { createSpaceStreams, dmDurableConfig, dlvDurableConfig, taskDurableConfig, fanoutDurableConfig, inboxReaderConfig,
|
|
30
|
+
import { createSpaceStreams, dmDurableConfig, dlvDurableConfig, taskDurableConfig, fanoutDurableConfig, inboxReaderConfig, MANAGER_LEASE_TTL_MS, MANAGER_LEASE_ATTEMPT_MS, TTL_RECONCILE_CANARY_KEY, PRESENCE_STORAGE } from "./streams.js";
|
|
31
|
+
import { PLANE3_MAX_MSGS_PER_SUBJECT } from "./backup-config.js";
|
|
29
32
|
import { jetstream, jetstreamManager, AckPolicy, DeliverPolicy, JetStreamApiCodes, JetStreamApiError, } from "@nats-io/jetstream";
|
|
30
33
|
import {} from "@nats-io/jetstream";
|
|
31
34
|
import { Kvm } from "@nats-io/kv";
|
|
@@ -35,7 +38,7 @@ import { openAclRegistry, readAcl, readAclForAlias, aclRetired, AmbiguousAclAlia
|
|
|
35
38
|
import { openDeliveryRegistry } from "./lease.js";
|
|
36
39
|
import { openChannelRegistry, effectiveReplay, effectiveReplayWindowMs, effectiveDeliveryClass, readChannelConfig, readChannelDefaults, decodeChannelEntry, } from "./channels.js";
|
|
37
40
|
import { isCasLoss } from "./endpoint-records.js";
|
|
38
|
-
import { anycastSubject, assertValidRole, CHANNEL_DEFAULTS_KEY, chatStream, chatHistDurable, chatSubject, historyReaderConsumer, calloutReaderId, HISTORY_READERS, controlServiceSubject, livenessSubject, livenessServeFilter, CONTROL_DELIVERY, CONTROL_DELIVERY_ADMIN, dmStream, dmDurable, dlvStream, dlvDurable, dlvSubject, dinboxSubject, inboxStream, parseDinboxPrincipal, FANOUT_DURABLE, INBOX_READER_DURABLE, leaseKey, managerBucket, MANAGER_LEASE_KEY, MANAGER_RENEWAL_LEASE_KEY, managerLeaseKey, chatWildcard, assertValidChannel, channelInAllow, isConcreteChannel, normalizeMentions, parseSubject, isPrincipalOwnerToken, assertInboxConnId, presenceBucket, isPresenceRow, assertPresenceRow, membershipBucket, MEMBERSHIP_FEED_KEY, principalKey, parsePrincipalKey, assertLifecycleToken, mintLifecycleUid, lifecycleNameKey, memberKey, DEV_OWNER, spacePrefix, spaceWildcard, subjectMatches, taskStream, taskDurable, token, unicastSubject, unicastRecvFilter, } from "./subjects.js";
|
|
41
|
+
import { anycastSubject, assertValidRole, CHANNEL_DEFAULTS_KEY, chatStream, chatHistDurable, chatSubject, historyReaderConsumer, calloutReaderId, HISTORY_READERS, controlServiceSubject, livenessSubject, livenessServeFilter, CONTROL_DELIVERY, CONTROL_DELIVERY_ADMIN, dmStream, dmDurable, dlvStream, dlvDurable, dlvSubject, dinboxSubject, inboxStream, parseDinboxPrincipal, FANOUT_DURABLE, INBOX_READER_DURABLE, leaseKey, managerBucket, MANAGER_LEASE_KEY, MANAGER_RENEWAL_LEASE_KEY, managerLeaseKey, chatWildcard, assertValidChannel, channelInAllow, membershipInAllow, isConcreteChannel, isChannelFamily, familiesCovering, normalizeMentions, parseSubject, isPrincipalOwnerToken, assertInboxConnId, presenceBucket, isPresenceRow, assertPresenceRow, membershipBucket, MEMBERSHIP_FEED_KEY, principalKey, parsePrincipalKey, assertLifecycleToken, mintLifecycleUid, lifecycleNameKey, memberKey, DEV_OWNER, spacePrefix, spaceWildcard, subjectMatches, taskStream, taskDurable, token, unicastSubject, unicastRecvFilter, } from "./subjects.js";
|
|
39
42
|
import { isPermissionDenied } from "./permission-denied.js";
|
|
40
43
|
export { isPermissionDenied };
|
|
41
44
|
export const DEFAULT_SERVER = "nats://127.0.0.1:4222";
|
|
@@ -122,7 +125,9 @@ export class PresenceWriteStuckError extends Error {
|
|
|
122
125
|
* terminal close). Lets an in-process agent track connectedness off the endpoint's own (re)binds
|
|
123
126
|
* instead of an imperative flag the self-heal path can't reach; "transport" ({ connected, server? })
|
|
124
127
|
* is the lower-level NATS socket edge, true before the full bind finishes and false during an internal
|
|
125
|
-
* nats.js reconnect without changing `connection` readiness.
|
|
128
|
+
* nats.js reconnect without changing `connection` readiness. "stored-copies-full" (no payload): a stored
|
|
129
|
+
* copy arrived with the store at its {@link PLANE3_MAX_MSGS_PER_SUBJECT}-copy limit, so the limit may have
|
|
130
|
+
* discarded older copies before this session read them; emitted for each such copy.
|
|
126
131
|
*
|
|
127
132
|
* Callers MUST attach an "error" listener before `start()`: async faults (incl. NATS
|
|
128
133
|
* permission denials, surfaced via `watchStatus`) are emitted as "error", and Node throws
|
|
@@ -212,6 +217,10 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
212
217
|
bearerSource;
|
|
213
218
|
/** The freshest bearer — what every (re)connect attempt presents. */
|
|
214
219
|
currentBearer;
|
|
220
|
+
/** The expiry, in seconds, of the credential the latest connect attempt presented. The issuer
|
|
221
|
+
* keys a listing generation on it (SPEC 13.15), so a listing reads the row bound to the
|
|
222
|
+
* credential the broker admitted, never one adopted since. */
|
|
223
|
+
presentedExp;
|
|
215
224
|
/** Cancels the armed bearer renewal or retry. */
|
|
216
225
|
cancelBearerRenewal;
|
|
217
226
|
/** Consecutive failed bearer renewals: the retry's step, and whether this run of failures has
|
|
@@ -245,6 +254,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
245
254
|
ackWaitMs;
|
|
246
255
|
inactiveThresholdMs;
|
|
247
256
|
nc;
|
|
257
|
+
requestsAwaitingReply = new Map();
|
|
248
258
|
onConnection;
|
|
249
259
|
js;
|
|
250
260
|
jsm;
|
|
@@ -863,6 +873,18 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
863
873
|
: "this endpoint's creds have expired and it holds no creds source to renew them - replace the credential and rebuild the endpoint (pass a creds FUNCTION for standing renewal)");
|
|
864
874
|
return creds;
|
|
865
875
|
}
|
|
876
|
+
/** The connect attempt's credential getters: what {@link credsForWire} and {@link bearerForWire}
|
|
877
|
+
* return, with its expiry recorded as {@link presentedExp}. */
|
|
878
|
+
presentCreds() {
|
|
879
|
+
const creds = this.credsForWire();
|
|
880
|
+
this.presentedExp = credsClaims(creds).exp;
|
|
881
|
+
return creds;
|
|
882
|
+
}
|
|
883
|
+
presentBearer() {
|
|
884
|
+
const bearer = this.bearerForWire();
|
|
885
|
+
this.presentedExp = bearerExpiryMs(bearer) / 1000;
|
|
886
|
+
return bearer;
|
|
887
|
+
}
|
|
866
888
|
/** The cached credential, checked. Handed to nats.js as the authenticator's source on EVERY auth
|
|
867
889
|
* mode (renewed or static), so each (re)connect attempt — ours or the library's — re-reads a
|
|
868
890
|
* CHECKED value. A refusal throws out of the authenticator, which nats-core turns into a closed
|
|
@@ -1269,7 +1291,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
1269
1291
|
// and dialed ANONYMOUSLY, so the broker answered `Authorization Violation` and the real fault
|
|
1270
1292
|
// (an empty credential) was never named. Routing it into the checked getter fails it loud
|
|
1271
1293
|
// instead. Anonymous access stays reachable the only way it should be: by passing no creds.
|
|
1272
|
-
...authOpts({ token: this.token, user: this.user, pass: this.pass, creds: this.currentCreds !== undefined || this.credsSource ? () => this.
|
|
1294
|
+
...authOpts({ token: this.token, user: this.user, pass: this.pass, creds: this.currentCreds !== undefined || this.credsSource ? () => this.presentCreds() : undefined, bearer: this.userMode ? () => this.presentBearer() : undefined, sentinelCreds: this.sentinelCreds, tls: this.tls,
|
|
1273
1295
|
onCreds: (creds) => {
|
|
1274
1296
|
watchDeletePermission = undefined;
|
|
1275
1297
|
try {
|
|
@@ -2293,6 +2315,38 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
2293
2315
|
}
|
|
2294
2316
|
}
|
|
2295
2317
|
}
|
|
2318
|
+
/** Keep native named-reply and NoResponders handling, but settle when its connection closes:
|
|
2319
|
+
* nats.js can close a no-mux subscription without settling its request. One close observer per
|
|
2320
|
+
* connection avoids retaining a closed-promise callback for every completed request. */
|
|
2321
|
+
async requestBoundReply(nc, subject, body, reply, timeoutMs) {
|
|
2322
|
+
let pending = this.requestsAwaitingReply.get(nc);
|
|
2323
|
+
if (!pending) {
|
|
2324
|
+
const waiting = new Set();
|
|
2325
|
+
this.requestsAwaitingReply.set(nc, waiting);
|
|
2326
|
+
void nc.closed().then((error) => {
|
|
2327
|
+
this.requestsAwaitingReply.delete(nc);
|
|
2328
|
+
const failure = error ?? new ClosedConnectionError();
|
|
2329
|
+
for (const reject of waiting)
|
|
2330
|
+
reject(failure);
|
|
2331
|
+
waiting.clear();
|
|
2332
|
+
});
|
|
2333
|
+
pending = waiting;
|
|
2334
|
+
}
|
|
2335
|
+
let rejectClosed;
|
|
2336
|
+
const closed = new Promise((_resolve, reject) => {
|
|
2337
|
+
rejectClosed = reject;
|
|
2338
|
+
pending.add(reject);
|
|
2339
|
+
});
|
|
2340
|
+
try {
|
|
2341
|
+
return await Promise.race([
|
|
2342
|
+
nc.request(subject, body, { timeout: timeoutMs, noMux: true, reply }),
|
|
2343
|
+
closed,
|
|
2344
|
+
]);
|
|
2345
|
+
}
|
|
2346
|
+
finally {
|
|
2347
|
+
pending.delete(rejectClosed);
|
|
2348
|
+
}
|
|
2349
|
+
}
|
|
2296
2350
|
/** Send a control request to a service and await its reply (client side). Like {@link requestDelivery},
|
|
2297
2351
|
* the reply rides a BOUNDED subject UNDER the request subject (`ctl.<service>.<id>.reply.<uuid>`), not
|
|
2298
2352
|
* the per-id `_INBOX`.
|
|
@@ -2305,7 +2359,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
2305
2359
|
const reqSubject = controlServiceSubject(this.space, service, this.owner, this.actor);
|
|
2306
2360
|
const reply = `${reqSubject}.reply.${randomUUID()}`;
|
|
2307
2361
|
const body = { ...req, from: req.from ?? this.ref() };
|
|
2308
|
-
const m = await this.nc
|
|
2362
|
+
const m = await this.requestBoundReply(this.nc, reqSubject, JSON.stringify(body), reply, timeoutMs);
|
|
2309
2363
|
return m.json();
|
|
2310
2364
|
}
|
|
2311
2365
|
/** The caller this connection's endpoint requests ride as: the legacy triple, or the issued
|
|
@@ -2400,6 +2454,46 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
2400
2454
|
this.resolvedServices.set(endpoint, svc);
|
|
2401
2455
|
return svc;
|
|
2402
2456
|
}
|
|
2457
|
+
/** The caller a registry listing rides (SPEC 13.15). An issued connection lists as itself. Any
|
|
2458
|
+
* other connection reads the listing generation the issuer bound to the credential it presented,
|
|
2459
|
+
* under the token its connection id and that credential's expiry derive. `undefined`: that
|
|
2460
|
+
* credential carries no listing binding, because it predates listing issuance or has no expiry. */
|
|
2461
|
+
async listingCaller() {
|
|
2462
|
+
if (!this.nc)
|
|
2463
|
+
throw new Error(this.notLiveMsg());
|
|
2464
|
+
return registryListingCaller(this.nc, this.space, this.serviceCaller(), { connId: this.connId, ...(this.presentedExp !== undefined ? { exp: this.presentedExp } : {}) });
|
|
2465
|
+
}
|
|
2466
|
+
/** One page of the registry's `endpoints` answer (SPEC 13.20) over this connection
|
|
2467
|
+
* ({@link listRegistryEndpoints}). The describe rides this connection's own caller; the question
|
|
2468
|
+
* rides the listing caller on an authed mesh. `unbound`: this credential carries no listing
|
|
2469
|
+
* binding. */
|
|
2470
|
+
async listEndpoints(input, opts = {}) {
|
|
2471
|
+
if (!this.nc)
|
|
2472
|
+
throw new Error(this.notLiveMsg());
|
|
2473
|
+
const nc = this.nc;
|
|
2474
|
+
const listing = this.authed ? await this.listingCaller() : undefined;
|
|
2475
|
+
if (this.authed && listing === undefined)
|
|
2476
|
+
return { unbound: true };
|
|
2477
|
+
return this.listEndpointsOver(nc, listing, input, opts);
|
|
2478
|
+
}
|
|
2479
|
+
/** Ask the registry over `nc` under `listing`: this connection, or another of this caller triple
|
|
2480
|
+
* such as a manager-call connection. The describe rides this connection, whose view holds the
|
|
2481
|
+
* registry's describe. Every endpoint the page lists has its cached describe dropped, so the next
|
|
2482
|
+
* describe of it reads the surface afresh. `unbound`: the broker refused the question under
|
|
2483
|
+
* `listing`, whose credential holds no listing row. */
|
|
2484
|
+
async listEndpointsOver(nc, listing, input, opts = {}) {
|
|
2485
|
+
if (!this.nc)
|
|
2486
|
+
throw new Error(this.notLiveMsg());
|
|
2487
|
+
const own = this.nc;
|
|
2488
|
+
const result = await listRegistryEndpoints(this.space, (deadlineMs, signal) => resolveService(own, this.space, REGISTRY_ENDPOINT, this.serviceCaller(), {
|
|
2489
|
+
deadlineMs, signal, commands: [REGISTRY_ENDPOINTS_COMMAND],
|
|
2490
|
+
}), { nc, ...(listing ? { caller: listing } : {}) }, input, opts);
|
|
2491
|
+
if (result.unbound)
|
|
2492
|
+
return result;
|
|
2493
|
+
for (const row of result.page?.rows ?? [])
|
|
2494
|
+
this.clearResolvedService(row.endpoint);
|
|
2495
|
+
return result;
|
|
2496
|
+
}
|
|
2403
2497
|
/** GENERIC v0.4 service invoke over this endpoint's own connection: resolve
|
|
2404
2498
|
* the named endpoint's registered surface — describe, §13.7 store fetch, digest-verified
|
|
2405
2499
|
* recompile ({@link resolveService}; cached per endpoint name) — and invoke one command. The
|
|
@@ -2626,7 +2720,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
2626
2720
|
// model. Keep both; don't regress to a counter. (Confirmed by the review panel's fact-check.)
|
|
2627
2721
|
const reply = `${reqSubject}.reply.${randomUUID()}`;
|
|
2628
2722
|
const body = { op, args, from: this.ref() };
|
|
2629
|
-
const m = await this.nc
|
|
2723
|
+
const m = await this.requestBoundReply(this.nc, reqSubject, JSON.stringify(body), reply, timeoutMs);
|
|
2630
2724
|
return m.json();
|
|
2631
2725
|
}
|
|
2632
2726
|
/** Send a PRIVILEGED delivery-admin request to the server-side delivery daemon and await its reply
|
|
@@ -2641,7 +2735,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
2641
2735
|
const reqSubject = controlServiceSubject(this.space, CONTROL_DELIVERY_ADMIN, this.owner, this.actor);
|
|
2642
2736
|
const reply = `${reqSubject}.reply.${randomUUID()}`;
|
|
2643
2737
|
const body = { op, args, from: this.ref() };
|
|
2644
|
-
const m = await this.nc
|
|
2738
|
+
const m = await this.requestBoundReply(this.nc, reqSubject, JSON.stringify(body), reply, timeoutMs);
|
|
2645
2739
|
return m.json();
|
|
2646
2740
|
}
|
|
2647
2741
|
// ---- presence ------------------------------------------------------------
|
|
@@ -2876,8 +2970,9 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
2876
2970
|
/**
|
|
2877
2971
|
* Join a channel mid-session: open a native core subscription (manager-free live read, broker-
|
|
2878
2972
|
* confirmed against `sub.allow`), capture the stream frontier as the join watermark, backfill its
|
|
2879
|
-
* history if replay is on, and — for a `durable`-class channel
|
|
2880
|
-
* request a Plane-3 durable backstop (via `ctl.delivery`).
|
|
2973
|
+
* history if replay is on, and — for a `durable`-class channel or a family such as `review.>` when a
|
|
2974
|
+
* delivery daemon is present — request a Plane-3 durable backstop (via `ctl.delivery`). Any other
|
|
2975
|
+
* pattern stays live only, with a `reason`. Idempotent: re-joining is a no-op (no
|
|
2881
2976
|
* re-backfill). Returns the backfill count + whether the durable backstop is active (+ a `reason`
|
|
2882
2977
|
* when a durable channel couldn't get one).
|
|
2883
2978
|
*/
|
|
@@ -2914,10 +3009,15 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
2914
3009
|
// channel, request a Plane-3 per-member backstop from the server-side delivery daemon (durableJoin via ctl.delivery) so a post reaches a
|
|
2915
3010
|
// busy/offline turn — the core-sub stays as the live wake-hint, dedup-coalesced with the Plane-3
|
|
2916
3011
|
// copy by id-dedup. No manager (open dev / manager-less) ⇒ joined LIVE only, surfaced via `reason`
|
|
2917
|
-
// (never silent). A `live`-class channel takes no backstop (joined live is the contract).
|
|
3012
|
+
// (never silent). A `live`-class channel takes no backstop (joined live is the contract). A family
|
|
3013
|
+
// (`review.>`) always asks for its row: each channel under it decides its own class at fan-out
|
|
3014
|
+
// (#5404). Any other pattern has no backstop, and says so.
|
|
2918
3015
|
let durable = false;
|
|
2919
3016
|
let reason;
|
|
2920
|
-
|
|
3017
|
+
const family = isChannelFamily(channel);
|
|
3018
|
+
if (!family && !isConcreteChannel(channel))
|
|
3019
|
+
reason = "a pattern with * or the root > has no stored copies; follow a concrete channel or a family such as review.>";
|
|
3020
|
+
else if (family || effectiveDeliveryClass(this.channelConfigs.get(channel), this.channelDefaults) === "durable") {
|
|
2921
3021
|
try {
|
|
2922
3022
|
const r = await this.durableJoinChannel(channel);
|
|
2923
3023
|
if (r.durable) {
|
|
@@ -2955,7 +3055,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
2955
3055
|
// tombstone can't be confirmed propagates (live sub stays up, mirror intact) for the caller to retry
|
|
2956
3056
|
// — reporting `left` while the trusted reader keeps transferring to DLV is the fail-open leak. A
|
|
2957
3057
|
// genuine no-responder (open / no delivery daemon, no Plane-3) means there is no membership to tombstone.
|
|
2958
|
-
if (this.authed && effectiveDeliveryClass(this.channelConfigs.get(channel), this.channelDefaults) === "durable") {
|
|
3058
|
+
if (this.authed && (isChannelFamily(channel) || effectiveDeliveryClass(this.channelConfigs.get(channel), this.channelDefaults) === "durable")) {
|
|
2959
3059
|
let generation = this.plane3Channels.get(channel);
|
|
2960
3060
|
if (generation === undefined)
|
|
2961
3061
|
generation = (await this.fetchMemberships())?.find((m) => m.channel === channel)?.generation;
|
|
@@ -2991,7 +3091,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
2991
3091
|
// As in listChannels: an absent CHAT stream holds nothing yet; any other failure stays loud.
|
|
2992
3092
|
if (!isJetStreamMissing(e, JetStreamApiCodes.StreamNotFound))
|
|
2993
3093
|
throw e;
|
|
2994
|
-
return { perSender: null, maxAgeMs: null, senders: [], stream: false, lastSeq: 0 };
|
|
3094
|
+
return { perSender: null, maxAgeMs: null, oldestAt: null, senders: [], stream: false, lastSeq: 0 };
|
|
2995
3095
|
}
|
|
2996
3096
|
const senders = [];
|
|
2997
3097
|
for (const [subject, messages] of Object.entries(info.state.subjects ?? {})) {
|
|
@@ -3004,6 +3104,8 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
3004
3104
|
return {
|
|
3005
3105
|
perSender: typeof cap === "number" && cap > 0 ? cap : null,
|
|
3006
3106
|
maxAgeMs: typeof age === "number" && age > 0 ? Math.round(age / 1_000_000) : null,
|
|
3107
|
+
// The stream's oldest post, of any channel; every post sent before it is gone.
|
|
3108
|
+
oldestAt: info.state.messages > 0 ? info.state.first_ts : null,
|
|
3007
3109
|
senders: senders.sort((a, b) => a.channel.localeCompare(b.channel) || a.sender.localeCompare(b.sender)),
|
|
3008
3110
|
stream: true,
|
|
3009
3111
|
lastSeq: info.state.last_seq,
|
|
@@ -3061,11 +3163,13 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
3061
3163
|
: { id, name: id, live: false };
|
|
3062
3164
|
};
|
|
3063
3165
|
const byName = (a, b) => a.name.localeCompare(b.name);
|
|
3064
|
-
if (channel !== undefined)
|
|
3065
|
-
|
|
3066
|
-
|
|
3067
|
-
.
|
|
3068
|
-
.
|
|
3166
|
+
if (channel !== undefined) {
|
|
3167
|
+
const stored = (await this.deliveryClassFresh(channel)) === "durable";
|
|
3168
|
+
const ids = new Set(members
|
|
3169
|
+
.filter((r) => r.channel === channel || (stored && isChannelFamily(r.channel) && subjectMatches(r.channel, channel)))
|
|
3170
|
+
.map((r) => r.owner));
|
|
3171
|
+
return [...ids].map(memberForId).sort(byName);
|
|
3172
|
+
}
|
|
3069
3173
|
const map = new Map();
|
|
3070
3174
|
for (const r of members) {
|
|
3071
3175
|
const arr = map.get(r.channel);
|
|
@@ -4853,7 +4957,8 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
4853
4957
|
.map((r) => ({ channel: r.channel, generation: r.generation, activated: r.activated === true }));
|
|
4854
4958
|
}
|
|
4855
4959
|
/** Effective delivery class read AUTHORITATIVELY from the registry KV (not the watch cache) — so a
|
|
4856
|
-
* `live`→`durable` flip is seen by fan-out without a cache-propagation gap (red-team MED-3)
|
|
4960
|
+
* `live`→`durable` flip is seen by fan-out without a cache-propagation gap (red-team MED-3), and a
|
|
4961
|
+
* reader just started need not wait for the cache to fill. */
|
|
4857
4962
|
async deliveryClassFresh(channel) {
|
|
4858
4963
|
if (!this.channelKv)
|
|
4859
4964
|
return effectiveDeliveryClass(undefined, undefined);
|
|
@@ -4870,17 +4975,21 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
4870
4975
|
const matches = [...this.roster.values()].filter((p) => p.card.name.toLowerCase() === name.toLowerCase());
|
|
4871
4976
|
return matches.length === 1 ? matches[0].card.id : undefined;
|
|
4872
4977
|
}
|
|
4873
|
-
/** Plane-3 publish-dedupe key for a fan-out/transfer entry
|
|
4874
|
-
*
|
|
4875
|
-
*
|
|
4876
|
-
*
|
|
4877
|
-
|
|
4878
|
-
|
|
4978
|
+
/** Plane-3 publish-dedupe key for a fan-out/transfer entry: the message id plus the member
|
|
4979
|
+
* lifecycle (`keySuffix`). One message reaches one lifecycle once, whichever membership row and
|
|
4980
|
+
* whichever writer (fan-out or any row's catch-up) produced the copy (#5404), within the stream's
|
|
4981
|
+
* duplicate window. An empty `msg.id` contributes NO msgID (#673, SPEC §4: two `id: ""` posts MUST
|
|
4982
|
+
* NOT collapse to one delivery) — a plain publish, never an empty or salted one — so an id-less
|
|
4983
|
+
* message is at-least-once on the durable plane (a redelivery may reach the member twice; the
|
|
4984
|
+
* receiver already treats each as its own delivery). */
|
|
4985
|
+
plane3MsgId(entry, keySuffix) {
|
|
4986
|
+
return entry.msg.id === "" ? {} : { msgID: `${entry.msg.id}:${keySuffix}` };
|
|
4879
4987
|
}
|
|
4880
4988
|
/** Publish one fan-out entry into a member LIFECYCLE's mixed inbox (`dinbox.<o>.<a>.<uid>`, SPEC
|
|
4881
4989
|
* §13.1: fan-out addresses the member row's RECORDED lifecycle, never the alias's current
|
|
4882
|
-
* occupant), idempotent via `Nats-Msg-Id` (`<msgId>:<principal>:<
|
|
4883
|
-
* and a racing fan-out copy collapse
|
|
4990
|
+
* occupant), idempotent via `Nats-Msg-Id` (`<msgId>:<principal>:<lifecycleUid>`) so a catch-up copy
|
|
4991
|
+
* and a racing fan-out copy collapse, from the same row or from two rows that both cover the post.
|
|
4992
|
+
* The `principal` is the member's owner+actor dot-form. */
|
|
4884
4993
|
async publishDinbox(principal, lifecycleUid, entry) {
|
|
4885
4994
|
if (!this.js)
|
|
4886
4995
|
return;
|
|
@@ -4889,9 +4998,10 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
4889
4998
|
throw new Error(`publishDinbox: "${principal}" is not a valid member principal <owner>.<actor>`);
|
|
4890
4999
|
await this.js.publish(dinboxSubject(this.space, p.owner, p.actor, lifecycleUid), JSON.stringify(entry), {
|
|
4891
5000
|
// JetStream dedupe is STREAM-WIDE, so the id must carry the LIFECYCLE too: with an alias-keyed
|
|
4892
|
-
// id, lifecycle A's copy would suppress a same-alias successor B's copy of the same message
|
|
4893
|
-
//
|
|
4894
|
-
|
|
5001
|
+
// id, lifecycle A's copy would suppress a same-alias successor B's copy of the same message —
|
|
5002
|
+
// cross-lifecycle suppression, not dedup. It carries no row generation: two rows of one
|
|
5003
|
+
// lifecycle covering the post must collapse to one copy.
|
|
5004
|
+
...this.plane3MsgId(entry, `${principal}:${lifecycleUid}`),
|
|
4895
5005
|
});
|
|
4896
5006
|
}
|
|
4897
5007
|
/** The fan-out consumer's delivered stream-seq — the activation-fence upper bound (red-team
|
|
@@ -4906,7 +5016,10 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
4906
5016
|
* record (CAS + generation bump), then ACTIVATION CATCH-UP idempotently copies `(joinCursor, fence]`
|
|
4907
5017
|
* into the owner inbox where `fence = max(frontier, fanoutDelivered)` — fan-out owns `seq > fence`.
|
|
4908
5018
|
* Idempotent against a timeout-retry (an already-activated membership no-ops). Returns `{durable:false}`
|
|
4909
|
-
*
|
|
5019
|
+
* only when a concurrent leave or rejoin superseded the activation. Catch-up copies what the stream
|
|
5020
|
+
* still holds and does not judge retention: a post the history limit removed before its copy was
|
|
5021
|
+
* taken is outside the durable guarantee (SPEC §4), and stream state cannot show which channel a
|
|
5022
|
+
* removed sequence belonged to.
|
|
4910
5023
|
*
|
|
4911
5024
|
* Runs on the daemon (which hosts the fan-out/reader loops + the members KV), so catch-up + the
|
|
4912
5025
|
* activation fence read are in-process — no cross-process cursor read.
|
|
@@ -4929,7 +5042,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
4929
5042
|
return { durable: true, generation: existing.record.generation }; // fully activated — idempotent
|
|
4930
5043
|
// Either a NEW join (no record / a tombstone to supersede) → fresh joinCursor + bumped generation,
|
|
4931
5044
|
// OR a retry of an INCOMPLETE activation (durable-active but not yet activated, from an earlier
|
|
4932
|
-
//
|
|
5045
|
+
// crash or failed catch-up) → re-run catch-up over the SAME join window, no bump. The record is committed
|
|
4933
5046
|
// `activated:false` first and routes IN-INTERVAL immediately (fan-out + reader deliver via the
|
|
4934
5047
|
// pure-interval durableEligible) so no live message published during catch-up is lost. `activated`
|
|
4935
5048
|
// gates only the REPORT — durableJoin returns true / channelMembers lists the owner only after the
|
|
@@ -4949,24 +5062,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
4949
5062
|
await this.keepRowIfAclAllows(kv, owner, lifecycleUid, channel, written.revision);
|
|
4950
5063
|
}
|
|
4951
5064
|
const fence = Math.max(await this.chatFrontier(), await this.fanoutDeliveredSeq());
|
|
4952
|
-
|
|
4953
|
-
if (cu.evicted) {
|
|
4954
|
-
// Catch-up window irreparably evicted (the oldest in-window message aged out) — this join can never
|
|
4955
|
-
// be a complete backstop. TOMBSTONE the just-committed record at `fence` so it does NOT route:
|
|
4956
|
-
// pure-interval durableEligible would otherwise keep delivering to a record the agent was told is
|
|
4957
|
-
// durable:false AND can't discover to leave (critic BLOCKER-1). Pass `generation` as the expected
|
|
4958
|
-
// generation (ux stale-write guard) so this cleanup can't tombstone a concurrent NEWER rejoin — if
|
|
4959
|
-
// one won, StaleMembershipWrite is the correct no-op (the rejoin is the live record). Then degrade
|
|
4960
|
-
// honestly — a retry is a fresh join (no longer `open`, so a current joinCursor is captured).
|
|
4961
|
-
try {
|
|
4962
|
-
await tombstoneMember(kv, channel, owner, lifecycleUid, fence, this.card.id, generation);
|
|
4963
|
-
}
|
|
4964
|
-
catch (e) {
|
|
4965
|
-
if (!(e instanceof StaleMembershipWrite))
|
|
4966
|
-
throw e;
|
|
4967
|
-
}
|
|
4968
|
-
return { durable: false, reason: "activation catch-up window partially evicted by retention", generation };
|
|
4969
|
-
}
|
|
5065
|
+
await this.catchupCopy(owner, lifecycleUid, channel, joinCursor, fence, generation);
|
|
4970
5066
|
// Flip → reported durable, ATOMICALLY: refuse if a concurrent SAME-generation leave (tombstone) or a
|
|
4971
5067
|
// rejoin superseded this pending join while catch-up ran. A blind same-gen commit would clobber the
|
|
4972
5068
|
// tombstone (clear leaveCursor) and resurrect the membership, reopening §7 (review-general-2 BLOCKER).
|
|
@@ -4975,11 +5071,12 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
4975
5071
|
return { durable: false, reason: "activation superseded by a concurrent leave or rejoin", generation };
|
|
4976
5072
|
return { durable: true, generation };
|
|
4977
5073
|
}
|
|
4978
|
-
/** Whether one lifecycle's exact ACL row allows `channel` now
|
|
4979
|
-
*
|
|
5074
|
+
/** Whether one lifecycle's exact ACL row allows a membership on `channel` now (a family must lie
|
|
5075
|
+
* inside one entry, {@link membershipInAllow}). A leader-served read: the handle is opened without
|
|
5076
|
+
* direct reads, so a row narrowed before this read began is the row it returns. */
|
|
4980
5077
|
async aclAllows(owner, lifecycleUid, channel) {
|
|
4981
5078
|
const row = await readAcl(await this.aclRegistry(), owner, lifecycleUid);
|
|
4982
|
-
return row !== undefined &&
|
|
5079
|
+
return row !== undefined && membershipInAllow(row.record.allowSubscribe, channel);
|
|
4983
5080
|
}
|
|
4984
5081
|
/** The post-write half of {@link durableJoinFor}'s `aclFence`: keep the row just written (at
|
|
4985
5082
|
* `revision`) only on proof that the lifecycle's ACL row still allows the channel, and refuse the
|
|
@@ -5025,18 +5122,28 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
5025
5122
|
}
|
|
5026
5123
|
/** Idempotently copy the eligible chat messages in `(fromSeqExcl, toSeqIncl]` for `channel` into the
|
|
5027
5124
|
* owner inbox, via a DEDICATED per-(owner,join) ephemeral consumer (NOT the agent-scoped
|
|
5028
|
-
* `chathist_<id>`/`histLock` — red-team HIGH-8).
|
|
5029
|
-
*
|
|
5125
|
+
* `chathist_<id>`/`histLock` — red-team HIGH-8). It copies what the stream still holds in that
|
|
5126
|
+
* window; a post the history limit removed first is not copied and is not reported (SPEC §4).
|
|
5127
|
+
*
|
|
5128
|
+
* A FAMILY row (`review.>`, #5404) reads the window on every channel under it and copies a post only
|
|
5129
|
+
* when that post's own channel is durable-class. Each copy names the post's own channel, never the
|
|
5130
|
+
* family, and it skips nothing another row of the lifecycle covers: two joins can both be pending
|
|
5131
|
+
* when a post lands, so neither proves the other copied it. The per-message, per-lifecycle dedupe id
|
|
5132
|
+
* keeps one copy. */
|
|
5030
5133
|
async catchupCopy(owner, lifecycleUid, channel, fromSeqExcl, toSeqIncl, generation) {
|
|
5031
5134
|
if (!this.js || !this.jsm || toSeqIncl <= fromSeqExcl)
|
|
5032
|
-
return
|
|
5135
|
+
return 0;
|
|
5033
5136
|
const subject = chatSubject(this.space, "*", "*", channel);
|
|
5034
|
-
|
|
5035
|
-
|
|
5036
|
-
|
|
5037
|
-
|
|
5038
|
-
|
|
5039
|
-
|
|
5137
|
+
const family = isChannelFamily(channel);
|
|
5138
|
+
const durableClass = new Map();
|
|
5139
|
+
const stored = (ch) => {
|
|
5140
|
+
if (!family)
|
|
5141
|
+
return Promise.resolve(true);
|
|
5142
|
+
let cls = durableClass.get(ch);
|
|
5143
|
+
if (cls === undefined)
|
|
5144
|
+
durableClass.set(ch, (cls = this.deliveryClassFresh(ch).then((c) => c === "durable")));
|
|
5145
|
+
return cls;
|
|
5146
|
+
};
|
|
5040
5147
|
// Consumer NAME must be JetStream-safe (no `.`) AND collision-free: use the principal DASH-form
|
|
5041
5148
|
// (`<owner>-<actor>`, `-` reserved as the sole separator), NOT `token(owner)` — `token()` maps the
|
|
5042
5149
|
// dot-form `.`→`_`, which is NOT collision-free (`_` is legal inside a token, so `a.b_c` and `a_b.c`
|
|
@@ -5045,6 +5152,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
5045
5152
|
// delete/recreate a predecessor's in-flight catch-up consumer — the uid disambiguates.
|
|
5046
5153
|
const cuP = parsePrincipalKey(owner);
|
|
5047
5154
|
const name = `cu_${cuP ? lifecycleNameKey(cuP.owner, cuP.actor, lifecycleUid) : `${token(owner)}-${lifecycleUid}`}_${generation}`;
|
|
5155
|
+
// A failed read throws, leaving the pending row to retry over the same window.
|
|
5048
5156
|
await this.deleteConsumerIfPresent(chatStream(this.space), name);
|
|
5049
5157
|
await this.jsm.consumers.add(chatStream(this.space), {
|
|
5050
5158
|
name, filter_subject: subject, ack_policy: AckPolicy.None, mem_storage: true,
|
|
@@ -5061,7 +5169,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
5061
5169
|
for await (const m of iter) {
|
|
5062
5170
|
got++;
|
|
5063
5171
|
if (m.seq > toSeqIncl)
|
|
5064
|
-
return
|
|
5172
|
+
return copied;
|
|
5065
5173
|
let msg;
|
|
5066
5174
|
try {
|
|
5067
5175
|
msg = m.json();
|
|
@@ -5072,7 +5180,11 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
5072
5180
|
const parsed = parseSubject(m.subject);
|
|
5073
5181
|
if (!parsed || !isFromSubjectSender(msg, parsed) || msg.from.id === owner)
|
|
5074
5182
|
continue;
|
|
5075
|
-
await
|
|
5183
|
+
if (!(await stored(parsed.rest)))
|
|
5184
|
+
continue;
|
|
5185
|
+
await this.publishDinbox(owner, lifecycleUid, {
|
|
5186
|
+
msg, channel: parsed.rest, seq: m.seq, reason: "durable-channel", generation, ...(family ? { row: channel } : {}),
|
|
5187
|
+
});
|
|
5076
5188
|
copied++;
|
|
5077
5189
|
}
|
|
5078
5190
|
if (got < want)
|
|
@@ -5083,7 +5195,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
5083
5195
|
finally {
|
|
5084
5196
|
await this.deleteConsumerIfPresent(chatStream(this.space), name);
|
|
5085
5197
|
}
|
|
5086
|
-
return
|
|
5198
|
+
return copied;
|
|
5087
5199
|
}
|
|
5088
5200
|
/** Start the Plane-3 fan-out writer + trusted reader on THIS (privileged, server-side delivery-daemon)
|
|
5089
5201
|
* endpoint, AND serve the `ctl.delivery` control service (runtime durable join/leave/list). `aclFor`
|
|
@@ -5154,8 +5266,9 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
5154
5266
|
return { ok: false, error: `op "${req.op}" not supported on the delivery control service` };
|
|
5155
5267
|
}
|
|
5156
5268
|
/** Validate the channel ARG shape only: non-blank, valid, concrete (NO ACL check, that is op-specific).
|
|
5269
|
+
* `family` also admits a channel family such as `review.>` (a membership join or leave, #5404).
|
|
5157
5270
|
* Returns the channel on success or a ControlReply error to short-circuit. */
|
|
5158
|
-
checkDurableChannelArg(args, op) {
|
|
5271
|
+
checkDurableChannelArg(args, op, opts = {}) {
|
|
5159
5272
|
const channel = typeof args.channel === "string" ? args.channel.trim() : "";
|
|
5160
5273
|
if (!channel)
|
|
5161
5274
|
return { ok: false, error: `${op}: channel must be a non-blank string` };
|
|
@@ -5165,14 +5278,22 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
5165
5278
|
catch (e) {
|
|
5166
5279
|
return { ok: false, error: e.message };
|
|
5167
5280
|
}
|
|
5281
|
+
if (opts.family && isChannelFamily(channel))
|
|
5282
|
+
return channel;
|
|
5168
5283
|
if (!isConcreteChannel(channel))
|
|
5169
|
-
return {
|
|
5284
|
+
return {
|
|
5285
|
+
ok: false,
|
|
5286
|
+
error: opts.family
|
|
5287
|
+
? `${op}: "${channel}" stays live only: a durable membership takes a concrete channel or a family such as review.>, never the root > or a pattern with *`
|
|
5288
|
+
: `${op}: "${channel}" must be a concrete channel (durable membership is per-concrete-channel, not wildcard)`,
|
|
5289
|
+
};
|
|
5170
5290
|
return channel;
|
|
5171
5291
|
}
|
|
5172
5292
|
/** JOIN requires the channel be within the caller's CURRENT read ACL (you can't durable-subscribe a
|
|
5173
|
-
* channel you may not read).
|
|
5293
|
+
* channel you may not read). A family must lie inside one ACL entry, so every channel under it is
|
|
5294
|
+
* readable. */
|
|
5174
5295
|
async deliveryJoin(caller, args) {
|
|
5175
|
-
const channel = this.checkDurableChannelArg(args, "durableJoin");
|
|
5296
|
+
const channel = this.checkDurableChannelArg(args, "durableJoin", { family: true });
|
|
5176
5297
|
if (typeof channel !== "string")
|
|
5177
5298
|
return channel; // a ControlReply error
|
|
5178
5299
|
// The caller arrives as an ALIAS (the control subject carries owner+actor, never a uid): resolve
|
|
@@ -5188,7 +5309,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
5188
5309
|
}
|
|
5189
5310
|
if (acl === undefined)
|
|
5190
5311
|
return { ok: false, error: `durableJoin: no read ACL on record for ${caller} (not provisioned for durable delivery)` };
|
|
5191
|
-
if (!
|
|
5312
|
+
if (!membershipInAllow(acl.allowSubscribe, channel))
|
|
5192
5313
|
return { ok: false, error: `channel "${channel}" is not within your read ACL [${acl.allowSubscribe.join(", ")}]` };
|
|
5193
5314
|
const uid = acl.lifecycleUid;
|
|
5194
5315
|
// The lookup above may be stale by the time the row is written (a retirement can fence and purge
|
|
@@ -5241,7 +5362,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
5241
5362
|
* (the join epoch — without it a stale/replayed leave could tombstone a newer rejoin), and an EXISTING
|
|
5242
5363
|
* own membership; `durableLeaveFor` → `tombstoneMember` then enforces the generation match. */
|
|
5243
5364
|
async deliveryLeave(caller, args) {
|
|
5244
|
-
const channel = this.checkDurableChannelArg(args, "durableLeave");
|
|
5365
|
+
const channel = this.checkDurableChannelArg(args, "durableLeave", { family: true });
|
|
5245
5366
|
if (typeof channel !== "string")
|
|
5246
5367
|
return channel; // a ControlReply error
|
|
5247
5368
|
if (typeof args.generation !== "number" || !Number.isFinite(args.generation))
|
|
@@ -5667,7 +5788,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
5667
5788
|
let answered;
|
|
5668
5789
|
let instance;
|
|
5669
5790
|
try {
|
|
5670
|
-
const m = await this.nc
|
|
5791
|
+
const m = await this.requestBoundReply(this.nc, reqSubject, "", reply, timeoutMs);
|
|
5671
5792
|
let body;
|
|
5672
5793
|
try {
|
|
5673
5794
|
body = m.json();
|
|
@@ -5982,7 +6103,9 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
5982
6103
|
if (!this.plane3MayAct())
|
|
5983
6104
|
return;
|
|
5984
6105
|
if ((await this.deliveryClassFresh(channel)) === "durable") {
|
|
5985
|
-
|
|
6106
|
+
// The channel's own rows and every family row above it (`review.>`). A lifecycle holding two of
|
|
6107
|
+
// them gets two publishes and keeps one copy: the dedupe id is per message and lifecycle (#5404).
|
|
6108
|
+
for (const rec of await listMembers(await this.membersRegistry(), { covering: channel })) {
|
|
5986
6109
|
if (rec.owner === msg.from.id)
|
|
5987
6110
|
continue; // never backstop the sender's own post
|
|
5988
6111
|
if (!durableEligible(rec, seq))
|
|
@@ -5997,7 +6120,10 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
5997
6120
|
// re-sends the whole set; publishDinbox is idempotent per (owner, lifecycle, msgID).
|
|
5998
6121
|
if (!this.plane3MayAct())
|
|
5999
6122
|
return;
|
|
6000
|
-
await this.publishDinbox(rec.owner, rec.lifecycleUid, {
|
|
6123
|
+
await this.publishDinbox(rec.owner, rec.lifecycleUid, {
|
|
6124
|
+
msg: normalizedMsg, channel, seq, reason: "durable-channel", generation: rec.generation,
|
|
6125
|
+
...(rec.channel !== channel ? { row: rec.channel } : {}),
|
|
6126
|
+
});
|
|
6001
6127
|
}
|
|
6002
6128
|
}
|
|
6003
6129
|
else {
|
|
@@ -6056,6 +6182,17 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
6056
6182
|
})().catch((e) => { if (!this.stopped)
|
|
6057
6183
|
this.emit("error", e); });
|
|
6058
6184
|
}
|
|
6185
|
+
/** Whether one of the lifecycle's rows covering concrete `channel`, the channel's own or a family
|
|
6186
|
+
* above it, is eligible for the post at `seq`. One exact read per row, the channel's own first. */
|
|
6187
|
+
async lifecycleCoversPost(owner, lifecycleUid, channel, seq) {
|
|
6188
|
+
const kv = await this.membersRegistry();
|
|
6189
|
+
for (const row of [channel, ...familiesCovering(channel).reverse()]) {
|
|
6190
|
+
const rec = await readMember(kv, row, owner, lifecycleUid);
|
|
6191
|
+
if (rec && durableEligible(rec.record, seq))
|
|
6192
|
+
return true;
|
|
6193
|
+
}
|
|
6194
|
+
return false;
|
|
6195
|
+
}
|
|
6059
6196
|
/** Remove a retired lifecycle's member row for `channel`, when one exists. Best effort: the row
|
|
6060
6197
|
* authorizes nothing (its ACL row is retired), and the next entry fan-out writes against it
|
|
6061
6198
|
* brings the reader back here. */
|
|
@@ -6141,7 +6278,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
6141
6278
|
// removes each copy. Remove the row where that cost is paid, by its exact key, only when it
|
|
6142
6279
|
// exists: a purge of an absent key would write a marker per dropped entry.
|
|
6143
6280
|
if (entry.reason === "durable-channel")
|
|
6144
|
-
await this.removeRetiredMemberRow(entry.channel, owner, pr.lifecycleUid);
|
|
6281
|
+
await this.removeRetiredMemberRow(entry.row ?? entry.channel, owner, pr.lifecycleUid);
|
|
6145
6282
|
if (!this.plane3MayAct())
|
|
6146
6283
|
return;
|
|
6147
6284
|
m.ack();
|
|
@@ -6167,14 +6304,12 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
6167
6304
|
m.ack();
|
|
6168
6305
|
return;
|
|
6169
6306
|
}
|
|
6170
|
-
|
|
6171
|
-
|
|
6172
|
-
|
|
6173
|
-
|
|
6174
|
-
|
|
6175
|
-
|
|
6176
|
-
return;
|
|
6177
|
-
}
|
|
6307
|
+
// INTERVAL re-auth (not a current-member boolean): a pre-leave entry (seq ≤ leaveCursor) stays
|
|
6308
|
+
// deliverable; seq > leaveCursor (or after a rejoin's newer joinCursor) is the hard cut. Any of the
|
|
6309
|
+
// lifecycle's rows that cover the channel may hold the post, whichever row addressed the copy.
|
|
6310
|
+
if (entry.reason === "durable-channel" && !(await this.lifecycleCoversPost(owner, pr.lifecycleUid, entry.channel, entry.seq))) {
|
|
6311
|
+
m.ack();
|
|
6312
|
+
return;
|
|
6178
6313
|
}
|
|
6179
6314
|
// COMMIT-POINT FENCE. Everything above this line is broker I/O - the ACL read and the member
|
|
6180
6315
|
// re-read - so the entry check at the top of the handler is stale by now if this unit was
|
|
@@ -6196,7 +6331,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
6196
6331
|
// subject (streams.ts filter_subject), and a predecessor's transferred copy must never suppress
|
|
6197
6332
|
// a successor's under the same alias (disjoint lifecycles, stream-wide dedupe).
|
|
6198
6333
|
await this.js.publish(dlvSubject(this.space, pr.owner, pr.actor, pr.lifecycleUid), JSON.stringify(frame), {
|
|
6199
|
-
...this.plane3MsgId(entry, `${owner}:${pr.lifecycleUid}
|
|
6334
|
+
...this.plane3MsgId(entry, `${owner}:${pr.lifecycleUid}`),
|
|
6200
6335
|
headers: frameHeaders,
|
|
6201
6336
|
});
|
|
6202
6337
|
}
|
|
@@ -6249,6 +6384,12 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
6249
6384
|
this.streamMsgs.push(msgs);
|
|
6250
6385
|
void (async () => {
|
|
6251
6386
|
for await (const m of msgs) {
|
|
6387
|
+
// A copy with a full store behind it means the store held its limit, so older copies may have
|
|
6388
|
+
// been discarded unread (#5404); NATS gives no notice of the discard itself. Checked on every
|
|
6389
|
+
// delivery, ahead of the checks below and the connector's id dedupe, so a copy the agent also
|
|
6390
|
+
// got live cannot hide it. The connector folds repeats into one notice.
|
|
6391
|
+
if (m.info.pending >= PLANE3_MAX_MSGS_PER_SUBJECT - 1)
|
|
6392
|
+
this.emit("stored-copies-full");
|
|
6252
6393
|
// The header is minted only by the trusted DLV writer. A body-only discriminator is
|
|
6253
6394
|
// insufficient because old Cotal envelopes permit unknown fields and could imitate it.
|
|
6254
6395
|
if (m.headers?.get(PLANE3_FRAME_HEADER) !== "1") {
|
|
@@ -6365,7 +6506,8 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
6365
6506
|
}
|
|
6366
6507
|
/** Agent-side, first connect (auth): SELF-JOIN this session's durable boot channels via the
|
|
6367
6508
|
* server-side delivery daemon — replacing the old manager-written boot membership. Each concrete
|
|
6368
|
-
* `durable`-class boot channel gets a `durableJoin` whose
|
|
6509
|
+
* `durable`-class boot channel, and each family such as `review.>`, gets a `durableJoin` whose
|
|
6510
|
+
* returned generation seeds the leave mirror
|
|
6369
6511
|
* + durable-state surface; an already-active membership (a relaunch) is idempotent (no re-catch-up).
|
|
6370
6512
|
* If the daemon is down/absent at first connect (or reports a transient `durable:false`), the channel
|
|
6371
6513
|
* is handed to {@link reconcileBootJoin} for capped-backoff retry — so the backstop is RESTORED once
|
|
@@ -6373,13 +6515,19 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
6373
6515
|
* degraded in `cotal_channels` ({@link hasDurableMembership}). */
|
|
6374
6516
|
async armBootDurableMemberships() {
|
|
6375
6517
|
for (const channel of this.channels) {
|
|
6376
|
-
if (
|
|
6377
|
-
continue;
|
|
6378
|
-
// A missing channel row legitimately resolves through the defaults. A failed/denied registry
|
|
6379
|
-
// read does not prove the channel is live-only, so let it fail startup loudly.
|
|
6380
|
-
const cls = await this.deliveryClassFresh(channel);
|
|
6381
|
-
if (cls !== "durable")
|
|
6518
|
+
if (this.plane3Channels.has(channel))
|
|
6382
6519
|
continue;
|
|
6520
|
+
// A family (`review.>`) asks for its row whatever the default class: each channel under it decides
|
|
6521
|
+
// its own at fan-out (#5404). Any other pattern has no durable membership.
|
|
6522
|
+
if (!isChannelFamily(channel)) {
|
|
6523
|
+
if (!isConcreteChannel(channel))
|
|
6524
|
+
continue;
|
|
6525
|
+
// A missing channel row legitimately resolves through the defaults. A failed/denied registry
|
|
6526
|
+
// read does not prove the channel is live-only, so let it fail startup loudly.
|
|
6527
|
+
const cls = await this.deliveryClassFresh(channel);
|
|
6528
|
+
if (cls !== "durable")
|
|
6529
|
+
continue;
|
|
6530
|
+
}
|
|
6383
6531
|
try {
|
|
6384
6532
|
const r = await this.durableJoinChannel(channel);
|
|
6385
6533
|
if (r.durable)
|
|
@@ -6948,10 +7096,11 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
6948
7096
|
/**
|
|
6949
7097
|
* Replay-gated pull of a channel's retained ambient from `sinceSeq` (exclusive) forward — the
|
|
6950
7098
|
* focus-recall read behind `cotal_inbox`. Returns the messages (NOT emitted — this is a pull,
|
|
6951
|
-
* not a push into context) plus `dropped: true` when the window
|
|
6952
|
-
*
|
|
6953
|
-
*
|
|
6954
|
-
* must say so rather than silently reporting an
|
|
7099
|
+
* not a push into context) plus `dropped: true` when the window may not be complete: the stream's
|
|
7100
|
+
* limits may have removed posts after the watermark ({@link chatMayHaveLost}: a sender at the
|
|
7101
|
+
* per-sender limit, or the stream's oldest post already past the watermark), or replay is off
|
|
7102
|
+
* for the channel below. Either way the caller must say so rather than silently reporting an
|
|
7103
|
+
* empty, complete window. `seqs[i]` is the chat
|
|
6955
7104
|
* stream sequence of `messages[i]`, the one identity two identical messages do not share.
|
|
6956
7105
|
* `unanswered` is true when no history read answered (replay is off, or the read failed), so
|
|
6957
7106
|
* `messages` says nothing about what the channel retains.
|
|
@@ -7002,49 +7151,43 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
7002
7151
|
collected.push(authenticatedMessage(msg, parsed));
|
|
7003
7152
|
seqs.push(sm.seq);
|
|
7004
7153
|
}
|
|
7005
|
-
|
|
7006
|
-
return { messages: collected, seqs, dropped, unanswered: false };
|
|
7007
|
-
}
|
|
7008
|
-
/** Did focus recall on `subject` miss ambient that aged out past the watermark? Ambient is only
|
|
7009
|
-
* ever discarded once a sender-subject reaches {@link MAX_MSGS_PER_SUBJECT} (`DiscardPolicy.Old`);
|
|
7010
|
-
* below the cap nothing was evicted, so the window is complete — return false without crying
|
|
7011
|
-
* wolf. At the cap, the surviving oldest seq decides: if it already postdates the watermark, the
|
|
7012
|
-
* eviction reached into the "since you focused" window. (Avoids the false positive of comparing a
|
|
7013
|
-
* per-subject oldest against the stream-global frontier, which fires on any other channel's
|
|
7014
|
-
* traffic.) */
|
|
7015
|
-
async channelDropped(subject, sinceSeq) {
|
|
7016
|
-
if (!this.jsm)
|
|
7017
|
-
return false;
|
|
7018
|
-
let maxPerSubject = 0;
|
|
7154
|
+
let dropped;
|
|
7019
7155
|
try {
|
|
7020
|
-
|
|
7021
|
-
for (const count of Object.values(info.state.subjects ?? {}))
|
|
7022
|
-
maxPerSubject = Math.max(maxPerSubject, count);
|
|
7156
|
+
dropped = await this.chatMayHaveLost(subject, sinceSeq);
|
|
7023
7157
|
}
|
|
7024
7158
|
catch (e) {
|
|
7025
|
-
|
|
7026
|
-
|
|
7027
|
-
|
|
7159
|
+
// Completeness could not be read, so the read is reported as unanswered: nothing settles on it.
|
|
7160
|
+
this.emit("error", e);
|
|
7161
|
+
if (isPermissionDenied(e))
|
|
7162
|
+
throw e;
|
|
7163
|
+
return { messages: [], seqs: [], dropped: true, unanswered: true };
|
|
7028
7164
|
}
|
|
7029
|
-
|
|
7030
|
-
return false; // never hit the cap ⇒ never evicted
|
|
7031
|
-
const oldest = await this.channelOldestSeq(subject);
|
|
7032
|
-
return oldest !== undefined && oldest > sinceSeq + 1;
|
|
7165
|
+
return { messages: collected, seqs, dropped, unanswered: false };
|
|
7033
7166
|
}
|
|
7034
|
-
/**
|
|
7035
|
-
*
|
|
7036
|
-
*
|
|
7037
|
-
|
|
7167
|
+
/** Whether the chat stream's limits may have removed posts on `subject` after `sinceSeq`, read from
|
|
7168
|
+
* the stream's own config rather than a compiled default. Throws when it cannot read, so no caller
|
|
7169
|
+
* takes an unread stream for a complete one. It reports possible loss, never proof: a removed
|
|
7170
|
+
* sequence does not record its subject, so the stream cannot say which channel lost it.
|
|
7171
|
+
*
|
|
7172
|
+
* Either rule is enough. The stream's oldest retained post is past the watermark, so every post up
|
|
7173
|
+
* to it is gone, from this channel or another. Or a sender on the channel is at the stream's
|
|
7174
|
+
* per-sender limit and the channel's oldest retained post is past the watermark, so that sender's
|
|
7175
|
+
* eviction may have reached into the window; a sender at exactly the limit may have lost nothing.
|
|
7176
|
+
* A hole behind a retained older post, as a channel-filtered purge leaves, sets neither. */
|
|
7177
|
+
async chatMayHaveLost(subject, sinceSeq) {
|
|
7038
7178
|
if (!this.jsm)
|
|
7039
|
-
|
|
7040
|
-
|
|
7041
|
-
|
|
7042
|
-
return
|
|
7043
|
-
|
|
7044
|
-
|
|
7045
|
-
|
|
7046
|
-
|
|
7047
|
-
|
|
7179
|
+
throw new Error(this.notLiveMsg());
|
|
7180
|
+
const info = await this.jsm.streams.info(chatStream(this.space), { subjects_filter: subject });
|
|
7181
|
+
if (info.state.first_seq > sinceSeq + 1)
|
|
7182
|
+
return true;
|
|
7183
|
+
const cap = info.config.max_msgs_per_subject;
|
|
7184
|
+
let maxPerSubject = 0;
|
|
7185
|
+
for (const count of Object.values(info.state.subjects ?? {}))
|
|
7186
|
+
maxPerSubject = Math.max(maxPerSubject, count);
|
|
7187
|
+
if (cap < 1 || maxPerSubject < cap)
|
|
7188
|
+
return false; // no sender at the cap ⇒ none evicted
|
|
7189
|
+
const oldest = (await this.collectHistory(subject, { seq: 1 }, { limit: 1 }))[0]?.seq;
|
|
7190
|
+
return oldest !== undefined && oldest > sinceSeq + 1;
|
|
7048
7191
|
}
|
|
7049
7192
|
async publishPresence() {
|
|
7050
7193
|
if (!this.doRegister || !this.kv)
|