@cotal-ai/core 0.153.1 → 0.155.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-file.d.ts +11 -4
- package/dist/agent-file.d.ts.map +1 -1
- package/dist/agent-file.js +10 -3
- package/dist/agent-file.js.map +1 -1
- package/dist/anycast-requests.d.ts +216 -0
- package/dist/anycast-requests.d.ts.map +1 -0
- package/dist/anycast-requests.js +335 -0
- package/dist/anycast-requests.js.map +1 -0
- package/dist/auth-provider.d.ts +5 -4
- package/dist/auth-provider.d.ts.map +1 -1
- package/dist/backup-config.d.ts.map +1 -1
- package/dist/backup-config.js +2 -1
- package/dist/backup-config.js.map +1 -1
- package/dist/connector.d.ts +4 -0
- package/dist/connector.d.ts.map +1 -1
- package/dist/connector.js.map +1 -1
- package/dist/endpoint.d.ts +52 -0
- package/dist/endpoint.d.ts.map +1 -1
- package/dist/endpoint.js +307 -6
- package/dist/endpoint.js.map +1 -1
- package/dist/grant-bound.d.ts +4 -1
- package/dist/grant-bound.d.ts.map +1 -1
- package/dist/grant-bound.js +3 -0
- package/dist/grant-bound.js.map +1 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/manager-contract.d.ts +47 -0
- package/dist/manager-contract.d.ts.map +1 -1
- package/dist/manager-contract.js +11 -0
- package/dist/manager-contract.js.map +1 -1
- package/dist/one-line.d.ts +3 -5
- package/dist/one-line.d.ts.map +1 -1
- package/dist/one-line.js +9 -6
- package/dist/one-line.js.map +1 -1
- package/dist/persona-endpoints.d.ts +30 -0
- package/dist/persona-endpoints.d.ts.map +1 -0
- package/dist/persona-endpoints.js +126 -0
- package/dist/persona-endpoints.js.map +1 -0
- package/dist/provision.d.ts.map +1 -1
- package/dist/provision.js +47 -7
- package/dist/provision.js.map +1 -1
- package/dist/space-deletion-targets.js +2 -2
- package/dist/space-deletion-targets.js.map +1 -1
- package/dist/space-resource-definitions.d.ts.map +1 -1
- package/dist/space-resource-definitions.js +2 -1
- package/dist/space-resource-definitions.js.map +1 -1
- package/dist/streams.d.ts +16 -0
- package/dist/streams.d.ts.map +1 -1
- package/dist/streams.js +14 -1
- package/dist/streams.js.map +1 -1
- package/dist/subjects.d.ts +26 -0
- package/dist/subjects.d.ts.map +1 -1
- package/dist/subjects.js +37 -0
- package/dist/subjects.js.map +1 -1
- package/dist/types.d.ts +21 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +1 -1
package/dist/endpoint.js
CHANGED
|
@@ -28,7 +28,7 @@ import { assertValidKind } from "./agent-file.js";
|
|
|
28
28
|
import { assertValidName } from "./resolve.js";
|
|
29
29
|
import { EVICT_PRINCIPALS_MAX } from "./evict.js";
|
|
30
30
|
import { nextHeartbeat } from "./presence-heartbeat.js";
|
|
31
|
-
import { createSpaceStreams, dmDurableConfig, dlvDurableConfig, taskDurableConfig, fanoutDurableConfig, inboxReaderConfig, MANAGER_LEASE_TTL_MS, MANAGER_LEASE_ATTEMPT_MS, TTL_RECONCILE_CANARY_KEY, PRESENCE_STORAGE } from "./streams.js";
|
|
31
|
+
import { anycastStoreDefinition, createSpaceStreams, dmDurableConfig, dlvDurableConfig, taskDurableConfig, fanoutDurableConfig, inboxReaderConfig, MANAGER_LEASE_TTL_MS, MANAGER_LEASE_ATTEMPT_MS, TTL_RECONCILE_CANARY_KEY, PRESENCE_STORAGE } from "./streams.js";
|
|
32
32
|
import { DELIVERY_LIMIT_HEADER, HISTORY_NO_COUNT, PLANE3_MAX_MSGS_PER_SUBJECT } from "./backup-config.js";
|
|
33
33
|
import { jetstream, jetstreamManager, AckPolicy, DeliverPolicy, JetStreamApiCodes, JetStreamApiError, } from "@nats-io/jetstream";
|
|
34
34
|
import {} from "@nats-io/jetstream";
|
|
@@ -39,11 +39,21 @@ import { openAclRegistry, readAcl, readAclForAlias, aclRetired, AmbiguousAclAlia
|
|
|
39
39
|
import { openDeliveryRegistry } from "./lease.js";
|
|
40
40
|
import { openChannelRegistry, effectiveReplay, effectiveReplayWindowMs, effectiveDeliveryClass, readChannelConfig, readChannelDefaults, decodeChannelEntry, } from "./channels.js";
|
|
41
41
|
import { isCasLoss } from "./endpoint-records.js";
|
|
42
|
-
import {
|
|
42
|
+
import { ANYCAST_CANCEL_OP, ANYCAST_NOTICE_KEY, ANYCAST_RECORDS_CREDENTIAL_OUTDATED, ANYCAST_STATUS_OP, ANYCAST_TAKES_OP, AnycastRequestAuthority, isAnycastSeq, } from "./anycast-requests.js";
|
|
43
|
+
import { anycastSubject, assertValidOwnerToken, assertValidRole, CHANNEL_DEFAULTS_KEY, chatStream, chatHistDurable, chatSubject, historyReaderConsumer, calloutReaderId, HISTORY_READERS, controlServiceSubject, livenessSubject, livenessServeFilter, CONTROL_DELIVERY, CONTROL_DELIVERY_ADMIN, dmStream, dmDurable, dlvStream, dlvDurable, dlvSubject, dinboxSubject, inboxStream, parseDinboxPrincipal, FANOUT_DURABLE, INBOX_READER_DURABLE, leaseKey, managerBucket, MANAGER_LEASE_KEY, MANAGER_RENEWAL_LEASE_KEY, managerLeaseKey, chatWildcard, assertValidChannel, channelInAllow, membershipInAllow, isConcreteChannel, isChannelFamily, familiesCovering, normalizeMentions, parseSubject, isPrincipalOwnerToken, assertInboxConnId, presenceBucket, channelBucket, isPresenceRow, assertPresenceRow, membershipBucket, MEMBERSHIP_FEED_KEY, principalKey, parsePrincipalKey, assertLifecycleToken, mintLifecycleUid, lifecycleNameKey, memberKey, DEV_OWNER, spacePrefix, spaceWildcard, subjectMatches, taskStream, taskDurable, takeServeFilter, takeSubject, parseTakeSubject, TAKE_ROUTE, token, unicastSubject, unicastRecvFilter, } from "./subjects.js";
|
|
43
44
|
import { isPermissionDenied } from "./permission-denied.js";
|
|
44
45
|
export { isPermissionDenied };
|
|
45
46
|
export const DEFAULT_SERVER = "nats://127.0.0.1:4222";
|
|
46
47
|
const PLANE3_FRAME_HEADER = "Cotal-Delivery-Frame";
|
|
48
|
+
/** The `ctl.delivery` ops a sender calls about one of its own anycast requests (SPEC §8.1). */
|
|
49
|
+
const ANYCAST_CONTROL_OPS = new Set([ANYCAST_STATUS_OP, ANYCAST_CANCEL_OP, ANYCAST_TAKES_OP]);
|
|
50
|
+
/** Takes a daemon answers at once; one more is refused as busy, so the holder keeps its copy and
|
|
51
|
+
* asks again rather than queueing work the daemon cannot bound. */
|
|
52
|
+
const MAX_TAKES_IN_FLIGHT = 64;
|
|
53
|
+
/** How often the daemon runs the anycast records maintenance pass. */
|
|
54
|
+
const ANYCAST_SWEEP_MS = 10 * 60 * 1000;
|
|
55
|
+
/** How long a holder waits for the daemon to answer a take. */
|
|
56
|
+
const ANYCAST_TAKE_TIMEOUT_MS = 5_000;
|
|
47
57
|
/** How long the trusted writer reuses its read of the DLV count before reading it again. */
|
|
48
58
|
const DELIVERY_LIMIT_READ_MS = 5_000;
|
|
49
59
|
/** The DLV count a stored copy was written under, from its {@link DELIVERY_LIMIT_HEADER} (SPEC §8):
|
|
@@ -322,6 +332,17 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
322
332
|
* {@link armDeliveryControl}; tracked so the stale one is dropped on reconnect. */
|
|
323
333
|
deliveryServeSub;
|
|
324
334
|
deliveryAdminServeSub;
|
|
335
|
+
/** The anycast take responder, the request records it writes, and their maintenance timer
|
|
336
|
+
* (delivery daemon, SPEC §8.1). Bound with the other Plane-3 responders and dropped with them. */
|
|
337
|
+
takeServeSub;
|
|
338
|
+
anycastAuthority;
|
|
339
|
+
/** The TASK stream's creation time for the role consumer this endpoint pumps, stamped on each copy
|
|
340
|
+
* it delivers and carried by that copy's take (SPEC §8.1). */
|
|
341
|
+
taskEpoch;
|
|
342
|
+
/** Why the records are not open, when this daemon's credential cannot open them. */
|
|
343
|
+
anycastUnavailable;
|
|
344
|
+
anycastSweep;
|
|
345
|
+
takesInFlight = 0;
|
|
325
346
|
/** The lifecycles whose membership inventory this daemon has been asked for, as principal to uids
|
|
326
347
|
* (#4965). A durable join or leave bound to one of them is refused, so no row of a retiring lifecycle
|
|
327
348
|
* is written after its inventory read starts. An entry goes when a later inventory for the same
|
|
@@ -1637,6 +1658,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
1637
1658
|
catch { /* dead with the old connection */ }
|
|
1638
1659
|
this.deliveryAdminServeSub = undefined;
|
|
1639
1660
|
}
|
|
1661
|
+
this.dropAnycastRequests();
|
|
1640
1662
|
for (const sub of this.subs) {
|
|
1641
1663
|
try {
|
|
1642
1664
|
sub.unsubscribe();
|
|
@@ -5396,6 +5418,10 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
5396
5418
|
return this.deliveryLeave(caller, args);
|
|
5397
5419
|
if (req.op === "readHistory")
|
|
5398
5420
|
return this.deliveryReadHistory(caller, args);
|
|
5421
|
+
if (req.op === ANYCAST_TAKES_OP)
|
|
5422
|
+
return this.anycastUnavailable ? { ok: false, error: `delivery: ${this.anycastUnavailable}` } : { ok: true, data: { takes: true } };
|
|
5423
|
+
if (req.op === ANYCAST_STATUS_OP || req.op === ANYCAST_CANCEL_OP)
|
|
5424
|
+
return this.deliveryAnycast(req.op, caller, args);
|
|
5399
5425
|
if (req.op === "listMemberships") {
|
|
5400
5426
|
if (typeof args.lifecycleUid !== "string")
|
|
5401
5427
|
return { ok: false, error: "listMemberships: the caller's lifecycleUid is required (membership rows are lifecycle-keyed, SPEC 13.1)" };
|
|
@@ -5674,6 +5700,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
5674
5700
|
catch { /* already dead */ }
|
|
5675
5701
|
this.deliveryAdminServeSub = undefined;
|
|
5676
5702
|
}
|
|
5703
|
+
this.dropAnycastRequests();
|
|
5677
5704
|
// Stopping the consumers ends the `for await` loops that drive fan-out and the reader. In-flight
|
|
5678
5705
|
// messages are NOT acked by a stopped consumer, so they redeliver to whoever holds the shard
|
|
5679
5706
|
// next: quiescing loses no message, it only stops this process from claiming them.
|
|
@@ -5733,6 +5760,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
5733
5760
|
return;
|
|
5734
5761
|
await this.manager(); // the manager runs consume:false, so this.jsm is lazy — ensure it
|
|
5735
5762
|
this.armDeliveryControl();
|
|
5763
|
+
await this.armAnycastRequests();
|
|
5736
5764
|
await this.runFanout();
|
|
5737
5765
|
await this.runReader();
|
|
5738
5766
|
}
|
|
@@ -5752,7 +5780,12 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
5752
5780
|
if (i >= 0)
|
|
5753
5781
|
this.subs.splice(i, 1);
|
|
5754
5782
|
}
|
|
5755
|
-
|
|
5783
|
+
// A request's status and cancel touch only the TASK queue and the anycast records, so they do
|
|
5784
|
+
// not wait behind a channel join on the in-order chain.
|
|
5785
|
+
this.deliveryServeSub = this.serveControl(CONTROL_DELIVERY, (req) => this.handleDeliveryControl(req), {
|
|
5786
|
+
boundReply: true,
|
|
5787
|
+
concurrent: (req) => ANYCAST_CONTROL_OPS.has(req.op),
|
|
5788
|
+
});
|
|
5756
5789
|
if (this.deliveryAdminServeSub) {
|
|
5757
5790
|
try {
|
|
5758
5791
|
this.deliveryAdminServeSub.unsubscribe();
|
|
@@ -5770,6 +5803,160 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
5770
5803
|
concurrent: (req) => req.op === "lifecycleMemberships",
|
|
5771
5804
|
});
|
|
5772
5805
|
}
|
|
5806
|
+
/** (Re)bind the anycast take responder and the request records maintenance on the current
|
|
5807
|
+
* connection (SPEC §8.1). The records bucket is created here as well as at space setup, so a mesh
|
|
5808
|
+
* that predates it gets one when its daemon restarts at this version. */
|
|
5809
|
+
async armAnycastRequests() {
|
|
5810
|
+
if (!this.nc || !this.jsm)
|
|
5811
|
+
return;
|
|
5812
|
+
this.dropAnycastRequests();
|
|
5813
|
+
const definition = anycastStoreDefinition(this.space);
|
|
5814
|
+
let kv;
|
|
5815
|
+
try {
|
|
5816
|
+
kv = await new Kvm(this.nc).create(definition.bucket, definition.createOptions);
|
|
5817
|
+
}
|
|
5818
|
+
catch (e) {
|
|
5819
|
+
// A credential minted before the records existed cannot open them. That costs the records and
|
|
5820
|
+
// nothing else: the rest of Plane-3 still serves, and every anycast op answers with the cause.
|
|
5821
|
+
if (!isPermissionDenied(e))
|
|
5822
|
+
throw e;
|
|
5823
|
+
this.anycastUnavailable = ANYCAST_RECORDS_CREDENTIAL_OUTDATED;
|
|
5824
|
+
this.emitRecoverable(new Error(`delivery: ${ANYCAST_RECORDS_CREDENTIAL_OUTDATED}`));
|
|
5825
|
+
return;
|
|
5826
|
+
}
|
|
5827
|
+
this.anycastUnavailable = undefined;
|
|
5828
|
+
const authority = new AnycastRequestAuthority({
|
|
5829
|
+
jsm: this.jsm,
|
|
5830
|
+
kv,
|
|
5831
|
+
stream: taskStream(this.space),
|
|
5832
|
+
send: (to, event) => this.sendAnycastNotice(to, event),
|
|
5833
|
+
mayAct: () => this.plane3MayAct(),
|
|
5834
|
+
onError: (e) => this.emitRecoverable(e),
|
|
5835
|
+
});
|
|
5836
|
+
this.anycastAuthority = authority;
|
|
5837
|
+
this.takeServeSub = this.serveTakes(authority);
|
|
5838
|
+
const sweep = () => void authority.sweep().catch((e) => this.emitRecoverable(e));
|
|
5839
|
+
this.anycastSweep = setInterval(sweep, ANYCAST_SWEEP_MS);
|
|
5840
|
+
this.anycastSweep.unref?.();
|
|
5841
|
+
sweep();
|
|
5842
|
+
}
|
|
5843
|
+
dropAnycastRequests() {
|
|
5844
|
+
if (this.takeServeSub) {
|
|
5845
|
+
try {
|
|
5846
|
+
this.takeServeSub.unsubscribe();
|
|
5847
|
+
}
|
|
5848
|
+
catch { /* dead with the old connection */ }
|
|
5849
|
+
const i = this.subs.indexOf(this.takeServeSub);
|
|
5850
|
+
if (i >= 0)
|
|
5851
|
+
this.subs.splice(i, 1);
|
|
5852
|
+
this.takeServeSub = undefined;
|
|
5853
|
+
}
|
|
5854
|
+
clearInterval(this.anycastSweep);
|
|
5855
|
+
this.anycastSweep = undefined;
|
|
5856
|
+
this.anycastAuthority = undefined;
|
|
5857
|
+
}
|
|
5858
|
+
/** Serve takes on `take.*.*.*`, each in its own time: a take reads one request and writes one row,
|
|
5859
|
+
* and shares nothing with any other take but the row's CAS. The reply is bound to the caller's own
|
|
5860
|
+
* request subtree, as `serveControl`'s `boundReply` binds it. */
|
|
5861
|
+
serveTakes(authority) {
|
|
5862
|
+
const sub = this.nc.subscribe(takeServeFilter(this.space), { queue: TAKE_ROUTE });
|
|
5863
|
+
this.subs.push(sub);
|
|
5864
|
+
void (async () => {
|
|
5865
|
+
for await (const m of sub) {
|
|
5866
|
+
if (!m.reply || !m.reply.startsWith(`${m.subject}.reply.`)) {
|
|
5867
|
+
this.emitRecoverable(new Error(`rejected take on ${m.subject}: reply target "${m.reply ?? "(none)"}" is not under the sender's own reply subtree`));
|
|
5868
|
+
continue;
|
|
5869
|
+
}
|
|
5870
|
+
void this.answerTake(authority, m).catch((e) => this.emitRecoverable(e));
|
|
5871
|
+
}
|
|
5872
|
+
})().catch((e) => this.emitRecoverable(e));
|
|
5873
|
+
return sub;
|
|
5874
|
+
}
|
|
5875
|
+
async answerTake(authority, m) {
|
|
5876
|
+
let reply;
|
|
5877
|
+
if (this.takesInFlight >= MAX_TAKES_IN_FLIGHT)
|
|
5878
|
+
reply = { ok: false, code: "busy", error: "the delivery daemon is answering too many takes; retry" };
|
|
5879
|
+
else {
|
|
5880
|
+
this.takesInFlight++;
|
|
5881
|
+
try {
|
|
5882
|
+
reply = await this.takeFor(authority, m);
|
|
5883
|
+
}
|
|
5884
|
+
catch (e) {
|
|
5885
|
+
reply = { ok: false, code: "busy", error: `the take could not be recorded: ${e.message}` };
|
|
5886
|
+
}
|
|
5887
|
+
finally {
|
|
5888
|
+
this.takesInFlight--;
|
|
5889
|
+
}
|
|
5890
|
+
}
|
|
5891
|
+
m.respond(JSON.stringify(reply));
|
|
5892
|
+
}
|
|
5893
|
+
/** Check one take's caller and arguments, then record it. The role and the caller come from the
|
|
5894
|
+
* broker-policed subject; the payload's `from.id` must match, as on every control rail. */
|
|
5895
|
+
async takeFor(authority, m) {
|
|
5896
|
+
if (!this.plane3MayAct())
|
|
5897
|
+
return { ok: false, code: "busy", error: "delivery: this daemon is not serving this shard (it is re-checking ownership); retry" };
|
|
5898
|
+
const parsed = parseTakeSubject(m.subject);
|
|
5899
|
+
let req;
|
|
5900
|
+
try {
|
|
5901
|
+
req = m.json();
|
|
5902
|
+
}
|
|
5903
|
+
catch {
|
|
5904
|
+
req = undefined;
|
|
5905
|
+
}
|
|
5906
|
+
if (!parsed || !isPrincipalOwnerToken(parsed.owner) || req?.from?.id !== parsed.sender || typeof req.from.name !== "string")
|
|
5907
|
+
return { ok: false, code: "mismatch", error: "sender mismatch - take rejected" };
|
|
5908
|
+
if (!isAnycastSeq(req.seq) || !isAnycastSeq(req.count) || typeof req.id !== "string" || req.id.length > 256 ||
|
|
5909
|
+
typeof req.epoch !== "string" || req.epoch.length > 64)
|
|
5910
|
+
return { ok: false, code: "mismatch", error: "a take names a seq, the request id, a delivery count and the stream's creation time" };
|
|
5911
|
+
// The name reaches the sender's status text, so it must pass the rule a presence card name does.
|
|
5912
|
+
try {
|
|
5913
|
+
assertValidName(req.from.name);
|
|
5914
|
+
}
|
|
5915
|
+
catch (e) {
|
|
5916
|
+
return { ok: false, code: "mismatch", error: `a take names its holder by a valid display name: ${e.message}` };
|
|
5917
|
+
}
|
|
5918
|
+
return authority.take(parsed.role, { from: { id: parsed.sender, name: req.from.name }, seq: req.seq, id: req.id, count: req.count, epoch: req.epoch });
|
|
5919
|
+
}
|
|
5920
|
+
/** DM one notice to a request's sender from this daemon's own principal. The event key is the
|
|
5921
|
+
* message id, so a resend after a crash is dropped by the DM stream's duplicate window and by
|
|
5922
|
+
* the receiving connector. */
|
|
5923
|
+
async sendAnycastNotice(to, event) {
|
|
5924
|
+
if (!this.js)
|
|
5925
|
+
throw new Error(this.notLiveMsg());
|
|
5926
|
+
const recipient = parsePrincipalKey(to);
|
|
5927
|
+
if (!recipient)
|
|
5928
|
+
throw new Error(`anycast notice: "${to}" is not a principal`);
|
|
5929
|
+
const msg = {
|
|
5930
|
+
id: event.key,
|
|
5931
|
+
ts: Date.now(),
|
|
5932
|
+
space: this.space,
|
|
5933
|
+
from: this.ref(),
|
|
5934
|
+
to,
|
|
5935
|
+
parts: [{ kind: "text", text: event.text }, { kind: "data", data: { [ANYCAST_NOTICE_KEY]: event.notice } }],
|
|
5936
|
+
};
|
|
5937
|
+
await this.js.publish(unicastSubject(this.space, recipient.owner, recipient.actor, this.owner, this.actor), JSON.stringify(msg), { msgID: msg.id });
|
|
5938
|
+
}
|
|
5939
|
+
/** Answer a sender's status or cancel for one of its own requests. The caller is the
|
|
5940
|
+
* authenticated subject sender, checked against the request's stored subject. */
|
|
5941
|
+
async deliveryAnycast(op, caller, args) {
|
|
5942
|
+
const authority = this.anycastAuthority;
|
|
5943
|
+
if (!authority)
|
|
5944
|
+
return { ok: false, error: `delivery: ${this.anycastUnavailable ?? "the anycast request records are not open on this daemon; retry"}` };
|
|
5945
|
+
if (!isAnycastSeq(args.seq))
|
|
5946
|
+
return { ok: false, error: `${op}: seq must be a whole number of at least 1` };
|
|
5947
|
+
if (args.id !== undefined && (typeof args.id !== "string" || args.id.length > 256))
|
|
5948
|
+
return { ok: false, error: `${op}: id must be a request id` };
|
|
5949
|
+
const q = { seq: args.seq, ...(typeof args.id === "string" ? { id: args.id } : {}) };
|
|
5950
|
+
try {
|
|
5951
|
+
const status = op === ANYCAST_CANCEL_OP ? await authority.cancel(caller, q) : await authority.status(caller, q);
|
|
5952
|
+
if (!this.plane3MayAct())
|
|
5953
|
+
return { ok: false, error: "delivery: this daemon stopped serving this shard while the request was in flight; retry" };
|
|
5954
|
+
return { ok: true, data: status };
|
|
5955
|
+
}
|
|
5956
|
+
catch (e) {
|
|
5957
|
+
return { ok: false, error: e.message };
|
|
5958
|
+
}
|
|
5959
|
+
}
|
|
5773
5960
|
// ---- the peer-readable liveness surface (#1577) --------------------------
|
|
5774
5961
|
/** Bind this endpoint as the liveness RESPONDER for one plane, answering presence and nothing
|
|
5775
5962
|
* else to any credentialed peer that asks (`live.<plane>.<owner>.<actor>`).
|
|
@@ -6663,6 +6850,92 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
6663
6850
|
isNoResponders(e) {
|
|
6664
6851
|
return e instanceof NoRespondersError || (e instanceof RequestError && e.isNoResponders());
|
|
6665
6852
|
}
|
|
6853
|
+
/** Holder side of a take (SPEC §8.1): ask the delivery daemon to record this endpoint's take of the
|
|
6854
|
+
* request at `seq`, for the copy with `count` deliveries. Never throws; a take the daemon could
|
|
6855
|
+
* not decide is reported as held, with what the operator has to do when that is anything. */
|
|
6856
|
+
async takeAnycast(role, seq, id, count, epoch, nak) {
|
|
6857
|
+
const nc = this.nc;
|
|
6858
|
+
if (!nc)
|
|
6859
|
+
return { taken: false, drop: false, reason: "daemon-down", detail: this.notLiveMsg() };
|
|
6860
|
+
const held = (reason, detail) => ({ taken: false, drop: false, reason, detail });
|
|
6861
|
+
const subject = takeSubject(this.space, role, this.owner, this.actor);
|
|
6862
|
+
const body = { from: { id: this.card.id, name: this.card.name }, seq, id, count, epoch };
|
|
6863
|
+
let reply;
|
|
6864
|
+
try {
|
|
6865
|
+
const m = await this.requestBoundReply(nc, subject, JSON.stringify(body), `${subject}.reply.${randomUUID()}`, ANYCAST_TAKE_TIMEOUT_MS);
|
|
6866
|
+
reply = m.json();
|
|
6867
|
+
}
|
|
6868
|
+
catch (e) {
|
|
6869
|
+
if (isPermissionDenied(e))
|
|
6870
|
+
return held("credential-outdated", "this seat's credential was minted before anycast takes existed, so it cannot take role requests; respawn the seat to get a current credential");
|
|
6871
|
+
if (this.isNoResponders(e))
|
|
6872
|
+
return this.takeResponderMissing();
|
|
6873
|
+
return held("daemon-down", `the delivery daemon did not answer the take: ${e.message}`);
|
|
6874
|
+
}
|
|
6875
|
+
if (reply.ok)
|
|
6876
|
+
return { taken: true };
|
|
6877
|
+
// A mismatch can mean the stream was recreated under this consumer, which resumes on the new
|
|
6878
|
+
// stream: read its creation time again, so the copies delivered from here on carry it. When the
|
|
6879
|
+
// time moved, this copy is nakked, so the broker delivers it again at once with the new time. A
|
|
6880
|
+
// redelivery that still mismatches carries the time read here, so it is forgotten, not nakked.
|
|
6881
|
+
if (reply.code === "mismatch") {
|
|
6882
|
+
const current = await this.readTaskEpoch().catch(() => undefined);
|
|
6883
|
+
if (current !== undefined && current !== epoch)
|
|
6884
|
+
nak();
|
|
6885
|
+
}
|
|
6886
|
+
if (reply.code === "cancelled" || reply.code === "taken" || reply.code === "gone" || reply.code === "mismatch")
|
|
6887
|
+
return { taken: false, drop: true, reason: reply.code, detail: reply.error };
|
|
6888
|
+
return held("daemon-busy", reply.error);
|
|
6889
|
+
}
|
|
6890
|
+
/** No responder on the take rail: an older daemon answers the probe op as unsupported, and a
|
|
6891
|
+
* stopped one does not answer at all. */
|
|
6892
|
+
async takeResponderMissing() {
|
|
6893
|
+
const held = (reason, detail) => ({ taken: false, drop: false, reason, detail });
|
|
6894
|
+
let reply;
|
|
6895
|
+
try {
|
|
6896
|
+
reply = await this.requestDelivery(ANYCAST_TAKES_OP, {}, ANYCAST_TAKE_TIMEOUT_MS);
|
|
6897
|
+
}
|
|
6898
|
+
catch {
|
|
6899
|
+
return held("daemon-down", "no delivery daemon is answering on this mesh, so role requests wait on the queue until one does");
|
|
6900
|
+
}
|
|
6901
|
+
if (!reply.ok && /not supported on the delivery control service/.test(reply.error ?? ""))
|
|
6902
|
+
return held("daemon-outdated", "the delivery daemon predates anycast takes, so this seat holds role requests until the daemon is restarted at this version (`cotal up` runs it)");
|
|
6903
|
+
if (!reply.ok && reply.error?.includes(ANYCAST_RECORDS_CREDENTIAL_OUTDATED))
|
|
6904
|
+
return held("daemon-outdated", `this seat holds role requests until the delivery daemon can record takes: ${ANYCAST_RECORDS_CREDENTIAL_OUTDATED}`);
|
|
6905
|
+
return held("daemon-down", "the delivery daemon is not answering takes, so role requests wait on the queue until it does");
|
|
6906
|
+
}
|
|
6907
|
+
/** The status of one of this endpoint's own anycast requests, from the delivery daemon's record
|
|
6908
|
+
* or, with no record, from the queue (SPEC §8.1). `id` narrows the answer to that request. Throws
|
|
6909
|
+
* on an open mesh, which runs no delivery daemon. */
|
|
6910
|
+
async anycastStatus(seq, id) {
|
|
6911
|
+
return this.anycastRequest(ANYCAST_STATUS_OP, seq, id);
|
|
6912
|
+
}
|
|
6913
|
+
/** Cancel one of this endpoint's own anycast requests while it waits. A request a holder already
|
|
6914
|
+
* took is refused, and the returned status names the taker. */
|
|
6915
|
+
async cancelAnycast(seq, id) {
|
|
6916
|
+
return this.anycastRequest(ANYCAST_CANCEL_OP, seq, id);
|
|
6917
|
+
}
|
|
6918
|
+
async anycastRequest(op, seq, id) {
|
|
6919
|
+
if (!this.authed)
|
|
6920
|
+
throw new Error("an open mesh runs no delivery daemon, so it keeps no record of anycast requests");
|
|
6921
|
+
if (!isAnycastSeq(seq))
|
|
6922
|
+
throw new Error(`seq must be a whole number of at least 1, got ${seq}`);
|
|
6923
|
+
let reply;
|
|
6924
|
+
try {
|
|
6925
|
+
reply = await this.requestDelivery(op, { seq, ...(id !== undefined ? { id } : {}) }, 10_000);
|
|
6926
|
+
}
|
|
6927
|
+
catch (e) {
|
|
6928
|
+
if (this.isNoResponders(e))
|
|
6929
|
+
throw new Error("no delivery daemon is answering on this mesh, so the request's record cannot be read");
|
|
6930
|
+
throw e;
|
|
6931
|
+
}
|
|
6932
|
+
if (!reply.ok) {
|
|
6933
|
+
if (/not supported on the delivery control service/.test(reply.error ?? ""))
|
|
6934
|
+
throw new Error("the delivery daemon predates anycast request status; restart it at this version (`cotal up` runs it)");
|
|
6935
|
+
throw new Error(reply.error ?? `${op} failed`);
|
|
6936
|
+
}
|
|
6937
|
+
return reply.data;
|
|
6938
|
+
}
|
|
6666
6939
|
/** Agent-side: this session's CURRENT durable memberships (channel + join generation) from the
|
|
6667
6940
|
* manager — the agent holds no read on the privileged members KV. `undefined` ⇒ NO control responder
|
|
6668
6941
|
* (open / no delivery daemon, so there is no Plane-3 and no memberships). THROWS on a responder-present RPC
|
|
@@ -6820,7 +7093,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
6820
7093
|
if (!this.authed) {
|
|
6821
7094
|
await this.jsm.consumers.add(taskStream(this.space), taskDurableConfig(this.space, this.card.role, { ackWaitMs: this.ackWaitMs }));
|
|
6822
7095
|
}
|
|
6823
|
-
await this.pump(taskStream(this.space), taskDurable(this.card.role));
|
|
7096
|
+
await this.pump(taskStream(this.space), taskDurable(this.card.role), this.authed ? { takeRole: this.card.role } : {});
|
|
6824
7097
|
}
|
|
6825
7098
|
}
|
|
6826
7099
|
/** Drive one consumer: decode and hand each message to listeners with ack control. Our own sends are
|
|
@@ -6829,7 +7102,9 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
6829
7102
|
async pump(stream, durable, opts = {}) {
|
|
6830
7103
|
if (!this.js)
|
|
6831
7104
|
throw new Error(this.notLiveMsg());
|
|
6832
|
-
const consumer =
|
|
7105
|
+
const consumer = opts.takeRole === undefined
|
|
7106
|
+
? await this.js.consumers.get(stream, durable)
|
|
7107
|
+
: await this.bindTaskConsumer(stream, durable);
|
|
6833
7108
|
const ackWaitMs = await ackWaitOf(consumer);
|
|
6834
7109
|
const msgs = await consumer.consume();
|
|
6835
7110
|
this.streamMsgs.push(msgs);
|
|
@@ -6841,7 +7116,16 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
6841
7116
|
if (!authed)
|
|
6842
7117
|
continue;
|
|
6843
7118
|
const { msg, parsed } = authed;
|
|
6844
|
-
|
|
7119
|
+
const delivery = durableDelivery(m, ackWaitMs);
|
|
7120
|
+
// On an auth mesh a role request carries its take: the holder records it with the delivery
|
|
7121
|
+
// daemon before showing the request, so its sender can cancel it until then (SPEC §8.1).
|
|
7122
|
+
// The copy carries the creation time of the stream it came from, read at bind.
|
|
7123
|
+
const role = opts.takeRole;
|
|
7124
|
+
if (role !== undefined) {
|
|
7125
|
+
const epoch = this.taskEpoch;
|
|
7126
|
+
delivery.take = () => this.takeAnycast(role, m.seq, msg.id, m.info.deliveryCount, epoch, () => m.nak());
|
|
7127
|
+
}
|
|
7128
|
+
this.emit("message", authenticatedMessage(msg, parsed), delivery, {
|
|
6845
7129
|
historical: false,
|
|
6846
7130
|
kind: kindFromParsed(parsed.kind),
|
|
6847
7131
|
});
|
|
@@ -6851,6 +7135,23 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
6851
7135
|
this.emit("error", e);
|
|
6852
7136
|
});
|
|
6853
7137
|
}
|
|
7138
|
+
/** Bind the role consumer and read the TASK stream's creation time on both sides of the bind, so
|
|
7139
|
+
* the time belongs to the stream the consumer was bound on (SPEC §8.1). */
|
|
7140
|
+
async bindTaskConsumer(stream, durable) {
|
|
7141
|
+
for (let attempt = 0; attempt < 3; attempt++) {
|
|
7142
|
+
const before = await this.readTaskEpoch();
|
|
7143
|
+
const consumer = await this.js.consumers.get(stream, durable);
|
|
7144
|
+
if (before === (await this.readTaskEpoch()))
|
|
7145
|
+
return consumer;
|
|
7146
|
+
}
|
|
7147
|
+
throw new Error(`the ${stream} stream was recreated while its role consumer was binding; reconnect to bind it again`);
|
|
7148
|
+
}
|
|
7149
|
+
/** Read the TASK stream's creation time into {@link taskEpoch}. */
|
|
7150
|
+
async readTaskEpoch() {
|
|
7151
|
+
const epoch = (await (await this.manager()).streams.info(taskStream(this.space))).created;
|
|
7152
|
+
this.taskEpoch = epoch;
|
|
7153
|
+
return epoch;
|
|
7154
|
+
}
|
|
6854
7155
|
/** Warn once when the broker reports this endpoint's own DM inbox missing while the connection is
|
|
6855
7156
|
* up, as after a store reset that the client's reconnect rode through, or after a manager restart
|
|
6856
7157
|
* deletes the inbox. nats.js keeps checking and resumes pulling if the inbox comes back, so this
|