@cotal-ai/core 0.166.3 → 0.168.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-endpoint.d.ts +242 -0
- package/dist/agent-endpoint.d.ts.map +1 -0
- package/dist/agent-endpoint.js +183 -0
- package/dist/agent-endpoint.js.map +1 -0
- package/dist/backup-config.d.ts +2 -2
- package/dist/endpoint-call.d.ts +36 -1
- package/dist/endpoint-call.d.ts.map +1 -1
- package/dist/endpoint-call.js +218 -116
- package/dist/endpoint-call.js.map +1 -1
- package/dist/endpoint-cluster.d.ts +6 -1
- package/dist/endpoint-cluster.d.ts.map +1 -1
- package/dist/endpoint-cluster.js +25 -1
- package/dist/endpoint-cluster.js.map +1 -1
- package/dist/endpoint-delivery-lease.d.ts +184 -0
- package/dist/endpoint-delivery-lease.d.ts.map +1 -0
- package/dist/endpoint-delivery-lease.js +270 -0
- package/dist/endpoint-delivery-lease.js.map +1 -0
- package/dist/endpoint-envelope.d.ts +4 -3
- package/dist/endpoint-envelope.d.ts.map +1 -1
- package/dist/endpoint-envelope.js +19 -2
- package/dist/endpoint-envelope.js.map +1 -1
- package/dist/endpoint-grants.d.ts +31 -1
- package/dist/endpoint-grants.d.ts.map +1 -1
- package/dist/endpoint-grants.js +62 -1
- package/dist/endpoint-grants.js.map +1 -1
- package/dist/endpoint-presence.d.ts +296 -0
- package/dist/endpoint-presence.d.ts.map +1 -0
- package/dist/endpoint-presence.js +784 -0
- package/dist/endpoint-presence.js.map +1 -0
- package/dist/endpoint-provenance.d.ts +11 -0
- package/dist/endpoint-provenance.d.ts.map +1 -0
- package/dist/endpoint-provenance.js +20 -0
- package/dist/endpoint-provenance.js.map +1 -0
- package/dist/endpoint-serve.d.ts +76 -2
- package/dist/endpoint-serve.d.ts.map +1 -1
- package/dist/endpoint-serve.js +332 -147
- package/dist/endpoint-serve.js.map +1 -1
- package/dist/endpoint-service.d.ts +5 -1
- package/dist/endpoint-service.d.ts.map +1 -1
- package/dist/endpoint-service.js +2 -1
- package/dist/endpoint-service.js.map +1 -1
- package/dist/endpoint-subjects.d.ts +76 -2
- package/dist/endpoint-subjects.d.ts.map +1 -1
- package/dist/endpoint-subjects.js +113 -2
- package/dist/endpoint-subjects.js.map +1 -1
- package/dist/endpoint-verbs.d.ts +1 -1
- package/dist/endpoint-verbs.d.ts.map +1 -1
- package/dist/endpoint-verbs.js +1 -1
- package/dist/endpoint-verbs.js.map +1 -1
- package/dist/endpoint.d.ts +132 -188
- package/dist/endpoint.d.ts.map +1 -1
- package/dist/endpoint.js +380 -790
- package/dist/endpoint.js.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/peer-label.d.ts +6 -5
- package/dist/peer-label.d.ts.map +1 -1
- package/dist/peer-label.js +7 -6
- package/dist/peer-label.js.map +1 -1
- package/dist/provision-jwt.d.ts +518 -0
- package/dist/provision-jwt.d.ts.map +1 -0
- package/dist/provision-jwt.js +641 -0
- package/dist/provision-jwt.js.map +1 -0
- package/dist/provision-permissions.d.ts +72 -0
- package/dist/provision-permissions.d.ts.map +1 -0
- package/dist/provision-permissions.js +2009 -0
- package/dist/provision-permissions.js.map +1 -0
- package/dist/provision-resources.d.ts +66 -0
- package/dist/provision-resources.d.ts.map +1 -0
- package/dist/provision-resources.js +61 -0
- package/dist/provision-resources.js.map +1 -0
- package/dist/provision-server-config.d.ts +115 -0
- package/dist/provision-server-config.d.ts.map +1 -0
- package/dist/provision-server-config.js +169 -0
- package/dist/provision-server-config.js.map +1 -0
- package/dist/provision.d.ts +19 -701
- package/dist/provision.d.ts.map +1 -1
- package/dist/provision.js +4 -2835
- package/dist/provision.js.map +1 -1
- package/dist/subjects.d.ts +3 -2
- package/dist/subjects.d.ts.map +1 -1
- package/dist/subjects.js +10 -2
- package/dist/subjects.js.map +1 -1
- package/dist/types.d.ts +11 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +1 -1
package/dist/endpoint.js
CHANGED
|
@@ -9,7 +9,8 @@ import { createConnection, isIP } from "node:net";
|
|
|
9
9
|
import { connect, headers, tokenAuthenticator, nanos, millis, AuthorizationError, PermissionViolationError, UserAuthenticationExpiredError, NoRespondersError, RequestError, TimeoutError, ClosedConnectionError, ConnectionError, } from "@nats-io/transport-node";
|
|
10
10
|
import { wsconnect } from "@nats-io/nats-core";
|
|
11
11
|
import { parseLivenessAnswer, responderFromLease, responderFromProbe, isLivenessPlane, LIVENESS_PLANES, } from "./liveness.js";
|
|
12
|
-
import {
|
|
12
|
+
import { EndpointPresence } from "./endpoint-presence.js";
|
|
13
|
+
export { PresenceWriteStuckError } from "./endpoint-presence.js";
|
|
13
14
|
import { credsClaims, credsFingerprint, credsRenewalDelayMs, idFromCreds } from "./identity.js";
|
|
14
15
|
import { nativeCredsAuthenticator } from "./creds-authenticator.js";
|
|
15
16
|
import { armRenewal, renewalDeadlineMs, jitteredBackoffMs } from "./renewal.js";
|
|
@@ -20,7 +21,11 @@ import { parseSecretStoreIdentity, } from "./secret-store.js";
|
|
|
20
21
|
import { BIND_SPLIT_REISSUES, resolveService, invokeCommand, goalFollowRefusal, submitAndFollowGoal } from "./endpoint-invoke.js";
|
|
21
22
|
import { EpEnvelopeError, respondedButUnbound, replyRefusedBeforeEffect, replyTargetUnmapped, EP_BIND_REFUSED } from "./endpoint-envelope.js";
|
|
22
23
|
import { isRepeatSafeCommand } from "./endpoint-grants.js";
|
|
23
|
-
import { assertIdToken, assertGeneration } from "./endpoint-subjects.js";
|
|
24
|
+
import { assertIdToken, assertGeneration, assertRuntimeToken, endpointToken } from "./endpoint-subjects.js";
|
|
25
|
+
import { servePrincipalRoute, describeContracts } from "./endpoint-serve.js";
|
|
26
|
+
import { epPrincipalCall } from "./endpoint-call.js";
|
|
27
|
+
import { AGENT_CONTRACTS, AGENT_ENDPOINT } from "./agent-endpoint.js";
|
|
28
|
+
import { processRuntime } from "./process-runtime.js";
|
|
24
29
|
import { importNativeSubjectPermissions, issuedPermitsSubject, readAcceptedRow } from "./issued-authority.js";
|
|
25
30
|
import { REGISTRY_ENDPOINT, REGISTRY_ENDPOINTS_COMMAND } from "./registry-contract.js";
|
|
26
31
|
import { listRegistryEndpoints, registryListingCaller } from "./registry-client.js";
|
|
@@ -32,7 +37,7 @@ import { assertValidKind } from "./agent-file.js";
|
|
|
32
37
|
import { assertValidName } from "./resolve.js";
|
|
33
38
|
import { EVICT_PRINCIPALS_MAX } from "./evict.js";
|
|
34
39
|
import { nextHeartbeat } from "./presence-heartbeat.js";
|
|
35
|
-
import { anycastStoreDefinition, createSpaceStreams, dmDurableConfig, dlvDurableConfig, taskDurableConfig, fanoutDurableConfig, inboxReaderConfig, MANAGER_LEASE_TTL_MS, MANAGER_LEASE_ATTEMPT_MS,
|
|
40
|
+
import { anycastStoreDefinition, createSpaceStreams, dmDurableConfig, dlvDurableConfig, taskDurableConfig, fanoutDurableConfig, inboxReaderConfig, MANAGER_LEASE_TTL_MS, MANAGER_LEASE_ATTEMPT_MS, PRESENCE_STORAGE } from "./streams.js";
|
|
36
41
|
import { DELIVERY_LIMIT_HEADER, HISTORY_NO_COUNT, PLANE3_MAX_MSGS_PER_SUBJECT } from "./backup-config.js";
|
|
37
42
|
import { jetstream, jetstreamManager, AckPolicy, DeliverPolicy, JetStreamApiCodes, JetStreamApiError, } from "@nats-io/jetstream";
|
|
38
43
|
import {} from "@nats-io/jetstream";
|
|
@@ -40,11 +45,11 @@ import { Kvm } from "@nats-io/kv";
|
|
|
40
45
|
import { Bucket, KvWatchInclude } from "@nats-io/kv/internal";
|
|
41
46
|
import { openMembersRegistry, commitMemberRevision, tombstoneMember, activateMember, readMember, deleteMember, listMembers, listLifecycleMemberChannels, durableEligible, } from "./members.js";
|
|
42
47
|
import { openAclRegistry, readAcl, readAclForAlias, aclRetired, AmbiguousAclAlias, commitAcl as writeAclRecord, reissueAcl as writeAclReissue } from "./acls.js";
|
|
43
|
-
import {
|
|
48
|
+
import { EndpointDeliveryLease } from "./endpoint-delivery-lease.js";
|
|
44
49
|
import { openChannelRegistry, effectiveReplay, effectiveReplayWindowMs, effectiveDeliveryClass, readChannelConfig, readChannelDefaults, decodeChannelEntry, } from "./channels.js";
|
|
45
50
|
import { isCasLoss } from "./endpoint-records.js";
|
|
46
51
|
import { ANYCAST_CANCEL_OP, ANYCAST_NOTICE_KEY, ANYCAST_RECORDS_CREDENTIAL_OUTDATED, ANYCAST_STATUS_OP, ANYCAST_TAKES_OP, AnycastRequestAuthority, isAnycastSeq, } from "./anycast-requests.js";
|
|
47
|
-
import { anycastSubject, assertValidOwnerToken, assertValidRole, CHANNEL_DEFAULTS_KEY, chatStream, chatHistDurable, chatSubject, historyReaderConsumer, calloutReaderId, HISTORY_READERS, controlServiceSubject, livenessSubject, livenessServeFilter, CONTROL_DELIVERY, CONTROL_DELIVERY_ADMIN, dmStream, dmDurable, dlvStream, dlvDurable, dlvSubject, dinboxSubject, inboxStream, parseDinboxPrincipal, FANOUT_DURABLE, INBOX_READER_DURABLE,
|
|
52
|
+
import { anycastSubject, assertValidOwnerToken, assertValidRole, CHANNEL_DEFAULTS_KEY, chatStream, chatHistDurable, chatSubject, historyReaderConsumer, calloutReaderId, HISTORY_READERS, controlServiceSubject, livenessSubject, livenessServeFilter, CONTROL_DELIVERY, CONTROL_DELIVERY_ADMIN, dmStream, dmDurable, dlvStream, dlvDurable, dlvSubject, dinboxSubject, inboxStream, parseDinboxPrincipal, FANOUT_DURABLE, INBOX_READER_DURABLE, managerBucket, MANAGER_LEASE_KEY, MANAGER_RENEWAL_LEASE_KEY, managerLeaseKey, assertValidChannel, channelInAllow, membershipInAllow, isConcreteChannel, isChannelFamily, familiesCovering, normalizeMentions, MENTION_NOTICE_KEY, parseSubject, isPrincipalOwnerToken, assertInboxConnId, presenceBucket, channelBucket, membershipBucket, MEMBERSHIP_FEED_KEY, principalKey, parsePrincipalKey, assertLifecycleToken, mintLifecycleUid, lifecycleNameKey, memberKey, DEV_OWNER, spacePrefix, spaceWildcard, subjectMatches, taskStream, taskDurable, takeServeFilter, takeSubject, parseTakeSubject, TAKE_ROUTE, token, unicastSubject, unicastRecvFilter, } from "./subjects.js";
|
|
48
53
|
import { isPermissionDenied } from "./permission-denied.js";
|
|
49
54
|
export { isPermissionDenied };
|
|
50
55
|
export const DEFAULT_SERVER = "nats://127.0.0.1:4222";
|
|
@@ -128,27 +133,6 @@ export class DmBindRefusedError extends Error {
|
|
|
128
133
|
this.name = "DmBindRefusedError";
|
|
129
134
|
}
|
|
130
135
|
}
|
|
131
|
-
/** A presence bucket that has refused consecutive writes for at least one full presence TTL.
|
|
132
|
-
* The condition is non-transient: ordinary heartbeat retry has already failed for the whole
|
|
133
|
-
* liveness window, so callers should report the roster as last-known until a write succeeds. */
|
|
134
|
-
export class PresenceWriteStuckError extends Error {
|
|
135
|
-
bucket;
|
|
136
|
-
since;
|
|
137
|
-
consecutiveFailures;
|
|
138
|
-
ttlMs;
|
|
139
|
-
lastError;
|
|
140
|
-
code = "presence-write-stuck";
|
|
141
|
-
transient = false;
|
|
142
|
-
constructor(bucket, since, consecutiveFailures, ttlMs, lastError) {
|
|
143
|
-
super(`presence writes to bucket ${JSON.stringify(bucket)} have failed ${consecutiveFailures} consecutive times for at least one TTL (${ttlMs}ms); the presence view is not live until a write succeeds or the broker store is repaired${lastError ? ` (last refusal: ${lastError})` : ""}`);
|
|
144
|
-
this.bucket = bucket;
|
|
145
|
-
this.since = since;
|
|
146
|
-
this.consecutiveFailures = consecutiveFailures;
|
|
147
|
-
this.ttlMs = ttlMs;
|
|
148
|
-
this.lastError = lastError;
|
|
149
|
-
this.name = "PresenceWriteStuckError";
|
|
150
|
-
}
|
|
151
|
-
}
|
|
152
136
|
/**
|
|
153
137
|
* Events: "message" (CotalMessage), "presence" (PresenceEvent), "roster" (Presence[]), "error" (Error),
|
|
154
138
|
* "connection" ({ connected: boolean }) — true on every successful (re)bind (initial start, manual
|
|
@@ -226,6 +210,13 @@ function membershipChange(e, update) {
|
|
|
226
210
|
}
|
|
227
211
|
return { type: "member", id: e.key, operation, revision: e.revision, update, member };
|
|
228
212
|
}
|
|
213
|
+
/** The highest `prompts` presence `seq` this process used under each runtime id (SPEC §6). A reader
|
|
214
|
+
* orders lists by `seq` within one `runtime.id`, and a runtime id lives as long as the process, so
|
|
215
|
+
* this does too: across a stopped serve, a stopped endpoint and a new endpoint for the lifecycle. */
|
|
216
|
+
const promptsSeqs = new Map();
|
|
217
|
+
/** The principal endpoint each `{space, runtime id, endpoint}` is served by in this process. Two
|
|
218
|
+
* would subscribe the same subject and both answer one request (SPEC 13.21). */
|
|
219
|
+
const principalServers = new Map();
|
|
229
220
|
export class CotalEndpoint extends EventEmitter {
|
|
230
221
|
card;
|
|
231
222
|
space;
|
|
@@ -295,11 +286,6 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
295
286
|
jsm;
|
|
296
287
|
kv;
|
|
297
288
|
channelKv;
|
|
298
|
-
/** The presence/channel-registry watches' own handles. Each is an ORDERED push consumer: its idle
|
|
299
|
-
* heartbeat monitor lives on a JS timer independent of the connection, so a drain that doesn't
|
|
300
|
-
* `.stop()` it first leaves the monitor to fire into a closing connection every 30s, throwing
|
|
301
|
-
* `DrainingConnectionError` out of `reset()` on a timer nothing awaits. */
|
|
302
|
-
presenceWatchIter;
|
|
303
289
|
channelWatchIter;
|
|
304
290
|
/** Consumer deletes of retired presence and channel-registry watches still in flight. A stop
|
|
305
291
|
* alone leaves each watch's ordered consumer on the broker for its five-minute inactive
|
|
@@ -310,23 +296,10 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
310
296
|
* {@link stop} or a rebuild moved the epoch retires its watch and starts that delete itself, so
|
|
311
297
|
* the same two paths wait for these first. */
|
|
312
298
|
kvWatchBinds = new Set();
|
|
313
|
-
/** The daemon's lease-loss watch (#1596): the INTENT that survives a connection rebuild, plus the
|
|
314
|
-
* live iterator. See {@link watchDeliveryLease}. */
|
|
315
|
-
leaseWatchIntent;
|
|
316
|
-
leaseWatchIter;
|
|
317
|
-
/** Consumer deletes of retired lease watches still in flight. The stop handle cannot await its
|
|
318
|
-
* delete, so {@link stop} and a live-epoch rebuild await these before they close the connection
|
|
319
|
-
* that carries them (#3966). */
|
|
320
|
-
leaseWatchDeletes = new Set();
|
|
321
|
-
/** Lease-watch binds still awaiting the broker. One that lands after {@link stop} or a rebuild
|
|
322
|
-
* moved the epoch retires its watch and starts that delete itself, so the same two paths wait for
|
|
323
|
-
* these first. Each is bounded by the JetStream request timeout. */
|
|
324
|
-
leaseWatchBinds = new Set();
|
|
325
299
|
/** Plane-3 durable-membership registry KV — lazily opened by the privileged delivery daemon (or a
|
|
326
300
|
* short-lived provisioner). */
|
|
327
301
|
membersKv;
|
|
328
302
|
aclKv;
|
|
329
|
-
deliveryKv;
|
|
330
303
|
managerLeaseKv;
|
|
331
304
|
/** Our revision of the per-space daemon-credential renewal lease (#1634), or undefined when we do
|
|
332
305
|
* not hold it. Cleared with the bound KV handle: a revision from a dead connection is not a lease
|
|
@@ -392,6 +365,12 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
392
365
|
* subscription on it, so a responder bound once would leave its plane to the broker's
|
|
393
366
|
* no-responders answer, which a peer grades `unbound`, while this endpoint is connected and serving. */
|
|
394
367
|
livenessResponders = new Map();
|
|
368
|
+
/** The principal endpoints {@link servePrincipal} serves, by endpoint name: INTENT, re-bound on
|
|
369
|
+
* every replaced connection. */
|
|
370
|
+
principalServes = new Map();
|
|
371
|
+
/** The `prompts` presence field {@link setPromptsPresence} last set. Shown only while the `agent`
|
|
372
|
+
* endpoint is served and its subscription was accepted on the current connection. */
|
|
373
|
+
promptsPresence;
|
|
395
374
|
streamMsgs = [];
|
|
396
375
|
/** Per-channel native core subscriptions (SPEC v0.3) — the manager-free live read path for boot +
|
|
397
376
|
* runtime channels (there is no per-instance chat durable). Keyed by channel so leave unsubscribes
|
|
@@ -439,71 +418,42 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
439
418
|
firstConnect = true;
|
|
440
419
|
heartbeatTimer;
|
|
441
420
|
sweepTimer;
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
*/
|
|
474
|
-
lastPresenceWatchAt = 0;
|
|
475
|
-
/** Last emitted presence-view state. Suppresses duplicate `presence-view` events. */
|
|
476
|
-
presenceViewState = "unpopulated";
|
|
477
|
-
/** A presence-watch rebind in flight (see {@link rebindStalePresenceWatch}); one at a time. */
|
|
478
|
-
presenceRebind;
|
|
421
|
+
deliveryLease = new EndpointDeliveryLease({
|
|
422
|
+
card: () => this.card,
|
|
423
|
+
space: () => this.space,
|
|
424
|
+
connection: () => this.nc,
|
|
425
|
+
epoch: () => this.presenceEpoch,
|
|
426
|
+
notLiveMsg: () => this.notLiveMsg(),
|
|
427
|
+
deleteWatchConsumer: (iter) => this.deleteOwnConsumer(kvWatchConsumer(iter)),
|
|
428
|
+
recoverable: (error) => this.emitRecoverable(error),
|
|
429
|
+
emit: (event, error) => this.emit(event, error),
|
|
430
|
+
});
|
|
431
|
+
presence = new EndpointPresence({
|
|
432
|
+
card: () => this.card,
|
|
433
|
+
space: () => this.space,
|
|
434
|
+
bucket: () => this.kv,
|
|
435
|
+
connection: () => this.nc,
|
|
436
|
+
lifecycleUid: () => this.ownLifecycleUid,
|
|
437
|
+
prompts: () => this.servedPrompts(),
|
|
438
|
+
ttlMs: () => this.ttlMs,
|
|
439
|
+
registers: () => this.doRegister,
|
|
440
|
+
watches: () => this.doWatch,
|
|
441
|
+
readsOnce: () => this.readOnce,
|
|
442
|
+
stopped: () => this.stopped,
|
|
443
|
+
reconnecting: () => this.reconnecting,
|
|
444
|
+
epoch: () => this.presenceEpoch,
|
|
445
|
+
notLiveMsg: () => this.notLiveMsg(),
|
|
446
|
+
bindWatch: (kv) => this.bindKvWatch(kv),
|
|
447
|
+
retireWatch: (iter) => this.retireKvWatch(iter),
|
|
448
|
+
emit: (event, value) => this.emit(event, value),
|
|
449
|
+
recoverable: (error) => this.emitRecoverable(error),
|
|
450
|
+
isPublishPermissionDenied,
|
|
451
|
+
});
|
|
479
452
|
/** Bumped by every connection-scoped teardown and by {@link stop}. A presence or lease-watch bind
|
|
480
453
|
* that was awaiting the broker when the epoch moved belongs to a retired epoch: it releases the
|
|
481
454
|
* iterator it got and installs nothing (see {@link startPresenceWatch} and
|
|
482
455
|
* {@link bindDeliveryLeaseWatch}). */
|
|
483
456
|
presenceEpoch = 0;
|
|
484
|
-
/** The current watch was bound onto a bucket with NO keys (see {@link markPresenceBucketEmpty}).
|
|
485
|
-
* Such a watch cannot deliver until someone writes, so its silence is not staleness. */
|
|
486
|
-
presenceWatchEmpty = false;
|
|
487
|
-
/** Wall-clock of the last rebind attempt, so a bucket that is silent because it is EMPTY (or a
|
|
488
|
-
* broker that keeps refusing the consumer create) is retried once per TTL, not per sweep tick. */
|
|
489
|
-
presenceRebindAt = 0;
|
|
490
|
-
status = "idle";
|
|
491
|
-
activity;
|
|
492
|
-
/** When {@link status} or {@link activity} last changed. Heartbeats republish it unchanged. */
|
|
493
|
-
statusSince = Date.now();
|
|
494
|
-
/** When {@link activity} last changed. A status change leaves it alone. */
|
|
495
|
-
activitySince;
|
|
496
|
-
/** Last harness-reported work progress. Carried by the next heartbeat, never published per event. */
|
|
497
|
-
activeAt;
|
|
498
|
-
condition;
|
|
499
|
-
/** Advances on every condition change so an older in-flight put cannot be the final KV state. */
|
|
500
|
-
conditionRevision = 0;
|
|
501
|
-
/** Read once at construction. Core publishes this opaque provider reference and never parses it. */
|
|
502
|
-
environment;
|
|
503
|
-
/** Mirror of the connector's authoritative attention state, published in presence (advisory). The
|
|
504
|
-
* endpoint never reads these back into delivery — they exist only to broadcast. */
|
|
505
|
-
attentionMode;
|
|
506
|
-
channelModes;
|
|
507
457
|
stopped = false;
|
|
508
458
|
stopPromise;
|
|
509
459
|
startPromise;
|
|
@@ -527,6 +477,8 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
527
477
|
* now instead of awaiting the full retryMs. */
|
|
528
478
|
backoffResolve;
|
|
529
479
|
backoffTimer;
|
|
480
|
+
/** Wakes for the membership retry loops' pauses (see {@link membershipPause}); stop() runs them all. */
|
|
481
|
+
membershipPauses = new Set();
|
|
530
482
|
retryMs = 3000;
|
|
531
483
|
/** Consecutive failed rebuilds, driving {@link nextRetryDelayMs}. Reset on every success. */
|
|
532
484
|
retryAttempt = 0;
|
|
@@ -572,9 +524,6 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
572
524
|
* only evidence the split rate exists. Always on, never behind a flag — a counter you have to
|
|
573
525
|
* enable is not there when the thing you needed it for happened. */
|
|
574
526
|
splitsRecovered = 0;
|
|
575
|
-
/** Presence rows rejected because their embedded id did not match the ACL-scoped KV key, or because
|
|
576
|
-
* a field the observer computes with was malformed. */
|
|
577
|
-
presenceBindingDrops = 0;
|
|
578
527
|
/** This endpoint's wire principal (owner + actor tokens, §13.2) — what its minted grant rows
|
|
579
528
|
* pin. Public so a caller can build owner-mode target blocks for {@link invokeService}. */
|
|
580
529
|
get principal() {
|
|
@@ -587,7 +536,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
587
536
|
}
|
|
588
537
|
/** Mis-keyed or malformed presence rows this reader rejected. The warning event may be missed; the count cannot. */
|
|
589
538
|
get presenceBindingDropCount() {
|
|
590
|
-
return this.
|
|
539
|
+
return this.presence.bindingDropCount;
|
|
591
540
|
}
|
|
592
541
|
/** The endpoint's own lifecycle UID, REQUIRED for every lifecycle-keyed messaging resource; absent
|
|
593
542
|
* ⇒ loud refusal naming the operation (the hard cut of SPEC §13.1 — no alias-keyed fallback). */
|
|
@@ -708,8 +657,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
708
657
|
// principalKey validates both tokens.
|
|
709
658
|
const principal = principalKey(this.owner, this.actor);
|
|
710
659
|
this.card = { ...opts.card, id: principal.key, owner: this.owner, actor: this.actor };
|
|
711
|
-
this.environment
|
|
712
|
-
(opts.environment === undefined ? process.env.COTAL_ENVIRONMENT : opts.environment)?.trim() || undefined;
|
|
660
|
+
this.presence.configureEnvironment((opts.environment === undefined ? process.env.COTAL_ENVIRONMENT : opts.environment)?.trim() || undefined);
|
|
713
661
|
this.servers = opts.servers ?? DEFAULT_SERVER;
|
|
714
662
|
this.token = opts.token;
|
|
715
663
|
this.user = opts.user;
|
|
@@ -756,7 +704,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
756
704
|
}
|
|
757
705
|
// Seed the presence mirror so file-default channel modes are visible from the first publish
|
|
758
706
|
// (not only after the first runtime toggle). Mirror only — delivery reads the connector's state.
|
|
759
|
-
this.
|
|
707
|
+
this.presence.seedChannelModes(opts.channelModes);
|
|
760
708
|
this.ackWaitMs = opts.ackWaitMs ?? 60_000;
|
|
761
709
|
this.inactiveThresholdMs = opts.inactiveThresholdMs ?? 600_000;
|
|
762
710
|
}
|
|
@@ -1520,7 +1468,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
1520
1468
|
// clearConnectionScoped; a bind that completes after a stop or another rebuild finds the epoch
|
|
1521
1469
|
// moved and retires itself. The replay re-delivers the row's current state, so a change that
|
|
1522
1470
|
// landed in the null window still reaches the trigger.
|
|
1523
|
-
if (this.
|
|
1471
|
+
if (this.deliveryLease.hasWatchIntent)
|
|
1524
1472
|
await this.bindDeliveryLeaseWatch();
|
|
1525
1473
|
// FAIL BEFORE PRESENCE (SPEC 13.1): an AUTHED endpoint that will register on the roster or
|
|
1526
1474
|
// bind lifecycle-keyed consumers must hold its launcher-supplied lifecycle uid BEFORE anything
|
|
@@ -1606,6 +1554,8 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
1606
1554
|
// the old connection and the responders with it.
|
|
1607
1555
|
for (const plane of this.livenessResponders.keys())
|
|
1608
1556
|
this.bindLiveness(plane);
|
|
1557
|
+
// The principal serves subscribe again on the new connection and run their check again.
|
|
1558
|
+
this.rebindPrincipalServes(false);
|
|
1609
1559
|
for (const tap of this.taps)
|
|
1610
1560
|
this.bindTap(tap);
|
|
1611
1561
|
// Bound and live — covers initial start, manual reconnect, AND background self-heal (every
|
|
@@ -1657,10 +1607,8 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
1657
1607
|
}
|
|
1658
1608
|
this.streamMsgs.length = 0;
|
|
1659
1609
|
this.presenceEpoch++;
|
|
1660
|
-
this.
|
|
1661
|
-
|
|
1662
|
-
this.retireKvWatch(this.presenceWatchIter);
|
|
1663
|
-
this.presenceWatchIter = undefined;
|
|
1610
|
+
this.presence.cancelRebind();
|
|
1611
|
+
this.presence.retireWatch();
|
|
1664
1612
|
if (this.channelWatchIter)
|
|
1665
1613
|
this.retireKvWatch(this.channelWatchIter);
|
|
1666
1614
|
this.channelWatchIter = undefined;
|
|
@@ -1676,22 +1624,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
1676
1624
|
this.chatSubs.clear();
|
|
1677
1625
|
this.chatSubDenied.clear();
|
|
1678
1626
|
this.ownWatchDeletePrefixes.clear();
|
|
1679
|
-
this.
|
|
1680
|
-
// A read in flight on the old connection must not land in the new connection's roster.
|
|
1681
|
-
this.presenceRead = undefined;
|
|
1682
|
-
this.presenceReadGeneration++;
|
|
1683
|
-
this.presenceReadApplied = this.presenceReadGeneration;
|
|
1684
|
-
// #1356: the presence-refusal record is connection-scoped like everything else torn down here.
|
|
1685
|
-
// It says "the broker on THIS connection refuses writes to this bucket", so it cannot outlive the
|
|
1686
|
-
// connection that observed it. Left behind, a later failure on a different connection inherits it
|
|
1687
|
-
// and is reported as a bucket refusal — measured: a failed bind against an unreachable server
|
|
1688
|
-
// still printed the refusal sentence while `connectionIssue` already said "connection refused".
|
|
1689
|
-
this.clearPresenceWriteFailure();
|
|
1690
|
-
this.lastPresenceWatchAt = 0;
|
|
1691
|
-
this.presenceRebindAt = 0;
|
|
1692
|
-
this.presenceWatchEmpty = false;
|
|
1693
|
-
this.presenceSnapshotPopulated = false;
|
|
1694
|
-
this.emitPresenceViewIfChanged();
|
|
1627
|
+
this.presence.resetConnection();
|
|
1695
1628
|
this.channelConfigs.clear();
|
|
1696
1629
|
this.channelDefaults = {};
|
|
1697
1630
|
// A read in flight on the old connection must not land in the new connection's cache.
|
|
@@ -1737,7 +1670,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
1737
1670
|
this.membersKv = undefined;
|
|
1738
1671
|
this.aclKv = undefined;
|
|
1739
1672
|
this.membershipFeedKv = undefined;
|
|
1740
|
-
this.
|
|
1673
|
+
this.deliveryLease.resetConnection();
|
|
1741
1674
|
this.managerLeaseKv = undefined;
|
|
1742
1675
|
this.daemonRenewalLeaseRevision = undefined;
|
|
1743
1676
|
// Handles that armDeliveryControl created on a previous connection: null them so a rebind
|
|
@@ -1891,7 +1824,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
1891
1824
|
this.membersKv = undefined;
|
|
1892
1825
|
this.aclKv = undefined;
|
|
1893
1826
|
this.membershipFeedKv = undefined;
|
|
1894
|
-
this.
|
|
1827
|
+
this.deliveryLease.resetConnection();
|
|
1895
1828
|
// The manager's liveness-lease handle too: left bound to the old connection, every renew and
|
|
1896
1829
|
// re-read after a reconnect times out, and the manager reports its lease unknown for good.
|
|
1897
1830
|
this.managerLeaseKv = undefined;
|
|
@@ -1978,6 +1911,16 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
1978
1911
|
this.reestablishing = false;
|
|
1979
1912
|
}
|
|
1980
1913
|
}
|
|
1914
|
+
/** Run the next rebuild attempt now if the endpoint is waiting out a {@link reestablishLoop}
|
|
1915
|
+
* backoff, for a caller with work waiting on the link. After a long outage that backoff can run
|
|
1916
|
+
* for up to a minute past the broker's return, and a send made in it waited for an attempt
|
|
1917
|
+
* nobody had scheduled (#7408). Idle endpoints keep their spread-out retries. A no-op otherwise:
|
|
1918
|
+
* a rebuild in flight is already the attempt, and a link nats.js is still reconnecting keeps
|
|
1919
|
+
* its buffered publishes. */
|
|
1920
|
+
retryNow() {
|
|
1921
|
+
if (!this.stopped && this.reestablishing && !this.rebuildPromise)
|
|
1922
|
+
this.kickBackoff();
|
|
1923
|
+
}
|
|
1981
1924
|
/** Cut an in-flight reestablish backoff short so the next attempt runs immediately, and
|
|
1982
1925
|
* clear its timer so it can't fire later on a stopped/restarted loop. */
|
|
1983
1926
|
kickBackoff() {
|
|
@@ -1987,6 +1930,21 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
1987
1930
|
this.backoffTimer = undefined;
|
|
1988
1931
|
}
|
|
1989
1932
|
}
|
|
1933
|
+
/** Capped exponential pause between membership retries. stop() ends it early, so a loop waiting
|
|
1934
|
+
* on an absent delivery daemon never holds a stopped endpoint's process open for the delay. */
|
|
1935
|
+
membershipPause(attempt) {
|
|
1936
|
+
if (this.stopped)
|
|
1937
|
+
return Promise.resolve();
|
|
1938
|
+
return new Promise((resolve) => {
|
|
1939
|
+
const wake = () => {
|
|
1940
|
+
clearTimeout(timer);
|
|
1941
|
+
this.membershipPauses.delete(wake);
|
|
1942
|
+
resolve();
|
|
1943
|
+
};
|
|
1944
|
+
const timer = setTimeout(wake, Math.min(30_000, 1000 * 2 ** attempt));
|
|
1945
|
+
this.membershipPauses.add(wake);
|
|
1946
|
+
});
|
|
1947
|
+
}
|
|
1990
1948
|
/** Manual reconnect: tear down the current connection and rebuild, WITHOUT the permanent
|
|
1991
1949
|
* stop (stopped/stopping stay false). Serialized with the self-heal supervisor via
|
|
1992
1950
|
* {@link rebuild}, and interruptible — if a backoff is in flight, kick it so the attempt
|
|
@@ -2020,10 +1978,15 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
2020
1978
|
}
|
|
2021
1979
|
this.activeGoalFollowers.clear();
|
|
2022
1980
|
this.presenceEpoch++;
|
|
2023
|
-
|
|
1981
|
+
// Every principal serve ends now, so a `ready` still waiting for the broker rejects and no new
|
|
1982
|
+
// request is taken. Its handlers in flight finish below, before the connection they reply on closes.
|
|
1983
|
+
const principalStops = [...this.principalServes.values()].map((intent) => this.stopPrincipalServe(intent, false));
|
|
1984
|
+
this.presence.cancelRebind();
|
|
2024
1985
|
// Wake a reestablishLoop sitting in backoff so it sees `stopped` and exits instead of
|
|
2025
1986
|
// sleeping out retryMs; also clears the timer so it can't fire later.
|
|
2026
1987
|
this.kickBackoff();
|
|
1988
|
+
for (const wake of [...this.membershipPauses])
|
|
1989
|
+
wake();
|
|
2027
1990
|
if (this.heartbeatTimer)
|
|
2028
1991
|
clearInterval(this.heartbeatTimer);
|
|
2029
1992
|
if (this.sweepTimer)
|
|
@@ -2060,17 +2023,15 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
2060
2023
|
/* already closed */
|
|
2061
2024
|
}
|
|
2062
2025
|
}
|
|
2063
|
-
|
|
2064
|
-
this.retireKvWatch(this.presenceWatchIter);
|
|
2065
|
-
this.presenceWatchIter = undefined;
|
|
2026
|
+
this.presence.retireWatch();
|
|
2066
2027
|
if (this.channelWatchIter)
|
|
2067
2028
|
this.retireKvWatch(this.channelWatchIter);
|
|
2068
2029
|
this.channelWatchIter = undefined;
|
|
2069
2030
|
this.stopDeliveryLeaseWatch();
|
|
2031
|
+
await Promise.all(principalStops);
|
|
2070
2032
|
try {
|
|
2071
2033
|
if (this.doRegister) {
|
|
2072
|
-
this.
|
|
2073
|
-
await this.publishPresence();
|
|
2034
|
+
await this.presence.publishOffline();
|
|
2074
2035
|
}
|
|
2075
2036
|
}
|
|
2076
2037
|
catch {
|
|
@@ -3092,7 +3053,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
3092
3053
|
}
|
|
3093
3054
|
// ---- presence ------------------------------------------------------------
|
|
3094
3055
|
getRoster() {
|
|
3095
|
-
return
|
|
3056
|
+
return this.presence.getRoster();
|
|
3096
3057
|
}
|
|
3097
3058
|
/**
|
|
3098
3059
|
* Read the roster from the broker now, for an endpoint that does not watch presence (SPEC §6). One
|
|
@@ -3106,49 +3067,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
3106
3067
|
* nothing. A credential that may not make the read is refused with the grant it lacks.
|
|
3107
3068
|
*/
|
|
3108
3069
|
async readPresence() {
|
|
3109
|
-
|
|
3110
|
-
throw new Error("readPresence is for an endpoint that does not watch presence; this one keeps its roster live (getRoster)");
|
|
3111
|
-
const kv = this.kv;
|
|
3112
|
-
if (!kv)
|
|
3113
|
-
throw new Error(this.nc ? "readPresence needs the presence bucket open (registerPresence or watchPresence)" : this.notLiveMsg());
|
|
3114
|
-
if (this.presenceRead)
|
|
3115
|
-
return this.presenceRead;
|
|
3116
|
-
const generation = ++this.presenceReadGeneration;
|
|
3117
|
-
const roster = (async () => {
|
|
3118
|
-
let entries;
|
|
3119
|
-
try {
|
|
3120
|
-
entries = await lastKvEntries(kv);
|
|
3121
|
-
}
|
|
3122
|
-
catch (e) {
|
|
3123
|
-
if (!isPublishPermissionDenied(e))
|
|
3124
|
-
throw e;
|
|
3125
|
-
throw new Error(`presence read refused for ${this.card.id}: this credential may not publish $JS.API.DIRECT.GET.KV_${presenceBucket(this.space)}, the on-demand roster read (SPEC Appendix B). A credential minted by an older issuer can lack this grant even when newly minted; upgrade the manager or auth service that issues credentials, then mint a new credential. Reusing credentials or reconnecting does not add the grant.`, { cause: e });
|
|
3126
|
-
}
|
|
3127
|
-
const at = Date.now();
|
|
3128
|
-
const read = new Map();
|
|
3129
|
-
for (const e of entries) {
|
|
3130
|
-
if (e.key === TTL_RECONCILE_CANARY_KEY || e.operation === "DEL" || e.operation === "PURGE")
|
|
3131
|
-
continue;
|
|
3132
|
-
const p = this.decodePresence(e);
|
|
3133
|
-
if (p)
|
|
3134
|
-
read.set(e.key, p.status === "offline" || at - p.ts > this.ttlMs ? this.toOffline(p) : p);
|
|
3135
|
-
}
|
|
3136
|
-
if (generation > this.presenceReadApplied) {
|
|
3137
|
-
this.presenceReadApplied = generation;
|
|
3138
|
-
for (const [id, p] of this.roster)
|
|
3139
|
-
if (!read.has(id))
|
|
3140
|
-
read.set(id, p.status === "offline" ? p : this.toOffline(p));
|
|
3141
|
-
this.roster.clear();
|
|
3142
|
-
for (const [id, p] of read)
|
|
3143
|
-
this.roster.set(id, p);
|
|
3144
|
-
}
|
|
3145
|
-
return this.getRoster();
|
|
3146
|
-
})();
|
|
3147
|
-
this.presenceRead = roster;
|
|
3148
|
-
const settled = () => { if (this.presenceRead === roster)
|
|
3149
|
-
this.presenceRead = undefined; };
|
|
3150
|
-
roster.then(settled, settled);
|
|
3151
|
-
return roster;
|
|
3070
|
+
return this.presence.readPresence();
|
|
3152
3071
|
}
|
|
3153
3072
|
/**
|
|
3154
3073
|
* Trust state of THIS observer's presence watch, not of any peer. `unpopulated` means the
|
|
@@ -3158,82 +3077,40 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
3158
3077
|
* unsafe states so consumers written before `state` was added degrade in the safe direction.
|
|
3159
3078
|
*/
|
|
3160
3079
|
presenceView() {
|
|
3161
|
-
|
|
3162
|
-
return { state: "current", fresh: true };
|
|
3163
|
-
if (!this.presenceSnapshotPopulated)
|
|
3164
|
-
return { state: "unpopulated", fresh: false };
|
|
3165
|
-
if (this.presenceWatchEmpty)
|
|
3166
|
-
return { state: "current", fresh: true };
|
|
3167
|
-
const staleSince = this.lastPresenceWatchAt + this.ttlMs;
|
|
3168
|
-
if (Date.now() < staleSince)
|
|
3169
|
-
return { state: "current", fresh: true };
|
|
3170
|
-
return { state: "stale", fresh: false, staleSince };
|
|
3080
|
+
return this.presence.presenceView();
|
|
3171
3081
|
}
|
|
3172
3082
|
/** Wait until the current presence watch has consumed its initial KV snapshot. An observer that
|
|
3173
3083
|
* does not register completes on an empty bucket at once. A registering one completes when its
|
|
3174
3084
|
* own record arrives, and the timeout bounds that wait. */
|
|
3175
3085
|
async waitForPresenceSnapshot(timeoutMs = 1_000) {
|
|
3176
|
-
|
|
3177
|
-
try {
|
|
3178
|
-
return await Promise.race([
|
|
3179
|
-
this.presenceSnapshot.then(() => "snapshot"),
|
|
3180
|
-
new Promise((resolve) => { timer = setTimeout(() => resolve("timeout"), timeoutMs); }),
|
|
3181
|
-
]);
|
|
3182
|
-
}
|
|
3183
|
-
finally {
|
|
3184
|
-
if (timer)
|
|
3185
|
-
clearTimeout(timer);
|
|
3186
|
-
}
|
|
3187
|
-
}
|
|
3188
|
-
assertPresenceUpdate(update) {
|
|
3189
|
-
assertPresenceRow({ card: this.card, status: this.status, activity: this.activity,
|
|
3190
|
-
condition: this.condition, attention: this.attentionMode, ts: Date.now(), ...update }, this.card.id);
|
|
3086
|
+
return this.presence.waitForPresenceSnapshot(timeoutMs);
|
|
3191
3087
|
}
|
|
3192
3088
|
async setActivity(activity) {
|
|
3193
|
-
this.
|
|
3194
|
-
if (activity !== this.activity)
|
|
3195
|
-
this.statusSince = this.activitySince = Date.now();
|
|
3196
|
-
this.activity = activity;
|
|
3197
|
-
await this.publishPresence();
|
|
3089
|
+
return this.presence.setActivity(activity);
|
|
3198
3090
|
}
|
|
3199
3091
|
/** Set the status, and with `activity` the activity line too, in one presence write, so no peer sees
|
|
3200
3092
|
* the new activity beside the old status. */
|
|
3201
3093
|
async setStatus(status, activity) {
|
|
3202
|
-
this.
|
|
3203
|
-
if (activity !== undefined && activity !== this.activity)
|
|
3204
|
-
this.statusSince = this.activitySince = Date.now();
|
|
3205
|
-
if (activity !== undefined)
|
|
3206
|
-
this.activity = activity;
|
|
3207
|
-
if (status !== this.status)
|
|
3208
|
-
this.statusSince = Date.now();
|
|
3209
|
-
this.status = status;
|
|
3210
|
-
await this.publishPresence();
|
|
3094
|
+
return this.presence.setStatus(status, activity);
|
|
3211
3095
|
}
|
|
3212
3096
|
/** Record harness-reported work progress as `activeAt`. It writes nothing itself: the next heartbeat
|
|
3213
3097
|
* carries it, so a stream of events costs no presence writes. */
|
|
3214
3098
|
noteActivity(at = Date.now()) {
|
|
3215
|
-
|
|
3216
|
-
this.activeAt = at;
|
|
3099
|
+
return this.presence.noteActivity(at);
|
|
3217
3100
|
}
|
|
3218
3101
|
/** Publish a harness-reported condition, or clear it. Core stores the relay without interpretation. */
|
|
3219
3102
|
async setCondition(condition) {
|
|
3220
|
-
this.
|
|
3221
|
-
this.condition = condition ?? undefined;
|
|
3222
|
-
this.conditionRevision++;
|
|
3223
|
-
await this.publishPresence();
|
|
3103
|
+
return this.presence.setCondition(condition);
|
|
3224
3104
|
}
|
|
3225
3105
|
/** Publish the agent's global attention mode into presence (advisory observability). Mirror only —
|
|
3226
3106
|
* delivery decisions stay in the connector's authoritative state. */
|
|
3227
3107
|
async setAttention(attention) {
|
|
3228
|
-
this.
|
|
3229
|
-
this.attentionMode = attention;
|
|
3230
|
-
await this.publishPresence();
|
|
3108
|
+
return this.presence.setAttention(attention);
|
|
3231
3109
|
}
|
|
3232
3110
|
/** Publish the agent's per-channel attention overrides into presence (advisory). An empty map drops
|
|
3233
3111
|
* the field. Mirror only — never read back into delivery. */
|
|
3234
3112
|
async setChannelModes(modes) {
|
|
3235
|
-
this.
|
|
3236
|
-
await this.publishPresence();
|
|
3113
|
+
return this.presence.setChannelModes(modes);
|
|
3237
3114
|
}
|
|
3238
3115
|
/** Overlay the host's live model and optional variant onto the card's display-only metadata, then
|
|
3239
3116
|
* republish presence. For connectors that learn their actual selection only after launch (e.g.
|
|
@@ -3581,7 +3458,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
3581
3458
|
async channelMembers(channel) {
|
|
3582
3459
|
const members = (await listMembers(await this.membersRegistry())).filter((r) => r.leaveCursor === undefined && r.activated === true);
|
|
3583
3460
|
const byId = new Map();
|
|
3584
|
-
for (const p of this.
|
|
3461
|
+
for (const p of this.presence.values())
|
|
3585
3462
|
byId.set(p.card.id, p);
|
|
3586
3463
|
const memberForId = (id) => {
|
|
3587
3464
|
const p = byId.get(id);
|
|
@@ -4682,6 +4559,8 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
4682
4559
|
}
|
|
4683
4560
|
if (s.type === "reconnect") {
|
|
4684
4561
|
this.armAuthExpiryReconnectFence(nc);
|
|
4562
|
+
// The library re-subscribed on a connection the broker authorized again: re-run the check.
|
|
4563
|
+
this.rebindPrincipalServes(true);
|
|
4685
4564
|
this.emit("transport", { connected: true, server: s.server });
|
|
4686
4565
|
for (const follower of this.activeGoalFollowers) {
|
|
4687
4566
|
follower.reconnected(nc);
|
|
@@ -4696,6 +4575,10 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
4696
4575
|
continue;
|
|
4697
4576
|
if (s.error instanceof PermissionViolationError && this.chatSubRefusals.has(s.error))
|
|
4698
4577
|
continue;
|
|
4578
|
+
// A principal-route refusal belongs to the call that throws it or the serve that ends on it
|
|
4579
|
+
// (SPEC 13.21); nothing else on this private connection publishes or subscribes there.
|
|
4580
|
+
if (s.error instanceof PermissionViolationError && this.isPrincipalRouteSubject(s.error.subject))
|
|
4581
|
+
continue;
|
|
4699
4582
|
if (s.error instanceof PermissionViolationError && handedToRequest.has(s.error))
|
|
4700
4583
|
continue;
|
|
4701
4584
|
// The predecessor nats.js deleted while rebuilding one of this connection's watches, scans or
|
|
@@ -4965,24 +4848,6 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
4965
4848
|
const issued = row.record.issuedAllowSubscribe ?? allow;
|
|
4966
4849
|
return { allowSubscribe: allow, issuedAllowSubscribe: issued, lifecycleUid: row.lifecycleUid };
|
|
4967
4850
|
}
|
|
4968
|
-
/** Lazily open the delivery lease/readiness KV (pre-created at `cotal up`; bind, never create). */
|
|
4969
|
-
async deliveryRegistry() {
|
|
4970
|
-
if (!this.nc)
|
|
4971
|
-
throw new Error(this.notLiveMsg());
|
|
4972
|
-
this.deliveryKv ??= await openDeliveryRegistry(this.nc, this.space);
|
|
4973
|
-
return this.deliveryKv;
|
|
4974
|
-
}
|
|
4975
|
-
/** This endpoint instance's LEASE INCARNATION: which run of this principal a lease row was written
|
|
4976
|
-
* by. Minted per construction and never re-derived, because the credential cannot supply it, the
|
|
4977
|
-
* daemon's cred is a file on disk that every restart re-reads, so `card.id` is stable across
|
|
4978
|
-
* processes by design. See {@link DeliveryLeaseInfo.incarnation}. */
|
|
4979
|
-
leaseIncarnation = randomUUID();
|
|
4980
|
-
/** When this endpoint last WON each shard's lease create. Every later write of that row carries it
|
|
4981
|
-
* unchanged, while `since` is re-stamped. See {@link DeliveryLeaseInfo.acquiredAt}. */
|
|
4982
|
-
leaseAcquiredAt = new Map();
|
|
4983
|
-
encodeLease(shardIndex, ready, acquiredAt = this.leaseAcquiredAt.get(shardIndex)) {
|
|
4984
|
-
return new TextEncoder().encode(JSON.stringify({ holder: this.card.id, incarnation: this.leaseIncarnation, acquiredAt, since: Date.now(), ready }));
|
|
4985
|
-
}
|
|
4986
4851
|
/** Is this shard's lease row one THIS ENDPOINT INSTANCE wrote? The question a daemon whose renew
|
|
4987
4852
|
* just failed has to answer before it decides whether it still owns the shard.
|
|
4988
4853
|
*
|
|
@@ -4993,7 +4858,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
4993
4858
|
* incarnation predates the field and cannot be proven ours, which is the safe reading: it leads
|
|
4994
4859
|
* to the takeover path rather than to serving on someone else's claim. */
|
|
4995
4860
|
ownsDeliveryLease(info) {
|
|
4996
|
-
return
|
|
4861
|
+
return this.deliveryLease.ownsDeliveryLease(info);
|
|
4997
4862
|
}
|
|
4998
4863
|
/** Acquire the single-flight delivery lease for a shard via an ATOMIC CAS create, marked NOT-ready.
|
|
4999
4864
|
* THROWS if a live lease exists — a loud refusal-to-bind (the daemon exits), never a retry, so two
|
|
@@ -5001,17 +4866,13 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
5001
4866
|
* freeing a re-acquire. Acquired BEFORE binding (single-flight gate); {@link markDeliveryLeaseReady}
|
|
5002
4867
|
* flips it ready AFTER the loops + `ctl.delivery` are bound. Returns the lease revision. */
|
|
5003
4868
|
async acquireDeliveryLease(shardIndex) {
|
|
5004
|
-
|
|
5005
|
-
const acquiredAt = Date.now();
|
|
5006
|
-
const revision = await (await this.deliveryRegistry()).create(leaseKey(shardIndex), this.encodeLease(shardIndex, false, acquiredAt));
|
|
5007
|
-
this.leaseAcquiredAt.set(shardIndex, acquiredAt);
|
|
5008
|
-
return revision;
|
|
4869
|
+
return this.deliveryLease.acquireDeliveryLease(shardIndex);
|
|
5009
4870
|
}
|
|
5010
4871
|
/** Flip the held lease to READY (CAS `kv.update`) AFTER `startPlane3` has bound the loops + the
|
|
5011
4872
|
* `ctl.delivery` responder — so "lease ready" proves the responder is up, not just that the slot was
|
|
5012
4873
|
* claimed. Returns the new revision. */
|
|
5013
4874
|
async markDeliveryLeaseReady(shardIndex, revision) {
|
|
5014
|
-
return
|
|
4875
|
+
return this.deliveryLease.markDeliveryLeaseReady(shardIndex, revision);
|
|
5015
4876
|
}
|
|
5016
4877
|
/** Flip the held lease back to NOT-ready, the counterpart to {@link markDeliveryLeaseReady}, for a
|
|
5017
4878
|
* holder that has UNBOUND its loops and control responder but has not given up the shard.
|
|
@@ -5023,13 +4884,13 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
5023
4884
|
* the row (rather than deleting it) is deliberate: the shard is still claimed, so no third daemon
|
|
5024
4885
|
* should be invited in; what is being withdrawn is only the claim to be answering. */
|
|
5025
4886
|
async markDeliveryLeaseNotReady(shardIndex, revision) {
|
|
5026
|
-
return
|
|
4887
|
+
return this.deliveryLease.markDeliveryLeaseNotReady(shardIndex, revision);
|
|
5027
4888
|
}
|
|
5028
4889
|
/** Renew the held lease (CAS `kv.update` against `revision`, keeping `ready:true`) to refresh it before
|
|
5029
4890
|
* the bucket TTL expires it. Returns the new revision. Throws if the revision moved (lost the lease —
|
|
5030
4891
|
* the daemon should exit). */
|
|
5031
4892
|
async renewDeliveryLease(shardIndex, revision) {
|
|
5032
|
-
return
|
|
4893
|
+
return this.deliveryLease.renewDeliveryLease(shardIndex, revision);
|
|
5033
4894
|
}
|
|
5034
4895
|
/** Release the held lease on clean shutdown so a replacement daemon re-acquires immediately (a
|
|
5035
4896
|
* crash just lets the bucket TTL expire it).
|
|
@@ -5055,15 +4916,13 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
5055
4916
|
* a red in `smoke:delivery-lease`, where a caller that genuinely held the lease released nothing
|
|
5056
4917
|
* and the next acquire was refused. A caller must now say which it means. */
|
|
5057
4918
|
async releaseDeliveryLease(shardIndex, revision) {
|
|
5058
|
-
|
|
5059
|
-
return;
|
|
5060
|
-
await (await this.deliveryRegistry()).delete(leaseKey(shardIndex), { previousSeq: revision });
|
|
4919
|
+
return this.deliveryLease.releaseDeliveryLease(shardIndex, revision);
|
|
5061
4920
|
}
|
|
5062
4921
|
/** Read a shard's delivery lease (the daemon-availability signal), or `undefined` if none is live.
|
|
5063
4922
|
* READ-ONLY surface — drives Component 6's `cotal_channels` delivery-health field (an agent reads it
|
|
5064
4923
|
* under its own cred, which holds lease-bucket read but no write). */
|
|
5065
4924
|
async readDeliveryLease(shardIndex) {
|
|
5066
|
-
return
|
|
4925
|
+
return this.deliveryLease.readDeliveryLease(shardIndex);
|
|
5067
4926
|
}
|
|
5068
4927
|
/** Watch THIS shard's lease key and nothing else, for the daemon's loss trigger (#1596).
|
|
5069
4928
|
*
|
|
@@ -5083,42 +4942,16 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
5083
4942
|
* bucket's current last-per-subject state, so a change that landed during the null window is
|
|
5084
4943
|
* delivered on the fresh watch. Resolves a stop handle. */
|
|
5085
4944
|
async watchDeliveryLease(shardIndex, onEvent) {
|
|
5086
|
-
this.
|
|
5087
|
-
await this.bindDeliveryLeaseWatch();
|
|
5088
|
-
return () => {
|
|
5089
|
-
if (this.leaseWatchIntent?.onEvent !== onEvent)
|
|
5090
|
-
return; // a later watch owns the slot
|
|
5091
|
-
this.leaseWatchIntent = undefined;
|
|
5092
|
-
this.stopDeliveryLeaseWatch();
|
|
5093
|
-
};
|
|
4945
|
+
return this.deliveryLease.watchDeliveryLease(shardIndex, onEvent);
|
|
5094
4946
|
}
|
|
5095
4947
|
stopDeliveryLeaseWatch() {
|
|
5096
|
-
|
|
5097
|
-
this.retireDeliveryLeaseWatch(this.leaseWatchIter);
|
|
5098
|
-
this.leaseWatchIter = undefined;
|
|
5099
|
-
}
|
|
5100
|
-
/** Stop a lease watch and delete its ordered consumer, which the stop alone leaves on the broker
|
|
5101
|
-
* until its five-minute inactive threshold (#3966). A connection that closed on its own cannot
|
|
5102
|
-
* carry the delete, so that consumer is left to the threshold. A failed delete is a warning: the
|
|
5103
|
-
* broker still reaps the consumer, and the daemon's synchronous stop handle has no caller to
|
|
5104
|
-
* reject. */
|
|
5105
|
-
retireDeliveryLeaseWatch(iter) {
|
|
5106
|
-
try {
|
|
5107
|
-
iter.stop();
|
|
5108
|
-
}
|
|
5109
|
-
catch { /* already closed with its connection */ }
|
|
5110
|
-
if (!this.nc || this.nc.isClosed())
|
|
5111
|
-
return;
|
|
5112
|
-
const deleting = this.deleteOwnConsumer(kvWatchConsumer(iter)).catch((e) => this.emitRecoverable(new Error(`a retired delivery lease watch consumer was not deleted (${e.message}); the broker reaps it at its inactive threshold`)));
|
|
5113
|
-
this.leaseWatchDeletes.add(deleting);
|
|
5114
|
-
void deleting.then(() => this.leaseWatchDeletes.delete(deleting));
|
|
4948
|
+
return this.deliveryLease.stopDeliveryLeaseWatch();
|
|
5115
4949
|
}
|
|
5116
4950
|
/** Wait for the lease-watch binds still in flight, then for every retired watch's consumer delete.
|
|
5117
4951
|
* {@link stop} and a live-epoch rebuild run this before they close the connection those deletes
|
|
5118
4952
|
* need (#3966). */
|
|
5119
4953
|
async settleDeliveryLeaseWatches() {
|
|
5120
|
-
|
|
5121
|
-
await Promise.all(this.leaseWatchDeletes);
|
|
4954
|
+
return this.deliveryLease.settleDeliveryLeaseWatches();
|
|
5122
4955
|
}
|
|
5123
4956
|
/** Stop a presence or channel-registry watch and delete its ordered consumer, which the stop
|
|
5124
4957
|
* alone leaves on the broker until its five-minute inactive threshold (#3914). A connection
|
|
@@ -5173,37 +5006,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
5173
5006
|
* its intent, or after a stop or connection teardown moved the epoch, retires its watch instead of
|
|
5174
5007
|
* installing it: the epoch it was bound for is gone, and a rebuilt one binds its own. */
|
|
5175
5008
|
bindDeliveryLeaseWatch() {
|
|
5176
|
-
|
|
5177
|
-
const settled = () => { this.leaseWatchBinds.delete(binding); };
|
|
5178
|
-
this.leaseWatchBinds.add(binding);
|
|
5179
|
-
void binding.then(settled, settled);
|
|
5180
|
-
return binding;
|
|
5181
|
-
}
|
|
5182
|
-
async bindDeliveryLeaseWatchOnce() {
|
|
5183
|
-
const intent = this.leaseWatchIntent;
|
|
5184
|
-
const epoch = this.presenceEpoch;
|
|
5185
|
-
const iter = await (await this.deliveryRegistry()).watch({ key: leaseKey(intent.shardIndex) });
|
|
5186
|
-
if (this.leaseWatchIntent !== intent || epoch !== this.presenceEpoch) {
|
|
5187
|
-
this.retireDeliveryLeaseWatch(iter);
|
|
5188
|
-
return;
|
|
5189
|
-
}
|
|
5190
|
-
this.leaseWatchIter = iter;
|
|
5191
|
-
void (async () => {
|
|
5192
|
-
for await (const e of iter) {
|
|
5193
|
-
if (this.leaseWatchIter !== iter)
|
|
5194
|
-
break;
|
|
5195
|
-
if (e.operation === "DEL" || e.operation === "PURGE") {
|
|
5196
|
-
intent.onEvent(undefined);
|
|
5197
|
-
continue;
|
|
5198
|
-
}
|
|
5199
|
-
try {
|
|
5200
|
-
intent.onEvent(e.json());
|
|
5201
|
-
}
|
|
5202
|
-
catch {
|
|
5203
|
-
intent.onEvent(undefined);
|
|
5204
|
-
}
|
|
5205
|
-
}
|
|
5206
|
-
})().catch((e) => this.emit("error", e));
|
|
5009
|
+
return this.deliveryLease.bindDeliveryLeaseWatch();
|
|
5207
5010
|
}
|
|
5208
5011
|
/** The principal recorded as holding a shard's delivery lease, read by THIS process from the row,
|
|
5209
5012
|
* or `undefined` when no row is live or the read failed.
|
|
@@ -5219,12 +5022,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
5219
5022
|
* that throws (a denied or unreadable bucket, a broker that will not answer) must not collapse
|
|
5220
5023
|
* into "nothing is there". Callers must treat it as a failure to determine (#1694). */
|
|
5221
5024
|
async deliveryLeaseHolder(shardIndex) {
|
|
5222
|
-
|
|
5223
|
-
return (await this.readDeliveryLeaseEntry(shardIndex))?.info.holder;
|
|
5224
|
-
}
|
|
5225
|
-
catch {
|
|
5226
|
-
return undefined;
|
|
5227
|
-
}
|
|
5025
|
+
return this.deliveryLease.deliveryLeaseHolder(shardIndex);
|
|
5228
5026
|
}
|
|
5229
5027
|
/** The lease row AND the KV revision it is at. The revision is the CAS token every renew and the
|
|
5230
5028
|
* CAS release are argued against, so a caller re-establishing ownership after a failed renew
|
|
@@ -5241,15 +5039,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
5241
5039
|
* (an owner testing "is this provably MINE") still gets its safe answer: an unparseable row is
|
|
5242
5040
|
* not provably anyone's, and catching here reads as `false` exactly as before. */
|
|
5243
5041
|
async readDeliveryLeaseEntry(shardIndex) {
|
|
5244
|
-
|
|
5245
|
-
if (!e || e.operation === "DEL" || e.operation === "PURGE")
|
|
5246
|
-
return undefined;
|
|
5247
|
-
try {
|
|
5248
|
-
return { info: e.json(), revision: e.revision };
|
|
5249
|
-
}
|
|
5250
|
-
catch (err) {
|
|
5251
|
-
throw new Error(`delivery lease row "${leaseKey(shardIndex)}" exists but does not parse as a lease record (${err.message}) - it is unreadable, not absent`);
|
|
5252
|
-
}
|
|
5042
|
+
return this.deliveryLease.readDeliveryLeaseEntry(shardIndex);
|
|
5253
5043
|
}
|
|
5254
5044
|
/** Ensure + bind the manager singleton-lease bucket. Mirrors the presence-bucket pattern (connectAndBind):
|
|
5255
5045
|
* AUTH mode OPENs the bucket pre-created at `cotal up` (the scoped `supervisor` cred holds no
|
|
@@ -5495,7 +5285,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
5495
5285
|
* peer wins; 0 or >1 matches drop (never fan a directed durable copy to an unrelated same-named
|
|
5496
5286
|
* bystander — red-team LOW; SPEC §4 unique instance id). */
|
|
5497
5287
|
resolveOwnerByName(name) {
|
|
5498
|
-
const matches = [...this.
|
|
5288
|
+
const matches = [...this.presence.values()].filter((p) => p.card.name.toLowerCase() === name.toLowerCase());
|
|
5499
5289
|
return matches.length === 1 ? matches[0].card.id : undefined;
|
|
5500
5290
|
}
|
|
5501
5291
|
/** Plane-3 publish-dedupe key for a fan-out/transfer entry: the message id plus the member
|
|
@@ -6548,6 +6338,241 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
6548
6338
|
// the plane, so it is graded like a refusal rather than guessed at.
|
|
6549
6339
|
return "refused";
|
|
6550
6340
|
}
|
|
6341
|
+
/**
|
|
6342
|
+
* Serve this process's own endpoint on the principal route (SPEC 13.21): a core-defined command
|
|
6343
|
+
* set about this principal, addressed by its own principal, lifecycle UID and runtime id, with no
|
|
6344
|
+
* registration. Revision 1 defines one endpoint, `agent` (SPEC 13.22).
|
|
6345
|
+
*
|
|
6346
|
+
* Throws on an open mesh and on a static credential, where no row names who may call, without a
|
|
6347
|
+
* lifecycle UID, before the endpoint has started, for an endpoint it already serves, and for a
|
|
6348
|
+
* cluster with a command that declares no `authorities` (see {@link servePrincipalRoute}). It
|
|
6349
|
+
* subscribes this process's own subject with no queue group.
|
|
6350
|
+
*
|
|
6351
|
+
* `ready` is a positive check: it resolves only after the broker accepted the subscription
|
|
6352
|
+
* (subscribe, flush, and no permission violation for that subject), and rejects with the
|
|
6353
|
+
* refusal, which a credential minted before the rows existed gets. The serve is INTENT: every
|
|
6354
|
+
* replaced connection subscribes again and runs the check again, and a refusal there ends the
|
|
6355
|
+
* serve, resolves `failed` and clears the presence field. `stop` unsubscribes and waits for the
|
|
6356
|
+
* handlers in flight.
|
|
6357
|
+
*/
|
|
6358
|
+
servePrincipal(serve) {
|
|
6359
|
+
if (!this.authed)
|
|
6360
|
+
throw new Error("the principal route is served only on an authenticated mesh: on an open mesh no row says who may call (SPEC 13.21)");
|
|
6361
|
+
if (!this.userMode)
|
|
6362
|
+
throw new Error("the principal route is served only under a user-auth credential: a static credential names no one who may call (SPEC 13.21)");
|
|
6363
|
+
const uid = this.ownLifecycleUid;
|
|
6364
|
+
if (uid === undefined)
|
|
6365
|
+
throw new Error("serving the principal route needs this endpoint's lifecycleUid: the subject names it (SPEC 13.21)");
|
|
6366
|
+
const nc = this.nc;
|
|
6367
|
+
if (!nc)
|
|
6368
|
+
throw new Error(this.notLiveMsg());
|
|
6369
|
+
endpointToken(serve.endpoint);
|
|
6370
|
+
if (this.principalServes.has(serve.endpoint))
|
|
6371
|
+
throw new Error(`this endpoint already serves "${serve.endpoint}" on the principal route`);
|
|
6372
|
+
const self = { owner: this.owner, actor: this.actor, uid, runtimeId: assertRuntimeToken(processRuntime(uid).id) };
|
|
6373
|
+
const serverKey = `${this.space}\n${self.runtimeId}\n${serve.endpoint}`;
|
|
6374
|
+
if (principalServers.has(serverKey))
|
|
6375
|
+
throw new Error(`another endpoint in this process already serves "${serve.endpoint}" for lifecycle ${uid}: both would subscribe the same principal-route subject and answer one request twice (SPEC 13.21); stop that one first`);
|
|
6376
|
+
let resolveReady, rejectReady, resolveFailed;
|
|
6377
|
+
const ready = new Promise((resolve, reject) => { resolveReady = resolve; rejectReady = reject; });
|
|
6378
|
+
ready.catch(() => { });
|
|
6379
|
+
const failed = new Promise((resolve) => { resolveFailed = resolve; });
|
|
6380
|
+
const intent = {
|
|
6381
|
+
serve, self, serverKey, handles: [], ended: false, readySettled: false, resolveReady, rejectReady, resolveFailed,
|
|
6382
|
+
};
|
|
6383
|
+
// Construction refuses an invalid cluster before anything is kept or subscribed.
|
|
6384
|
+
const handle = this.bindPrincipalHandle(intent, nc);
|
|
6385
|
+
this.principalServes.set(serve.endpoint, intent);
|
|
6386
|
+
principalServers.set(serverKey, intent);
|
|
6387
|
+
void this.armPrincipalServe(intent, nc, handle);
|
|
6388
|
+
return { ready, failed, stop: () => this.stopPrincipalServe(intent) };
|
|
6389
|
+
}
|
|
6390
|
+
/** Subscribe `intent` on `nc`. */
|
|
6391
|
+
bindPrincipalHandle(intent, nc) {
|
|
6392
|
+
return servePrincipalRoute(nc, this.space, intent.self, intent.serve);
|
|
6393
|
+
}
|
|
6394
|
+
/** A request or reply subject (or filter) of the principal route in this space. */
|
|
6395
|
+
isPrincipalRouteSubject(subject) {
|
|
6396
|
+
const p = `${spacePrefix(this.space)}.ep.`;
|
|
6397
|
+
return subject.startsWith(`${p}prin.`) || subject.startsWith(`${p}prin-reply.`);
|
|
6398
|
+
}
|
|
6399
|
+
/** Track one binding: a refusal ends the serve, and an accepted subscription on the current
|
|
6400
|
+
* connection shows the presence field. */
|
|
6401
|
+
async armPrincipalServe(intent, nc, handle) {
|
|
6402
|
+
intent.binding = { nc, handle };
|
|
6403
|
+
intent.handles.push(handle);
|
|
6404
|
+
intent.readyOn = undefined;
|
|
6405
|
+
void handle.refused.then((err) => {
|
|
6406
|
+
if (!intent.ended && intent.binding?.handle === handle)
|
|
6407
|
+
this.endPrincipalServe(intent, err);
|
|
6408
|
+
});
|
|
6409
|
+
try {
|
|
6410
|
+
await handle.ready;
|
|
6411
|
+
}
|
|
6412
|
+
catch {
|
|
6413
|
+
// A refusal ends the serve through `refused`. A flush that failed because the connection
|
|
6414
|
+
// went away is answered by the binding the next connection makes.
|
|
6415
|
+
return;
|
|
6416
|
+
}
|
|
6417
|
+
this.principalServeConfirmed(intent, nc, handle);
|
|
6418
|
+
}
|
|
6419
|
+
/** The broker accepted `handle` on `nc`. When that is still the current binding, the serve is
|
|
6420
|
+
* ready there. */
|
|
6421
|
+
principalServeConfirmed(intent, nc, handle) {
|
|
6422
|
+
if (intent.ended || intent.binding?.handle !== handle || this.nc !== nc)
|
|
6423
|
+
return;
|
|
6424
|
+
intent.readyOn = nc;
|
|
6425
|
+
if (!intent.readySettled) {
|
|
6426
|
+
intent.readySettled = true;
|
|
6427
|
+
intent.resolveReady();
|
|
6428
|
+
}
|
|
6429
|
+
if (intent.serve.endpoint === AGENT_ENDPOINT)
|
|
6430
|
+
this.publishPresence().catch((e) => this.emitRecoverable(e));
|
|
6431
|
+
}
|
|
6432
|
+
/** Re-bind every principal serve on a replaced connection (a rebuild), or re-run the check on the
|
|
6433
|
+
* connection the library reconnected and re-subscribed itself. */
|
|
6434
|
+
rebindPrincipalServes(reconnected) {
|
|
6435
|
+
const nc = this.nc;
|
|
6436
|
+
if (!nc)
|
|
6437
|
+
return;
|
|
6438
|
+
for (const intent of this.principalServes.values()) {
|
|
6439
|
+
if (intent.ended)
|
|
6440
|
+
continue;
|
|
6441
|
+
const bound = intent.binding;
|
|
6442
|
+
if (bound && bound.nc === nc) {
|
|
6443
|
+
if (!reconnected)
|
|
6444
|
+
continue;
|
|
6445
|
+
intent.readyOn = undefined;
|
|
6446
|
+
void bound.handle.check().then(() => this.principalServeConfirmed(intent, nc, bound.handle), () => { });
|
|
6447
|
+
continue;
|
|
6448
|
+
}
|
|
6449
|
+
let handle;
|
|
6450
|
+
try {
|
|
6451
|
+
handle = this.bindPrincipalHandle(intent, nc);
|
|
6452
|
+
}
|
|
6453
|
+
catch (e) {
|
|
6454
|
+
this.endPrincipalServe(intent, e);
|
|
6455
|
+
continue;
|
|
6456
|
+
}
|
|
6457
|
+
void this.armPrincipalServe(intent, nc, handle);
|
|
6458
|
+
}
|
|
6459
|
+
}
|
|
6460
|
+
/** End a serve the broker refused: no more serving, `failed` resolves, the presence field goes. */
|
|
6461
|
+
endPrincipalServe(intent, err) {
|
|
6462
|
+
if (intent.ended)
|
|
6463
|
+
return;
|
|
6464
|
+
this.forgetPrincipalServe(intent);
|
|
6465
|
+
const wasReady = intent.readySettled;
|
|
6466
|
+
if (!intent.readySettled) {
|
|
6467
|
+
intent.readySettled = true;
|
|
6468
|
+
intent.rejectReady(err);
|
|
6469
|
+
}
|
|
6470
|
+
intent.resolveFailed(err);
|
|
6471
|
+
for (const handle of intent.handles)
|
|
6472
|
+
void handle.stop();
|
|
6473
|
+
if (intent.serve.endpoint === AGENT_ENDPOINT) {
|
|
6474
|
+
this.promptsPresence = undefined;
|
|
6475
|
+
this.publishPresence().catch((e) => this.emitRecoverable(e));
|
|
6476
|
+
}
|
|
6477
|
+
// `ready` already carried a refusal that came before it settled; a later one is reported here.
|
|
6478
|
+
if (wasReady)
|
|
6479
|
+
this.emitRecoverable(new Error(`stopped serving "${intent.serve.endpoint}" on the principal route: ${err.message}`, { cause: err }));
|
|
6480
|
+
}
|
|
6481
|
+
/** Mark `intent` ended and release what it holds in this endpoint and in this process. */
|
|
6482
|
+
forgetPrincipalServe(intent) {
|
|
6483
|
+
intent.ended = true;
|
|
6484
|
+
intent.readyOn = undefined;
|
|
6485
|
+
if (this.principalServes.get(intent.serve.endpoint) === intent)
|
|
6486
|
+
this.principalServes.delete(intent.serve.endpoint);
|
|
6487
|
+
if (principalServers.get(intent.serverKey) === intent)
|
|
6488
|
+
principalServers.delete(intent.serverKey);
|
|
6489
|
+
}
|
|
6490
|
+
/** Stop serving `intent` and wait for its handlers in flight. The ending is synchronous, so a
|
|
6491
|
+
* `ready` still waiting for the broker rejects before the caller's next step. `republish` writes
|
|
6492
|
+
* presence without the `prompts` field; the endpoint's own stop writes it offline instead. */
|
|
6493
|
+
async stopPrincipalServe(intent, republish = true) {
|
|
6494
|
+
if (!intent.ended) {
|
|
6495
|
+
this.forgetPrincipalServe(intent);
|
|
6496
|
+
if (!intent.readySettled) {
|
|
6497
|
+
intent.readySettled = true;
|
|
6498
|
+
intent.rejectReady(new Error(`stopped serving "${intent.serve.endpoint}" before the broker confirmed the subscription`));
|
|
6499
|
+
}
|
|
6500
|
+
if (intent.serve.endpoint === AGENT_ENDPOINT) {
|
|
6501
|
+
this.promptsPresence = undefined;
|
|
6502
|
+
if (republish)
|
|
6503
|
+
this.publishPresence().catch((e) => this.emitRecoverable(e));
|
|
6504
|
+
}
|
|
6505
|
+
}
|
|
6506
|
+
await Promise.all(intent.handles.map((handle) => handle.stop()));
|
|
6507
|
+
}
|
|
6508
|
+
/**
|
|
6509
|
+
* Set this process's `prompts` presence field and write presence at once (SPEC §6): how many
|
|
6510
|
+
* prompts it holds `open` or `sent`, and a counter that goes up by one each time a prompt opens or
|
|
6511
|
+
* changes state. The field shows only while the `agent` endpoint is served and the broker accepted
|
|
6512
|
+
* its subscription on the current connection. Until this is called it shows `open: 0` with the
|
|
6513
|
+
* highest `seq` this process used under its runtime id, or 0. Throws when the `agent` endpoint is
|
|
6514
|
+
* not served, and a RangeError for a `seq` below one this runtime already used.
|
|
6515
|
+
*/
|
|
6516
|
+
async setPromptsPresence(prompts) {
|
|
6517
|
+
for (const k of ["open", "seq"])
|
|
6518
|
+
if (!Number.isSafeInteger(prompts?.[k]) || prompts[k] < 0)
|
|
6519
|
+
throw new TypeError(`prompts.${k} must be a non-negative safe integer`);
|
|
6520
|
+
const intent = this.principalServes.get(AGENT_ENDPOINT);
|
|
6521
|
+
if (!intent)
|
|
6522
|
+
throw new Error(`the prompts presence field belongs to a served "${AGENT_ENDPOINT}" endpoint: call servePrincipal first (SPEC §6)`);
|
|
6523
|
+
const used = promptsSeqs.get(intent.self.runtimeId) ?? 0;
|
|
6524
|
+
if (prompts.seq < used)
|
|
6525
|
+
throw new RangeError(`prompts.seq ${prompts.seq} is below ${used}, which this process already used under runtime ${intent.self.runtimeId}: seq never goes back under one runtime id (SPEC §6)`);
|
|
6526
|
+
promptsSeqs.set(intent.self.runtimeId, prompts.seq);
|
|
6527
|
+
this.promptsPresence = { open: prompts.open, seq: prompts.seq };
|
|
6528
|
+
await this.publishPresence();
|
|
6529
|
+
}
|
|
6530
|
+
/**
|
|
6531
|
+
* Take the next `prompts` `seq` for this process's runtime (SPEC §6): 0 the first time, then one
|
|
6532
|
+
* above every `seq` this process used under its runtime id, whichever endpoint object used it. A
|
|
6533
|
+
* prompt desk takes every `seq` it shows from here. Throws without a lifecycle UID.
|
|
6534
|
+
*/
|
|
6535
|
+
nextPromptsSeq() {
|
|
6536
|
+
const runtimeId = processRuntime(this.requireLifecycleUid("nextPromptsSeq")).id;
|
|
6537
|
+
const used = promptsSeqs.get(runtimeId);
|
|
6538
|
+
const next = used === undefined ? 0 : used + 1;
|
|
6539
|
+
promptsSeqs.set(runtimeId, next);
|
|
6540
|
+
return next;
|
|
6541
|
+
}
|
|
6542
|
+
/** The `prompts` field this process's presence carries now, or none. */
|
|
6543
|
+
servedPrompts() {
|
|
6544
|
+
const intent = this.principalServes.get(AGENT_ENDPOINT);
|
|
6545
|
+
if (!intent || intent.ended || !this.nc || intent.readyOn !== this.nc)
|
|
6546
|
+
return undefined;
|
|
6547
|
+
return this.promptsPresence ?? { open: 0, seq: promptsSeqs.get(intent.self.runtimeId) ?? 0 };
|
|
6548
|
+
}
|
|
6549
|
+
/**
|
|
6550
|
+
* Call one command of a principal's own endpoint on the principal route (SPEC 13.21), as this
|
|
6551
|
+
* endpoint's caller triple with the authority word its row covers. `principal` is the addressed
|
|
6552
|
+
* process: owner, actor, lifecycle UID and the `runtime.id` its presence publishes. Presence is
|
|
6553
|
+
* advisory, and a wrong value reaches a subject nobody serves or one this caller's row does not
|
|
6554
|
+
* cover. It cannot widen authority.
|
|
6555
|
+
*
|
|
6556
|
+
* The `agent` endpoint's contracts and `describe`'s are built in; any other endpoint needs
|
|
6557
|
+
* `contract`. Resolves the reply, whose `ok: false` is the agent's own refusal; throws
|
|
6558
|
+
* `unavailable` (`not-executed`) when nothing serves that process, `permission-denied`
|
|
6559
|
+
* (`not-executed`) when this credential holds no row for the request, and `deadline-exceeded`
|
|
6560
|
+
* (`unknown`) when no reply came. See {@link epPrincipalCall}.
|
|
6561
|
+
*/
|
|
6562
|
+
async callPrincipal(call) {
|
|
6563
|
+
const nc = this.nc;
|
|
6564
|
+
if (!nc)
|
|
6565
|
+
throw new Error(this.notLiveMsg());
|
|
6566
|
+
const contract = call.contract
|
|
6567
|
+
?? (call.command === "describe" ? describeContracts() : call.endpoint === AGENT_ENDPOINT ? AGENT_CONTRACTS[call.command] : undefined);
|
|
6568
|
+
if (contract === undefined)
|
|
6569
|
+
throw new EpEnvelopeError("bad-request", `no contract is built in for ${call.endpoint}.${call.command}; pass one`, undefined, "not-executed");
|
|
6570
|
+
const caller = { owner: this.owner, actor: this.actor, uid: this.requireLifecycleUid("callPrincipal") };
|
|
6571
|
+
return epPrincipalCall(nc, this.space, { principal: call.principal, authority: call.authority }, {
|
|
6572
|
+
endpoint: call.endpoint, command: call.command, contract, caller, name: this.card.name,
|
|
6573
|
+
...(call.args !== undefined ? { args: call.args } : {}),
|
|
6574
|
+
}, { deadlineMs: call.deadlineMs, ...(call.signal ? { signal: call.signal } : {}) });
|
|
6575
|
+
}
|
|
6551
6576
|
/** Serve one PRIVILEGED delivery-admin request (the D5 rail-split). The cred layer is the caller
|
|
6552
6577
|
* boundary — only the supervisor profile can publish here — and `serveControl`'s sender check +
|
|
6553
6578
|
* bounded reply still apply on top. `reloadCreds` is the class-2 renewal ADOPTION step: re-read
|
|
@@ -7252,7 +7277,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
7252
7277
|
catch (e) {
|
|
7253
7278
|
if (attempt === 0)
|
|
7254
7279
|
this.emitRecoverable(new Error(`channel "${channel}": Plane-3 durable membership (generation ${generation}) not yet tombstoned after a refused live sub - retrying; §7 boundary may be open until it succeeds (${e.message})`));
|
|
7255
|
-
await
|
|
7280
|
+
await this.membershipPause(attempt);
|
|
7256
7281
|
}
|
|
7257
7282
|
}
|
|
7258
7283
|
}
|
|
@@ -7421,7 +7446,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
7421
7446
|
return; // already reconciling
|
|
7422
7447
|
this.pendingBootJoins.add(channel);
|
|
7423
7448
|
for (let attempt = 0;; attempt++) {
|
|
7424
|
-
await
|
|
7449
|
+
await this.membershipPause(attempt);
|
|
7425
7450
|
if (this.stopped || !this.channels.includes(channel) || this.plane3Channels.has(channel)) {
|
|
7426
7451
|
this.pendingBootJoins.delete(channel);
|
|
7427
7452
|
return; // stopped, left, or another path established it
|
|
@@ -7982,7 +8007,10 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
7982
8007
|
if (floorSeq !== undefined && floorSeq >= upToSeq)
|
|
7983
8008
|
return 0;
|
|
7984
8009
|
const subject = chatSubject(this.space, "*", "*", channel);
|
|
7985
|
-
|
|
8010
|
+
// A window that reaches back before 1970 covers the whole retained window (#7323). Its start time
|
|
8011
|
+
// would be a date JetStream does not read, or not a valid date at all, and the read replayed nothing.
|
|
8012
|
+
const since = sinceMs === undefined ? undefined : Date.now() - sinceMs;
|
|
8013
|
+
const start = floorSeq !== undefined ? { seq: floorSeq + 1 } : since === undefined || !(since > 0) ? { seq: 1 } : { time: new Date(since) };
|
|
7986
8014
|
let msgs;
|
|
7987
8015
|
try {
|
|
7988
8016
|
msgs = await this.collectHistory(subject, start, { untilSeq: upToSeq });
|
|
@@ -8112,91 +8140,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
8112
8140
|
return oldest !== undefined && oldest > sinceSeq + 1;
|
|
8113
8141
|
}
|
|
8114
8142
|
async publishPresence() {
|
|
8115
|
-
|
|
8116
|
-
return; // observers watch but never publish their own record
|
|
8117
|
-
const p = {
|
|
8118
|
-
card: this.card,
|
|
8119
|
-
// SPEC §6: presence carries the incarnation's lifecycle UID (MUST in auth mode from v0.4);
|
|
8120
|
-
// omitted only where the endpoint has none (a pure operator/daemon connection never registers).
|
|
8121
|
-
...(this.ownLifecycleUid !== undefined ? { lifecycleUid: this.ownLifecycleUid } : {}),
|
|
8122
|
-
runtime: processRuntime(this.ownLifecycleUid),
|
|
8123
|
-
status: this.status,
|
|
8124
|
-
condition: this.condition,
|
|
8125
|
-
environment: this.environment,
|
|
8126
|
-
activity: this.activity,
|
|
8127
|
-
statusSince: this.statusSince,
|
|
8128
|
-
activitySince: this.activitySince,
|
|
8129
|
-
activeAt: this.activeAt,
|
|
8130
|
-
attention: this.attentionMode,
|
|
8131
|
-
channelModes: this.channelModes,
|
|
8132
|
-
ts: Date.now(),
|
|
8133
|
-
};
|
|
8134
|
-
// Wire contract (SPEC §6): an OFFLINE record must not carry the advisory attention fields or
|
|
8135
|
-
// `statusSince`. Scrub at the publisher — this covers stop(), setStatus("offline"), and any future
|
|
8136
|
-
// offline publish site, so the raw KV record is compliant, not only the observer-side roster
|
|
8137
|
-
// materialization.
|
|
8138
|
-
const record = this.status === "offline" ? this.toOffline(p) : p;
|
|
8139
|
-
assertPresenceRow(record, this.card.id);
|
|
8140
|
-
// #1356: this put can still be in flight when a teardown runs, because a rebind reaches
|
|
8141
|
-
// publishPresence through onPresenceBucketEmpty and no teardown awaits that flight. Take the same
|
|
8142
|
-
// epoch fence {@link startPresenceWatch} takes, so a put that outlives its epoch still throws to
|
|
8143
|
-
// its caller but no longer writes presence-refusal state belonging to a later connection. Without
|
|
8144
|
-
// it, a heartbeat put (default 2s) whose ~5s JetStream timeout elapses after a rebuild plants a
|
|
8145
|
-
// refusal on the connection that just published successfully.
|
|
8146
|
-
const epoch = this.presenceEpoch;
|
|
8147
|
-
const conditionRevision = this.conditionRevision;
|
|
8148
|
-
// #1461: snapshot the put's generation BEFORE the put. A settle (success or rejection) is the
|
|
8149
|
-
// latest evidence on its epoch only while no put that started after it has ALREADY settled —
|
|
8150
|
-
// a newer put merely in flight is not evidence yet. Same-epoch puts overlap routinely
|
|
8151
|
-
// (heartbeat 2s against a ~5s JetStream put timeout), so "succeeded" alone is not "newest".
|
|
8152
|
-
const generation = ++this.presencePutGeneration;
|
|
8153
|
-
try {
|
|
8154
|
-
await this.kv.put(this.card.id, JSON.stringify(record));
|
|
8155
|
-
}
|
|
8156
|
-
catch (e) {
|
|
8157
|
-
// #1356: a broker can put this bucket into a state where it refuses every write and never
|
|
8158
|
-
// recovers, and the write is the ONLY thing that fails — open and watch both still succeed, so
|
|
8159
|
-
// nothing else here notices. Record WHEN the refusals started, at the one site that knows the
|
|
8160
|
-
// failing write was a presence write; a caller cannot infer that from the generic `warning`
|
|
8161
|
-
// stream, which carries any recoverable error.
|
|
8162
|
-
//
|
|
8163
|
-
// #1461: a rejection speaks when it is the newest SETTLED evidence — a newer put that has
|
|
8164
|
-
// merely started does not silence it, but a newer put that already settled (either way)
|
|
8165
|
-
// does.
|
|
8166
|
-
const superseded = (this.presencePutSettled ?? 0) > generation;
|
|
8167
|
-
this.presencePutSettled = Math.max(this.presencePutSettled ?? 0, generation);
|
|
8168
|
-
if (epoch === this.presenceEpoch && !this.stopped && !superseded) {
|
|
8169
|
-
this.presenceWriteFailingSince ??= Date.now();
|
|
8170
|
-
this.lastPresenceWriteError = e?.message ?? String(e);
|
|
8171
|
-
this.presenceWriteFailures++;
|
|
8172
|
-
this.escalatePresenceWriteFailureIfStuck();
|
|
8173
|
-
}
|
|
8174
|
-
if (e instanceof TimeoutError)
|
|
8175
|
-
throw new Error(`${e.message}. The presence write outcome is unknown: the broker may have stored the update. Read the roster before retrying.`, { cause: e });
|
|
8176
|
-
throw e;
|
|
8177
|
-
}
|
|
8178
|
-
// A late SUCCESS cannot erase a refusal from newer evidence. Two fences, each answering half:
|
|
8179
|
-
// the epoch fence (#1356) keeps a put that outlives its connection from writing or erasing a
|
|
8180
|
-
// record that belongs to a later connection; the generation fence (#1461) orders puts WITHIN
|
|
8181
|
-
// one epoch by SETTLE order, not start order: a settle is the latest evidence only while no
|
|
8182
|
-
// put that started after it has already settled, whichever way that newer put settled. So an
|
|
8183
|
-
// earlier put settling AFTER a later put settled no longer speaks — success cannot clear a
|
|
8184
|
-
// newer refusal (#1461's original case), and a newer settle is not blocked by being merely
|
|
8185
|
-
// started (panel round 1). Without the fence, an earlier put succeeding late cleared a later
|
|
8186
|
-
// put's refusal while the bucket still refused writes (heartbeat 2s against a ~5s put timeout,
|
|
8187
|
-
// so the overlap is routine rather than a corner).
|
|
8188
|
-
const superseded = (this.presencePutSettled ?? 0) > generation;
|
|
8189
|
-
this.presencePutSettled = Math.max(this.presencePutSettled ?? 0, generation);
|
|
8190
|
-
if (epoch !== this.presenceEpoch || this.stopped || superseded)
|
|
8191
|
-
return;
|
|
8192
|
-
// Presence writes may overlap. If this put carried an older condition, repair the KV with the
|
|
8193
|
-
// latest local state before returning so a late failure publish cannot resurrect after a new
|
|
8194
|
-
// turn cleared it. The revision is condition-specific; unrelated heartbeat overlap is unchanged.
|
|
8195
|
-
if (conditionRevision !== this.conditionRevision) {
|
|
8196
|
-
await this.publishPresence();
|
|
8197
|
-
return;
|
|
8198
|
-
}
|
|
8199
|
-
this.clearPresenceWriteFailure();
|
|
8143
|
+
return this.presence.publishPresence();
|
|
8200
8144
|
}
|
|
8201
8145
|
/** #1356: drop the presence-refusal record when the connection that OBSERVED those refusals goes
|
|
8202
8146
|
* away. "This bucket is refusing writes" is a claim about a specific broker connection; once that
|
|
@@ -8208,23 +8152,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
8208
8152
|
* failed bind against an unreachable server still reported the presence-refusal sentence while the
|
|
8209
8153
|
* endpoint's own `connectionIssue` already said "connection refused". */
|
|
8210
8154
|
clearPresenceWriteFailure() {
|
|
8211
|
-
this.
|
|
8212
|
-
this.lastPresenceWriteError = undefined;
|
|
8213
|
-
this.presenceWriteFailures = 0;
|
|
8214
|
-
this.presenceWriteEscalated = false;
|
|
8215
|
-
}
|
|
8216
|
-
/** A heartbeat retry remains recoverable inside one TTL. Once two or more consecutive writes have
|
|
8217
|
-
* failed across the full window, ordinary retry has exhausted the roster's liveness budget: raise
|
|
8218
|
-
* one named, non-transient warning. Later failures keep the duration/count current without flooding
|
|
8219
|
-
* the operator log; the next successful write clears and re-arms the condition. */
|
|
8220
|
-
escalatePresenceWriteFailureIfStuck() {
|
|
8221
|
-
if (this.presenceWriteEscalated ||
|
|
8222
|
-
this.presenceWriteFailingSince === undefined ||
|
|
8223
|
-
this.presenceWriteFailures < 2 ||
|
|
8224
|
-
Date.now() - this.presenceWriteFailingSince < this.ttlMs)
|
|
8225
|
-
return;
|
|
8226
|
-
this.presenceWriteEscalated = true;
|
|
8227
|
-
this.emitRecoverable(new PresenceWriteStuckError(presenceBucket(this.space), this.presenceWriteFailingSince, this.presenceWriteFailures, this.ttlMs, this.lastPresenceWriteError));
|
|
8155
|
+
return this.presence.clearPresenceWriteFailure();
|
|
8228
8156
|
}
|
|
8229
8157
|
/** #1356: the presence bucket has been refusing writes since this time, or `undefined` when the
|
|
8230
8158
|
* last publish succeeded. Cleared by a successful write that is the latest evidence on its
|
|
@@ -8235,16 +8163,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
8235
8163
|
* while the NATS connection stays up, so a caller must not read this as "only presence is
|
|
8236
8164
|
* affected". It reports what was observed, not how far the fault extends. */
|
|
8237
8165
|
presenceWriteFailure() {
|
|
8238
|
-
|
|
8239
|
-
return undefined;
|
|
8240
|
-
return {
|
|
8241
|
-
since: this.presenceWriteFailingSince,
|
|
8242
|
-
forMs: Date.now() - this.presenceWriteFailingSince,
|
|
8243
|
-
error: this.lastPresenceWriteError,
|
|
8244
|
-
bucket: presenceBucket(this.space),
|
|
8245
|
-
consecutiveFailures: this.presenceWriteFailures,
|
|
8246
|
-
stuck: this.presenceWriteEscalated,
|
|
8247
|
-
};
|
|
8166
|
+
return this.presence.presenceWriteFailure();
|
|
8248
8167
|
}
|
|
8249
8168
|
/** Bind a presence watch on the current connection. Resolves the bound consumer's info when
|
|
8250
8169
|
* the watch was installed, undefined when the endpoint stopped or rebuilt while the bind was
|
|
@@ -8254,117 +8173,12 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
8254
8173
|
* {@link stop} would resurrect a watch on a stopped endpoint, and one that completes after a
|
|
8255
8174
|
* rebuild would overwrite the fresh epoch's watch with a dead-connection iterator. */
|
|
8256
8175
|
async startPresenceWatch() {
|
|
8257
|
-
|
|
8258
|
-
return undefined;
|
|
8259
|
-
let hydrated;
|
|
8260
|
-
this.presenceSnapshot = new Promise((resolve) => { hydrated = resolve; });
|
|
8261
|
-
const bound = await this.bindKvWatch(this.kv);
|
|
8262
|
-
if (!bound) {
|
|
8263
|
-
hydrated();
|
|
8264
|
-
return undefined;
|
|
8265
|
-
}
|
|
8266
|
-
const { iter, info } = bound;
|
|
8267
|
-
this.presenceWatchIter = iter;
|
|
8268
|
-
// A bucket with no keys replays nothing, so no entry carries the end-of-replay marker. For an
|
|
8269
|
-
// observer that does not register, that empty replay is the whole snapshot: nobody is present.
|
|
8270
|
-
// A registering observer is populated by its own record instead (see onPresenceBucketEmpty).
|
|
8271
|
-
const empty = info.num_pending === 0 && !this.doRegister;
|
|
8272
|
-
if (empty) {
|
|
8273
|
-
this.presenceSnapshotPopulated = true;
|
|
8274
|
-
hydrated();
|
|
8275
|
-
this.markPresenceBucketEmpty();
|
|
8276
|
-
}
|
|
8277
|
-
void (async () => {
|
|
8278
|
-
let ready = empty;
|
|
8279
|
-
for await (const e of iter) {
|
|
8280
|
-
// A rebind bound a newer watch: this one's remaining entries belong to a retired epoch.
|
|
8281
|
-
// Stop consuming them (the rebind stops the iterator too; this ends the loop first).
|
|
8282
|
-
if (this.presenceWatchIter !== iter)
|
|
8283
|
-
break;
|
|
8284
|
-
this.handleKvEntry(e);
|
|
8285
|
-
// @nats-io/kv marks the final initial replay entry isUpdate=true. Later updates stay true.
|
|
8286
|
-
if (!ready && e.isUpdate) {
|
|
8287
|
-
ready = true;
|
|
8288
|
-
this.presenceSnapshotPopulated = true;
|
|
8289
|
-
hydrated();
|
|
8290
|
-
this.emitPresenceViewIfChanged();
|
|
8291
|
-
}
|
|
8292
|
-
}
|
|
8293
|
-
hydrated();
|
|
8294
|
-
})().catch((e) => this.emit("error", e));
|
|
8295
|
-
return info;
|
|
8176
|
+
return this.presence.startPresenceWatch();
|
|
8296
8177
|
}
|
|
8297
8178
|
/** {@link EndpointOptions.readOnce}: fill the roster from one read of the presence bucket. Stale
|
|
8298
8179
|
* and offline records resolve as they do in a watch's initial replay. */
|
|
8299
8180
|
async readPresenceOnce() {
|
|
8300
|
-
|
|
8301
|
-
return;
|
|
8302
|
-
for (const e of await lastKvEntries(this.kv))
|
|
8303
|
-
this.handleKvEntry(e);
|
|
8304
|
-
this.presenceSnapshotPopulated = true;
|
|
8305
|
-
// No record reached the roster (an empty bucket, or only the TTL canary): nobody is present.
|
|
8306
|
-
if (this.lastPresenceWatchAt === 0)
|
|
8307
|
-
this.markPresenceBucketEmpty();
|
|
8308
|
-
else
|
|
8309
|
-
this.emitPresenceViewIfChanged();
|
|
8310
|
-
}
|
|
8311
|
-
/**
|
|
8312
|
-
* Replace a presence watch that has gone silent past TTL while the connection is up. The new
|
|
8313
|
-
* ordered consumer starts from the bucket's current last-per-subject state, so a peer that is
|
|
8314
|
-
* heartbeating is re-observed within one replay and a peer that is gone is aged out by the
|
|
8315
|
-
* next sweep exactly as if the watch had never stalled. Rate-limited to one attempt per TTL
|
|
8316
|
-
* per observer, never overlapping, never on a stopped or rebuilding endpoint (those own their
|
|
8317
|
-
* watch through {@link connectAndBind}). A stop or rebuild that lands while the bind is in
|
|
8318
|
-
* flight retires it: {@link startPresenceWatch} retires the late watch and reports
|
|
8319
|
-
* nothing, since the epoch that was silent no longer exists. A failed bind is reported and
|
|
8320
|
-
* the view stays stale.
|
|
8321
|
-
*/
|
|
8322
|
-
rebindStalePresenceWatch(now) {
|
|
8323
|
-
if (this.stopped || this.reconnecting || !this.kv || !this.nc || this.nc.isClosed())
|
|
8324
|
-
return;
|
|
8325
|
-
if (this.presenceRebind || now - this.presenceRebindAt < this.ttlMs)
|
|
8326
|
-
return;
|
|
8327
|
-
this.presenceRebindAt = now;
|
|
8328
|
-
const old = this.presenceWatchIter;
|
|
8329
|
-
const epoch = this.presenceEpoch;
|
|
8330
|
-
this.presenceRebind = (async () => {
|
|
8331
|
-
try {
|
|
8332
|
-
const silentMs = now - this.lastPresenceWatchAt;
|
|
8333
|
-
// Bind the replacement FIRST. A rebind the broker refuses (stream gone, create denied) or
|
|
8334
|
-
// one a held link never answers must leave the old watch in place: on a plain stall that
|
|
8335
|
-
// watch is the one that recovers by itself, and its replay is still guarded against
|
|
8336
|
-
// expired PUTs. Only a successfully bound watch retires its predecessor.
|
|
8337
|
-
const info = await this.startPresenceWatch();
|
|
8338
|
-
// Retired mid-bind (stop or rebuild moved the epoch): the late watch is already
|
|
8339
|
-
// retired and the old watch was torn down by whoever moved the epoch. Nothing to
|
|
8340
|
-
// retire, nothing to report.
|
|
8341
|
-
if (!info)
|
|
8342
|
-
return;
|
|
8343
|
-
// The predecessor's consumer goes with it: left alone, every rebind of a silent bucket
|
|
8344
|
-
// would add one idle consumer to the presence stream per TTL (#3914).
|
|
8345
|
-
if (old && old !== this.presenceWatchIter)
|
|
8346
|
-
this.retireKvWatch(old);
|
|
8347
|
-
// A bucket with no keys replays nothing, so the new watch cannot refresh
|
|
8348
|
-
// `lastPresenceWatchAt` by delivering. It IS current knowledge: nobody is present. The
|
|
8349
|
-
// bind's consumer info carries the initial pending count for that one fact; nats.js's KV
|
|
8350
|
-
// watch placed the isUpdate marker from the same `info(true)`.
|
|
8351
|
-
if (info.num_pending === 0)
|
|
8352
|
-
await this.onPresenceBucketEmpty();
|
|
8353
|
-
this.emit("warning", new Error(`presence watch silent for ${silentMs}ms with the connection up; rebound it from the bucket's current state`));
|
|
8354
|
-
}
|
|
8355
|
-
catch (e) {
|
|
8356
|
-
// A bind the epoch swap itself rejected (connection drained under it) is not a fault of
|
|
8357
|
-
// the epoch that replaced it; only a refusal on a still-current epoch is reported.
|
|
8358
|
-
if (epoch === this.presenceEpoch)
|
|
8359
|
-
this.emit("error", e);
|
|
8360
|
-
}
|
|
8361
|
-
finally {
|
|
8362
|
-
// An epoch swap already disowned this flight (and may own a successor's by now); only
|
|
8363
|
-
// a flight still in its own epoch clears the slot. Within one epoch there is one flight.
|
|
8364
|
-
if (epoch === this.presenceEpoch)
|
|
8365
|
-
this.presenceRebind = undefined;
|
|
8366
|
-
}
|
|
8367
|
-
})();
|
|
8181
|
+
return this.presence.readPresenceOnce();
|
|
8368
8182
|
}
|
|
8369
8183
|
/** Watch the channel registry: replay existing keys, then stream updates, into the local
|
|
8370
8184
|
* cache. A failed bind throws out of connect; an entry that does not decode or validate is
|
|
@@ -8409,214 +8223,8 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
8409
8223
|
this.emitRecoverable(new Error(`dropped channel registry entry for key ${JSON.stringify(e.key)}: ${err.message}`));
|
|
8410
8224
|
}
|
|
8411
8225
|
}
|
|
8412
|
-
/** The watch was just bound onto a bucket with no keys.
|
|
8413
|
-
*
|
|
8414
|
-
* A NON-REGISTERING observer (a `cotal status` probe, a lease checker) has real knowledge:
|
|
8415
|
-
* nobody is present. Every peer still in its roster is known gone (its key is not there to
|
|
8416
|
-
* replay), so it is marked offline now rather than aged out against a delivery that cannot
|
|
8417
|
-
* come; the silence gate is disarmed and the view reads current until the first write lands.
|
|
8418
|
-
* Without that the empty-bucket view relapsed to stale one window later and rebound again on
|
|
8419
|
-
* every window, one consumer create and one warning per TTL for as long as the mesh was empty.
|
|
8420
|
-
*
|
|
8421
|
-
* A REGISTERING observer (the manager) is itself one of the keys that should be there. An
|
|
8422
|
-
* empty bucket under it means the bucket was wiped since its last heartbeat (the stream
|
|
8423
|
-
* recreation), and the same wipe took every peer's record: their absence says the bucket is
|
|
8424
|
-
* new, not that they left. a reviewer reproduced the previous behaviour at default timing:
|
|
8425
|
-
* the rebind landed ~0.9s after the recreation, the observer marked every peer AND ITSELF
|
|
8426
|
-
* offline, and held the view current for up to one heartbeat, a false verdict `cotal ps`
|
|
8427
|
-
* would print as `mesh offline`. So a registering observer re-publishes its own record NOW,
|
|
8428
|
-
* which the new watch delivers, and lets the ordinary per-peer age-out run from that delivery:
|
|
8429
|
-
* a peer that is still heartbeating rewrites its key within its own heartbeat interval and is
|
|
8430
|
-
* re-observed live; one that is gone ages out exactly as after a plain rebind. The roster is
|
|
8431
|
-
* not touched here and the view is not held; the delivery is what makes it current.
|
|
8432
|
-
*
|
|
8433
|
-
* What neither branch covers: a consumer that dies again while the bucket is still empty is
|
|
8434
|
-
* not detectable by silence, so the first write after that is missed until the observer
|
|
8435
|
-
* restarts. */
|
|
8436
|
-
async onPresenceBucketEmpty() {
|
|
8437
|
-
if (this.doRegister) {
|
|
8438
|
-
// Our own key is missing from a bucket we write to: put it back. publishPresence throws on
|
|
8439
|
-
// a refused write; that surfaces through the rebind flight's error path with the epoch
|
|
8440
|
-
// fence intact, and the view stays stale, which is the honest state for an observer that
|
|
8441
|
-
// cannot even record itself.
|
|
8442
|
-
await this.publishPresence();
|
|
8443
|
-
return;
|
|
8444
|
-
}
|
|
8445
|
-
this.markPresenceBucketEmpty();
|
|
8446
|
-
}
|
|
8447
|
-
/** See {@link onPresenceBucketEmpty}: the non-registering branch. */
|
|
8448
|
-
markPresenceBucketEmpty() {
|
|
8449
|
-
this.presenceWatchEmpty = true;
|
|
8450
|
-
this.lastPresenceWatchAt = 0;
|
|
8451
|
-
let changed = false;
|
|
8452
|
-
for (const [id, p] of this.roster) {
|
|
8453
|
-
if (p.status === "offline")
|
|
8454
|
-
continue;
|
|
8455
|
-
const offline = this.toOffline(p);
|
|
8456
|
-
this.roster.set(id, offline);
|
|
8457
|
-
this.emit("presence", { type: "offline", presence: offline });
|
|
8458
|
-
changed = true;
|
|
8459
|
-
}
|
|
8460
|
-
if (changed)
|
|
8461
|
-
this.emit("roster", this.getRoster());
|
|
8462
|
-
this.emitPresenceViewIfChanged();
|
|
8463
|
-
}
|
|
8464
|
-
handleKvEntry(e) {
|
|
8465
|
-
// Provisioning uses this one reserved key to prove the bucket's max_age is enforced (#404).
|
|
8466
|
-
// It is maintenance traffic, never a peer identity, an offline transition, or view freshness.
|
|
8467
|
-
if (e.key === TTL_RECONCILE_CANARY_KEY)
|
|
8468
|
-
return;
|
|
8469
|
-
this.lastPresenceWatchAt = Date.now();
|
|
8470
|
-
this.presenceWatchEmpty = false;
|
|
8471
|
-
if (e.operation === "DEL" || e.operation === "PURGE") {
|
|
8472
|
-
this.markOffline(e.key);
|
|
8473
|
-
return;
|
|
8474
|
-
}
|
|
8475
|
-
const p = this.decodePresence(e);
|
|
8476
|
-
if (p)
|
|
8477
|
-
this.applyPresence(e.key, p);
|
|
8478
|
-
}
|
|
8479
|
-
/** The record a presence PUT carries, or undefined when a reader must drop it (SPEC §6). The watch
|
|
8480
|
-
* and {@link readPresence} decode through this one gate, so they drop the same records. */
|
|
8481
|
-
decodePresence(e) {
|
|
8482
|
-
let p;
|
|
8483
|
-
try {
|
|
8484
|
-
p = e.json();
|
|
8485
|
-
}
|
|
8486
|
-
catch {
|
|
8487
|
-
return undefined;
|
|
8488
|
-
}
|
|
8489
|
-
if (!isPresenceRow(p, e.key)) {
|
|
8490
|
-
this.presenceBindingDrops++;
|
|
8491
|
-
this.emitRecoverable(new Error(`dropped presence entry for key ${JSON.stringify(e.key)}: not a usable presence row (a record with card.id equal to its KV key, a non-blank string card.name, a known status, a finite ts, string display fields and a condition with a string code)`));
|
|
8492
|
-
return undefined;
|
|
8493
|
-
}
|
|
8494
|
-
// isPresenceRow is the one gate: it refuses a row whose card.id disagrees with the KV key, so a
|
|
8495
|
-
// forged or mis-keyed record never surfaces a spoofed roster identity. The write-side scoping
|
|
8496
|
-
// ($KV.<presenceBucket>.<own-id>) is the primary guard.
|
|
8497
|
-
return p;
|
|
8498
|
-
}
|
|
8499
|
-
applyPresence(id, raw) {
|
|
8500
|
-
const prev = this.roster.get(id);
|
|
8501
|
-
const stale = Date.now() - raw.ts > this.ttlMs;
|
|
8502
|
-
// A watch recovering from a stall replays the bucket. Those PUTs still carry the publisher's
|
|
8503
|
-
// pre-stall timestamps, so they look TTL-expired even though the peers kept heartbeating on
|
|
8504
|
-
// the broker. Materializing them as offline is the empty-to-full flicker #1045 named: the
|
|
8505
|
-
// observer's catch-up, not N deaths. Keep last-known rows, stay view-stale, and wait for a
|
|
8506
|
-
// live timestamp (or an explicit offline/delete) before changing a known peer.
|
|
8507
|
-
if (stale && raw.status !== "offline") {
|
|
8508
|
-
if (prev)
|
|
8509
|
-
return;
|
|
8510
|
-
this.roster.set(id, this.toOffline(raw));
|
|
8511
|
-
this.emit("roster", this.getRoster());
|
|
8512
|
-
return;
|
|
8513
|
-
}
|
|
8514
|
-
if (this.presenceSnapshotPopulated)
|
|
8515
|
-
this.emitPresenceViewIfChanged();
|
|
8516
|
-
// Any offline materialization (a stale snapshot OR a graceful-leave record) drops the advisory
|
|
8517
|
-
// attention fields — an offline peer must not carry a stale `[focus]`/`locally muted` hint.
|
|
8518
|
-
const p = stale || raw.status === "offline" ? this.toOffline(raw) : raw;
|
|
8519
|
-
// First time we hear about an already-offline peer (stale snapshot): record quietly.
|
|
8520
|
-
if (!prev && p.status === "offline") {
|
|
8521
|
-
this.roster.set(id, p);
|
|
8522
|
-
this.emit("roster", this.getRoster());
|
|
8523
|
-
return;
|
|
8524
|
-
}
|
|
8525
|
-
// Heartbeat refresh with no real change: bump liveness quietly and don't
|
|
8526
|
-
// emit — otherwise the periodic keep-alive looks like a stream of "updates".
|
|
8527
|
-
// A CHANGED lifecycleUid is never a heartbeat: it is a NEW LIFECYCLE reusing the same principal (a
|
|
8528
|
-
// terminal same-name recreation — in user mode a retired lifecycle and its same-name successor
|
|
8529
|
-
// share `<owner>.<actor>`; a supervised respawn keeps the uid and is unaffected), so it MUST notify
|
|
8530
|
-
// watchers even when its status/activity match the prior lifecycle's lingering record. Omitting this
|
|
8531
|
-
// left a same-principal successor invisible to presence-event consumers (e.g. the manager's
|
|
8532
|
-
// readiness race), which then timed out "uncertain" on a healthy agent (#29).
|
|
8533
|
-
if (prev &&
|
|
8534
|
-
prev.status !== "offline" &&
|
|
8535
|
-
p.status !== "offline" &&
|
|
8536
|
-
prev.lifecycleUid === p.lifecycleUid &&
|
|
8537
|
-
sameRuntime(prev.runtime, p.runtime) &&
|
|
8538
|
-
prev.status === p.status &&
|
|
8539
|
-
prev.activity === p.activity &&
|
|
8540
|
-
sameCondition(prev.condition, p.condition) &&
|
|
8541
|
-
prev.environment === p.environment &&
|
|
8542
|
-
prev.attention === p.attention &&
|
|
8543
|
-
sameChannelModes(prev.channelModes, p.channelModes)) {
|
|
8544
|
-
this.roster.set(id, p);
|
|
8545
|
-
return;
|
|
8546
|
-
}
|
|
8547
|
-
this.roster.set(id, p);
|
|
8548
|
-
const type = p.status === "offline"
|
|
8549
|
-
? "offline"
|
|
8550
|
-
: !prev || prev.status === "offline"
|
|
8551
|
-
? "join"
|
|
8552
|
-
: "update";
|
|
8553
|
-
this.emit("presence", { type, presence: p });
|
|
8554
|
-
this.emit("roster", this.getRoster());
|
|
8555
|
-
}
|
|
8556
|
-
/** Materialize an OFFLINE presence record: drop the advisory attention fields. An offline peer must
|
|
8557
|
-
* not show a stale `[focus]` or "locally muted #x" hint — SPEC: attention removed on offline sweep,
|
|
8558
|
-
* channel modes reset on restart. `statusSince` goes too: it dated the live status, and an observer
|
|
8559
|
-
* that derives offline from a stale heartbeat does not know when the peer left. card/activity/ts
|
|
8560
|
-
* are kept. */
|
|
8561
|
-
toOffline(p) {
|
|
8562
|
-
return { ...p, status: "offline", statusSince: undefined, attention: undefined, channelModes: undefined };
|
|
8563
|
-
}
|
|
8564
|
-
/** Mark a known peer offline (on KV delete/purge), keeping it in the roster. */
|
|
8565
|
-
markOffline(id) {
|
|
8566
|
-
const prev = this.roster.get(id);
|
|
8567
|
-
if (!prev || prev.status === "offline")
|
|
8568
|
-
return;
|
|
8569
|
-
const offline = this.toOffline(prev);
|
|
8570
|
-
this.roster.set(id, offline);
|
|
8571
|
-
this.emit("presence", { type: "offline", presence: offline });
|
|
8572
|
-
this.emit("roster", this.getRoster());
|
|
8573
|
-
}
|
|
8574
|
-
emitPresenceViewIfChanged() {
|
|
8575
|
-
const view = this.presenceView();
|
|
8576
|
-
if (view.state === this.presenceViewState)
|
|
8577
|
-
return;
|
|
8578
|
-
this.presenceViewState = view.state;
|
|
8579
|
-
this.emit("presence-view", view);
|
|
8580
|
-
}
|
|
8581
8226
|
sweep() {
|
|
8582
|
-
|
|
8583
|
-
// Whole-bucket silence past TTL is THIS observer going deaf. Real rosters do not lose every
|
|
8584
|
-
// peer in one window, and treating that silence as N offline verdicts empties an online-only
|
|
8585
|
-
// sidebar with nothing saying the window went blind (#1045). Gate the per-peer age-out on
|
|
8586
|
-
// watch freshness; surface the view as stale instead.
|
|
8587
|
-
if (this.lastPresenceWatchAt !== 0 && now - this.lastPresenceWatchAt > this.ttlMs) {
|
|
8588
|
-
this.emitPresenceViewIfChanged();
|
|
8589
|
-
// Staying stale is the right verdict for a held link (#1045), and the wrong END STATE when
|
|
8590
|
-
// the transport is up and the watch's own consumer is what died. Measured on a live deployment
|
|
8591
|
-
// 2026-09-09: the presence stream was deleted and recreated, its sequence restarted, and
|
|
8592
|
-
// every observer's ORDERED consumer re-created itself at the OLD start sequence (nats.js
|
|
8593
|
-
// 3.4.0 resets from its cursor). The broker kept sending idle heartbeats, so the client
|
|
8594
|
-
// never reset again, the iterator never closed, and the manager's roster stayed frozen at
|
|
8595
|
-
// the pre-recreation snapshot for hours: `cotal ps` read every older seat "mesh offline"
|
|
8596
|
-
// and every newer seat "not in roster" while all of them were heartbeating. The same
|
|
8597
|
-
// end state follows a plain consumer delete (an operator, or the 5-minute inactive
|
|
8598
|
-
// threshold after a long stall). Rebind the watch from the bucket's CURRENT state; a held
|
|
8599
|
-
// link's rebind fails or stays silent and the view simply stays stale, as before.
|
|
8600
|
-
this.rebindStalePresenceWatch(now);
|
|
8601
|
-
return;
|
|
8602
|
-
}
|
|
8603
|
-
let changed = false;
|
|
8604
|
-
for (const [id, p] of this.roster) {
|
|
8605
|
-
// A peer's own `ts` always trails the observer's last delivery, so "older than TTL by the
|
|
8606
|
-
// wall clock" alone ages peers out on the tick just before the whole-bucket gate above
|
|
8607
|
-
// trips (#1311's flap; measured as 3 offline verdicts per silence in the rebind suite).
|
|
8608
|
-
// Require that the watch itself delivered for a full TTL after this peer's last heartbeat:
|
|
8609
|
-
// then other peers were heard and this one was not, which is the only silence that is the
|
|
8610
|
-
// peer's rather than the observer's.
|
|
8611
|
-
if (p.status !== "offline" && now - p.ts > this.ttlMs && this.lastPresenceWatchAt - p.ts > this.ttlMs) {
|
|
8612
|
-
const offline = this.toOffline(p);
|
|
8613
|
-
this.roster.set(id, offline);
|
|
8614
|
-
this.emit("presence", { type: "offline", presence: offline });
|
|
8615
|
-
changed = true;
|
|
8616
|
-
}
|
|
8617
|
-
}
|
|
8618
|
-
if (changed)
|
|
8619
|
-
this.emit("roster", this.getRoster());
|
|
8227
|
+
return this.presence.sweep();
|
|
8620
8228
|
}
|
|
8621
8229
|
}
|
|
8622
8230
|
/** What a history read failure NAMES when it could not finish. One filter subject is the useful
|
|
@@ -8652,7 +8260,7 @@ function kindFromParsed(kind) {
|
|
|
8652
8260
|
* an old-shape alias. Every receive path asks this one predicate. What a path does with a refusal
|
|
8653
8261
|
* (drop, skip, ack, term or reject) and its own extra checks (a usable id, its own echo) stay with
|
|
8654
8262
|
* the path. */
|
|
8655
|
-
function isFromSubjectSender(msg, parsed) {
|
|
8263
|
+
export function isFromSubjectSender(msg, parsed) {
|
|
8656
8264
|
return isRecord(msg) && isRecord(msg.from) && msg.from.id === parsed.sender && isPrincipalOwnerToken(parsed.owner);
|
|
8657
8265
|
}
|
|
8658
8266
|
/** Routing fields in the envelope are advisory. The broker forge-locks sender (and for DMs,
|
|
@@ -8930,24 +8538,6 @@ function assertPartsSerializable(parts) {
|
|
|
8930
8538
|
}
|
|
8931
8539
|
}
|
|
8932
8540
|
}
|
|
8933
|
-
/** Shallow-equal two per-channel-mode maps (presence dedup): a change must re-emit, so an attention
|
|
8934
|
-
* toggle isn't swallowed as a quiet heartbeat. Absent and empty compare equal. */
|
|
8935
|
-
function sameChannelModes(a, b) {
|
|
8936
|
-
const ak = a ? Object.keys(a) : [];
|
|
8937
|
-
const bk = b ? Object.keys(b) : [];
|
|
8938
|
-
if (ak.length !== bk.length)
|
|
8939
|
-
return false;
|
|
8940
|
-
return ak.every((k) => a[k] === b?.[k]);
|
|
8941
|
-
}
|
|
8942
|
-
function sameRuntime(a, b) {
|
|
8943
|
-
return a?.id === b?.id && a?.startedAt === b?.startedAt && a?.source === b?.source;
|
|
8944
|
-
}
|
|
8945
|
-
function sameCondition(a, b) {
|
|
8946
|
-
return a?.code === b?.code
|
|
8947
|
-
&& a?.source === b?.source
|
|
8948
|
-
&& a?.message === b?.message
|
|
8949
|
-
&& a?.since === b?.since;
|
|
8950
|
-
}
|
|
8951
8541
|
function authOpts(a) {
|
|
8952
8542
|
const tls = a.tls ? {} : undefined;
|
|
8953
8543
|
// USER MODE: present the shared auth-account sentinel creds AND the user bearer as `auth_token` (an
|