@cotal-ai/core 0.166.2 → 0.167.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-endpoint.d.ts +242 -0
- package/dist/agent-endpoint.d.ts.map +1 -0
- package/dist/agent-endpoint.js +183 -0
- package/dist/agent-endpoint.js.map +1 -0
- package/dist/backup-config.d.ts +2 -2
- package/dist/endpoint-call.d.ts +34 -1
- package/dist/endpoint-call.d.ts.map +1 -1
- package/dist/endpoint-call.js +209 -116
- package/dist/endpoint-call.js.map +1 -1
- package/dist/endpoint-cluster.d.ts +6 -1
- package/dist/endpoint-cluster.d.ts.map +1 -1
- package/dist/endpoint-cluster.js +25 -1
- package/dist/endpoint-cluster.js.map +1 -1
- package/dist/endpoint-delivery-lease.d.ts +184 -0
- package/dist/endpoint-delivery-lease.d.ts.map +1 -0
- package/dist/endpoint-delivery-lease.js +270 -0
- package/dist/endpoint-delivery-lease.js.map +1 -0
- package/dist/endpoint-envelope.d.ts +4 -3
- package/dist/endpoint-envelope.d.ts.map +1 -1
- package/dist/endpoint-envelope.js +19 -2
- package/dist/endpoint-envelope.js.map +1 -1
- package/dist/endpoint-grants.d.ts +31 -1
- package/dist/endpoint-grants.d.ts.map +1 -1
- package/dist/endpoint-grants.js +62 -1
- package/dist/endpoint-grants.js.map +1 -1
- package/dist/endpoint-presence.d.ts +296 -0
- package/dist/endpoint-presence.d.ts.map +1 -0
- package/dist/endpoint-presence.js +784 -0
- package/dist/endpoint-presence.js.map +1 -0
- package/dist/endpoint-publish-denial.d.ts +6 -0
- package/dist/endpoint-publish-denial.d.ts.map +1 -1
- package/dist/endpoint-publish-denial.js +13 -0
- package/dist/endpoint-publish-denial.js.map +1 -1
- package/dist/endpoint-serve.d.ts +76 -2
- package/dist/endpoint-serve.d.ts.map +1 -1
- package/dist/endpoint-serve.js +340 -147
- package/dist/endpoint-serve.js.map +1 -1
- package/dist/endpoint-service.d.ts +5 -1
- package/dist/endpoint-service.d.ts.map +1 -1
- package/dist/endpoint-service.js +2 -1
- package/dist/endpoint-service.js.map +1 -1
- package/dist/endpoint-subjects.d.ts +76 -2
- package/dist/endpoint-subjects.d.ts.map +1 -1
- package/dist/endpoint-subjects.js +113 -2
- package/dist/endpoint-subjects.js.map +1 -1
- package/dist/endpoint-verbs.d.ts +1 -1
- package/dist/endpoint-verbs.d.ts.map +1 -1
- package/dist/endpoint-verbs.js +1 -1
- package/dist/endpoint-verbs.js.map +1 -1
- package/dist/endpoint.d.ts +118 -190
- package/dist/endpoint.d.ts.map +1 -1
- package/dist/endpoint.js +323 -791
- package/dist/endpoint.js.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/kv-scan.d.ts +2 -1
- package/dist/kv-scan.d.ts.map +1 -1
- package/dist/kv-scan.js +4 -2
- package/dist/kv-scan.js.map +1 -1
- package/dist/peer-label.d.ts +6 -5
- package/dist/peer-label.d.ts.map +1 -1
- package/dist/peer-label.js +7 -6
- package/dist/peer-label.js.map +1 -1
- package/dist/provision.d.ts +11 -1
- package/dist/provision.d.ts.map +1 -1
- package/dist/provision.js +41 -11
- package/dist/provision.js.map +1 -1
- package/dist/subjects.d.ts +3 -2
- package/dist/subjects.d.ts.map +1 -1
- package/dist/subjects.js +10 -2
- package/dist/subjects.js.map +1 -1
- package/dist/types.d.ts +11 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +1 -1
package/dist/endpoint.js
CHANGED
|
@@ -9,7 +9,8 @@ import { createConnection, isIP } from "node:net";
|
|
|
9
9
|
import { connect, headers, tokenAuthenticator, nanos, millis, AuthorizationError, PermissionViolationError, UserAuthenticationExpiredError, NoRespondersError, RequestError, TimeoutError, ClosedConnectionError, ConnectionError, } from "@nats-io/transport-node";
|
|
10
10
|
import { wsconnect } from "@nats-io/nats-core";
|
|
11
11
|
import { parseLivenessAnswer, responderFromLease, responderFromProbe, isLivenessPlane, LIVENESS_PLANES, } from "./liveness.js";
|
|
12
|
-
import {
|
|
12
|
+
import { EndpointPresence } from "./endpoint-presence.js";
|
|
13
|
+
export { PresenceWriteStuckError } from "./endpoint-presence.js";
|
|
13
14
|
import { credsClaims, credsFingerprint, credsRenewalDelayMs, idFromCreds } from "./identity.js";
|
|
14
15
|
import { nativeCredsAuthenticator } from "./creds-authenticator.js";
|
|
15
16
|
import { armRenewal, renewalDeadlineMs, jitteredBackoffMs } from "./renewal.js";
|
|
@@ -20,7 +21,11 @@ import { parseSecretStoreIdentity, } from "./secret-store.js";
|
|
|
20
21
|
import { BIND_SPLIT_REISSUES, resolveService, invokeCommand, goalFollowRefusal, submitAndFollowGoal } from "./endpoint-invoke.js";
|
|
21
22
|
import { EpEnvelopeError, respondedButUnbound, replyRefusedBeforeEffect, replyTargetUnmapped, EP_BIND_REFUSED } from "./endpoint-envelope.js";
|
|
22
23
|
import { isRepeatSafeCommand } from "./endpoint-grants.js";
|
|
23
|
-
import { assertIdToken, assertGeneration } from "./endpoint-subjects.js";
|
|
24
|
+
import { assertIdToken, assertGeneration, assertRuntimeToken, endpointToken } from "./endpoint-subjects.js";
|
|
25
|
+
import { servePrincipalRoute, describeContracts } from "./endpoint-serve.js";
|
|
26
|
+
import { epPrincipalCall } from "./endpoint-call.js";
|
|
27
|
+
import { AGENT_CONTRACTS, AGENT_ENDPOINT } from "./agent-endpoint.js";
|
|
28
|
+
import { processRuntime } from "./process-runtime.js";
|
|
24
29
|
import { importNativeSubjectPermissions, issuedPermitsSubject, readAcceptedRow } from "./issued-authority.js";
|
|
25
30
|
import { REGISTRY_ENDPOINT, REGISTRY_ENDPOINTS_COMMAND } from "./registry-contract.js";
|
|
26
31
|
import { listRegistryEndpoints, registryListingCaller } from "./registry-client.js";
|
|
@@ -32,7 +37,7 @@ import { assertValidKind } from "./agent-file.js";
|
|
|
32
37
|
import { assertValidName } from "./resolve.js";
|
|
33
38
|
import { EVICT_PRINCIPALS_MAX } from "./evict.js";
|
|
34
39
|
import { nextHeartbeat } from "./presence-heartbeat.js";
|
|
35
|
-
import { anycastStoreDefinition, createSpaceStreams, dmDurableConfig, dlvDurableConfig, taskDurableConfig, fanoutDurableConfig, inboxReaderConfig, MANAGER_LEASE_TTL_MS, MANAGER_LEASE_ATTEMPT_MS,
|
|
40
|
+
import { anycastStoreDefinition, createSpaceStreams, dmDurableConfig, dlvDurableConfig, taskDurableConfig, fanoutDurableConfig, inboxReaderConfig, MANAGER_LEASE_TTL_MS, MANAGER_LEASE_ATTEMPT_MS, PRESENCE_STORAGE } from "./streams.js";
|
|
36
41
|
import { DELIVERY_LIMIT_HEADER, HISTORY_NO_COUNT, PLANE3_MAX_MSGS_PER_SUBJECT } from "./backup-config.js";
|
|
37
42
|
import { jetstream, jetstreamManager, AckPolicy, DeliverPolicy, JetStreamApiCodes, JetStreamApiError, } from "@nats-io/jetstream";
|
|
38
43
|
import {} from "@nats-io/jetstream";
|
|
@@ -40,11 +45,11 @@ import { Kvm } from "@nats-io/kv";
|
|
|
40
45
|
import { Bucket, KvWatchInclude } from "@nats-io/kv/internal";
|
|
41
46
|
import { openMembersRegistry, commitMemberRevision, tombstoneMember, activateMember, readMember, deleteMember, listMembers, listLifecycleMemberChannels, durableEligible, } from "./members.js";
|
|
42
47
|
import { openAclRegistry, readAcl, readAclForAlias, aclRetired, AmbiguousAclAlias, commitAcl as writeAclRecord, reissueAcl as writeAclReissue } from "./acls.js";
|
|
43
|
-
import {
|
|
48
|
+
import { EndpointDeliveryLease } from "./endpoint-delivery-lease.js";
|
|
44
49
|
import { openChannelRegistry, effectiveReplay, effectiveReplayWindowMs, effectiveDeliveryClass, readChannelConfig, readChannelDefaults, decodeChannelEntry, } from "./channels.js";
|
|
45
50
|
import { isCasLoss } from "./endpoint-records.js";
|
|
46
51
|
import { ANYCAST_CANCEL_OP, ANYCAST_NOTICE_KEY, ANYCAST_RECORDS_CREDENTIAL_OUTDATED, ANYCAST_STATUS_OP, ANYCAST_TAKES_OP, AnycastRequestAuthority, isAnycastSeq, } from "./anycast-requests.js";
|
|
47
|
-
import { anycastSubject, assertValidOwnerToken, assertValidRole, CHANNEL_DEFAULTS_KEY, chatStream, chatHistDurable, chatSubject, historyReaderConsumer, calloutReaderId, HISTORY_READERS, controlServiceSubject, livenessSubject, livenessServeFilter, CONTROL_DELIVERY, CONTROL_DELIVERY_ADMIN, dmStream, dmDurable, dlvStream, dlvDurable, dlvSubject, dinboxSubject, inboxStream, parseDinboxPrincipal, FANOUT_DURABLE, INBOX_READER_DURABLE,
|
|
52
|
+
import { anycastSubject, assertValidOwnerToken, assertValidRole, CHANNEL_DEFAULTS_KEY, chatStream, chatHistDurable, chatSubject, historyReaderConsumer, calloutReaderId, HISTORY_READERS, controlServiceSubject, livenessSubject, livenessServeFilter, CONTROL_DELIVERY, CONTROL_DELIVERY_ADMIN, dmStream, dmDurable, dlvStream, dlvDurable, dlvSubject, dinboxSubject, inboxStream, parseDinboxPrincipal, FANOUT_DURABLE, INBOX_READER_DURABLE, managerBucket, MANAGER_LEASE_KEY, MANAGER_RENEWAL_LEASE_KEY, managerLeaseKey, assertValidChannel, channelInAllow, membershipInAllow, isConcreteChannel, isChannelFamily, familiesCovering, normalizeMentions, MENTION_NOTICE_KEY, parseSubject, isPrincipalOwnerToken, assertInboxConnId, presenceBucket, channelBucket, membershipBucket, MEMBERSHIP_FEED_KEY, principalKey, parsePrincipalKey, assertLifecycleToken, mintLifecycleUid, lifecycleNameKey, memberKey, DEV_OWNER, spacePrefix, spaceWildcard, subjectMatches, taskStream, taskDurable, takeServeFilter, takeSubject, parseTakeSubject, TAKE_ROUTE, token, unicastSubject, unicastRecvFilter, } from "./subjects.js";
|
|
48
53
|
import { isPermissionDenied } from "./permission-denied.js";
|
|
49
54
|
export { isPermissionDenied };
|
|
50
55
|
export const DEFAULT_SERVER = "nats://127.0.0.1:4222";
|
|
@@ -128,27 +133,6 @@ export class DmBindRefusedError extends Error {
|
|
|
128
133
|
this.name = "DmBindRefusedError";
|
|
129
134
|
}
|
|
130
135
|
}
|
|
131
|
-
/** A presence bucket that has refused consecutive writes for at least one full presence TTL.
|
|
132
|
-
* The condition is non-transient: ordinary heartbeat retry has already failed for the whole
|
|
133
|
-
* liveness window, so callers should report the roster as last-known until a write succeeds. */
|
|
134
|
-
export class PresenceWriteStuckError extends Error {
|
|
135
|
-
bucket;
|
|
136
|
-
since;
|
|
137
|
-
consecutiveFailures;
|
|
138
|
-
ttlMs;
|
|
139
|
-
lastError;
|
|
140
|
-
code = "presence-write-stuck";
|
|
141
|
-
transient = false;
|
|
142
|
-
constructor(bucket, since, consecutiveFailures, ttlMs, lastError) {
|
|
143
|
-
super(`presence writes to bucket ${JSON.stringify(bucket)} have failed ${consecutiveFailures} consecutive times for at least one TTL (${ttlMs}ms); the presence view is not live until a write succeeds or the broker store is repaired${lastError ? ` (last refusal: ${lastError})` : ""}`);
|
|
144
|
-
this.bucket = bucket;
|
|
145
|
-
this.since = since;
|
|
146
|
-
this.consecutiveFailures = consecutiveFailures;
|
|
147
|
-
this.ttlMs = ttlMs;
|
|
148
|
-
this.lastError = lastError;
|
|
149
|
-
this.name = "PresenceWriteStuckError";
|
|
150
|
-
}
|
|
151
|
-
}
|
|
152
136
|
/**
|
|
153
137
|
* Events: "message" (CotalMessage), "presence" (PresenceEvent), "roster" (Presence[]), "error" (Error),
|
|
154
138
|
* "connection" ({ connected: boolean }) — true on every successful (re)bind (initial start, manual
|
|
@@ -295,11 +279,6 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
295
279
|
jsm;
|
|
296
280
|
kv;
|
|
297
281
|
channelKv;
|
|
298
|
-
/** The presence/channel-registry watches' own handles. Each is an ORDERED push consumer: its idle
|
|
299
|
-
* heartbeat monitor lives on a JS timer independent of the connection, so a drain that doesn't
|
|
300
|
-
* `.stop()` it first leaves the monitor to fire into a closing connection every 30s, throwing
|
|
301
|
-
* `DrainingConnectionError` out of `reset()` on a timer nothing awaits. */
|
|
302
|
-
presenceWatchIter;
|
|
303
282
|
channelWatchIter;
|
|
304
283
|
/** Consumer deletes of retired presence and channel-registry watches still in flight. A stop
|
|
305
284
|
* alone leaves each watch's ordered consumer on the broker for its five-minute inactive
|
|
@@ -310,23 +289,10 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
310
289
|
* {@link stop} or a rebuild moved the epoch retires its watch and starts that delete itself, so
|
|
311
290
|
* the same two paths wait for these first. */
|
|
312
291
|
kvWatchBinds = new Set();
|
|
313
|
-
/** The daemon's lease-loss watch (#1596): the INTENT that survives a connection rebuild, plus the
|
|
314
|
-
* live iterator. See {@link watchDeliveryLease}. */
|
|
315
|
-
leaseWatchIntent;
|
|
316
|
-
leaseWatchIter;
|
|
317
|
-
/** Consumer deletes of retired lease watches still in flight. The stop handle cannot await its
|
|
318
|
-
* delete, so {@link stop} and a live-epoch rebuild await these before they close the connection
|
|
319
|
-
* that carries them (#3966). */
|
|
320
|
-
leaseWatchDeletes = new Set();
|
|
321
|
-
/** Lease-watch binds still awaiting the broker. One that lands after {@link stop} or a rebuild
|
|
322
|
-
* moved the epoch retires its watch and starts that delete itself, so the same two paths wait for
|
|
323
|
-
* these first. Each is bounded by the JetStream request timeout. */
|
|
324
|
-
leaseWatchBinds = new Set();
|
|
325
292
|
/** Plane-3 durable-membership registry KV — lazily opened by the privileged delivery daemon (or a
|
|
326
293
|
* short-lived provisioner). */
|
|
327
294
|
membersKv;
|
|
328
295
|
aclKv;
|
|
329
|
-
deliveryKv;
|
|
330
296
|
managerLeaseKv;
|
|
331
297
|
/** Our revision of the per-space daemon-credential renewal lease (#1634), or undefined when we do
|
|
332
298
|
* not hold it. Cleared with the bound KV handle: a revision from a dead connection is not a lease
|
|
@@ -392,6 +358,12 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
392
358
|
* subscription on it, so a responder bound once would leave its plane to the broker's
|
|
393
359
|
* no-responders answer, which a peer grades `unbound`, while this endpoint is connected and serving. */
|
|
394
360
|
livenessResponders = new Map();
|
|
361
|
+
/** The principal endpoints {@link servePrincipal} serves, by endpoint name: INTENT, re-bound on
|
|
362
|
+
* every replaced connection. */
|
|
363
|
+
principalServes = new Map();
|
|
364
|
+
/** The `prompts` presence field {@link setPromptsPresence} last set. Shown only while the `agent`
|
|
365
|
+
* endpoint is served and its subscription was accepted on the current connection. */
|
|
366
|
+
promptsPresence;
|
|
395
367
|
streamMsgs = [];
|
|
396
368
|
/** Per-channel native core subscriptions (SPEC v0.3) — the manager-free live read path for boot +
|
|
397
369
|
* runtime channels (there is no per-instance chat durable). Keyed by channel so leave unsubscribes
|
|
@@ -439,71 +411,42 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
439
411
|
firstConnect = true;
|
|
440
412
|
heartbeatTimer;
|
|
441
413
|
sweepTimer;
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
*/
|
|
474
|
-
lastPresenceWatchAt = 0;
|
|
475
|
-
/** Last emitted presence-view state. Suppresses duplicate `presence-view` events. */
|
|
476
|
-
presenceViewState = "unpopulated";
|
|
477
|
-
/** A presence-watch rebind in flight (see {@link rebindStalePresenceWatch}); one at a time. */
|
|
478
|
-
presenceRebind;
|
|
414
|
+
deliveryLease = new EndpointDeliveryLease({
|
|
415
|
+
card: () => this.card,
|
|
416
|
+
space: () => this.space,
|
|
417
|
+
connection: () => this.nc,
|
|
418
|
+
epoch: () => this.presenceEpoch,
|
|
419
|
+
notLiveMsg: () => this.notLiveMsg(),
|
|
420
|
+
deleteWatchConsumer: (iter) => this.deleteOwnConsumer(kvWatchConsumer(iter)),
|
|
421
|
+
recoverable: (error) => this.emitRecoverable(error),
|
|
422
|
+
emit: (event, error) => this.emit(event, error),
|
|
423
|
+
});
|
|
424
|
+
presence = new EndpointPresence({
|
|
425
|
+
card: () => this.card,
|
|
426
|
+
space: () => this.space,
|
|
427
|
+
bucket: () => this.kv,
|
|
428
|
+
connection: () => this.nc,
|
|
429
|
+
lifecycleUid: () => this.ownLifecycleUid,
|
|
430
|
+
prompts: () => this.servedPrompts(),
|
|
431
|
+
ttlMs: () => this.ttlMs,
|
|
432
|
+
registers: () => this.doRegister,
|
|
433
|
+
watches: () => this.doWatch,
|
|
434
|
+
readsOnce: () => this.readOnce,
|
|
435
|
+
stopped: () => this.stopped,
|
|
436
|
+
reconnecting: () => this.reconnecting,
|
|
437
|
+
epoch: () => this.presenceEpoch,
|
|
438
|
+
notLiveMsg: () => this.notLiveMsg(),
|
|
439
|
+
bindWatch: (kv) => this.bindKvWatch(kv),
|
|
440
|
+
retireWatch: (iter) => this.retireKvWatch(iter),
|
|
441
|
+
emit: (event, value) => this.emit(event, value),
|
|
442
|
+
recoverable: (error) => this.emitRecoverable(error),
|
|
443
|
+
isPublishPermissionDenied,
|
|
444
|
+
});
|
|
479
445
|
/** Bumped by every connection-scoped teardown and by {@link stop}. A presence or lease-watch bind
|
|
480
446
|
* that was awaiting the broker when the epoch moved belongs to a retired epoch: it releases the
|
|
481
447
|
* iterator it got and installs nothing (see {@link startPresenceWatch} and
|
|
482
448
|
* {@link bindDeliveryLeaseWatch}). */
|
|
483
449
|
presenceEpoch = 0;
|
|
484
|
-
/** The current watch was bound onto a bucket with NO keys (see {@link markPresenceBucketEmpty}).
|
|
485
|
-
* Such a watch cannot deliver until someone writes, so its silence is not staleness. */
|
|
486
|
-
presenceWatchEmpty = false;
|
|
487
|
-
/** Wall-clock of the last rebind attempt, so a bucket that is silent because it is EMPTY (or a
|
|
488
|
-
* broker that keeps refusing the consumer create) is retried once per TTL, not per sweep tick. */
|
|
489
|
-
presenceRebindAt = 0;
|
|
490
|
-
status = "idle";
|
|
491
|
-
activity;
|
|
492
|
-
/** When {@link status} or {@link activity} last changed. Heartbeats republish it unchanged. */
|
|
493
|
-
statusSince = Date.now();
|
|
494
|
-
/** When {@link activity} last changed. A status change leaves it alone. */
|
|
495
|
-
activitySince;
|
|
496
|
-
/** Last harness-reported work progress. Carried by the next heartbeat, never published per event. */
|
|
497
|
-
activeAt;
|
|
498
|
-
condition;
|
|
499
|
-
/** Advances on every condition change so an older in-flight put cannot be the final KV state. */
|
|
500
|
-
conditionRevision = 0;
|
|
501
|
-
/** Read once at construction. Core publishes this opaque provider reference and never parses it. */
|
|
502
|
-
environment;
|
|
503
|
-
/** Mirror of the connector's authoritative attention state, published in presence (advisory). The
|
|
504
|
-
* endpoint never reads these back into delivery — they exist only to broadcast. */
|
|
505
|
-
attentionMode;
|
|
506
|
-
channelModes;
|
|
507
450
|
stopped = false;
|
|
508
451
|
stopPromise;
|
|
509
452
|
startPromise;
|
|
@@ -572,9 +515,6 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
572
515
|
* only evidence the split rate exists. Always on, never behind a flag — a counter you have to
|
|
573
516
|
* enable is not there when the thing you needed it for happened. */
|
|
574
517
|
splitsRecovered = 0;
|
|
575
|
-
/** Presence rows rejected because their embedded id did not match the ACL-scoped KV key, or because
|
|
576
|
-
* a field the observer computes with was malformed. */
|
|
577
|
-
presenceBindingDrops = 0;
|
|
578
518
|
/** This endpoint's wire principal (owner + actor tokens, §13.2) — what its minted grant rows
|
|
579
519
|
* pin. Public so a caller can build owner-mode target blocks for {@link invokeService}. */
|
|
580
520
|
get principal() {
|
|
@@ -587,7 +527,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
587
527
|
}
|
|
588
528
|
/** Mis-keyed or malformed presence rows this reader rejected. The warning event may be missed; the count cannot. */
|
|
589
529
|
get presenceBindingDropCount() {
|
|
590
|
-
return this.
|
|
530
|
+
return this.presence.bindingDropCount;
|
|
591
531
|
}
|
|
592
532
|
/** The endpoint's own lifecycle UID, REQUIRED for every lifecycle-keyed messaging resource; absent
|
|
593
533
|
* ⇒ loud refusal naming the operation (the hard cut of SPEC §13.1 — no alias-keyed fallback). */
|
|
@@ -708,8 +648,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
708
648
|
// principalKey validates both tokens.
|
|
709
649
|
const principal = principalKey(this.owner, this.actor);
|
|
710
650
|
this.card = { ...opts.card, id: principal.key, owner: this.owner, actor: this.actor };
|
|
711
|
-
this.environment
|
|
712
|
-
(opts.environment === undefined ? process.env.COTAL_ENVIRONMENT : opts.environment)?.trim() || undefined;
|
|
651
|
+
this.presence.configureEnvironment((opts.environment === undefined ? process.env.COTAL_ENVIRONMENT : opts.environment)?.trim() || undefined);
|
|
713
652
|
this.servers = opts.servers ?? DEFAULT_SERVER;
|
|
714
653
|
this.token = opts.token;
|
|
715
654
|
this.user = opts.user;
|
|
@@ -756,7 +695,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
756
695
|
}
|
|
757
696
|
// Seed the presence mirror so file-default channel modes are visible from the first publish
|
|
758
697
|
// (not only after the first runtime toggle). Mirror only — delivery reads the connector's state.
|
|
759
|
-
this.
|
|
698
|
+
this.presence.seedChannelModes(opts.channelModes);
|
|
760
699
|
this.ackWaitMs = opts.ackWaitMs ?? 60_000;
|
|
761
700
|
this.inactiveThresholdMs = opts.inactiveThresholdMs ?? 600_000;
|
|
762
701
|
}
|
|
@@ -1520,7 +1459,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
1520
1459
|
// clearConnectionScoped; a bind that completes after a stop or another rebuild finds the epoch
|
|
1521
1460
|
// moved and retires itself. The replay re-delivers the row's current state, so a change that
|
|
1522
1461
|
// landed in the null window still reaches the trigger.
|
|
1523
|
-
if (this.
|
|
1462
|
+
if (this.deliveryLease.hasWatchIntent)
|
|
1524
1463
|
await this.bindDeliveryLeaseWatch();
|
|
1525
1464
|
// FAIL BEFORE PRESENCE (SPEC 13.1): an AUTHED endpoint that will register on the roster or
|
|
1526
1465
|
// bind lifecycle-keyed consumers must hold its launcher-supplied lifecycle uid BEFORE anything
|
|
@@ -1606,6 +1545,8 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
1606
1545
|
// the old connection and the responders with it.
|
|
1607
1546
|
for (const plane of this.livenessResponders.keys())
|
|
1608
1547
|
this.bindLiveness(plane);
|
|
1548
|
+
// The principal serves subscribe again on the new connection and run their check again.
|
|
1549
|
+
this.rebindPrincipalServes(false);
|
|
1609
1550
|
for (const tap of this.taps)
|
|
1610
1551
|
this.bindTap(tap);
|
|
1611
1552
|
// Bound and live — covers initial start, manual reconnect, AND background self-heal (every
|
|
@@ -1657,10 +1598,8 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
1657
1598
|
}
|
|
1658
1599
|
this.streamMsgs.length = 0;
|
|
1659
1600
|
this.presenceEpoch++;
|
|
1660
|
-
this.
|
|
1661
|
-
|
|
1662
|
-
this.retireKvWatch(this.presenceWatchIter);
|
|
1663
|
-
this.presenceWatchIter = undefined;
|
|
1601
|
+
this.presence.cancelRebind();
|
|
1602
|
+
this.presence.retireWatch();
|
|
1664
1603
|
if (this.channelWatchIter)
|
|
1665
1604
|
this.retireKvWatch(this.channelWatchIter);
|
|
1666
1605
|
this.channelWatchIter = undefined;
|
|
@@ -1676,22 +1615,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
1676
1615
|
this.chatSubs.clear();
|
|
1677
1616
|
this.chatSubDenied.clear();
|
|
1678
1617
|
this.ownWatchDeletePrefixes.clear();
|
|
1679
|
-
this.
|
|
1680
|
-
// A read in flight on the old connection must not land in the new connection's roster.
|
|
1681
|
-
this.presenceRead = undefined;
|
|
1682
|
-
this.presenceReadGeneration++;
|
|
1683
|
-
this.presenceReadApplied = this.presenceReadGeneration;
|
|
1684
|
-
// #1356: the presence-refusal record is connection-scoped like everything else torn down here.
|
|
1685
|
-
// It says "the broker on THIS connection refuses writes to this bucket", so it cannot outlive the
|
|
1686
|
-
// connection that observed it. Left behind, a later failure on a different connection inherits it
|
|
1687
|
-
// and is reported as a bucket refusal — measured: a failed bind against an unreachable server
|
|
1688
|
-
// still printed the refusal sentence while `connectionIssue` already said "connection refused".
|
|
1689
|
-
this.clearPresenceWriteFailure();
|
|
1690
|
-
this.lastPresenceWatchAt = 0;
|
|
1691
|
-
this.presenceRebindAt = 0;
|
|
1692
|
-
this.presenceWatchEmpty = false;
|
|
1693
|
-
this.presenceSnapshotPopulated = false;
|
|
1694
|
-
this.emitPresenceViewIfChanged();
|
|
1618
|
+
this.presence.resetConnection();
|
|
1695
1619
|
this.channelConfigs.clear();
|
|
1696
1620
|
this.channelDefaults = {};
|
|
1697
1621
|
// A read in flight on the old connection must not land in the new connection's cache.
|
|
@@ -1737,7 +1661,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
1737
1661
|
this.membersKv = undefined;
|
|
1738
1662
|
this.aclKv = undefined;
|
|
1739
1663
|
this.membershipFeedKv = undefined;
|
|
1740
|
-
this.
|
|
1664
|
+
this.deliveryLease.resetConnection();
|
|
1741
1665
|
this.managerLeaseKv = undefined;
|
|
1742
1666
|
this.daemonRenewalLeaseRevision = undefined;
|
|
1743
1667
|
// Handles that armDeliveryControl created on a previous connection: null them so a rebind
|
|
@@ -1891,7 +1815,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
1891
1815
|
this.membersKv = undefined;
|
|
1892
1816
|
this.aclKv = undefined;
|
|
1893
1817
|
this.membershipFeedKv = undefined;
|
|
1894
|
-
this.
|
|
1818
|
+
this.deliveryLease.resetConnection();
|
|
1895
1819
|
// The manager's liveness-lease handle too: left bound to the old connection, every renew and
|
|
1896
1820
|
// re-read after a reconnect times out, and the manager reports its lease unknown for good.
|
|
1897
1821
|
this.managerLeaseKv = undefined;
|
|
@@ -1978,6 +1902,16 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
1978
1902
|
this.reestablishing = false;
|
|
1979
1903
|
}
|
|
1980
1904
|
}
|
|
1905
|
+
/** Run the next rebuild attempt now if the endpoint is waiting out a {@link reestablishLoop}
|
|
1906
|
+
* backoff, for a caller with work waiting on the link. After a long outage that backoff can run
|
|
1907
|
+
* for up to a minute past the broker's return, and a send made in it waited for an attempt
|
|
1908
|
+
* nobody had scheduled (#7408). Idle endpoints keep their spread-out retries. A no-op otherwise:
|
|
1909
|
+
* a rebuild in flight is already the attempt, and a link nats.js is still reconnecting keeps
|
|
1910
|
+
* its buffered publishes. */
|
|
1911
|
+
retryNow() {
|
|
1912
|
+
if (!this.stopped && this.reestablishing && !this.rebuildPromise)
|
|
1913
|
+
this.kickBackoff();
|
|
1914
|
+
}
|
|
1981
1915
|
/** Cut an in-flight reestablish backoff short so the next attempt runs immediately, and
|
|
1982
1916
|
* clear its timer so it can't fire later on a stopped/restarted loop. */
|
|
1983
1917
|
kickBackoff() {
|
|
@@ -2020,7 +1954,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
2020
1954
|
}
|
|
2021
1955
|
this.activeGoalFollowers.clear();
|
|
2022
1956
|
this.presenceEpoch++;
|
|
2023
|
-
this.
|
|
1957
|
+
this.presence.cancelRebind();
|
|
2024
1958
|
// Wake a reestablishLoop sitting in backoff so it sees `stopped` and exits instead of
|
|
2025
1959
|
// sleeping out retryMs; also clears the timer so it can't fire later.
|
|
2026
1960
|
this.kickBackoff();
|
|
@@ -2060,17 +1994,14 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
2060
1994
|
/* already closed */
|
|
2061
1995
|
}
|
|
2062
1996
|
}
|
|
2063
|
-
|
|
2064
|
-
this.retireKvWatch(this.presenceWatchIter);
|
|
2065
|
-
this.presenceWatchIter = undefined;
|
|
1997
|
+
this.presence.retireWatch();
|
|
2066
1998
|
if (this.channelWatchIter)
|
|
2067
1999
|
this.retireKvWatch(this.channelWatchIter);
|
|
2068
2000
|
this.channelWatchIter = undefined;
|
|
2069
2001
|
this.stopDeliveryLeaseWatch();
|
|
2070
2002
|
try {
|
|
2071
2003
|
if (this.doRegister) {
|
|
2072
|
-
this.
|
|
2073
|
-
await this.publishPresence();
|
|
2004
|
+
await this.presence.publishOffline();
|
|
2074
2005
|
}
|
|
2075
2006
|
}
|
|
2076
2007
|
catch {
|
|
@@ -3092,7 +3023,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
3092
3023
|
}
|
|
3093
3024
|
// ---- presence ------------------------------------------------------------
|
|
3094
3025
|
getRoster() {
|
|
3095
|
-
return
|
|
3026
|
+
return this.presence.getRoster();
|
|
3096
3027
|
}
|
|
3097
3028
|
/**
|
|
3098
3029
|
* Read the roster from the broker now, for an endpoint that does not watch presence (SPEC §6). One
|
|
@@ -3106,49 +3037,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
3106
3037
|
* nothing. A credential that may not make the read is refused with the grant it lacks.
|
|
3107
3038
|
*/
|
|
3108
3039
|
async readPresence() {
|
|
3109
|
-
|
|
3110
|
-
throw new Error("readPresence is for an endpoint that does not watch presence; this one keeps its roster live (getRoster)");
|
|
3111
|
-
const kv = this.kv;
|
|
3112
|
-
if (!kv)
|
|
3113
|
-
throw new Error(this.nc ? "readPresence needs the presence bucket open (registerPresence or watchPresence)" : this.notLiveMsg());
|
|
3114
|
-
if (this.presenceRead)
|
|
3115
|
-
return this.presenceRead;
|
|
3116
|
-
const generation = ++this.presenceReadGeneration;
|
|
3117
|
-
const roster = (async () => {
|
|
3118
|
-
let entries;
|
|
3119
|
-
try {
|
|
3120
|
-
entries = await lastKvEntries(kv);
|
|
3121
|
-
}
|
|
3122
|
-
catch (e) {
|
|
3123
|
-
if (!isPublishPermissionDenied(e))
|
|
3124
|
-
throw e;
|
|
3125
|
-
throw new Error(`presence read refused for ${this.card.id}: this credential may not publish $JS.API.DIRECT.GET.KV_${presenceBucket(this.space)}, the on-demand roster read (SPEC Appendix B). A credential minted by an older issuer can lack this grant even when newly minted; upgrade the manager or auth service that issues credentials, then mint a new credential. Reusing credentials or reconnecting does not add the grant.`, { cause: e });
|
|
3126
|
-
}
|
|
3127
|
-
const at = Date.now();
|
|
3128
|
-
const read = new Map();
|
|
3129
|
-
for (const e of entries) {
|
|
3130
|
-
if (e.key === TTL_RECONCILE_CANARY_KEY || e.operation === "DEL" || e.operation === "PURGE")
|
|
3131
|
-
continue;
|
|
3132
|
-
const p = this.decodePresence(e);
|
|
3133
|
-
if (p)
|
|
3134
|
-
read.set(e.key, p.status === "offline" || at - p.ts > this.ttlMs ? this.toOffline(p) : p);
|
|
3135
|
-
}
|
|
3136
|
-
if (generation > this.presenceReadApplied) {
|
|
3137
|
-
this.presenceReadApplied = generation;
|
|
3138
|
-
for (const [id, p] of this.roster)
|
|
3139
|
-
if (!read.has(id))
|
|
3140
|
-
read.set(id, p.status === "offline" ? p : this.toOffline(p));
|
|
3141
|
-
this.roster.clear();
|
|
3142
|
-
for (const [id, p] of read)
|
|
3143
|
-
this.roster.set(id, p);
|
|
3144
|
-
}
|
|
3145
|
-
return this.getRoster();
|
|
3146
|
-
})();
|
|
3147
|
-
this.presenceRead = roster;
|
|
3148
|
-
const settled = () => { if (this.presenceRead === roster)
|
|
3149
|
-
this.presenceRead = undefined; };
|
|
3150
|
-
roster.then(settled, settled);
|
|
3151
|
-
return roster;
|
|
3040
|
+
return this.presence.readPresence();
|
|
3152
3041
|
}
|
|
3153
3042
|
/**
|
|
3154
3043
|
* Trust state of THIS observer's presence watch, not of any peer. `unpopulated` means the
|
|
@@ -3158,82 +3047,40 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
3158
3047
|
* unsafe states so consumers written before `state` was added degrade in the safe direction.
|
|
3159
3048
|
*/
|
|
3160
3049
|
presenceView() {
|
|
3161
|
-
|
|
3162
|
-
return { state: "current", fresh: true };
|
|
3163
|
-
if (!this.presenceSnapshotPopulated)
|
|
3164
|
-
return { state: "unpopulated", fresh: false };
|
|
3165
|
-
if (this.presenceWatchEmpty)
|
|
3166
|
-
return { state: "current", fresh: true };
|
|
3167
|
-
const staleSince = this.lastPresenceWatchAt + this.ttlMs;
|
|
3168
|
-
if (Date.now() < staleSince)
|
|
3169
|
-
return { state: "current", fresh: true };
|
|
3170
|
-
return { state: "stale", fresh: false, staleSince };
|
|
3050
|
+
return this.presence.presenceView();
|
|
3171
3051
|
}
|
|
3172
3052
|
/** Wait until the current presence watch has consumed its initial KV snapshot. An observer that
|
|
3173
3053
|
* does not register completes on an empty bucket at once. A registering one completes when its
|
|
3174
3054
|
* own record arrives, and the timeout bounds that wait. */
|
|
3175
3055
|
async waitForPresenceSnapshot(timeoutMs = 1_000) {
|
|
3176
|
-
|
|
3177
|
-
try {
|
|
3178
|
-
return await Promise.race([
|
|
3179
|
-
this.presenceSnapshot.then(() => "snapshot"),
|
|
3180
|
-
new Promise((resolve) => { timer = setTimeout(() => resolve("timeout"), timeoutMs); }),
|
|
3181
|
-
]);
|
|
3182
|
-
}
|
|
3183
|
-
finally {
|
|
3184
|
-
if (timer)
|
|
3185
|
-
clearTimeout(timer);
|
|
3186
|
-
}
|
|
3187
|
-
}
|
|
3188
|
-
assertPresenceUpdate(update) {
|
|
3189
|
-
assertPresenceRow({ card: this.card, status: this.status, activity: this.activity,
|
|
3190
|
-
condition: this.condition, attention: this.attentionMode, ts: Date.now(), ...update }, this.card.id);
|
|
3056
|
+
return this.presence.waitForPresenceSnapshot(timeoutMs);
|
|
3191
3057
|
}
|
|
3192
3058
|
async setActivity(activity) {
|
|
3193
|
-
this.
|
|
3194
|
-
if (activity !== this.activity)
|
|
3195
|
-
this.statusSince = this.activitySince = Date.now();
|
|
3196
|
-
this.activity = activity;
|
|
3197
|
-
await this.publishPresence();
|
|
3059
|
+
return this.presence.setActivity(activity);
|
|
3198
3060
|
}
|
|
3199
3061
|
/** Set the status, and with `activity` the activity line too, in one presence write, so no peer sees
|
|
3200
3062
|
* the new activity beside the old status. */
|
|
3201
3063
|
async setStatus(status, activity) {
|
|
3202
|
-
this.
|
|
3203
|
-
if (activity !== undefined && activity !== this.activity)
|
|
3204
|
-
this.statusSince = this.activitySince = Date.now();
|
|
3205
|
-
if (activity !== undefined)
|
|
3206
|
-
this.activity = activity;
|
|
3207
|
-
if (status !== this.status)
|
|
3208
|
-
this.statusSince = Date.now();
|
|
3209
|
-
this.status = status;
|
|
3210
|
-
await this.publishPresence();
|
|
3064
|
+
return this.presence.setStatus(status, activity);
|
|
3211
3065
|
}
|
|
3212
3066
|
/** Record harness-reported work progress as `activeAt`. It writes nothing itself: the next heartbeat
|
|
3213
3067
|
* carries it, so a stream of events costs no presence writes. */
|
|
3214
3068
|
noteActivity(at = Date.now()) {
|
|
3215
|
-
|
|
3216
|
-
this.activeAt = at;
|
|
3069
|
+
return this.presence.noteActivity(at);
|
|
3217
3070
|
}
|
|
3218
3071
|
/** Publish a harness-reported condition, or clear it. Core stores the relay without interpretation. */
|
|
3219
3072
|
async setCondition(condition) {
|
|
3220
|
-
this.
|
|
3221
|
-
this.condition = condition ?? undefined;
|
|
3222
|
-
this.conditionRevision++;
|
|
3223
|
-
await this.publishPresence();
|
|
3073
|
+
return this.presence.setCondition(condition);
|
|
3224
3074
|
}
|
|
3225
3075
|
/** Publish the agent's global attention mode into presence (advisory observability). Mirror only —
|
|
3226
3076
|
* delivery decisions stay in the connector's authoritative state. */
|
|
3227
3077
|
async setAttention(attention) {
|
|
3228
|
-
this.
|
|
3229
|
-
this.attentionMode = attention;
|
|
3230
|
-
await this.publishPresence();
|
|
3078
|
+
return this.presence.setAttention(attention);
|
|
3231
3079
|
}
|
|
3232
3080
|
/** Publish the agent's per-channel attention overrides into presence (advisory). An empty map drops
|
|
3233
3081
|
* the field. Mirror only — never read back into delivery. */
|
|
3234
3082
|
async setChannelModes(modes) {
|
|
3235
|
-
this.
|
|
3236
|
-
await this.publishPresence();
|
|
3083
|
+
return this.presence.setChannelModes(modes);
|
|
3237
3084
|
}
|
|
3238
3085
|
/** Overlay the host's live model and optional variant onto the card's display-only metadata, then
|
|
3239
3086
|
* republish presence. For connectors that learn their actual selection only after launch (e.g.
|
|
@@ -3444,8 +3291,9 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
3444
3291
|
* a hard read boundary for the backstop — a pre-leave entry stays deliverable, `seq > leaveCursor` is
|
|
3445
3292
|
* denied). FAIL-CLOSED: if the tombstone can't be confirmed the call throws and the leave is NOT
|
|
3446
3293
|
* applied (live sub stays up, local mirror intact) so the caller can retry — never close the live
|
|
3447
|
-
* read while the backstop keeps delivering.
|
|
3448
|
-
|
|
3294
|
+
* read while the backstop keeps delivering. `legacyDiscoveryMessage` lets the caller supply the
|
|
3295
|
+
* remedy when membership discovery predates lifecycle support, without changing the refusal. */
|
|
3296
|
+
async leaveChannel(channel, legacyDiscoveryMessage) {
|
|
3449
3297
|
if (!this.jsm)
|
|
3450
3298
|
throw new Error(this.notLiveMsg());
|
|
3451
3299
|
let left = this.channels.includes(channel);
|
|
@@ -3466,7 +3314,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
3466
3314
|
// A daemon before 0.166.0 answers with this lifecycle's rows and no lifecycleUid, so it cannot
|
|
3467
3315
|
// show an earlier lifecycle's row. Refuse with the remedy rather than leave that row open.
|
|
3468
3316
|
if (member.lifecycleUid === undefined)
|
|
3469
|
-
throw new Error("the delivery daemon predates lifecycle-aware membership discovery, so this durable leave was not applied and the channel stays joined; restart the daemon at this version (`cotal up` runs it)");
|
|
3317
|
+
throw new Error(legacyDiscoveryMessage ?? "the delivery daemon predates lifecycle-aware membership discovery, so this durable leave was not applied and the channel stays joined; restart the daemon at this version (`cotal up` runs it)");
|
|
3470
3318
|
await this.durableLeaveChannel(channel, member.generation, member.lifecycleUid);
|
|
3471
3319
|
left = true;
|
|
3472
3320
|
}
|
|
@@ -3580,7 +3428,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
3580
3428
|
async channelMembers(channel) {
|
|
3581
3429
|
const members = (await listMembers(await this.membersRegistry())).filter((r) => r.leaveCursor === undefined && r.activated === true);
|
|
3582
3430
|
const byId = new Map();
|
|
3583
|
-
for (const p of this.
|
|
3431
|
+
for (const p of this.presence.values())
|
|
3584
3432
|
byId.set(p.card.id, p);
|
|
3585
3433
|
const memberForId = (id) => {
|
|
3586
3434
|
const p = byId.get(id);
|
|
@@ -4681,6 +4529,8 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
4681
4529
|
}
|
|
4682
4530
|
if (s.type === "reconnect") {
|
|
4683
4531
|
this.armAuthExpiryReconnectFence(nc);
|
|
4532
|
+
// The library re-subscribed on a connection the broker authorized again: re-run the check.
|
|
4533
|
+
this.rebindPrincipalServes(true);
|
|
4684
4534
|
this.emit("transport", { connected: true, server: s.server });
|
|
4685
4535
|
for (const follower of this.activeGoalFollowers) {
|
|
4686
4536
|
follower.reconnected(nc);
|
|
@@ -4695,6 +4545,10 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
4695
4545
|
continue;
|
|
4696
4546
|
if (s.error instanceof PermissionViolationError && this.chatSubRefusals.has(s.error))
|
|
4697
4547
|
continue;
|
|
4548
|
+
// A principal-route refusal belongs to the call that throws it or the serve that ends on it
|
|
4549
|
+
// (SPEC 13.21); nothing else on this private connection publishes or subscribes there.
|
|
4550
|
+
if (s.error instanceof PermissionViolationError && this.isPrincipalRouteSubject(s.error.subject))
|
|
4551
|
+
continue;
|
|
4698
4552
|
if (s.error instanceof PermissionViolationError && handedToRequest.has(s.error))
|
|
4699
4553
|
continue;
|
|
4700
4554
|
// The predecessor nats.js deleted while rebuilding one of this connection's watches, scans or
|
|
@@ -4964,24 +4818,6 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
4964
4818
|
const issued = row.record.issuedAllowSubscribe ?? allow;
|
|
4965
4819
|
return { allowSubscribe: allow, issuedAllowSubscribe: issued, lifecycleUid: row.lifecycleUid };
|
|
4966
4820
|
}
|
|
4967
|
-
/** Lazily open the delivery lease/readiness KV (pre-created at `cotal up`; bind, never create). */
|
|
4968
|
-
async deliveryRegistry() {
|
|
4969
|
-
if (!this.nc)
|
|
4970
|
-
throw new Error(this.notLiveMsg());
|
|
4971
|
-
this.deliveryKv ??= await openDeliveryRegistry(this.nc, this.space);
|
|
4972
|
-
return this.deliveryKv;
|
|
4973
|
-
}
|
|
4974
|
-
/** This endpoint instance's LEASE INCARNATION: which run of this principal a lease row was written
|
|
4975
|
-
* by. Minted per construction and never re-derived, because the credential cannot supply it, the
|
|
4976
|
-
* daemon's cred is a file on disk that every restart re-reads, so `card.id` is stable across
|
|
4977
|
-
* processes by design. See {@link DeliveryLeaseInfo.incarnation}. */
|
|
4978
|
-
leaseIncarnation = randomUUID();
|
|
4979
|
-
/** When this endpoint last WON each shard's lease create. Every later write of that row carries it
|
|
4980
|
-
* unchanged, while `since` is re-stamped. See {@link DeliveryLeaseInfo.acquiredAt}. */
|
|
4981
|
-
leaseAcquiredAt = new Map();
|
|
4982
|
-
encodeLease(shardIndex, ready, acquiredAt = this.leaseAcquiredAt.get(shardIndex)) {
|
|
4983
|
-
return new TextEncoder().encode(JSON.stringify({ holder: this.card.id, incarnation: this.leaseIncarnation, acquiredAt, since: Date.now(), ready }));
|
|
4984
|
-
}
|
|
4985
4821
|
/** Is this shard's lease row one THIS ENDPOINT INSTANCE wrote? The question a daemon whose renew
|
|
4986
4822
|
* just failed has to answer before it decides whether it still owns the shard.
|
|
4987
4823
|
*
|
|
@@ -4992,7 +4828,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
4992
4828
|
* incarnation predates the field and cannot be proven ours, which is the safe reading: it leads
|
|
4993
4829
|
* to the takeover path rather than to serving on someone else's claim. */
|
|
4994
4830
|
ownsDeliveryLease(info) {
|
|
4995
|
-
return
|
|
4831
|
+
return this.deliveryLease.ownsDeliveryLease(info);
|
|
4996
4832
|
}
|
|
4997
4833
|
/** Acquire the single-flight delivery lease for a shard via an ATOMIC CAS create, marked NOT-ready.
|
|
4998
4834
|
* THROWS if a live lease exists — a loud refusal-to-bind (the daemon exits), never a retry, so two
|
|
@@ -5000,17 +4836,13 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
5000
4836
|
* freeing a re-acquire. Acquired BEFORE binding (single-flight gate); {@link markDeliveryLeaseReady}
|
|
5001
4837
|
* flips it ready AFTER the loops + `ctl.delivery` are bound. Returns the lease revision. */
|
|
5002
4838
|
async acquireDeliveryLease(shardIndex) {
|
|
5003
|
-
|
|
5004
|
-
const acquiredAt = Date.now();
|
|
5005
|
-
const revision = await (await this.deliveryRegistry()).create(leaseKey(shardIndex), this.encodeLease(shardIndex, false, acquiredAt));
|
|
5006
|
-
this.leaseAcquiredAt.set(shardIndex, acquiredAt);
|
|
5007
|
-
return revision;
|
|
4839
|
+
return this.deliveryLease.acquireDeliveryLease(shardIndex);
|
|
5008
4840
|
}
|
|
5009
4841
|
/** Flip the held lease to READY (CAS `kv.update`) AFTER `startPlane3` has bound the loops + the
|
|
5010
4842
|
* `ctl.delivery` responder — so "lease ready" proves the responder is up, not just that the slot was
|
|
5011
4843
|
* claimed. Returns the new revision. */
|
|
5012
4844
|
async markDeliveryLeaseReady(shardIndex, revision) {
|
|
5013
|
-
return
|
|
4845
|
+
return this.deliveryLease.markDeliveryLeaseReady(shardIndex, revision);
|
|
5014
4846
|
}
|
|
5015
4847
|
/** Flip the held lease back to NOT-ready, the counterpart to {@link markDeliveryLeaseReady}, for a
|
|
5016
4848
|
* holder that has UNBOUND its loops and control responder but has not given up the shard.
|
|
@@ -5022,13 +4854,13 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
5022
4854
|
* the row (rather than deleting it) is deliberate: the shard is still claimed, so no third daemon
|
|
5023
4855
|
* should be invited in; what is being withdrawn is only the claim to be answering. */
|
|
5024
4856
|
async markDeliveryLeaseNotReady(shardIndex, revision) {
|
|
5025
|
-
return
|
|
4857
|
+
return this.deliveryLease.markDeliveryLeaseNotReady(shardIndex, revision);
|
|
5026
4858
|
}
|
|
5027
4859
|
/** Renew the held lease (CAS `kv.update` against `revision`, keeping `ready:true`) to refresh it before
|
|
5028
4860
|
* the bucket TTL expires it. Returns the new revision. Throws if the revision moved (lost the lease —
|
|
5029
4861
|
* the daemon should exit). */
|
|
5030
4862
|
async renewDeliveryLease(shardIndex, revision) {
|
|
5031
|
-
return
|
|
4863
|
+
return this.deliveryLease.renewDeliveryLease(shardIndex, revision);
|
|
5032
4864
|
}
|
|
5033
4865
|
/** Release the held lease on clean shutdown so a replacement daemon re-acquires immediately (a
|
|
5034
4866
|
* crash just lets the bucket TTL expire it).
|
|
@@ -5054,15 +4886,13 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
5054
4886
|
* a red in `smoke:delivery-lease`, where a caller that genuinely held the lease released nothing
|
|
5055
4887
|
* and the next acquire was refused. A caller must now say which it means. */
|
|
5056
4888
|
async releaseDeliveryLease(shardIndex, revision) {
|
|
5057
|
-
|
|
5058
|
-
return;
|
|
5059
|
-
await (await this.deliveryRegistry()).delete(leaseKey(shardIndex), { previousSeq: revision });
|
|
4889
|
+
return this.deliveryLease.releaseDeliveryLease(shardIndex, revision);
|
|
5060
4890
|
}
|
|
5061
4891
|
/** Read a shard's delivery lease (the daemon-availability signal), or `undefined` if none is live.
|
|
5062
4892
|
* READ-ONLY surface — drives Component 6's `cotal_channels` delivery-health field (an agent reads it
|
|
5063
4893
|
* under its own cred, which holds lease-bucket read but no write). */
|
|
5064
4894
|
async readDeliveryLease(shardIndex) {
|
|
5065
|
-
return
|
|
4895
|
+
return this.deliveryLease.readDeliveryLease(shardIndex);
|
|
5066
4896
|
}
|
|
5067
4897
|
/** Watch THIS shard's lease key and nothing else, for the daemon's loss trigger (#1596).
|
|
5068
4898
|
*
|
|
@@ -5082,42 +4912,16 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
5082
4912
|
* bucket's current last-per-subject state, so a change that landed during the null window is
|
|
5083
4913
|
* delivered on the fresh watch. Resolves a stop handle. */
|
|
5084
4914
|
async watchDeliveryLease(shardIndex, onEvent) {
|
|
5085
|
-
this.
|
|
5086
|
-
await this.bindDeliveryLeaseWatch();
|
|
5087
|
-
return () => {
|
|
5088
|
-
if (this.leaseWatchIntent?.onEvent !== onEvent)
|
|
5089
|
-
return; // a later watch owns the slot
|
|
5090
|
-
this.leaseWatchIntent = undefined;
|
|
5091
|
-
this.stopDeliveryLeaseWatch();
|
|
5092
|
-
};
|
|
4915
|
+
return this.deliveryLease.watchDeliveryLease(shardIndex, onEvent);
|
|
5093
4916
|
}
|
|
5094
4917
|
stopDeliveryLeaseWatch() {
|
|
5095
|
-
|
|
5096
|
-
this.retireDeliveryLeaseWatch(this.leaseWatchIter);
|
|
5097
|
-
this.leaseWatchIter = undefined;
|
|
5098
|
-
}
|
|
5099
|
-
/** Stop a lease watch and delete its ordered consumer, which the stop alone leaves on the broker
|
|
5100
|
-
* until its five-minute inactive threshold (#3966). A connection that closed on its own cannot
|
|
5101
|
-
* carry the delete, so that consumer is left to the threshold. A failed delete is a warning: the
|
|
5102
|
-
* broker still reaps the consumer, and the daemon's synchronous stop handle has no caller to
|
|
5103
|
-
* reject. */
|
|
5104
|
-
retireDeliveryLeaseWatch(iter) {
|
|
5105
|
-
try {
|
|
5106
|
-
iter.stop();
|
|
5107
|
-
}
|
|
5108
|
-
catch { /* already closed with its connection */ }
|
|
5109
|
-
if (!this.nc || this.nc.isClosed())
|
|
5110
|
-
return;
|
|
5111
|
-
const deleting = this.deleteOwnConsumer(kvWatchConsumer(iter)).catch((e) => this.emitRecoverable(new Error(`a retired delivery lease watch consumer was not deleted (${e.message}); the broker reaps it at its inactive threshold`)));
|
|
5112
|
-
this.leaseWatchDeletes.add(deleting);
|
|
5113
|
-
void deleting.then(() => this.leaseWatchDeletes.delete(deleting));
|
|
4918
|
+
return this.deliveryLease.stopDeliveryLeaseWatch();
|
|
5114
4919
|
}
|
|
5115
4920
|
/** Wait for the lease-watch binds still in flight, then for every retired watch's consumer delete.
|
|
5116
4921
|
* {@link stop} and a live-epoch rebuild run this before they close the connection those deletes
|
|
5117
4922
|
* need (#3966). */
|
|
5118
4923
|
async settleDeliveryLeaseWatches() {
|
|
5119
|
-
|
|
5120
|
-
await Promise.all(this.leaseWatchDeletes);
|
|
4924
|
+
return this.deliveryLease.settleDeliveryLeaseWatches();
|
|
5121
4925
|
}
|
|
5122
4926
|
/** Stop a presence or channel-registry watch and delete its ordered consumer, which the stop
|
|
5123
4927
|
* alone leaves on the broker until its five-minute inactive threshold (#3914). A connection
|
|
@@ -5172,37 +4976,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
5172
4976
|
* its intent, or after a stop or connection teardown moved the epoch, retires its watch instead of
|
|
5173
4977
|
* installing it: the epoch it was bound for is gone, and a rebuilt one binds its own. */
|
|
5174
4978
|
bindDeliveryLeaseWatch() {
|
|
5175
|
-
|
|
5176
|
-
const settled = () => { this.leaseWatchBinds.delete(binding); };
|
|
5177
|
-
this.leaseWatchBinds.add(binding);
|
|
5178
|
-
void binding.then(settled, settled);
|
|
5179
|
-
return binding;
|
|
5180
|
-
}
|
|
5181
|
-
async bindDeliveryLeaseWatchOnce() {
|
|
5182
|
-
const intent = this.leaseWatchIntent;
|
|
5183
|
-
const epoch = this.presenceEpoch;
|
|
5184
|
-
const iter = await (await this.deliveryRegistry()).watch({ key: leaseKey(intent.shardIndex) });
|
|
5185
|
-
if (this.leaseWatchIntent !== intent || epoch !== this.presenceEpoch) {
|
|
5186
|
-
this.retireDeliveryLeaseWatch(iter);
|
|
5187
|
-
return;
|
|
5188
|
-
}
|
|
5189
|
-
this.leaseWatchIter = iter;
|
|
5190
|
-
void (async () => {
|
|
5191
|
-
for await (const e of iter) {
|
|
5192
|
-
if (this.leaseWatchIter !== iter)
|
|
5193
|
-
break;
|
|
5194
|
-
if (e.operation === "DEL" || e.operation === "PURGE") {
|
|
5195
|
-
intent.onEvent(undefined);
|
|
5196
|
-
continue;
|
|
5197
|
-
}
|
|
5198
|
-
try {
|
|
5199
|
-
intent.onEvent(e.json());
|
|
5200
|
-
}
|
|
5201
|
-
catch {
|
|
5202
|
-
intent.onEvent(undefined);
|
|
5203
|
-
}
|
|
5204
|
-
}
|
|
5205
|
-
})().catch((e) => this.emit("error", e));
|
|
4979
|
+
return this.deliveryLease.bindDeliveryLeaseWatch();
|
|
5206
4980
|
}
|
|
5207
4981
|
/** The principal recorded as holding a shard's delivery lease, read by THIS process from the row,
|
|
5208
4982
|
* or `undefined` when no row is live or the read failed.
|
|
@@ -5218,12 +4992,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
5218
4992
|
* that throws (a denied or unreadable bucket, a broker that will not answer) must not collapse
|
|
5219
4993
|
* into "nothing is there". Callers must treat it as a failure to determine (#1694). */
|
|
5220
4994
|
async deliveryLeaseHolder(shardIndex) {
|
|
5221
|
-
|
|
5222
|
-
return (await this.readDeliveryLeaseEntry(shardIndex))?.info.holder;
|
|
5223
|
-
}
|
|
5224
|
-
catch {
|
|
5225
|
-
return undefined;
|
|
5226
|
-
}
|
|
4995
|
+
return this.deliveryLease.deliveryLeaseHolder(shardIndex);
|
|
5227
4996
|
}
|
|
5228
4997
|
/** The lease row AND the KV revision it is at. The revision is the CAS token every renew and the
|
|
5229
4998
|
* CAS release are argued against, so a caller re-establishing ownership after a failed renew
|
|
@@ -5240,15 +5009,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
5240
5009
|
* (an owner testing "is this provably MINE") still gets its safe answer: an unparseable row is
|
|
5241
5010
|
* not provably anyone's, and catching here reads as `false` exactly as before. */
|
|
5242
5011
|
async readDeliveryLeaseEntry(shardIndex) {
|
|
5243
|
-
|
|
5244
|
-
if (!e || e.operation === "DEL" || e.operation === "PURGE")
|
|
5245
|
-
return undefined;
|
|
5246
|
-
try {
|
|
5247
|
-
return { info: e.json(), revision: e.revision };
|
|
5248
|
-
}
|
|
5249
|
-
catch (err) {
|
|
5250
|
-
throw new Error(`delivery lease row "${leaseKey(shardIndex)}" exists but does not parse as a lease record (${err.message}) - it is unreadable, not absent`);
|
|
5251
|
-
}
|
|
5012
|
+
return this.deliveryLease.readDeliveryLeaseEntry(shardIndex);
|
|
5252
5013
|
}
|
|
5253
5014
|
/** Ensure + bind the manager singleton-lease bucket. Mirrors the presence-bucket pattern (connectAndBind):
|
|
5254
5015
|
* AUTH mode OPENs the bucket pre-created at `cotal up` (the scoped `supervisor` cred holds no
|
|
@@ -5494,7 +5255,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
5494
5255
|
* peer wins; 0 or >1 matches drop (never fan a directed durable copy to an unrelated same-named
|
|
5495
5256
|
* bystander — red-team LOW; SPEC §4 unique instance id). */
|
|
5496
5257
|
resolveOwnerByName(name) {
|
|
5497
|
-
const matches = [...this.
|
|
5258
|
+
const matches = [...this.presence.values()].filter((p) => p.card.name.toLowerCase() === name.toLowerCase());
|
|
5498
5259
|
return matches.length === 1 ? matches[0].card.id : undefined;
|
|
5499
5260
|
}
|
|
5500
5261
|
/** Plane-3 publish-dedupe key for a fan-out/transfer entry: the message id plus the member
|
|
@@ -6547,6 +6308,212 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
6547
6308
|
// the plane, so it is graded like a refusal rather than guessed at.
|
|
6548
6309
|
return "refused";
|
|
6549
6310
|
}
|
|
6311
|
+
/**
|
|
6312
|
+
* Serve this process's own endpoint on the principal route (SPEC 13.21): a core-defined command
|
|
6313
|
+
* set about this principal, addressed by its own principal, lifecycle UID and runtime id, with no
|
|
6314
|
+
* registration. Revision 1 defines one endpoint, `agent` (SPEC 13.22).
|
|
6315
|
+
*
|
|
6316
|
+
* Throws on an open mesh and on a static credential, where no row names who may call, without a
|
|
6317
|
+
* lifecycle UID, before the endpoint has started, for an endpoint it already serves, and for a
|
|
6318
|
+
* cluster with a command that declares no `authorities` (see {@link servePrincipalRoute}). It
|
|
6319
|
+
* subscribes this process's own subject with no queue group.
|
|
6320
|
+
*
|
|
6321
|
+
* `ready` is a positive check: it resolves only after the broker accepted the subscription
|
|
6322
|
+
* (subscribe, flush, and no permission violation for that subject), and rejects with the
|
|
6323
|
+
* refusal, which a credential minted before the rows existed gets. The serve is INTENT: every
|
|
6324
|
+
* replaced connection subscribes again and runs the check again, and a refusal there ends the
|
|
6325
|
+
* serve, resolves `failed` and clears the presence field. `stop` unsubscribes and waits for the
|
|
6326
|
+
* handlers in flight.
|
|
6327
|
+
*/
|
|
6328
|
+
servePrincipal(serve) {
|
|
6329
|
+
if (!this.authed)
|
|
6330
|
+
throw new Error("the principal route is served only on an authenticated mesh: on an open mesh no row says who may call (SPEC 13.21)");
|
|
6331
|
+
if (!this.userMode)
|
|
6332
|
+
throw new Error("the principal route is served only under a user-auth credential: a static credential names no one who may call (SPEC 13.21)");
|
|
6333
|
+
const uid = this.ownLifecycleUid;
|
|
6334
|
+
if (uid === undefined)
|
|
6335
|
+
throw new Error("serving the principal route needs this endpoint's lifecycleUid: the subject names it (SPEC 13.21)");
|
|
6336
|
+
const nc = this.nc;
|
|
6337
|
+
if (!nc)
|
|
6338
|
+
throw new Error(this.notLiveMsg());
|
|
6339
|
+
endpointToken(serve.endpoint);
|
|
6340
|
+
if (this.principalServes.has(serve.endpoint))
|
|
6341
|
+
throw new Error(`this endpoint already serves "${serve.endpoint}" on the principal route`);
|
|
6342
|
+
const self = { owner: this.owner, actor: this.actor, uid, runtimeId: assertRuntimeToken(processRuntime(uid).id) };
|
|
6343
|
+
let resolveReady, rejectReady, resolveFailed;
|
|
6344
|
+
const ready = new Promise((resolve, reject) => { resolveReady = resolve; rejectReady = reject; });
|
|
6345
|
+
ready.catch(() => { });
|
|
6346
|
+
const failed = new Promise((resolve) => { resolveFailed = resolve; });
|
|
6347
|
+
const intent = {
|
|
6348
|
+
serve, self, handles: [], ended: false, readySettled: false, resolveReady, rejectReady, resolveFailed,
|
|
6349
|
+
};
|
|
6350
|
+
// Construction refuses an invalid cluster before anything is kept or subscribed.
|
|
6351
|
+
const handle = this.bindPrincipalHandle(intent, nc);
|
|
6352
|
+
this.principalServes.set(serve.endpoint, intent);
|
|
6353
|
+
void this.armPrincipalServe(intent, nc, handle);
|
|
6354
|
+
return { ready, failed, stop: () => this.stopPrincipalServe(intent) };
|
|
6355
|
+
}
|
|
6356
|
+
/** Subscribe `intent` on `nc`. */
|
|
6357
|
+
bindPrincipalHandle(intent, nc) {
|
|
6358
|
+
return servePrincipalRoute(nc, this.space, intent.self, intent.serve);
|
|
6359
|
+
}
|
|
6360
|
+
/** A request or reply subject (or filter) of the principal route in this space. */
|
|
6361
|
+
isPrincipalRouteSubject(subject) {
|
|
6362
|
+
const p = `${spacePrefix(this.space)}.ep.`;
|
|
6363
|
+
return subject.startsWith(`${p}prin.`) || subject.startsWith(`${p}prin-reply.`);
|
|
6364
|
+
}
|
|
6365
|
+
/** Track one binding: a refusal ends the serve, and an accepted subscription on the current
|
|
6366
|
+
* connection shows the presence field. */
|
|
6367
|
+
async armPrincipalServe(intent, nc, handle) {
|
|
6368
|
+
intent.binding = { nc, handle };
|
|
6369
|
+
intent.handles.push(handle);
|
|
6370
|
+
intent.readyOn = undefined;
|
|
6371
|
+
void handle.refused.then((err) => {
|
|
6372
|
+
if (!intent.ended && intent.binding?.handle === handle)
|
|
6373
|
+
this.endPrincipalServe(intent, err);
|
|
6374
|
+
});
|
|
6375
|
+
try {
|
|
6376
|
+
await handle.ready;
|
|
6377
|
+
}
|
|
6378
|
+
catch {
|
|
6379
|
+
// A refusal ends the serve through `refused`. A flush that failed because the connection
|
|
6380
|
+
// went away is answered by the binding the next connection makes.
|
|
6381
|
+
return;
|
|
6382
|
+
}
|
|
6383
|
+
this.principalServeConfirmed(intent, nc, handle);
|
|
6384
|
+
}
|
|
6385
|
+
/** The broker accepted `handle` on `nc`. When that is still the current binding, the serve is
|
|
6386
|
+
* ready there. */
|
|
6387
|
+
principalServeConfirmed(intent, nc, handle) {
|
|
6388
|
+
if (intent.ended || intent.binding?.handle !== handle || this.nc !== nc)
|
|
6389
|
+
return;
|
|
6390
|
+
intent.readyOn = nc;
|
|
6391
|
+
if (!intent.readySettled) {
|
|
6392
|
+
intent.readySettled = true;
|
|
6393
|
+
intent.resolveReady();
|
|
6394
|
+
}
|
|
6395
|
+
if (intent.serve.endpoint === AGENT_ENDPOINT)
|
|
6396
|
+
this.publishPresence().catch((e) => this.emitRecoverable(e));
|
|
6397
|
+
}
|
|
6398
|
+
/** Re-bind every principal serve on a replaced connection (a rebuild), or re-run the check on the
|
|
6399
|
+
* connection the library reconnected and re-subscribed itself. */
|
|
6400
|
+
rebindPrincipalServes(reconnected) {
|
|
6401
|
+
const nc = this.nc;
|
|
6402
|
+
if (!nc)
|
|
6403
|
+
return;
|
|
6404
|
+
for (const intent of this.principalServes.values()) {
|
|
6405
|
+
if (intent.ended)
|
|
6406
|
+
continue;
|
|
6407
|
+
const bound = intent.binding;
|
|
6408
|
+
if (bound && bound.nc === nc) {
|
|
6409
|
+
if (!reconnected)
|
|
6410
|
+
continue;
|
|
6411
|
+
intent.readyOn = undefined;
|
|
6412
|
+
void bound.handle.check().then(() => this.principalServeConfirmed(intent, nc, bound.handle), () => { });
|
|
6413
|
+
continue;
|
|
6414
|
+
}
|
|
6415
|
+
let handle;
|
|
6416
|
+
try {
|
|
6417
|
+
handle = this.bindPrincipalHandle(intent, nc);
|
|
6418
|
+
}
|
|
6419
|
+
catch (e) {
|
|
6420
|
+
this.endPrincipalServe(intent, e);
|
|
6421
|
+
continue;
|
|
6422
|
+
}
|
|
6423
|
+
void this.armPrincipalServe(intent, nc, handle);
|
|
6424
|
+
}
|
|
6425
|
+
}
|
|
6426
|
+
/** End a serve the broker refused: no more serving, `failed` resolves, the presence field goes. */
|
|
6427
|
+
endPrincipalServe(intent, err) {
|
|
6428
|
+
if (intent.ended)
|
|
6429
|
+
return;
|
|
6430
|
+
intent.ended = true;
|
|
6431
|
+
intent.readyOn = undefined;
|
|
6432
|
+
if (this.principalServes.get(intent.serve.endpoint) === intent)
|
|
6433
|
+
this.principalServes.delete(intent.serve.endpoint);
|
|
6434
|
+
const wasReady = intent.readySettled;
|
|
6435
|
+
if (!intent.readySettled) {
|
|
6436
|
+
intent.readySettled = true;
|
|
6437
|
+
intent.rejectReady(err);
|
|
6438
|
+
}
|
|
6439
|
+
intent.resolveFailed(err);
|
|
6440
|
+
for (const handle of intent.handles)
|
|
6441
|
+
void handle.stop();
|
|
6442
|
+
if (intent.serve.endpoint === AGENT_ENDPOINT) {
|
|
6443
|
+
this.promptsPresence = undefined;
|
|
6444
|
+
this.publishPresence().catch((e) => this.emitRecoverable(e));
|
|
6445
|
+
}
|
|
6446
|
+
// `ready` already carried a refusal that came before it settled; a later one is reported here.
|
|
6447
|
+
if (wasReady)
|
|
6448
|
+
this.emitRecoverable(new Error(`stopped serving "${intent.serve.endpoint}" on the principal route: ${err.message}`, { cause: err }));
|
|
6449
|
+
}
|
|
6450
|
+
async stopPrincipalServe(intent) {
|
|
6451
|
+
if (!intent.ended) {
|
|
6452
|
+
intent.ended = true;
|
|
6453
|
+
intent.readyOn = undefined;
|
|
6454
|
+
if (this.principalServes.get(intent.serve.endpoint) === intent)
|
|
6455
|
+
this.principalServes.delete(intent.serve.endpoint);
|
|
6456
|
+
if (!intent.readySettled) {
|
|
6457
|
+
intent.readySettled = true;
|
|
6458
|
+
intent.rejectReady(new Error(`stopped serving "${intent.serve.endpoint}" before the broker confirmed the subscription`));
|
|
6459
|
+
}
|
|
6460
|
+
if (intent.serve.endpoint === AGENT_ENDPOINT) {
|
|
6461
|
+
this.promptsPresence = undefined;
|
|
6462
|
+
this.publishPresence().catch((e) => this.emitRecoverable(e));
|
|
6463
|
+
}
|
|
6464
|
+
}
|
|
6465
|
+
await Promise.all(intent.handles.map((handle) => handle.stop()));
|
|
6466
|
+
}
|
|
6467
|
+
/**
|
|
6468
|
+
* Set this process's `prompts` presence field and write presence at once (SPEC §6): how many
|
|
6469
|
+
* prompts it holds `open` or `sent`, and a counter that goes up by one each time a prompt opens or
|
|
6470
|
+
* changes state. The field shows only while the `agent` endpoint is served and the broker accepted
|
|
6471
|
+
* its subscription on the current connection, and it shows `{ open: 0, seq: 0 }` until this is
|
|
6472
|
+
* called. Throws when the `agent` endpoint is not served.
|
|
6473
|
+
*/
|
|
6474
|
+
async setPromptsPresence(prompts) {
|
|
6475
|
+
for (const k of ["open", "seq"])
|
|
6476
|
+
if (!Number.isSafeInteger(prompts?.[k]) || prompts[k] < 0)
|
|
6477
|
+
throw new TypeError(`prompts.${k} must be a non-negative safe integer`);
|
|
6478
|
+
if (!this.principalServes.has(AGENT_ENDPOINT))
|
|
6479
|
+
throw new Error(`the prompts presence field belongs to a served "${AGENT_ENDPOINT}" endpoint: call servePrincipal first (SPEC §6)`);
|
|
6480
|
+
this.promptsPresence = { open: prompts.open, seq: prompts.seq };
|
|
6481
|
+
await this.publishPresence();
|
|
6482
|
+
}
|
|
6483
|
+
/** The `prompts` field this process's presence carries now, or none. */
|
|
6484
|
+
servedPrompts() {
|
|
6485
|
+
const intent = this.principalServes.get(AGENT_ENDPOINT);
|
|
6486
|
+
if (!intent || intent.ended || !this.nc || intent.readyOn !== this.nc)
|
|
6487
|
+
return undefined;
|
|
6488
|
+
return this.promptsPresence ?? { open: 0, seq: 0 };
|
|
6489
|
+
}
|
|
6490
|
+
/**
|
|
6491
|
+
* Call one command of a principal's own endpoint on the principal route (SPEC 13.21), as this
|
|
6492
|
+
* endpoint's caller triple with the authority word its row covers. `principal` is the addressed
|
|
6493
|
+
* process: owner, actor, lifecycle UID and the `runtime.id` its presence publishes. Presence is
|
|
6494
|
+
* advisory, and a wrong value reaches a subject nobody serves or one this caller's row does not
|
|
6495
|
+
* cover. It cannot widen authority.
|
|
6496
|
+
*
|
|
6497
|
+
* The `agent` endpoint's contracts and `describe`'s are built in; any other endpoint needs
|
|
6498
|
+
* `contract`. Resolves the reply, whose `ok: false` is the agent's own refusal; throws
|
|
6499
|
+
* `unavailable` (`not-executed`) when nothing serves that process, `permission-denied`
|
|
6500
|
+
* (`not-executed`) when this credential holds no row for the request, and `deadline-exceeded`
|
|
6501
|
+
* (`unknown`) when no reply came. See {@link epPrincipalCall}.
|
|
6502
|
+
*/
|
|
6503
|
+
async callPrincipal(call) {
|
|
6504
|
+
const nc = this.nc;
|
|
6505
|
+
if (!nc)
|
|
6506
|
+
throw new Error(this.notLiveMsg());
|
|
6507
|
+
const contract = call.contract
|
|
6508
|
+
?? (call.command === "describe" ? describeContracts() : call.endpoint === AGENT_ENDPOINT ? AGENT_CONTRACTS[call.command] : undefined);
|
|
6509
|
+
if (contract === undefined)
|
|
6510
|
+
throw new EpEnvelopeError("bad-request", `no contract is built in for ${call.endpoint}.${call.command}; pass one`, undefined, "not-executed");
|
|
6511
|
+
const caller = { owner: this.owner, actor: this.actor, uid: this.requireLifecycleUid("callPrincipal") };
|
|
6512
|
+
return epPrincipalCall(nc, this.space, { principal: call.principal, authority: call.authority }, {
|
|
6513
|
+
endpoint: call.endpoint, command: call.command, contract, caller, name: this.card.name,
|
|
6514
|
+
...(call.args !== undefined ? { args: call.args } : {}),
|
|
6515
|
+
}, { deadlineMs: call.deadlineMs, ...(call.signal ? { signal: call.signal } : {}) });
|
|
6516
|
+
}
|
|
6550
6517
|
/** Serve one PRIVILEGED delivery-admin request (the D5 rail-split). The cred layer is the caller
|
|
6551
6518
|
* boundary — only the supervisor profile can publish here — and `serveControl`'s sender check +
|
|
6552
6519
|
* bounded reply still apply on top. `reloadCreds` is the class-2 renewal ADOPTION step: re-read
|
|
@@ -7981,7 +7948,10 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
7981
7948
|
if (floorSeq !== undefined && floorSeq >= upToSeq)
|
|
7982
7949
|
return 0;
|
|
7983
7950
|
const subject = chatSubject(this.space, "*", "*", channel);
|
|
7984
|
-
|
|
7951
|
+
// A window that reaches back before 1970 covers the whole retained window (#7323). Its start time
|
|
7952
|
+
// would be a date JetStream does not read, or not a valid date at all, and the read replayed nothing.
|
|
7953
|
+
const since = sinceMs === undefined ? undefined : Date.now() - sinceMs;
|
|
7954
|
+
const start = floorSeq !== undefined ? { seq: floorSeq + 1 } : since === undefined || !(since > 0) ? { seq: 1 } : { time: new Date(since) };
|
|
7985
7955
|
let msgs;
|
|
7986
7956
|
try {
|
|
7987
7957
|
msgs = await this.collectHistory(subject, start, { untilSeq: upToSeq });
|
|
@@ -8111,91 +8081,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
8111
8081
|
return oldest !== undefined && oldest > sinceSeq + 1;
|
|
8112
8082
|
}
|
|
8113
8083
|
async publishPresence() {
|
|
8114
|
-
|
|
8115
|
-
return; // observers watch but never publish their own record
|
|
8116
|
-
const p = {
|
|
8117
|
-
card: this.card,
|
|
8118
|
-
// SPEC §6: presence carries the incarnation's lifecycle UID (MUST in auth mode from v0.4);
|
|
8119
|
-
// omitted only where the endpoint has none (a pure operator/daemon connection never registers).
|
|
8120
|
-
...(this.ownLifecycleUid !== undefined ? { lifecycleUid: this.ownLifecycleUid } : {}),
|
|
8121
|
-
runtime: processRuntime(this.ownLifecycleUid),
|
|
8122
|
-
status: this.status,
|
|
8123
|
-
condition: this.condition,
|
|
8124
|
-
environment: this.environment,
|
|
8125
|
-
activity: this.activity,
|
|
8126
|
-
statusSince: this.statusSince,
|
|
8127
|
-
activitySince: this.activitySince,
|
|
8128
|
-
activeAt: this.activeAt,
|
|
8129
|
-
attention: this.attentionMode,
|
|
8130
|
-
channelModes: this.channelModes,
|
|
8131
|
-
ts: Date.now(),
|
|
8132
|
-
};
|
|
8133
|
-
// Wire contract (SPEC §6): an OFFLINE record must not carry the advisory attention fields or
|
|
8134
|
-
// `statusSince`. Scrub at the publisher — this covers stop(), setStatus("offline"), and any future
|
|
8135
|
-
// offline publish site, so the raw KV record is compliant, not only the observer-side roster
|
|
8136
|
-
// materialization.
|
|
8137
|
-
const record = this.status === "offline" ? this.toOffline(p) : p;
|
|
8138
|
-
assertPresenceRow(record, this.card.id);
|
|
8139
|
-
// #1356: this put can still be in flight when a teardown runs, because a rebind reaches
|
|
8140
|
-
// publishPresence through onPresenceBucketEmpty and no teardown awaits that flight. Take the same
|
|
8141
|
-
// epoch fence {@link startPresenceWatch} takes, so a put that outlives its epoch still throws to
|
|
8142
|
-
// its caller but no longer writes presence-refusal state belonging to a later connection. Without
|
|
8143
|
-
// it, a heartbeat put (default 2s) whose ~5s JetStream timeout elapses after a rebuild plants a
|
|
8144
|
-
// refusal on the connection that just published successfully.
|
|
8145
|
-
const epoch = this.presenceEpoch;
|
|
8146
|
-
const conditionRevision = this.conditionRevision;
|
|
8147
|
-
// #1461: snapshot the put's generation BEFORE the put. A settle (success or rejection) is the
|
|
8148
|
-
// latest evidence on its epoch only while no put that started after it has ALREADY settled —
|
|
8149
|
-
// a newer put merely in flight is not evidence yet. Same-epoch puts overlap routinely
|
|
8150
|
-
// (heartbeat 2s against a ~5s JetStream put timeout), so "succeeded" alone is not "newest".
|
|
8151
|
-
const generation = ++this.presencePutGeneration;
|
|
8152
|
-
try {
|
|
8153
|
-
await this.kv.put(this.card.id, JSON.stringify(record));
|
|
8154
|
-
}
|
|
8155
|
-
catch (e) {
|
|
8156
|
-
// #1356: a broker can put this bucket into a state where it refuses every write and never
|
|
8157
|
-
// recovers, and the write is the ONLY thing that fails — open and watch both still succeed, so
|
|
8158
|
-
// nothing else here notices. Record WHEN the refusals started, at the one site that knows the
|
|
8159
|
-
// failing write was a presence write; a caller cannot infer that from the generic `warning`
|
|
8160
|
-
// stream, which carries any recoverable error.
|
|
8161
|
-
//
|
|
8162
|
-
// #1461: a rejection speaks when it is the newest SETTLED evidence — a newer put that has
|
|
8163
|
-
// merely started does not silence it, but a newer put that already settled (either way)
|
|
8164
|
-
// does.
|
|
8165
|
-
const superseded = (this.presencePutSettled ?? 0) > generation;
|
|
8166
|
-
this.presencePutSettled = Math.max(this.presencePutSettled ?? 0, generation);
|
|
8167
|
-
if (epoch === this.presenceEpoch && !this.stopped && !superseded) {
|
|
8168
|
-
this.presenceWriteFailingSince ??= Date.now();
|
|
8169
|
-
this.lastPresenceWriteError = e?.message ?? String(e);
|
|
8170
|
-
this.presenceWriteFailures++;
|
|
8171
|
-
this.escalatePresenceWriteFailureIfStuck();
|
|
8172
|
-
}
|
|
8173
|
-
if (e instanceof TimeoutError)
|
|
8174
|
-
throw new Error(`${e.message}. The presence write outcome is unknown: the broker may have stored the update. Read the roster before retrying.`, { cause: e });
|
|
8175
|
-
throw e;
|
|
8176
|
-
}
|
|
8177
|
-
// A late SUCCESS cannot erase a refusal from newer evidence. Two fences, each answering half:
|
|
8178
|
-
// the epoch fence (#1356) keeps a put that outlives its connection from writing or erasing a
|
|
8179
|
-
// record that belongs to a later connection; the generation fence (#1461) orders puts WITHIN
|
|
8180
|
-
// one epoch by SETTLE order, not start order: a settle is the latest evidence only while no
|
|
8181
|
-
// put that started after it has already settled, whichever way that newer put settled. So an
|
|
8182
|
-
// earlier put settling AFTER a later put settled no longer speaks — success cannot clear a
|
|
8183
|
-
// newer refusal (#1461's original case), and a newer settle is not blocked by being merely
|
|
8184
|
-
// started (panel round 1). Without the fence, an earlier put succeeding late cleared a later
|
|
8185
|
-
// put's refusal while the bucket still refused writes (heartbeat 2s against a ~5s put timeout,
|
|
8186
|
-
// so the overlap is routine rather than a corner).
|
|
8187
|
-
const superseded = (this.presencePutSettled ?? 0) > generation;
|
|
8188
|
-
this.presencePutSettled = Math.max(this.presencePutSettled ?? 0, generation);
|
|
8189
|
-
if (epoch !== this.presenceEpoch || this.stopped || superseded)
|
|
8190
|
-
return;
|
|
8191
|
-
// Presence writes may overlap. If this put carried an older condition, repair the KV with the
|
|
8192
|
-
// latest local state before returning so a late failure publish cannot resurrect after a new
|
|
8193
|
-
// turn cleared it. The revision is condition-specific; unrelated heartbeat overlap is unchanged.
|
|
8194
|
-
if (conditionRevision !== this.conditionRevision) {
|
|
8195
|
-
await this.publishPresence();
|
|
8196
|
-
return;
|
|
8197
|
-
}
|
|
8198
|
-
this.clearPresenceWriteFailure();
|
|
8084
|
+
return this.presence.publishPresence();
|
|
8199
8085
|
}
|
|
8200
8086
|
/** #1356: drop the presence-refusal record when the connection that OBSERVED those refusals goes
|
|
8201
8087
|
* away. "This bucket is refusing writes" is a claim about a specific broker connection; once that
|
|
@@ -8207,23 +8093,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
8207
8093
|
* failed bind against an unreachable server still reported the presence-refusal sentence while the
|
|
8208
8094
|
* endpoint's own `connectionIssue` already said "connection refused". */
|
|
8209
8095
|
clearPresenceWriteFailure() {
|
|
8210
|
-
this.
|
|
8211
|
-
this.lastPresenceWriteError = undefined;
|
|
8212
|
-
this.presenceWriteFailures = 0;
|
|
8213
|
-
this.presenceWriteEscalated = false;
|
|
8214
|
-
}
|
|
8215
|
-
/** A heartbeat retry remains recoverable inside one TTL. Once two or more consecutive writes have
|
|
8216
|
-
* failed across the full window, ordinary retry has exhausted the roster's liveness budget: raise
|
|
8217
|
-
* one named, non-transient warning. Later failures keep the duration/count current without flooding
|
|
8218
|
-
* the operator log; the next successful write clears and re-arms the condition. */
|
|
8219
|
-
escalatePresenceWriteFailureIfStuck() {
|
|
8220
|
-
if (this.presenceWriteEscalated ||
|
|
8221
|
-
this.presenceWriteFailingSince === undefined ||
|
|
8222
|
-
this.presenceWriteFailures < 2 ||
|
|
8223
|
-
Date.now() - this.presenceWriteFailingSince < this.ttlMs)
|
|
8224
|
-
return;
|
|
8225
|
-
this.presenceWriteEscalated = true;
|
|
8226
|
-
this.emitRecoverable(new PresenceWriteStuckError(presenceBucket(this.space), this.presenceWriteFailingSince, this.presenceWriteFailures, this.ttlMs, this.lastPresenceWriteError));
|
|
8096
|
+
return this.presence.clearPresenceWriteFailure();
|
|
8227
8097
|
}
|
|
8228
8098
|
/** #1356: the presence bucket has been refusing writes since this time, or `undefined` when the
|
|
8229
8099
|
* last publish succeeded. Cleared by a successful write that is the latest evidence on its
|
|
@@ -8234,16 +8104,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
8234
8104
|
* while the NATS connection stays up, so a caller must not read this as "only presence is
|
|
8235
8105
|
* affected". It reports what was observed, not how far the fault extends. */
|
|
8236
8106
|
presenceWriteFailure() {
|
|
8237
|
-
|
|
8238
|
-
return undefined;
|
|
8239
|
-
return {
|
|
8240
|
-
since: this.presenceWriteFailingSince,
|
|
8241
|
-
forMs: Date.now() - this.presenceWriteFailingSince,
|
|
8242
|
-
error: this.lastPresenceWriteError,
|
|
8243
|
-
bucket: presenceBucket(this.space),
|
|
8244
|
-
consecutiveFailures: this.presenceWriteFailures,
|
|
8245
|
-
stuck: this.presenceWriteEscalated,
|
|
8246
|
-
};
|
|
8107
|
+
return this.presence.presenceWriteFailure();
|
|
8247
8108
|
}
|
|
8248
8109
|
/** Bind a presence watch on the current connection. Resolves the bound consumer's info when
|
|
8249
8110
|
* the watch was installed, undefined when the endpoint stopped or rebuilt while the bind was
|
|
@@ -8253,117 +8114,12 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
8253
8114
|
* {@link stop} would resurrect a watch on a stopped endpoint, and one that completes after a
|
|
8254
8115
|
* rebuild would overwrite the fresh epoch's watch with a dead-connection iterator. */
|
|
8255
8116
|
async startPresenceWatch() {
|
|
8256
|
-
|
|
8257
|
-
return undefined;
|
|
8258
|
-
let hydrated;
|
|
8259
|
-
this.presenceSnapshot = new Promise((resolve) => { hydrated = resolve; });
|
|
8260
|
-
const bound = await this.bindKvWatch(this.kv);
|
|
8261
|
-
if (!bound) {
|
|
8262
|
-
hydrated();
|
|
8263
|
-
return undefined;
|
|
8264
|
-
}
|
|
8265
|
-
const { iter, info } = bound;
|
|
8266
|
-
this.presenceWatchIter = iter;
|
|
8267
|
-
// A bucket with no keys replays nothing, so no entry carries the end-of-replay marker. For an
|
|
8268
|
-
// observer that does not register, that empty replay is the whole snapshot: nobody is present.
|
|
8269
|
-
// A registering observer is populated by its own record instead (see onPresenceBucketEmpty).
|
|
8270
|
-
const empty = info.num_pending === 0 && !this.doRegister;
|
|
8271
|
-
if (empty) {
|
|
8272
|
-
this.presenceSnapshotPopulated = true;
|
|
8273
|
-
hydrated();
|
|
8274
|
-
this.markPresenceBucketEmpty();
|
|
8275
|
-
}
|
|
8276
|
-
void (async () => {
|
|
8277
|
-
let ready = empty;
|
|
8278
|
-
for await (const e of iter) {
|
|
8279
|
-
// A rebind bound a newer watch: this one's remaining entries belong to a retired epoch.
|
|
8280
|
-
// Stop consuming them (the rebind stops the iterator too; this ends the loop first).
|
|
8281
|
-
if (this.presenceWatchIter !== iter)
|
|
8282
|
-
break;
|
|
8283
|
-
this.handleKvEntry(e);
|
|
8284
|
-
// @nats-io/kv marks the final initial replay entry isUpdate=true. Later updates stay true.
|
|
8285
|
-
if (!ready && e.isUpdate) {
|
|
8286
|
-
ready = true;
|
|
8287
|
-
this.presenceSnapshotPopulated = true;
|
|
8288
|
-
hydrated();
|
|
8289
|
-
this.emitPresenceViewIfChanged();
|
|
8290
|
-
}
|
|
8291
|
-
}
|
|
8292
|
-
hydrated();
|
|
8293
|
-
})().catch((e) => this.emit("error", e));
|
|
8294
|
-
return info;
|
|
8117
|
+
return this.presence.startPresenceWatch();
|
|
8295
8118
|
}
|
|
8296
8119
|
/** {@link EndpointOptions.readOnce}: fill the roster from one read of the presence bucket. Stale
|
|
8297
8120
|
* and offline records resolve as they do in a watch's initial replay. */
|
|
8298
8121
|
async readPresenceOnce() {
|
|
8299
|
-
|
|
8300
|
-
return;
|
|
8301
|
-
for (const e of await lastKvEntries(this.kv))
|
|
8302
|
-
this.handleKvEntry(e);
|
|
8303
|
-
this.presenceSnapshotPopulated = true;
|
|
8304
|
-
// No record reached the roster (an empty bucket, or only the TTL canary): nobody is present.
|
|
8305
|
-
if (this.lastPresenceWatchAt === 0)
|
|
8306
|
-
this.markPresenceBucketEmpty();
|
|
8307
|
-
else
|
|
8308
|
-
this.emitPresenceViewIfChanged();
|
|
8309
|
-
}
|
|
8310
|
-
/**
|
|
8311
|
-
* Replace a presence watch that has gone silent past TTL while the connection is up. The new
|
|
8312
|
-
* ordered consumer starts from the bucket's current last-per-subject state, so a peer that is
|
|
8313
|
-
* heartbeating is re-observed within one replay and a peer that is gone is aged out by the
|
|
8314
|
-
* next sweep exactly as if the watch had never stalled. Rate-limited to one attempt per TTL
|
|
8315
|
-
* per observer, never overlapping, never on a stopped or rebuilding endpoint (those own their
|
|
8316
|
-
* watch through {@link connectAndBind}). A stop or rebuild that lands while the bind is in
|
|
8317
|
-
* flight retires it: {@link startPresenceWatch} retires the late watch and reports
|
|
8318
|
-
* nothing, since the epoch that was silent no longer exists. A failed bind is reported and
|
|
8319
|
-
* the view stays stale.
|
|
8320
|
-
*/
|
|
8321
|
-
rebindStalePresenceWatch(now) {
|
|
8322
|
-
if (this.stopped || this.reconnecting || !this.kv || !this.nc || this.nc.isClosed())
|
|
8323
|
-
return;
|
|
8324
|
-
if (this.presenceRebind || now - this.presenceRebindAt < this.ttlMs)
|
|
8325
|
-
return;
|
|
8326
|
-
this.presenceRebindAt = now;
|
|
8327
|
-
const old = this.presenceWatchIter;
|
|
8328
|
-
const epoch = this.presenceEpoch;
|
|
8329
|
-
this.presenceRebind = (async () => {
|
|
8330
|
-
try {
|
|
8331
|
-
const silentMs = now - this.lastPresenceWatchAt;
|
|
8332
|
-
// Bind the replacement FIRST. A rebind the broker refuses (stream gone, create denied) or
|
|
8333
|
-
// one a held link never answers must leave the old watch in place: on a plain stall that
|
|
8334
|
-
// watch is the one that recovers by itself, and its replay is still guarded against
|
|
8335
|
-
// expired PUTs. Only a successfully bound watch retires its predecessor.
|
|
8336
|
-
const info = await this.startPresenceWatch();
|
|
8337
|
-
// Retired mid-bind (stop or rebuild moved the epoch): the late watch is already
|
|
8338
|
-
// retired and the old watch was torn down by whoever moved the epoch. Nothing to
|
|
8339
|
-
// retire, nothing to report.
|
|
8340
|
-
if (!info)
|
|
8341
|
-
return;
|
|
8342
|
-
// The predecessor's consumer goes with it: left alone, every rebind of a silent bucket
|
|
8343
|
-
// would add one idle consumer to the presence stream per TTL (#3914).
|
|
8344
|
-
if (old && old !== this.presenceWatchIter)
|
|
8345
|
-
this.retireKvWatch(old);
|
|
8346
|
-
// A bucket with no keys replays nothing, so the new watch cannot refresh
|
|
8347
|
-
// `lastPresenceWatchAt` by delivering. It IS current knowledge: nobody is present. The
|
|
8348
|
-
// bind's consumer info carries the initial pending count for that one fact; nats.js's KV
|
|
8349
|
-
// watch placed the isUpdate marker from the same `info(true)`.
|
|
8350
|
-
if (info.num_pending === 0)
|
|
8351
|
-
await this.onPresenceBucketEmpty();
|
|
8352
|
-
this.emit("warning", new Error(`presence watch silent for ${silentMs}ms with the connection up; rebound it from the bucket's current state`));
|
|
8353
|
-
}
|
|
8354
|
-
catch (e) {
|
|
8355
|
-
// A bind the epoch swap itself rejected (connection drained under it) is not a fault of
|
|
8356
|
-
// the epoch that replaced it; only a refusal on a still-current epoch is reported.
|
|
8357
|
-
if (epoch === this.presenceEpoch)
|
|
8358
|
-
this.emit("error", e);
|
|
8359
|
-
}
|
|
8360
|
-
finally {
|
|
8361
|
-
// An epoch swap already disowned this flight (and may own a successor's by now); only
|
|
8362
|
-
// a flight still in its own epoch clears the slot. Within one epoch there is one flight.
|
|
8363
|
-
if (epoch === this.presenceEpoch)
|
|
8364
|
-
this.presenceRebind = undefined;
|
|
8365
|
-
}
|
|
8366
|
-
})();
|
|
8122
|
+
return this.presence.readPresenceOnce();
|
|
8367
8123
|
}
|
|
8368
8124
|
/** Watch the channel registry: replay existing keys, then stream updates, into the local
|
|
8369
8125
|
* cache. A failed bind throws out of connect; an entry that does not decode or validate is
|
|
@@ -8408,214 +8164,8 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
8408
8164
|
this.emitRecoverable(new Error(`dropped channel registry entry for key ${JSON.stringify(e.key)}: ${err.message}`));
|
|
8409
8165
|
}
|
|
8410
8166
|
}
|
|
8411
|
-
/** The watch was just bound onto a bucket with no keys.
|
|
8412
|
-
*
|
|
8413
|
-
* A NON-REGISTERING observer (a `cotal status` probe, a lease checker) has real knowledge:
|
|
8414
|
-
* nobody is present. Every peer still in its roster is known gone (its key is not there to
|
|
8415
|
-
* replay), so it is marked offline now rather than aged out against a delivery that cannot
|
|
8416
|
-
* come; the silence gate is disarmed and the view reads current until the first write lands.
|
|
8417
|
-
* Without that the empty-bucket view relapsed to stale one window later and rebound again on
|
|
8418
|
-
* every window, one consumer create and one warning per TTL for as long as the mesh was empty.
|
|
8419
|
-
*
|
|
8420
|
-
* A REGISTERING observer (the manager) is itself one of the keys that should be there. An
|
|
8421
|
-
* empty bucket under it means the bucket was wiped since its last heartbeat (the stream
|
|
8422
|
-
* recreation), and the same wipe took every peer's record: their absence says the bucket is
|
|
8423
|
-
* new, not that they left. a reviewer reproduced the previous behaviour at default timing:
|
|
8424
|
-
* the rebind landed ~0.9s after the recreation, the observer marked every peer AND ITSELF
|
|
8425
|
-
* offline, and held the view current for up to one heartbeat, a false verdict `cotal ps`
|
|
8426
|
-
* would print as `mesh offline`. So a registering observer re-publishes its own record NOW,
|
|
8427
|
-
* which the new watch delivers, and lets the ordinary per-peer age-out run from that delivery:
|
|
8428
|
-
* a peer that is still heartbeating rewrites its key within its own heartbeat interval and is
|
|
8429
|
-
* re-observed live; one that is gone ages out exactly as after a plain rebind. The roster is
|
|
8430
|
-
* not touched here and the view is not held; the delivery is what makes it current.
|
|
8431
|
-
*
|
|
8432
|
-
* What neither branch covers: a consumer that dies again while the bucket is still empty is
|
|
8433
|
-
* not detectable by silence, so the first write after that is missed until the observer
|
|
8434
|
-
* restarts. */
|
|
8435
|
-
async onPresenceBucketEmpty() {
|
|
8436
|
-
if (this.doRegister) {
|
|
8437
|
-
// Our own key is missing from a bucket we write to: put it back. publishPresence throws on
|
|
8438
|
-
// a refused write; that surfaces through the rebind flight's error path with the epoch
|
|
8439
|
-
// fence intact, and the view stays stale, which is the honest state for an observer that
|
|
8440
|
-
// cannot even record itself.
|
|
8441
|
-
await this.publishPresence();
|
|
8442
|
-
return;
|
|
8443
|
-
}
|
|
8444
|
-
this.markPresenceBucketEmpty();
|
|
8445
|
-
}
|
|
8446
|
-
/** See {@link onPresenceBucketEmpty}: the non-registering branch. */
|
|
8447
|
-
markPresenceBucketEmpty() {
|
|
8448
|
-
this.presenceWatchEmpty = true;
|
|
8449
|
-
this.lastPresenceWatchAt = 0;
|
|
8450
|
-
let changed = false;
|
|
8451
|
-
for (const [id, p] of this.roster) {
|
|
8452
|
-
if (p.status === "offline")
|
|
8453
|
-
continue;
|
|
8454
|
-
const offline = this.toOffline(p);
|
|
8455
|
-
this.roster.set(id, offline);
|
|
8456
|
-
this.emit("presence", { type: "offline", presence: offline });
|
|
8457
|
-
changed = true;
|
|
8458
|
-
}
|
|
8459
|
-
if (changed)
|
|
8460
|
-
this.emit("roster", this.getRoster());
|
|
8461
|
-
this.emitPresenceViewIfChanged();
|
|
8462
|
-
}
|
|
8463
|
-
handleKvEntry(e) {
|
|
8464
|
-
// Provisioning uses this one reserved key to prove the bucket's max_age is enforced (#404).
|
|
8465
|
-
// It is maintenance traffic, never a peer identity, an offline transition, or view freshness.
|
|
8466
|
-
if (e.key === TTL_RECONCILE_CANARY_KEY)
|
|
8467
|
-
return;
|
|
8468
|
-
this.lastPresenceWatchAt = Date.now();
|
|
8469
|
-
this.presenceWatchEmpty = false;
|
|
8470
|
-
if (e.operation === "DEL" || e.operation === "PURGE") {
|
|
8471
|
-
this.markOffline(e.key);
|
|
8472
|
-
return;
|
|
8473
|
-
}
|
|
8474
|
-
const p = this.decodePresence(e);
|
|
8475
|
-
if (p)
|
|
8476
|
-
this.applyPresence(e.key, p);
|
|
8477
|
-
}
|
|
8478
|
-
/** The record a presence PUT carries, or undefined when a reader must drop it (SPEC §6). The watch
|
|
8479
|
-
* and {@link readPresence} decode through this one gate, so they drop the same records. */
|
|
8480
|
-
decodePresence(e) {
|
|
8481
|
-
let p;
|
|
8482
|
-
try {
|
|
8483
|
-
p = e.json();
|
|
8484
|
-
}
|
|
8485
|
-
catch {
|
|
8486
|
-
return undefined;
|
|
8487
|
-
}
|
|
8488
|
-
if (!isPresenceRow(p, e.key)) {
|
|
8489
|
-
this.presenceBindingDrops++;
|
|
8490
|
-
this.emitRecoverable(new Error(`dropped presence entry for key ${JSON.stringify(e.key)}: not a usable presence row (a record with card.id equal to its KV key, a non-blank string card.name, a known status, a finite ts, string display fields and a condition with a string code)`));
|
|
8491
|
-
return undefined;
|
|
8492
|
-
}
|
|
8493
|
-
// isPresenceRow is the one gate: it refuses a row whose card.id disagrees with the KV key, so a
|
|
8494
|
-
// forged or mis-keyed record never surfaces a spoofed roster identity. The write-side scoping
|
|
8495
|
-
// ($KV.<presenceBucket>.<own-id>) is the primary guard.
|
|
8496
|
-
return p;
|
|
8497
|
-
}
|
|
8498
|
-
applyPresence(id, raw) {
|
|
8499
|
-
const prev = this.roster.get(id);
|
|
8500
|
-
const stale = Date.now() - raw.ts > this.ttlMs;
|
|
8501
|
-
// A watch recovering from a stall replays the bucket. Those PUTs still carry the publisher's
|
|
8502
|
-
// pre-stall timestamps, so they look TTL-expired even though the peers kept heartbeating on
|
|
8503
|
-
// the broker. Materializing them as offline is the empty-to-full flicker #1045 named: the
|
|
8504
|
-
// observer's catch-up, not N deaths. Keep last-known rows, stay view-stale, and wait for a
|
|
8505
|
-
// live timestamp (or an explicit offline/delete) before changing a known peer.
|
|
8506
|
-
if (stale && raw.status !== "offline") {
|
|
8507
|
-
if (prev)
|
|
8508
|
-
return;
|
|
8509
|
-
this.roster.set(id, this.toOffline(raw));
|
|
8510
|
-
this.emit("roster", this.getRoster());
|
|
8511
|
-
return;
|
|
8512
|
-
}
|
|
8513
|
-
if (this.presenceSnapshotPopulated)
|
|
8514
|
-
this.emitPresenceViewIfChanged();
|
|
8515
|
-
// Any offline materialization (a stale snapshot OR a graceful-leave record) drops the advisory
|
|
8516
|
-
// attention fields — an offline peer must not carry a stale `[focus]`/`locally muted` hint.
|
|
8517
|
-
const p = stale || raw.status === "offline" ? this.toOffline(raw) : raw;
|
|
8518
|
-
// First time we hear about an already-offline peer (stale snapshot): record quietly.
|
|
8519
|
-
if (!prev && p.status === "offline") {
|
|
8520
|
-
this.roster.set(id, p);
|
|
8521
|
-
this.emit("roster", this.getRoster());
|
|
8522
|
-
return;
|
|
8523
|
-
}
|
|
8524
|
-
// Heartbeat refresh with no real change: bump liveness quietly and don't
|
|
8525
|
-
// emit — otherwise the periodic keep-alive looks like a stream of "updates".
|
|
8526
|
-
// A CHANGED lifecycleUid is never a heartbeat: it is a NEW LIFECYCLE reusing the same principal (a
|
|
8527
|
-
// terminal same-name recreation — in user mode a retired lifecycle and its same-name successor
|
|
8528
|
-
// share `<owner>.<actor>`; a supervised respawn keeps the uid and is unaffected), so it MUST notify
|
|
8529
|
-
// watchers even when its status/activity match the prior lifecycle's lingering record. Omitting this
|
|
8530
|
-
// left a same-principal successor invisible to presence-event consumers (e.g. the manager's
|
|
8531
|
-
// readiness race), which then timed out "uncertain" on a healthy agent (#29).
|
|
8532
|
-
if (prev &&
|
|
8533
|
-
prev.status !== "offline" &&
|
|
8534
|
-
p.status !== "offline" &&
|
|
8535
|
-
prev.lifecycleUid === p.lifecycleUid &&
|
|
8536
|
-
sameRuntime(prev.runtime, p.runtime) &&
|
|
8537
|
-
prev.status === p.status &&
|
|
8538
|
-
prev.activity === p.activity &&
|
|
8539
|
-
sameCondition(prev.condition, p.condition) &&
|
|
8540
|
-
prev.environment === p.environment &&
|
|
8541
|
-
prev.attention === p.attention &&
|
|
8542
|
-
sameChannelModes(prev.channelModes, p.channelModes)) {
|
|
8543
|
-
this.roster.set(id, p);
|
|
8544
|
-
return;
|
|
8545
|
-
}
|
|
8546
|
-
this.roster.set(id, p);
|
|
8547
|
-
const type = p.status === "offline"
|
|
8548
|
-
? "offline"
|
|
8549
|
-
: !prev || prev.status === "offline"
|
|
8550
|
-
? "join"
|
|
8551
|
-
: "update";
|
|
8552
|
-
this.emit("presence", { type, presence: p });
|
|
8553
|
-
this.emit("roster", this.getRoster());
|
|
8554
|
-
}
|
|
8555
|
-
/** Materialize an OFFLINE presence record: drop the advisory attention fields. An offline peer must
|
|
8556
|
-
* not show a stale `[focus]` or "locally muted #x" hint — SPEC: attention removed on offline sweep,
|
|
8557
|
-
* channel modes reset on restart. `statusSince` goes too: it dated the live status, and an observer
|
|
8558
|
-
* that derives offline from a stale heartbeat does not know when the peer left. card/activity/ts
|
|
8559
|
-
* are kept. */
|
|
8560
|
-
toOffline(p) {
|
|
8561
|
-
return { ...p, status: "offline", statusSince: undefined, attention: undefined, channelModes: undefined };
|
|
8562
|
-
}
|
|
8563
|
-
/** Mark a known peer offline (on KV delete/purge), keeping it in the roster. */
|
|
8564
|
-
markOffline(id) {
|
|
8565
|
-
const prev = this.roster.get(id);
|
|
8566
|
-
if (!prev || prev.status === "offline")
|
|
8567
|
-
return;
|
|
8568
|
-
const offline = this.toOffline(prev);
|
|
8569
|
-
this.roster.set(id, offline);
|
|
8570
|
-
this.emit("presence", { type: "offline", presence: offline });
|
|
8571
|
-
this.emit("roster", this.getRoster());
|
|
8572
|
-
}
|
|
8573
|
-
emitPresenceViewIfChanged() {
|
|
8574
|
-
const view = this.presenceView();
|
|
8575
|
-
if (view.state === this.presenceViewState)
|
|
8576
|
-
return;
|
|
8577
|
-
this.presenceViewState = view.state;
|
|
8578
|
-
this.emit("presence-view", view);
|
|
8579
|
-
}
|
|
8580
8167
|
sweep() {
|
|
8581
|
-
|
|
8582
|
-
// Whole-bucket silence past TTL is THIS observer going deaf. Real rosters do not lose every
|
|
8583
|
-
// peer in one window, and treating that silence as N offline verdicts empties an online-only
|
|
8584
|
-
// sidebar with nothing saying the window went blind (#1045). Gate the per-peer age-out on
|
|
8585
|
-
// watch freshness; surface the view as stale instead.
|
|
8586
|
-
if (this.lastPresenceWatchAt !== 0 && now - this.lastPresenceWatchAt > this.ttlMs) {
|
|
8587
|
-
this.emitPresenceViewIfChanged();
|
|
8588
|
-
// Staying stale is the right verdict for a held link (#1045), and the wrong END STATE when
|
|
8589
|
-
// the transport is up and the watch's own consumer is what died. Measured on a live deployment
|
|
8590
|
-
// 2026-09-09: the presence stream was deleted and recreated, its sequence restarted, and
|
|
8591
|
-
// every observer's ORDERED consumer re-created itself at the OLD start sequence (nats.js
|
|
8592
|
-
// 3.4.0 resets from its cursor). The broker kept sending idle heartbeats, so the client
|
|
8593
|
-
// never reset again, the iterator never closed, and the manager's roster stayed frozen at
|
|
8594
|
-
// the pre-recreation snapshot for hours: `cotal ps` read every older seat "mesh offline"
|
|
8595
|
-
// and every newer seat "not in roster" while all of them were heartbeating. The same
|
|
8596
|
-
// end state follows a plain consumer delete (an operator, or the 5-minute inactive
|
|
8597
|
-
// threshold after a long stall). Rebind the watch from the bucket's CURRENT state; a held
|
|
8598
|
-
// link's rebind fails or stays silent and the view simply stays stale, as before.
|
|
8599
|
-
this.rebindStalePresenceWatch(now);
|
|
8600
|
-
return;
|
|
8601
|
-
}
|
|
8602
|
-
let changed = false;
|
|
8603
|
-
for (const [id, p] of this.roster) {
|
|
8604
|
-
// A peer's own `ts` always trails the observer's last delivery, so "older than TTL by the
|
|
8605
|
-
// wall clock" alone ages peers out on the tick just before the whole-bucket gate above
|
|
8606
|
-
// trips (#1311's flap; measured as 3 offline verdicts per silence in the rebind suite).
|
|
8607
|
-
// Require that the watch itself delivered for a full TTL after this peer's last heartbeat:
|
|
8608
|
-
// then other peers were heard and this one was not, which is the only silence that is the
|
|
8609
|
-
// peer's rather than the observer's.
|
|
8610
|
-
if (p.status !== "offline" && now - p.ts > this.ttlMs && this.lastPresenceWatchAt - p.ts > this.ttlMs) {
|
|
8611
|
-
const offline = this.toOffline(p);
|
|
8612
|
-
this.roster.set(id, offline);
|
|
8613
|
-
this.emit("presence", { type: "offline", presence: offline });
|
|
8614
|
-
changed = true;
|
|
8615
|
-
}
|
|
8616
|
-
}
|
|
8617
|
-
if (changed)
|
|
8618
|
-
this.emit("roster", this.getRoster());
|
|
8168
|
+
return this.presence.sweep();
|
|
8619
8169
|
}
|
|
8620
8170
|
}
|
|
8621
8171
|
/** What a history read failure NAMES when it could not finish. One filter subject is the useful
|
|
@@ -8651,7 +8201,7 @@ function kindFromParsed(kind) {
|
|
|
8651
8201
|
* an old-shape alias. Every receive path asks this one predicate. What a path does with a refusal
|
|
8652
8202
|
* (drop, skip, ack, term or reject) and its own extra checks (a usable id, its own echo) stay with
|
|
8653
8203
|
* the path. */
|
|
8654
|
-
function isFromSubjectSender(msg, parsed) {
|
|
8204
|
+
export function isFromSubjectSender(msg, parsed) {
|
|
8655
8205
|
return isRecord(msg) && isRecord(msg.from) && msg.from.id === parsed.sender && isPrincipalOwnerToken(parsed.owner);
|
|
8656
8206
|
}
|
|
8657
8207
|
/** Routing fields in the envelope are advisory. The broker forge-locks sender (and for DMs,
|
|
@@ -8929,24 +8479,6 @@ function assertPartsSerializable(parts) {
|
|
|
8929
8479
|
}
|
|
8930
8480
|
}
|
|
8931
8481
|
}
|
|
8932
|
-
/** Shallow-equal two per-channel-mode maps (presence dedup): a change must re-emit, so an attention
|
|
8933
|
-
* toggle isn't swallowed as a quiet heartbeat. Absent and empty compare equal. */
|
|
8934
|
-
function sameChannelModes(a, b) {
|
|
8935
|
-
const ak = a ? Object.keys(a) : [];
|
|
8936
|
-
const bk = b ? Object.keys(b) : [];
|
|
8937
|
-
if (ak.length !== bk.length)
|
|
8938
|
-
return false;
|
|
8939
|
-
return ak.every((k) => a[k] === b?.[k]);
|
|
8940
|
-
}
|
|
8941
|
-
function sameRuntime(a, b) {
|
|
8942
|
-
return a?.id === b?.id && a?.startedAt === b?.startedAt && a?.source === b?.source;
|
|
8943
|
-
}
|
|
8944
|
-
function sameCondition(a, b) {
|
|
8945
|
-
return a?.code === b?.code
|
|
8946
|
-
&& a?.source === b?.source
|
|
8947
|
-
&& a?.message === b?.message
|
|
8948
|
-
&& a?.since === b?.since;
|
|
8949
|
-
}
|
|
8950
8482
|
function authOpts(a) {
|
|
8951
8483
|
const tls = a.tls ? {} : undefined;
|
|
8952
8484
|
// USER MODE: present the shared auth-account sentinel creds AND the user bearer as `auth_token` (an
|