@cotal-ai/core 0.166.2 → 0.167.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. package/dist/agent-endpoint.d.ts +242 -0
  2. package/dist/agent-endpoint.d.ts.map +1 -0
  3. package/dist/agent-endpoint.js +183 -0
  4. package/dist/agent-endpoint.js.map +1 -0
  5. package/dist/backup-config.d.ts +2 -2
  6. package/dist/endpoint-call.d.ts +34 -1
  7. package/dist/endpoint-call.d.ts.map +1 -1
  8. package/dist/endpoint-call.js +209 -116
  9. package/dist/endpoint-call.js.map +1 -1
  10. package/dist/endpoint-cluster.d.ts +6 -1
  11. package/dist/endpoint-cluster.d.ts.map +1 -1
  12. package/dist/endpoint-cluster.js +25 -1
  13. package/dist/endpoint-cluster.js.map +1 -1
  14. package/dist/endpoint-delivery-lease.d.ts +184 -0
  15. package/dist/endpoint-delivery-lease.d.ts.map +1 -0
  16. package/dist/endpoint-delivery-lease.js +270 -0
  17. package/dist/endpoint-delivery-lease.js.map +1 -0
  18. package/dist/endpoint-envelope.d.ts +4 -3
  19. package/dist/endpoint-envelope.d.ts.map +1 -1
  20. package/dist/endpoint-envelope.js +19 -2
  21. package/dist/endpoint-envelope.js.map +1 -1
  22. package/dist/endpoint-grants.d.ts +31 -1
  23. package/dist/endpoint-grants.d.ts.map +1 -1
  24. package/dist/endpoint-grants.js +62 -1
  25. package/dist/endpoint-grants.js.map +1 -1
  26. package/dist/endpoint-presence.d.ts +296 -0
  27. package/dist/endpoint-presence.d.ts.map +1 -0
  28. package/dist/endpoint-presence.js +784 -0
  29. package/dist/endpoint-presence.js.map +1 -0
  30. package/dist/endpoint-publish-denial.d.ts +6 -0
  31. package/dist/endpoint-publish-denial.d.ts.map +1 -1
  32. package/dist/endpoint-publish-denial.js +13 -0
  33. package/dist/endpoint-publish-denial.js.map +1 -1
  34. package/dist/endpoint-serve.d.ts +76 -2
  35. package/dist/endpoint-serve.d.ts.map +1 -1
  36. package/dist/endpoint-serve.js +340 -147
  37. package/dist/endpoint-serve.js.map +1 -1
  38. package/dist/endpoint-service.d.ts +5 -1
  39. package/dist/endpoint-service.d.ts.map +1 -1
  40. package/dist/endpoint-service.js +2 -1
  41. package/dist/endpoint-service.js.map +1 -1
  42. package/dist/endpoint-subjects.d.ts +76 -2
  43. package/dist/endpoint-subjects.d.ts.map +1 -1
  44. package/dist/endpoint-subjects.js +113 -2
  45. package/dist/endpoint-subjects.js.map +1 -1
  46. package/dist/endpoint-verbs.d.ts +1 -1
  47. package/dist/endpoint-verbs.d.ts.map +1 -1
  48. package/dist/endpoint-verbs.js +1 -1
  49. package/dist/endpoint-verbs.js.map +1 -1
  50. package/dist/endpoint.d.ts +118 -190
  51. package/dist/endpoint.d.ts.map +1 -1
  52. package/dist/endpoint.js +323 -791
  53. package/dist/endpoint.js.map +1 -1
  54. package/dist/index.d.ts +1 -0
  55. package/dist/index.d.ts.map +1 -1
  56. package/dist/index.js +1 -0
  57. package/dist/index.js.map +1 -1
  58. package/dist/kv-scan.d.ts +2 -1
  59. package/dist/kv-scan.d.ts.map +1 -1
  60. package/dist/kv-scan.js +4 -2
  61. package/dist/kv-scan.js.map +1 -1
  62. package/dist/peer-label.d.ts +6 -5
  63. package/dist/peer-label.d.ts.map +1 -1
  64. package/dist/peer-label.js +7 -6
  65. package/dist/peer-label.js.map +1 -1
  66. package/dist/provision.d.ts +11 -1
  67. package/dist/provision.d.ts.map +1 -1
  68. package/dist/provision.js +41 -11
  69. package/dist/provision.js.map +1 -1
  70. package/dist/subjects.d.ts +3 -2
  71. package/dist/subjects.d.ts.map +1 -1
  72. package/dist/subjects.js +10 -2
  73. package/dist/subjects.js.map +1 -1
  74. package/dist/types.d.ts +11 -0
  75. package/dist/types.d.ts.map +1 -1
  76. package/package.json +1 -1
@@ -0,0 +1,784 @@
1
+ import { TimeoutError } from "@nats-io/transport-node";
2
+ import { processRuntime } from "./process-runtime.js";
3
+ import { lastKvEntries } from "./kv-scan.js";
4
+ import { TTL_RECONCILE_CANARY_KEY } from "./streams.js";
5
+ import { presenceBucket } from "./subjects.js";
6
+ import { isPresenceRow, assertPresenceRow } from "./subjects.js";
7
+ /** A presence bucket that has refused consecutive writes for at least one full presence TTL.
8
+ * The condition is non-transient: ordinary heartbeat retry has already failed for the whole
9
+ * liveness window, so callers should report the roster as last-known until a write succeeds. */
10
+ export class PresenceWriteStuckError extends Error {
11
+ bucket;
12
+ since;
13
+ consecutiveFailures;
14
+ ttlMs;
15
+ lastError;
16
+ code = "presence-write-stuck";
17
+ transient = false;
18
+ constructor(bucket, since, consecutiveFailures, ttlMs, lastError) {
19
+ super(`presence writes to bucket ${JSON.stringify(bucket)} have failed ${consecutiveFailures} consecutive times for at least one TTL (${ttlMs}ms); the presence view is not live until a write succeeds or the broker store is repaired${lastError ? ` (last refusal: ${lastError})` : ""}`);
20
+ this.bucket = bucket;
21
+ this.since = since;
22
+ this.consecutiveFailures = consecutiveFailures;
23
+ this.ttlMs = ttlMs;
24
+ this.lastError = lastError;
25
+ this.name = "PresenceWriteStuckError";
26
+ }
27
+ }
28
+ /** Presence publication, local roster materialization and observer freshness for one endpoint. */
29
+ export class EndpointPresence {
30
+ deps;
31
+ /** #1356: when the presence bucket started refusing writes; undefined once one succeeds. */
32
+ presenceWriteFailingSince;
33
+ /** #1356: the broker's last refusal message, kept alongside the start time for diagnosis. */
34
+ lastPresenceWriteError;
35
+ /** Consecutive latest-evidence refusals on the current connection epoch. A success resets it. */
36
+ presenceWriteFailures = 0;
37
+ /** One named non-transient warning per failed run. A success re-arms it. */
38
+ presenceWriteEscalated = false;
39
+ /** #1461: monotonic per-put generation on the presence path, so a settle can tell whether it is
40
+ * the latest evidence on its epoch. {@link publishPresence} snapshots it at put start and every
41
+ * settle records its generation here: a settle is the latest evidence only while no put that
42
+ * started after it has already settled. */
43
+ presencePutGeneration = 0;
44
+ /** #1461: the generation of the newest presence put that has SETTLED (succeeded or rejected),
45
+ * or `undefined` when none ever has. A put that is merely in flight is not evidence yet. */
46
+ presencePutSettled;
47
+ roster = new Map();
48
+ /** The {@link readPresence} in flight; calls made before it settles share it. */
49
+ presenceRead;
50
+ /** Orders {@link readPresence} merges: a read that settles after a newer one was applied is dropped. */
51
+ presenceReadGeneration = 0;
52
+ presenceReadApplied = 0;
53
+ /** Resolves when the current presence watch has consumed its complete initial KV snapshot. */
54
+ presenceSnapshot = Promise.resolve();
55
+ /** False from connection reset until the watch marks the last entry in its initial replay. */
56
+ presenceSnapshotPopulated = false;
57
+ /**
58
+ * Observer-local age of the last presence-KV delivery (any key, including DEL/PURGE). Distinct
59
+ * from each peer's `ts`: that is the publisher's heartbeat. Whole-bucket silence past TTL is
60
+ * the observer going deaf, not N simultaneous deaths, and sweep must not treat it as the
61
+ * latter (#1045).
62
+ */
63
+ lastPresenceWatchAt = 0;
64
+ /** Last emitted presence-view state. Suppresses duplicate `presence-view` events. */
65
+ presenceViewState = "unpopulated";
66
+ /** A presence-watch rebind in flight (see {@link rebindStalePresenceWatch}); one at a time. */
67
+ presenceRebind;
68
+ /** The current watch was bound onto a bucket with NO keys (see {@link markPresenceBucketEmpty}).
69
+ * Such a watch cannot deliver until someone writes, so its silence is not staleness. */
70
+ presenceWatchEmpty = false;
71
+ /** Wall-clock of the last rebind attempt, so a bucket that is silent because it is EMPTY (or a
72
+ * broker that keeps refusing the consumer create) is retried once per TTL, not per sweep tick. */
73
+ presenceRebindAt = 0;
74
+ status = "idle";
75
+ activity;
76
+ /** When {@link status} or {@link activity} last changed. Heartbeats republish it unchanged. */
77
+ statusSince = Date.now();
78
+ /** When {@link activity} last changed. A status change leaves it alone. */
79
+ activitySince;
80
+ /** Last harness-reported work progress. Carried by the next heartbeat, never published per event. */
81
+ activeAt;
82
+ condition;
83
+ /** Advances on every condition change so an older in-flight put cannot be the final KV state. */
84
+ conditionRevision = 0;
85
+ /** Read once at construction. Core publishes this opaque provider reference and never parses it. */
86
+ environment;
87
+ /** Mirror of the connector's authoritative attention state, published in presence (advisory). The
88
+ * endpoint never reads these back into delivery - they exist only to broadcast. */
89
+ attentionMode;
90
+ channelModes;
91
+ /** Presence rows rejected because their embedded id did not match the ACL-scoped KV key, or because
92
+ * a field the observer computes with was malformed. */
93
+ presenceBindingDrops = 0;
94
+ /** The presence/channel-registry watches' own handles. Each is an ORDERED push consumer: its idle
95
+ * heartbeat monitor lives on a JS timer independent of the connection, so a drain that doesn't
96
+ * `.stop()` it first leaves the monitor to fire into a closing connection every 30s, throwing
97
+ * `DrainingConnectionError` out of `reset()` on a timer nothing awaits. */
98
+ presenceWatchIter;
99
+ constructor(deps) {
100
+ this.deps = deps;
101
+ }
102
+ get card() { return this.deps.card(); }
103
+ get space() { return this.deps.space(); }
104
+ get kv() { return this.deps.bucket(); }
105
+ get nc() { return this.deps.connection(); }
106
+ get ownLifecycleUid() { return this.deps.lifecycleUid(); }
107
+ get ttlMs() { return this.deps.ttlMs(); }
108
+ get doRegister() { return this.deps.registers(); }
109
+ get doWatch() { return this.deps.watches(); }
110
+ get readOnce() { return this.deps.readsOnce(); }
111
+ get stopped() { return this.deps.stopped(); }
112
+ get reconnecting() { return this.deps.reconnecting(); }
113
+ get presenceEpoch() { return this.deps.epoch(); }
114
+ notLiveMsg() { return this.deps.notLiveMsg(); }
115
+ bindKvWatch(kv) { return this.deps.bindWatch(kv); }
116
+ retireKvWatch(iter) { this.deps.retireWatch(iter); }
117
+ emit(event, value) { return this.deps.emit(event, value); }
118
+ emitRecoverable(error) { this.deps.recoverable(error); }
119
+ configureEnvironment(environment) { this.environment = environment; }
120
+ seedChannelModes(modes) {
121
+ this.channelModes = modes && Object.keys(modes).length ? modes : undefined;
122
+ }
123
+ get bindingDropCount() { return this.presenceBindingDrops; }
124
+ values() { return this.roster.values(); }
125
+ cancelRebind() { this.presenceRebind = undefined; }
126
+ retireWatch() {
127
+ if (this.presenceWatchIter)
128
+ this.retireKvWatch(this.presenceWatchIter);
129
+ this.presenceWatchIter = undefined;
130
+ }
131
+ async publishOffline() {
132
+ this.status = "offline";
133
+ await this.publishPresence();
134
+ }
135
+ resetConnection() {
136
+ this.roster.clear();
137
+ // A read in flight on the old connection must not land in the new connection's roster.
138
+ this.presenceRead = undefined;
139
+ this.presenceReadGeneration++;
140
+ this.presenceReadApplied = this.presenceReadGeneration;
141
+ // #1356: the presence-refusal record is connection-scoped like everything else torn down here.
142
+ // It says "the broker on THIS connection refuses writes to this bucket", so it cannot outlive the
143
+ // connection that observed it. Left behind, a later failure on a different connection inherits it
144
+ // and is reported as a bucket refusal - measured: a failed bind against an unreachable server
145
+ // still printed the refusal sentence while `connectionIssue` already said "connection refused".
146
+ this.clearPresenceWriteFailure();
147
+ this.lastPresenceWatchAt = 0;
148
+ this.presenceRebindAt = 0;
149
+ this.presenceWatchEmpty = false;
150
+ this.presenceSnapshotPopulated = false;
151
+ this.emitPresenceViewIfChanged();
152
+ }
153
+ // ---- presence ------------------------------------------------------------
154
+ getRoster() {
155
+ return [...this.roster.values()].sort((a, b) => a.card.name.localeCompare(b.card.name));
156
+ }
157
+ /**
158
+ * Read the roster from the broker now, for an endpoint that does not watch presence (SPEC §6). One
159
+ * batched Direct Get of the bucket that binds no consumer, decoded and dropped as the watch decodes.
160
+ * A record that says `offline` or whose `ts` is past the liveness window reads offline. A peer an
161
+ * earlier read saw that is gone from the bucket stays in the roster as offline, as a watch keeps
162
+ * it. The result is also what {@link getRoster} returns until the next read.
163
+ *
164
+ * Calls made while a read is in flight share it, and every other call reads anew, so a verdict
165
+ * never rests on a read that settled before it was asked. A failed read throws and changes
166
+ * nothing. A credential that may not make the read is refused with the grant it lacks.
167
+ */
168
+ async readPresence() {
169
+ if (this.doWatch && !this.readOnce)
170
+ throw new Error("readPresence is for an endpoint that does not watch presence; this one keeps its roster live (getRoster)");
171
+ const kv = this.kv;
172
+ if (!kv)
173
+ throw new Error(this.nc ? "readPresence needs the presence bucket open (registerPresence or watchPresence)" : this.notLiveMsg());
174
+ if (this.presenceRead)
175
+ return this.presenceRead;
176
+ const generation = ++this.presenceReadGeneration;
177
+ const roster = (async () => {
178
+ let entries;
179
+ try {
180
+ entries = await lastKvEntries(kv);
181
+ }
182
+ catch (e) {
183
+ if (!this.deps.isPublishPermissionDenied(e))
184
+ throw e;
185
+ throw new Error(`presence read refused for ${this.card.id}: this credential may not publish $JS.API.DIRECT.GET.KV_${presenceBucket(this.space)}, the on-demand roster read (SPEC Appendix B). A credential minted by an older issuer can lack this grant even when newly minted; upgrade the manager or auth service that issues credentials, then mint a new credential. Reusing credentials or reconnecting does not add the grant.`, { cause: e });
186
+ }
187
+ const at = Date.now();
188
+ const read = new Map();
189
+ for (const e of entries) {
190
+ if (e.key === TTL_RECONCILE_CANARY_KEY || e.operation === "DEL" || e.operation === "PURGE")
191
+ continue;
192
+ const p = this.decodePresence(e);
193
+ if (p)
194
+ read.set(e.key, p.status === "offline" || at - p.ts > this.ttlMs ? this.toOffline(p) : p);
195
+ }
196
+ if (generation > this.presenceReadApplied) {
197
+ this.presenceReadApplied = generation;
198
+ for (const [id, p] of this.roster)
199
+ if (!read.has(id))
200
+ read.set(id, p.status === "offline" ? p : this.toOffline(p));
201
+ this.roster.clear();
202
+ for (const [id, p] of read)
203
+ this.roster.set(id, p);
204
+ }
205
+ return this.getRoster();
206
+ })();
207
+ this.presenceRead = roster;
208
+ const settled = () => { if (this.presenceRead === roster)
209
+ this.presenceRead = undefined; };
210
+ roster.then(settled, settled);
211
+ return roster;
212
+ }
213
+ /**
214
+ * Trust state of THIS observer's presence watch, not of any peer. `unpopulated` means the
215
+ * current watch has not completed its initial snapshot, so {@link getRoster} may be partial and
216
+ * cannot support an absence verdict. `stale` means the whole bucket has been silent past the
217
+ * liveness window, so the roster is last-known as of `staleSince`. `fresh` is false for both
218
+ * unsafe states so consumers written before `state` was added degrade in the safe direction.
219
+ */
220
+ presenceView() {
221
+ if (!this.doWatch)
222
+ return { state: "current", fresh: true };
223
+ if (!this.presenceSnapshotPopulated)
224
+ return { state: "unpopulated", fresh: false };
225
+ if (this.presenceWatchEmpty)
226
+ return { state: "current", fresh: true };
227
+ const staleSince = this.lastPresenceWatchAt + this.ttlMs;
228
+ if (Date.now() < staleSince)
229
+ return { state: "current", fresh: true };
230
+ return { state: "stale", fresh: false, staleSince };
231
+ }
232
+ /** Wait until the current presence watch has consumed its initial KV snapshot. An observer that
233
+ * does not register completes on an empty bucket at once. A registering one completes when its
234
+ * own record arrives, and the timeout bounds that wait. */
235
+ async waitForPresenceSnapshot(timeoutMs = 1_000) {
236
+ let timer;
237
+ try {
238
+ return await Promise.race([
239
+ this.presenceSnapshot.then(() => "snapshot"),
240
+ new Promise((resolve) => { timer = setTimeout(() => resolve("timeout"), timeoutMs); }),
241
+ ]);
242
+ }
243
+ finally {
244
+ if (timer)
245
+ clearTimeout(timer);
246
+ }
247
+ }
248
+ assertPresenceUpdate(update) {
249
+ assertPresenceRow({ card: this.card, status: this.status, activity: this.activity,
250
+ condition: this.condition, attention: this.attentionMode, ts: Date.now(), ...update }, this.card.id);
251
+ }
252
+ async setActivity(activity) {
253
+ this.assertPresenceUpdate({ activity });
254
+ if (activity !== this.activity)
255
+ this.statusSince = this.activitySince = Date.now();
256
+ this.activity = activity;
257
+ await this.publishPresence();
258
+ }
259
+ /** Set the status, and with `activity` the activity line too, in one presence write, so no peer sees
260
+ * the new activity beside the old status. */
261
+ async setStatus(status, activity) {
262
+ this.assertPresenceUpdate({ status, ...(activity === undefined ? {} : { activity }) });
263
+ if (activity !== undefined && activity !== this.activity)
264
+ this.statusSince = this.activitySince = Date.now();
265
+ if (activity !== undefined)
266
+ this.activity = activity;
267
+ if (status !== this.status)
268
+ this.statusSince = Date.now();
269
+ this.status = status;
270
+ await this.publishPresence();
271
+ }
272
+ /** Record harness-reported work progress as `activeAt`. It writes nothing itself: the next heartbeat
273
+ * carries it, so a stream of events costs no presence writes. */
274
+ noteActivity(at = Date.now()) {
275
+ if (Number.isFinite(at) && at > (this.activeAt ?? 0))
276
+ this.activeAt = at;
277
+ }
278
+ /** Publish a harness-reported condition, or clear it. Core stores the relay without interpretation. */
279
+ async setCondition(condition) {
280
+ this.assertPresenceUpdate({ condition: condition ?? undefined });
281
+ this.condition = condition ?? undefined;
282
+ this.conditionRevision++;
283
+ await this.publishPresence();
284
+ }
285
+ /** Publish the agent's global attention mode into presence (advisory observability). Mirror only -
286
+ * delivery decisions stay in the connector's authoritative state. */
287
+ async setAttention(attention) {
288
+ this.assertPresenceUpdate({ attention });
289
+ this.attentionMode = attention;
290
+ await this.publishPresence();
291
+ }
292
+ /** Publish the agent's per-channel attention overrides into presence (advisory). An empty map drops
293
+ * the field. Mirror only - never read back into delivery. */
294
+ async setChannelModes(modes) {
295
+ this.channelModes = Object.keys(modes).length ? modes : undefined;
296
+ await this.publishPresence();
297
+ }
298
+ async publishPresence() {
299
+ if (!this.doRegister || !this.kv)
300
+ return; // observers watch but never publish their own record
301
+ const p = {
302
+ card: this.card,
303
+ // SPEC §6: presence carries the incarnation's lifecycle UID (MUST in auth mode from v0.4);
304
+ // omitted only where the endpoint has none (a pure operator/daemon connection never registers).
305
+ ...(this.ownLifecycleUid !== undefined ? { lifecycleUid: this.ownLifecycleUid } : {}),
306
+ runtime: processRuntime(this.ownLifecycleUid),
307
+ status: this.status,
308
+ condition: this.condition,
309
+ environment: this.environment,
310
+ activity: this.activity,
311
+ statusSince: this.statusSince,
312
+ activitySince: this.activitySince,
313
+ activeAt: this.activeAt,
314
+ attention: this.attentionMode,
315
+ channelModes: this.channelModes,
316
+ prompts: this.deps.prompts(),
317
+ ts: Date.now(),
318
+ };
319
+ // Wire contract (SPEC §6): an OFFLINE record must not carry the advisory attention fields or
320
+ // `statusSince`. Scrub at the publisher - this covers stop(), setStatus("offline"), and any future
321
+ // offline publish site, so the raw KV record is compliant, not only the observer-side roster
322
+ // materialization.
323
+ const record = this.status === "offline" ? this.toOffline(p) : p;
324
+ assertPresenceRow(record, this.card.id);
325
+ // #1356: this put can still be in flight when a teardown runs, because a rebind reaches
326
+ // publishPresence through onPresenceBucketEmpty and no teardown awaits that flight. Take the same
327
+ // epoch fence {@link startPresenceWatch} takes, so a put that outlives its epoch still throws to
328
+ // its caller but no longer writes presence-refusal state belonging to a later connection. Without
329
+ // it, a heartbeat put (default 2s) whose ~5s JetStream timeout elapses after a rebuild plants a
330
+ // refusal on the connection that just published successfully.
331
+ const epoch = this.presenceEpoch;
332
+ const conditionRevision = this.conditionRevision;
333
+ // #1461: snapshot the put's generation BEFORE the put. A settle (success or rejection) is the
334
+ // latest evidence on its epoch only while no put that started after it has ALREADY settled -
335
+ // a newer put merely in flight is not evidence yet. Same-epoch puts overlap routinely
336
+ // (heartbeat 2s against a ~5s JetStream put timeout), so "succeeded" alone is not "newest".
337
+ const generation = ++this.presencePutGeneration;
338
+ try {
339
+ await this.kv.put(this.card.id, JSON.stringify(record));
340
+ }
341
+ catch (e) {
342
+ // #1356: a broker can put this bucket into a state where it refuses every write and never
343
+ // recovers, and the write is the ONLY thing that fails - open and watch both still succeed, so
344
+ // nothing else here notices. Record WHEN the refusals started, at the one site that knows the
345
+ // failing write was a presence write; a caller cannot infer that from the generic `warning`
346
+ // stream, which carries any recoverable error.
347
+ //
348
+ // #1461: a rejection speaks when it is the newest SETTLED evidence - a newer put that has
349
+ // merely started does not silence it, but a newer put that already settled (either way)
350
+ // does.
351
+ const superseded = (this.presencePutSettled ?? 0) > generation;
352
+ this.presencePutSettled = Math.max(this.presencePutSettled ?? 0, generation);
353
+ if (epoch === this.presenceEpoch && !this.stopped && !superseded) {
354
+ this.presenceWriteFailingSince ??= Date.now();
355
+ this.lastPresenceWriteError = e?.message ?? String(e);
356
+ this.presenceWriteFailures++;
357
+ this.escalatePresenceWriteFailureIfStuck();
358
+ }
359
+ if (e instanceof TimeoutError)
360
+ throw new Error(`${e.message}. The presence write outcome is unknown: the broker may have stored the update. Read the roster before retrying.`, { cause: e });
361
+ throw e;
362
+ }
363
+ // A late SUCCESS cannot erase a refusal from newer evidence. Two fences, each answering half:
364
+ // the epoch fence (#1356) keeps a put that outlives its connection from writing or erasing a
365
+ // record that belongs to a later connection; the generation fence (#1461) orders puts WITHIN
366
+ // one epoch by SETTLE order, not start order: a settle is the latest evidence only while no
367
+ // put that started after it has already settled, whichever way that newer put settled. So an
368
+ // earlier put settling AFTER a later put settled no longer speaks - success cannot clear a
369
+ // newer refusal (#1461's original case), and a newer settle is not blocked by being merely
370
+ // started (panel round 1). Without the fence, an earlier put succeeding late cleared a later
371
+ // put's refusal while the bucket still refused writes (heartbeat 2s against a ~5s put timeout,
372
+ // so the overlap is routine rather than a corner).
373
+ const superseded = (this.presencePutSettled ?? 0) > generation;
374
+ this.presencePutSettled = Math.max(this.presencePutSettled ?? 0, generation);
375
+ if (epoch !== this.presenceEpoch || this.stopped || superseded)
376
+ return;
377
+ // Presence writes may overlap. If this put carried an older condition, repair the KV with the
378
+ // latest local state before returning so a late failure publish cannot resurrect after a new
379
+ // turn cleared it. The revision is condition-specific; unrelated heartbeat overlap is unchanged.
380
+ if (conditionRevision !== this.conditionRevision) {
381
+ await this.publishPresence();
382
+ return;
383
+ }
384
+ this.clearPresenceWriteFailure();
385
+ }
386
+ /** #1356: drop the presence-refusal record when the connection that OBSERVED those refusals goes
387
+ * away. "This bucket is refusing writes" is a claim about a specific broker connection; once that
388
+ * connection is torn down or rebuilt the claim has no remaining basis, and a later failure on a
389
+ * fresh connection must establish it again from its own evidence.
390
+ *
391
+ * Cleared at the SOURCE rather than guarded at each reader, because a guard protects one consumer
392
+ * and clearing protects every consumer, including ones not yet written. Measured: without this, a
393
+ * failed bind against an unreachable server still reported the presence-refusal sentence while the
394
+ * endpoint's own `connectionIssue` already said "connection refused". */
395
+ clearPresenceWriteFailure() {
396
+ this.presenceWriteFailingSince = undefined;
397
+ this.lastPresenceWriteError = undefined;
398
+ this.presenceWriteFailures = 0;
399
+ this.presenceWriteEscalated = false;
400
+ }
401
+ /** A heartbeat retry remains recoverable inside one TTL. Once two or more consecutive writes have
402
+ * failed across the full window, ordinary retry has exhausted the roster's liveness budget: raise
403
+ * one named, non-transient warning. Later failures keep the duration/count current without flooding
404
+ * the operator log; the next successful write clears and re-arms the condition. */
405
+ escalatePresenceWriteFailureIfStuck() {
406
+ if (this.presenceWriteEscalated ||
407
+ this.presenceWriteFailingSince === undefined ||
408
+ this.presenceWriteFailures < 2 ||
409
+ Date.now() - this.presenceWriteFailingSince < this.ttlMs)
410
+ return;
411
+ this.presenceWriteEscalated = true;
412
+ this.emitRecoverable(new PresenceWriteStuckError(presenceBucket(this.space), this.presenceWriteFailingSince, this.presenceWriteFailures, this.ttlMs, this.lastPresenceWriteError));
413
+ }
414
+ /** #1356: the presence bucket has been refusing writes since this time, or `undefined` when the
415
+ * last publish succeeded. Cleared by a successful write that is the latest evidence on its
416
+ * epoch (#1461), so a survived blip reads as healthy and only a SUSTAINED failure carries a
417
+ * duration.
418
+ *
419
+ * Presence writes are the CANARY, not the scope: the broker can disable JetStream account-wide
420
+ * while the NATS connection stays up, so a caller must not read this as "only presence is
421
+ * affected". It reports what was observed, not how far the fault extends. */
422
+ presenceWriteFailure() {
423
+ if (this.presenceWriteFailingSince === undefined)
424
+ return undefined;
425
+ return {
426
+ since: this.presenceWriteFailingSince,
427
+ forMs: Date.now() - this.presenceWriteFailingSince,
428
+ error: this.lastPresenceWriteError,
429
+ bucket: presenceBucket(this.space),
430
+ consecutiveFailures: this.presenceWriteFailures,
431
+ stuck: this.presenceWriteEscalated,
432
+ };
433
+ }
434
+ /** Bind a presence watch on the current connection. Resolves the bound consumer's info when
435
+ * the watch was installed, undefined when the endpoint stopped or rebuilt while the bind was
436
+ * in flight: that bind's watch is retired by {@link bindKvWatch} and nothing is installed,
437
+ * because the epoch that asked for it is gone and the epoch that replaced it binds its own
438
+ * watch through the endpoint connection bind. Without this fence a bind that completes after
439
+ * endpoint shutdown would resurrect a watch on a stopped endpoint, and one that completes after a
440
+ * rebuild would overwrite the fresh epoch's watch with a dead-connection iterator. */
441
+ async startPresenceWatch() {
442
+ if (!this.kv)
443
+ return undefined;
444
+ let hydrated;
445
+ this.presenceSnapshot = new Promise((resolve) => { hydrated = resolve; });
446
+ const bound = await this.bindKvWatch(this.kv);
447
+ if (!bound) {
448
+ hydrated();
449
+ return undefined;
450
+ }
451
+ const { iter, info } = bound;
452
+ this.presenceWatchIter = iter;
453
+ // A bucket with no keys replays nothing, so no entry carries the end-of-replay marker. For an
454
+ // observer that does not register, that empty replay is the whole snapshot: nobody is present.
455
+ // A registering observer is populated by its own record instead (see onPresenceBucketEmpty).
456
+ const empty = info.num_pending === 0 && !this.doRegister;
457
+ if (empty) {
458
+ this.presenceSnapshotPopulated = true;
459
+ hydrated();
460
+ this.markPresenceBucketEmpty();
461
+ }
462
+ void (async () => {
463
+ let ready = empty;
464
+ for await (const e of iter) {
465
+ // A rebind bound a newer watch: this one's remaining entries belong to a retired epoch.
466
+ // Stop consuming them (the rebind stops the iterator too; this ends the loop first).
467
+ if (this.presenceWatchIter !== iter)
468
+ break;
469
+ this.handleKvEntry(e);
470
+ // @nats-io/kv marks the final initial replay entry isUpdate=true. Later updates stay true.
471
+ if (!ready && e.isUpdate) {
472
+ ready = true;
473
+ this.presenceSnapshotPopulated = true;
474
+ hydrated();
475
+ this.emitPresenceViewIfChanged();
476
+ }
477
+ }
478
+ hydrated();
479
+ })().catch((e) => this.emit("error", e));
480
+ return info;
481
+ }
482
+ /** `readOnce`: fill the roster from one read of the presence bucket. Stale
483
+ * and offline records resolve as they do in a watch's initial replay. */
484
+ async readPresenceOnce() {
485
+ if (!this.kv)
486
+ return;
487
+ for (const e of await lastKvEntries(this.kv))
488
+ this.handleKvEntry(e);
489
+ this.presenceSnapshotPopulated = true;
490
+ // No record reached the roster (an empty bucket, or only the TTL canary): nobody is present.
491
+ if (this.lastPresenceWatchAt === 0)
492
+ this.markPresenceBucketEmpty();
493
+ else
494
+ this.emitPresenceViewIfChanged();
495
+ }
496
+ /**
497
+ * Replace a presence watch that has gone silent past TTL while the connection is up. The new
498
+ * ordered consumer starts from the bucket's current last-per-subject state, so a peer that is
499
+ * heartbeating is re-observed within one replay and a peer that is gone is aged out by the
500
+ * next sweep exactly as if the watch had never stalled. Rate-limited to one attempt per TTL
501
+ * per observer, never overlapping, never on a stopped or rebuilding endpoint (those own their
502
+ * watch through the endpoint connection bind). A stop or rebuild that lands while the bind is in
503
+ * flight retires it: {@link startPresenceWatch} retires the late watch and reports
504
+ * nothing, since the epoch that was silent no longer exists. A failed bind is reported and
505
+ * the view stays stale.
506
+ */
507
+ rebindStalePresenceWatch(now) {
508
+ if (this.stopped || this.reconnecting || !this.kv || !this.nc || this.nc.isClosed())
509
+ return;
510
+ if (this.presenceRebind || now - this.presenceRebindAt < this.ttlMs)
511
+ return;
512
+ this.presenceRebindAt = now;
513
+ const old = this.presenceWatchIter;
514
+ const epoch = this.presenceEpoch;
515
+ this.presenceRebind = (async () => {
516
+ try {
517
+ const silentMs = now - this.lastPresenceWatchAt;
518
+ // Bind the replacement FIRST. A rebind the broker refuses (stream gone, create denied) or
519
+ // one a held link never answers must leave the old watch in place: on a plain stall that
520
+ // watch is the one that recovers by itself, and its replay is still guarded against
521
+ // expired PUTs. Only a successfully bound watch retires its predecessor.
522
+ const info = await this.startPresenceWatch();
523
+ // Retired mid-bind (stop or rebuild moved the epoch): the late watch is already
524
+ // retired and the old watch was torn down by whoever moved the epoch. Nothing to
525
+ // retire, nothing to report.
526
+ if (!info)
527
+ return;
528
+ // The predecessor's consumer goes with it: left alone, every rebind of a silent bucket
529
+ // would add one idle consumer to the presence stream per TTL (#3914).
530
+ if (old && old !== this.presenceWatchIter)
531
+ this.retireKvWatch(old);
532
+ // A bucket with no keys replays nothing, so the new watch cannot refresh
533
+ // `lastPresenceWatchAt` by delivering. It IS current knowledge: nobody is present. The
534
+ // bind's consumer info carries the initial pending count for that one fact; nats.js's KV
535
+ // watch placed the isUpdate marker from the same `info(true)`.
536
+ if (info.num_pending === 0)
537
+ await this.onPresenceBucketEmpty();
538
+ this.emit("warning", new Error(`presence watch silent for ${silentMs}ms with the connection up; rebound it from the bucket's current state`));
539
+ }
540
+ catch (e) {
541
+ // A bind the epoch swap itself rejected (connection drained under it) is not a fault of
542
+ // the epoch that replaced it; only a refusal on a still-current epoch is reported.
543
+ if (epoch === this.presenceEpoch)
544
+ this.emit("error", e);
545
+ }
546
+ finally {
547
+ // An epoch swap already disowned this flight (and may own a successor's by now); only
548
+ // a flight still in its own epoch clears the slot. Within one epoch there is one flight.
549
+ if (epoch === this.presenceEpoch)
550
+ this.presenceRebind = undefined;
551
+ }
552
+ })();
553
+ }
554
+ /** The watch was just bound onto a bucket with no keys.
555
+ *
556
+ * A NON-REGISTERING observer (a `cotal status` probe, a lease checker) has real knowledge:
557
+ * nobody is present. Every peer still in its roster is known gone (its key is not there to
558
+ * replay), so it is marked offline now rather than aged out against a delivery that cannot
559
+ * come; the silence gate is disarmed and the view reads current until the first write lands.
560
+ * Without that the empty-bucket view relapsed to stale one window later and rebound again on
561
+ * every window, one consumer create and one warning per TTL for as long as the mesh was empty.
562
+ *
563
+ * A REGISTERING observer (the manager) is itself one of the keys that should be there. An
564
+ * empty bucket under it means the bucket was wiped since its last heartbeat (the stream
565
+ * recreation), and the same wipe took every peer's record: their absence says the bucket is
566
+ * new, not that they left. a reviewer reproduced the previous behaviour at default timing:
567
+ * the rebind landed ~0.9s after the recreation, the observer marked every peer AND ITSELF
568
+ * offline, and held the view current for up to one heartbeat, a false verdict `cotal ps`
569
+ * would print as `mesh offline`. So a registering observer re-publishes its own record NOW,
570
+ * which the new watch delivers, and lets the ordinary per-peer age-out run from that delivery:
571
+ * a peer that is still heartbeating rewrites its key within its own heartbeat interval and is
572
+ * re-observed live; one that is gone ages out exactly as after a plain rebind. The roster is
573
+ * not touched here and the view is not held; the delivery is what makes it current.
574
+ *
575
+ * What neither branch covers: a consumer that dies again while the bucket is still empty is
576
+ * not detectable by silence, so the first write after that is missed until the observer
577
+ * restarts. */
578
+ async onPresenceBucketEmpty() {
579
+ if (this.doRegister) {
580
+ // Our own key is missing from a bucket we write to: put it back. publishPresence throws on
581
+ // a refused write; that surfaces through the rebind flight's error path with the epoch
582
+ // fence intact, and the view stays stale, which is the honest state for an observer that
583
+ // cannot even record itself.
584
+ await this.publishPresence();
585
+ return;
586
+ }
587
+ this.markPresenceBucketEmpty();
588
+ }
589
+ /** See {@link onPresenceBucketEmpty}: the non-registering branch. */
590
+ markPresenceBucketEmpty() {
591
+ this.presenceWatchEmpty = true;
592
+ this.lastPresenceWatchAt = 0;
593
+ let changed = false;
594
+ for (const [id, p] of this.roster) {
595
+ if (p.status === "offline")
596
+ continue;
597
+ const offline = this.toOffline(p);
598
+ this.roster.set(id, offline);
599
+ this.emit("presence", { type: "offline", presence: offline });
600
+ changed = true;
601
+ }
602
+ if (changed)
603
+ this.emit("roster", this.getRoster());
604
+ this.emitPresenceViewIfChanged();
605
+ }
606
+ handleKvEntry(e) {
607
+ // Provisioning uses this one reserved key to prove the bucket's max_age is enforced (#404).
608
+ // It is maintenance traffic, never a peer identity, an offline transition, or view freshness.
609
+ if (e.key === TTL_RECONCILE_CANARY_KEY)
610
+ return;
611
+ this.lastPresenceWatchAt = Date.now();
612
+ this.presenceWatchEmpty = false;
613
+ if (e.operation === "DEL" || e.operation === "PURGE") {
614
+ this.markOffline(e.key);
615
+ return;
616
+ }
617
+ const p = this.decodePresence(e);
618
+ if (p)
619
+ this.applyPresence(e.key, p);
620
+ }
621
+ /** The record a presence PUT carries, or undefined when a reader must drop it (SPEC §6). The watch
622
+ * and {@link readPresence} decode through this one gate, so they drop the same records. */
623
+ decodePresence(e) {
624
+ let p;
625
+ try {
626
+ p = e.json();
627
+ }
628
+ catch {
629
+ return undefined;
630
+ }
631
+ if (!isPresenceRow(p, e.key)) {
632
+ this.presenceBindingDrops++;
633
+ this.emitRecoverable(new Error(`dropped presence entry for key ${JSON.stringify(e.key)}: not a usable presence row (a record with card.id equal to its KV key, a non-blank string card.name, a known status, a finite ts, string display fields and a condition with a string code)`));
634
+ return undefined;
635
+ }
636
+ // isPresenceRow is the one gate: it refuses a row whose card.id disagrees with the KV key, so a
637
+ // forged or mis-keyed record never surfaces a spoofed roster identity. The write-side scoping
638
+ // ($KV.<presenceBucket>.<own-id>) is the primary guard.
639
+ return p;
640
+ }
641
+ applyPresence(id, raw) {
642
+ const prev = this.roster.get(id);
643
+ const stale = Date.now() - raw.ts > this.ttlMs;
644
+ // A watch recovering from a stall replays the bucket. Those PUTs still carry the publisher's
645
+ // pre-stall timestamps, so they look TTL-expired even though the peers kept heartbeating on
646
+ // the broker. Materializing them as offline is the empty-to-full flicker #1045 named: the
647
+ // observer's catch-up, not N deaths. Keep last-known rows, stay view-stale, and wait for a
648
+ // live timestamp (or an explicit offline/delete) before changing a known peer.
649
+ if (stale && raw.status !== "offline") {
650
+ if (prev)
651
+ return;
652
+ this.roster.set(id, this.toOffline(raw));
653
+ this.emit("roster", this.getRoster());
654
+ return;
655
+ }
656
+ if (this.presenceSnapshotPopulated)
657
+ this.emitPresenceViewIfChanged();
658
+ // Any offline materialization (a stale snapshot OR a graceful-leave record) drops the advisory
659
+ // attention fields - an offline peer must not carry a stale `[focus]`/`locally muted` hint.
660
+ const p = stale || raw.status === "offline" ? this.toOffline(raw) : raw;
661
+ // First time we hear about an already-offline peer (stale snapshot): record quietly.
662
+ if (!prev && p.status === "offline") {
663
+ this.roster.set(id, p);
664
+ this.emit("roster", this.getRoster());
665
+ return;
666
+ }
667
+ // Heartbeat refresh with no real change: bump liveness quietly and don't
668
+ // emit - otherwise the periodic keep-alive looks like a stream of "updates".
669
+ // A CHANGED lifecycleUid is never a heartbeat: it is a NEW LIFECYCLE reusing the same principal (a
670
+ // terminal same-name recreation - in user mode a retired lifecycle and its same-name successor
671
+ // share `<owner>.<actor>`; a supervised respawn keeps the uid and is unaffected), so it MUST notify
672
+ // watchers even when its status/activity match the prior lifecycle's lingering record. Omitting this
673
+ // left a same-principal successor invisible to presence-event consumers (e.g. the manager's
674
+ // readiness race), which then timed out "uncertain" on a healthy agent (#29).
675
+ if (prev &&
676
+ prev.status !== "offline" &&
677
+ p.status !== "offline" &&
678
+ prev.lifecycleUid === p.lifecycleUid &&
679
+ sameRuntime(prev.runtime, p.runtime) &&
680
+ prev.status === p.status &&
681
+ prev.activity === p.activity &&
682
+ sameCondition(prev.condition, p.condition) &&
683
+ prev.environment === p.environment &&
684
+ prev.attention === p.attention &&
685
+ sameChannelModes(prev.channelModes, p.channelModes) &&
686
+ prev.prompts?.open === p.prompts?.open &&
687
+ prev.prompts?.seq === p.prompts?.seq) {
688
+ this.roster.set(id, p);
689
+ return;
690
+ }
691
+ this.roster.set(id, p);
692
+ const type = p.status === "offline"
693
+ ? "offline"
694
+ : !prev || prev.status === "offline"
695
+ ? "join"
696
+ : "update";
697
+ this.emit("presence", { type, presence: p });
698
+ this.emit("roster", this.getRoster());
699
+ }
700
+ /** Materialize an OFFLINE presence record: drop the advisory attention fields. An offline peer must
701
+ * not show a stale `[focus]` or "locally muted #x" hint - SPEC: attention removed on offline sweep,
702
+ * channel modes reset on restart. `statusSince` goes too: it dated the live status, and an observer
703
+ * that derives offline from a stale heartbeat does not know when the peer left. `prompts` goes as
704
+ * well: an offline process serves nothing. card/activity/ts are kept. */
705
+ toOffline(p) {
706
+ return { ...p, status: "offline", statusSince: undefined, attention: undefined, channelModes: undefined, prompts: undefined };
707
+ }
708
+ /** Mark a known peer offline (on KV delete/purge), keeping it in the roster. */
709
+ markOffline(id) {
710
+ const prev = this.roster.get(id);
711
+ if (!prev || prev.status === "offline")
712
+ return;
713
+ const offline = this.toOffline(prev);
714
+ this.roster.set(id, offline);
715
+ this.emit("presence", { type: "offline", presence: offline });
716
+ this.emit("roster", this.getRoster());
717
+ }
718
+ emitPresenceViewIfChanged() {
719
+ const view = this.presenceView();
720
+ if (view.state === this.presenceViewState)
721
+ return;
722
+ this.presenceViewState = view.state;
723
+ this.emit("presence-view", view);
724
+ }
725
+ sweep() {
726
+ const now = Date.now();
727
+ // Whole-bucket silence past TTL is THIS observer going deaf. Real rosters do not lose every
728
+ // peer in one window, and treating that silence as N offline verdicts empties an online-only
729
+ // sidebar with nothing saying the window went blind (#1045). Gate the per-peer age-out on
730
+ // watch freshness; surface the view as stale instead.
731
+ if (this.lastPresenceWatchAt !== 0 && now - this.lastPresenceWatchAt > this.ttlMs) {
732
+ this.emitPresenceViewIfChanged();
733
+ // Staying stale is the right verdict for a held link (#1045), and the wrong END STATE when
734
+ // the transport is up and the watch's own consumer is what died. Measured on a live deployment
735
+ // 2026-09-09: the presence stream was deleted and recreated, its sequence restarted, and
736
+ // every observer's ORDERED consumer re-created itself at the OLD start sequence (nats.js
737
+ // 3.4.0 resets from its cursor). The broker kept sending idle heartbeats, so the client
738
+ // never reset again, the iterator never closed, and the manager's roster stayed frozen at
739
+ // the pre-recreation snapshot for hours: `cotal ps` read every older seat "mesh offline"
740
+ // and every newer seat "not in roster" while all of them were heartbeating. The same
741
+ // end state follows a plain consumer delete (an operator, or the 5-minute inactive
742
+ // threshold after a long stall). Rebind the watch from the bucket's CURRENT state; a held
743
+ // link's rebind fails or stays silent and the view simply stays stale, as before.
744
+ this.rebindStalePresenceWatch(now);
745
+ return;
746
+ }
747
+ let changed = false;
748
+ for (const [id, p] of this.roster) {
749
+ // A peer's own `ts` always trails the observer's last delivery, so "older than TTL by the
750
+ // wall clock" alone ages peers out on the tick just before the whole-bucket gate above
751
+ // trips (#1311's flap; measured as 3 offline verdicts per silence in the rebind suite).
752
+ // Require that the watch itself delivered for a full TTL after this peer's last heartbeat:
753
+ // then other peers were heard and this one was not, which is the only silence that is the
754
+ // peer's rather than the observer's.
755
+ if (p.status !== "offline" && now - p.ts > this.ttlMs && this.lastPresenceWatchAt - p.ts > this.ttlMs) {
756
+ const offline = this.toOffline(p);
757
+ this.roster.set(id, offline);
758
+ this.emit("presence", { type: "offline", presence: offline });
759
+ changed = true;
760
+ }
761
+ }
762
+ if (changed)
763
+ this.emit("roster", this.getRoster());
764
+ }
765
+ }
766
+ /** Shallow-equal two per-channel-mode maps (presence dedup): a change must re-emit, so an attention
767
+ * toggle isn't swallowed as a quiet heartbeat. Absent and empty compare equal. */
768
+ function sameChannelModes(a, b) {
769
+ const ak = a ? Object.keys(a) : [];
770
+ const bk = b ? Object.keys(b) : [];
771
+ if (ak.length !== bk.length)
772
+ return false;
773
+ return ak.every((k) => a[k] === b?.[k]);
774
+ }
775
+ function sameRuntime(a, b) {
776
+ return a?.id === b?.id && a?.startedAt === b?.startedAt && a?.source === b?.source;
777
+ }
778
+ function sameCondition(a, b) {
779
+ return a?.code === b?.code
780
+ && a?.source === b?.source
781
+ && a?.message === b?.message
782
+ && a?.since === b?.since;
783
+ }
784
+ //# sourceMappingURL=endpoint-presence.js.map