@cotal-ai/core 0.36.0 → 0.38.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/dist/agent-file.d.ts +18 -0
  2. package/dist/agent-file.d.ts.map +1 -1
  3. package/dist/agent-file.js +37 -1
  4. package/dist/agent-file.js.map +1 -1
  5. package/dist/connector-setup.d.ts +30 -0
  6. package/dist/connector-setup.d.ts.map +1 -0
  7. package/dist/connector-setup.js +2 -0
  8. package/dist/connector-setup.js.map +1 -0
  9. package/dist/connector.d.ts +31 -3
  10. package/dist/connector.d.ts.map +1 -1
  11. package/dist/connector.js +12 -1
  12. package/dist/connector.js.map +1 -1
  13. package/dist/endpoint-action.d.ts +6 -1
  14. package/dist/endpoint-action.d.ts.map +1 -1
  15. package/dist/endpoint-action.js +6 -2
  16. package/dist/endpoint-action.js.map +1 -1
  17. package/dist/endpoint-envelope.d.ts +1 -1
  18. package/dist/endpoint-envelope.d.ts.map +1 -1
  19. package/dist/endpoint-envelope.js +1 -1
  20. package/dist/endpoint-envelope.js.map +1 -1
  21. package/dist/endpoint-error.d.ts +26 -0
  22. package/dist/endpoint-error.d.ts.map +1 -1
  23. package/dist/endpoint-error.js +41 -0
  24. package/dist/endpoint-error.js.map +1 -1
  25. package/dist/endpoint-grants.d.ts +8 -7
  26. package/dist/endpoint-grants.d.ts.map +1 -1
  27. package/dist/endpoint-grants.js +9 -8
  28. package/dist/endpoint-grants.js.map +1 -1
  29. package/dist/endpoint-invoke.d.ts.map +1 -1
  30. package/dist/endpoint-invoke.js +58 -7
  31. package/dist/endpoint-invoke.js.map +1 -1
  32. package/dist/endpoint-serve-kv.d.ts +17 -0
  33. package/dist/endpoint-serve-kv.d.ts.map +1 -1
  34. package/dist/endpoint-serve-kv.js +59 -1
  35. package/dist/endpoint-serve-kv.js.map +1 -1
  36. package/dist/endpoint-service.d.ts +6 -0
  37. package/dist/endpoint-service.d.ts.map +1 -1
  38. package/dist/endpoint-service.js +12 -3
  39. package/dist/endpoint-service.js.map +1 -1
  40. package/dist/endpoint.d.ts +78 -14
  41. package/dist/endpoint.d.ts.map +1 -1
  42. package/dist/endpoint.js +439 -118
  43. package/dist/endpoint.js.map +1 -1
  44. package/dist/index.d.ts +1 -0
  45. package/dist/index.d.ts.map +1 -1
  46. package/dist/index.js +1 -0
  47. package/dist/index.js.map +1 -1
  48. package/dist/lifecycle-saga.d.ts.map +1 -1
  49. package/dist/lifecycle-saga.js +24 -5
  50. package/dist/lifecycle-saga.js.map +1 -1
  51. package/dist/lifecycle-state.d.ts +15 -0
  52. package/dist/lifecycle-state.d.ts.map +1 -1
  53. package/dist/lifecycle-state.js +60 -0
  54. package/dist/lifecycle-state.js.map +1 -1
  55. package/dist/provision.d.ts.map +1 -1
  56. package/dist/provision.js +19 -7
  57. package/dist/provision.js.map +1 -1
  58. package/dist/types.d.ts +7 -0
  59. package/dist/types.d.ts.map +1 -1
  60. package/package.json +1 -1
package/dist/endpoint.js CHANGED
@@ -13,7 +13,7 @@ import { liveKvEntries } from "./kv-scan.js";
13
13
  import { ARTIFACT_PART_KIND, isArtifactPart } from "./artifact.js";
14
14
  import { assertValidName } from "./resolve.js";
15
15
  import { createSpaceStreams, dmDurableConfig, dlvDurableConfig, taskDurableConfig, fanoutDurableConfig, inboxReaderConfig, MAX_MSGS_PER_SUBJECT, MANAGER_LEASE_TTL_MS, MANAGER_LEASE_ATTEMPT_MS } from "./streams.js";
16
- import { jetstream, jetstreamManager, AckPolicy, DeliverPolicy, } from "@nats-io/jetstream";
16
+ import { jetstream, jetstreamManager, AckPolicy, DeliverPolicy, JetStreamApiCodes, JetStreamApiError, } from "@nats-io/jetstream";
17
17
  import {} from "@nats-io/jetstream";
18
18
  import { Kvm } from "@nats-io/kv";
19
19
  import { Bucket, KvWatchInclude } from "@nats-io/kv/internal";
@@ -31,7 +31,9 @@ export const DEFAULT_SPACE = "main";
31
31
  * "connection" ({ connected: boolean }) — true on every successful (re)bind (initial start, manual
32
32
  * reconnect, AND background self-heal), false the moment the connection drops (rebuild null window /
33
33
  * terminal close). Lets an in-process agent track connectedness off the endpoint's own (re)binds
34
- * instead of an imperative flag the self-heal path can't reach.
34
+ * instead of an imperative flag the self-heal path can't reach; "transport" ({ connected, server? })
35
+ * is the lower-level NATS socket edge, true before the full bind finishes and false during an internal
36
+ * nats.js reconnect without changing `connection` readiness.
35
37
  *
36
38
  * Callers MUST attach an "error" listener before `start()`: async faults (incl. NATS
37
39
  * permission denials, surfaced via `watchStatus`) are emitted as "error", and Node throws
@@ -105,6 +107,12 @@ export class CotalEndpoint extends EventEmitter {
105
107
  jsm;
106
108
  kv;
107
109
  channelKv;
110
+ /** The presence/channel-registry watches' own handles. Each is an ORDERED push consumer: its idle
111
+ * heartbeat monitor lives on a JS timer independent of the connection, so a drain that doesn't
112
+ * `.stop()` it first leaves the monitor to fire into a closing connection every 30s, throwing
113
+ * `DrainingConnectionError` out of `reset()` on a timer nothing awaits. */
114
+ presenceWatchIter;
115
+ channelWatchIter;
108
116
  /** Plane-3 durable-membership registry KV — lazily opened by the privileged delivery daemon (or a
109
117
  * short-lived provisioner). */
110
118
  membersKv;
@@ -177,6 +185,15 @@ export class CotalEndpoint extends EventEmitter {
177
185
  roster = new Map();
178
186
  /** Resolves when the current presence watch has consumed its complete initial KV snapshot. */
179
187
  presenceSnapshot = Promise.resolve();
188
+ /**
189
+ * Observer-local age of the last presence-KV delivery (any key, including DEL/PURGE). Distinct
190
+ * from each peer's `ts`: that is the publisher's heartbeat. Whole-bucket silence past TTL is
191
+ * the observer going deaf, not N simultaneous deaths, and sweep must not treat it as the
192
+ * latter (#1045).
193
+ */
194
+ lastPresenceWatchAt = 0;
195
+ /** Last emitted presence-view freshness. Suppresses duplicate `presence-view` events. */
196
+ presenceViewFresh = true;
180
197
  status = "idle";
181
198
  activity;
182
199
  /** Mirror of the connector's authoritative attention state, published in presence (advisory). The
@@ -371,10 +388,15 @@ export class CotalEndpoint extends EventEmitter {
371
388
  }
372
389
  async start() {
373
390
  await this.connectAndBind();
374
- // nats.js auto-reconnects transient drops; when it exhausts its attempts and the
375
- // connection closes for good, rebuild from scratch so an in-process agent (e.g. the
376
- // OpenCode plugin) recovers without a host respawn. Armed only after a successful first
377
- // connect a first-connect failure throws to the caller's connect-retry loop instead.
391
+ // stop() can finish while the INITIAL connectAndBind is still awaiting its broker work. The
392
+ // rebuild path already closes that race; initial start needs the same fence or the late bind
393
+ // leaves a fresh nc, heartbeat, consumers, and presence live on an endpoint already stopped.
394
+ // superviseConnection below: nats.js auto-reconnects transient drops, and when it exhausts its
395
+ // attempts and the connection closes for good we rebuild from scratch, so an in-process agent
396
+ // (e.g. the OpenCode plugin) recovers without a host respawn. Armed only after a successful
397
+ // first connect; a first-connect failure throws to the caller's connect-retry loop instead.
398
+ if (await this.tearDownIfStopped())
399
+ return;
378
400
  this.superviseConnection();
379
401
  }
380
402
  /** How far ahead of the current bearer's `exp` a refresh fires, and how soon a FAILED refresh
@@ -382,9 +404,17 @@ export class CotalEndpoint extends EventEmitter {
382
404
  * so whatever `currentBearer` holds is what every attempt presents). */
383
405
  static BEARER_REFRESH_MARGIN_MS = 60_000;
384
406
  static BEARER_RETRY_MS = 15_000;
407
+ /**
408
+ * A condition the endpoint is already surviving. Node rethrows `error` when no listener is
409
+ * attached, so emitting retry notices on `error` killed hosts that the endpoint intended to
410
+ * keep running (#891). `warning` is observable and never fatal without a listener.
411
+ */
412
+ emitRecoverable(err) {
413
+ this.emit("warning", err);
414
+ }
385
415
  /** Fetch a fresh bearer from the source, pin its principal to ours, arm the next refresh. On a
386
416
  * fetch/principal failure: THROWS when `initial` (start() must fail loud before first connect);
387
- * otherwise emits "error" and retries — the live connection keeps working until its current JWT
417
+ * otherwise emits "warning" and retries — the live connection keeps working until its current JWT
388
418
  * expiry, so a dead auth service is loud without instantly dropping the mesh. */
389
419
  async refreshBearer(initial = false) {
390
420
  try {
@@ -398,7 +428,7 @@ export class CotalEndpoint extends EventEmitter {
398
428
  catch (e) {
399
429
  if (initial)
400
430
  throw e;
401
- this.emit("error", new Error(`bearer refresh failed (${e instanceof Error ? e.message : String(e)}) - retrying; this connection dies at its current token's expiry if the auth service stays down`));
431
+ this.emitRecoverable(new Error(`bearer refresh failed (${e instanceof Error ? e.message : String(e)}) - retrying; this connection dies at its current token's expiry if the auth service stays down`));
402
432
  this.armBearerRefresh(CotalEndpoint.BEARER_RETRY_MS);
403
433
  }
404
434
  }
@@ -466,7 +496,7 @@ export class CotalEndpoint extends EventEmitter {
466
496
  catch (e) {
467
497
  if (initial)
468
498
  throw e;
469
- this.emit("error", new Error(`creds refresh failed (${e instanceof Error ? e.message : String(e)}) - retrying; this connection dies at its current JWT's expiry if renewal keeps failing`));
499
+ this.emitRecoverable(new Error(`creds refresh failed (${e instanceof Error ? e.message : String(e)}) - retrying; this connection dies at its current JWT's expiry if renewal keeps failing`));
470
500
  this.armCredsRefresh(CotalEndpoint.CREDS_RETRY_MS);
471
501
  }
472
502
  }
@@ -546,7 +576,7 @@ export class CotalEndpoint extends EventEmitter {
546
576
  await this.swapConnectionOntoFreshCreds();
547
577
  }
548
578
  catch (e) {
549
- this.emit("error", new Error(`creds refresh failed (${e instanceof Error ? e.message : String(e)}) - retrying; this connection dies at its current JWT's expiry if renewal keeps failing`));
579
+ this.emitRecoverable(new Error(`creds refresh failed (${e instanceof Error ? e.message : String(e)}) - retrying; this connection dies at its current JWT's expiry if renewal keeps failing`));
550
580
  this.armCredsRefresh(CotalEndpoint.CREDS_RETRY_MS);
551
581
  }
552
582
  }
@@ -633,6 +663,11 @@ export class CotalEndpoint extends EventEmitter {
633
663
  throw new Error(this.bearerSource
634
664
  ? "this endpoint's user bearer has expired and renewal through the auth exchange is failing - not presenting the expired token to the broker; retrying with backoff"
635
665
  : "this endpoint's user bearer has expired and it holds no bearer source to renew it - re-authenticate and rebuild the endpoint (construct it with a bearer FUNCTION for standing renewal)");
666
+ const credsExp = this.currentCreds && credsClaims(this.currentCreds).exp;
667
+ if (typeof credsExp === "number" && credsExp * 1000 <= Date.now())
668
+ throw new Error(this.credsSource
669
+ ? "this endpoint's creds have expired and renewal is failing - not presenting the expired credential to the broker; retrying with backoff"
670
+ : "this endpoint's creds have expired and it holds no creds source to renew them - replace the credential and rebuild the endpoint (pass a creds FUNCTION for standing renewal)");
636
671
  this.nc = await dialerFor(this.servers)({
637
672
  servers: this.servers,
638
673
  // In USER MODE the connection `name` carries the client-chosen inbox nonce (= connId) the callout
@@ -719,7 +754,7 @@ export class CotalEndpoint extends EventEmitter {
719
754
  if (this.doRegister) {
720
755
  await this.publishPresence();
721
756
  this.heartbeatTimer = setInterval(() => {
722
- this.publishPresence().catch((e) => this.emit("error", e));
757
+ this.publishPresence().catch((e) => this.emitRecoverable(e));
723
758
  }, this.heartbeatMs);
724
759
  }
725
760
  // Caller-owned membership watches are INTENT rather than one-connection iterators. Re-open them
@@ -731,6 +766,20 @@ export class CotalEndpoint extends EventEmitter {
731
766
  await this.armPlane3();
732
767
  // Bound and live — covers initial start, manual reconnect, AND background self-heal (every
733
768
  // path lands here). The single signal an in-process agent's connected flag tracks.
769
+ //
770
+ // The stopped guard: stop() can land in any await above. Both callers tear the fresh
771
+ // connection back down (tearDownIfStopped), but an event has no undo, so a late
772
+ // `connection: true` would be the last edge a listener ever sees on a stopped endpoint and
773
+ // nothing follows it to correct the record. It belongs here rather than in a consumer because
774
+ // every listener reads the same edge; MeshAgent carries its own `stopping` guard and so was
775
+ // never the one exposed, which is the point.
776
+ //
777
+ // Measured for the start() caller by the broker suite's mid-bind cell. doRebuild is
778
+ // also measured, by a separate cell that holds the rebuild's connectAndBind at
779
+ // armPlane3 after a successful first bind and lands stop() in that window (#1028).
780
+ // Shared-line reasoning is no longer the rebuild proof.
781
+ if (this.stopped)
782
+ return;
734
783
  this.emit("connection", { connected: true });
735
784
  }
736
785
  /** Tear down everything {@link connectAndBind} (re)creates, so a rebind can't leak a
@@ -754,6 +803,20 @@ export class CotalEndpoint extends EventEmitter {
754
803
  }
755
804
  }
756
805
  this.streamMsgs.length = 0;
806
+ try {
807
+ this.presenceWatchIter?.stop();
808
+ }
809
+ catch {
810
+ /* already closed with the connection */
811
+ }
812
+ this.presenceWatchIter = undefined;
813
+ try {
814
+ this.channelWatchIter?.stop();
815
+ }
816
+ catch {
817
+ /* already closed with the connection */
818
+ }
819
+ this.channelWatchIter = undefined;
757
820
  for (const sub of this.chatSubs.values()) {
758
821
  try {
759
822
  sub.unsubscribe();
@@ -766,11 +829,13 @@ export class CotalEndpoint extends EventEmitter {
766
829
  this.chatSubDenied.clear();
767
830
  this.confirmingChatSubs.clear();
768
831
  this.roster.clear();
832
+ this.lastPresenceWatchAt = 0;
833
+ this.presenceViewFresh = true;
769
834
  this.joinSeq.clear();
770
835
  this.channelConfigs.clear();
771
836
  this.channelDefaults = {};
772
837
  for (const watch of this.membershipFeedWatches)
773
- watch.arm = watch.arm.catch(() => { }).then(() => this.disarmMembershipWatch(watch));
838
+ watch.arm = watch.arm.catch(() => { }).then(() => this.disarmMembershipWatch(watch)).catch((err) => { this.emit("error", err); });
774
839
  }
775
840
  /** If stop() ran during a rebuild's `await connectAndBind`, the just-bound connection +
776
841
  * heartbeat + supervisor would be left live on a stopped endpoint. Tear that fresh
@@ -806,7 +871,10 @@ export class CotalEndpoint extends EventEmitter {
806
871
  return;
807
872
  if (this.nc !== nc)
808
873
  return; // epoch-stale — a rebuild already swapped this connection
809
- this.emit("connection", { connected: false }); // dropped report it before the rebuild kicks in
874
+ // ORDER IS PART OF THE DIAGNOSTIC CONTRACT. MeshAgent retains endpoint errors only while it is
875
+ // not bound, so readiness must turn false before the matching terminal-close error is emitted.
876
+ // Reversing these two lines silently loses the only post-drop reason an agent can report.
877
+ this.emit("connection", { connected: false });
810
878
  this.emit("error", new Error(`mesh connection closed${err ? `: ${err.message}` : ""} - re-establishing`));
811
879
  void this.reestablishLoop();
812
880
  });
@@ -853,7 +921,11 @@ export class CotalEndpoint extends EventEmitter {
853
921
  // The manager's liveness-lease handle too: left bound to the old connection, every renew and
854
922
  // re-read after a reconnect times out, and the manager reports its lease unknown for good.
855
923
  this.managerLeaseKv = undefined;
856
- this.emit("connection", { connected: false }); // null window opened not live until the rebind below
924
+ // This is an application-requested epoch teardown, not a transient nats.js blip. The old
925
+ // status iterator is now stale by construction and its close is epoch-dropped, so this line is
926
+ // the authoritative raw-liveness edge for the no-nc window until the new watcher seeds true.
927
+ this.emit("transport", { connected: false });
928
+ this.emit("connection", { connected: false });
857
929
  try {
858
930
  await oldNc?.drain();
859
931
  }
@@ -864,9 +936,10 @@ export class CotalEndpoint extends EventEmitter {
864
936
  // stop() may have run during the await — don't leave a live connection + heartbeat +
865
937
  // supervisor on a stopped endpoint. (Reads this.nc in its own scope — a bare `this.nc`
866
938
  // here in doRebuild narrows to `never` via TS inlining connectAndBind's assignment.)
939
+ // Re-arm on the fresh nc only after this stopped fence accepts it.
867
940
  if (await this.tearDownIfStopped())
868
941
  return;
869
- this.superviseConnection(); // re-arm on the fresh nc
942
+ this.superviseConnection();
870
943
  }
871
944
  finally {
872
945
  this.reconnecting = false;
@@ -902,7 +975,7 @@ export class CotalEndpoint extends EventEmitter {
902
975
  }
903
976
  catch (e) {
904
977
  if (!this.stopped)
905
- this.emit("error", e);
978
+ this.emitRecoverable(e);
906
979
  const delay = this.nextRetryDelayMs();
907
980
  await new Promise((resolve) => {
908
981
  this.backoffResolve = resolve;
@@ -984,6 +1057,20 @@ export class CotalEndpoint extends EventEmitter {
984
1057
  /* already closed */
985
1058
  }
986
1059
  }
1060
+ try {
1061
+ this.presenceWatchIter?.stop();
1062
+ }
1063
+ catch {
1064
+ /* already closed */
1065
+ }
1066
+ this.presenceWatchIter = undefined;
1067
+ try {
1068
+ this.channelWatchIter?.stop();
1069
+ }
1070
+ catch {
1071
+ /* already closed */
1072
+ }
1073
+ this.channelWatchIter = undefined;
987
1074
  try {
988
1075
  if (this.doRegister) {
989
1076
  this.status = "offline";
@@ -1520,6 +1607,22 @@ export class CotalEndpoint extends EventEmitter {
1520
1607
  getRoster() {
1521
1608
  return [...this.roster.values()].sort((a, b) => a.card.name.localeCompare(b.card.name));
1522
1609
  }
1610
+ /**
1611
+ * Freshness of THIS observer's presence watch, not of any peer. `fresh: false` means the
1612
+ * whole bucket has been silent past the liveness window — the view is stale as of
1613
+ * `staleSince`, and {@link getRoster} is last-known rather than a current offline verdict.
1614
+ * A watch that has not yet delivered anything is not stale (there is no T to name).
1615
+ */
1616
+ presenceView() {
1617
+ if (!this.doWatch)
1618
+ return { fresh: true };
1619
+ if (this.lastPresenceWatchAt === 0)
1620
+ return { fresh: true };
1621
+ const staleSince = this.lastPresenceWatchAt + this.ttlMs;
1622
+ if (Date.now() < staleSince)
1623
+ return { fresh: true };
1624
+ return { fresh: false, staleSince };
1625
+ }
1523
1626
  /** Wait until the current presence watch has consumed its initial KV snapshot. An empty bucket
1524
1627
  * emits no watch entry, so the timeout keeps a genuinely empty mesh bounded. */
1525
1628
  async waitForPresenceSnapshot(timeoutMs = 1_000) {
@@ -1723,8 +1826,12 @@ export class CotalEndpoint extends EventEmitter {
1723
1826
  }
1724
1827
  }
1725
1828
  }
1726
- catch {
1727
- /* stream missing fall through to registry-only channels */
1829
+ catch (e) {
1830
+ // A genuinely absent CHAT stream means there are no retained message counts yet. Every other
1831
+ // failure, especially a STREAM.INFO permission denial, means the count view could not be read
1832
+ // and must stay loud rather than returning a valid-looking registry-only subset.
1833
+ if (!isJetStreamMissing(e, JetStreamApiCodes.StreamNotFound))
1834
+ throw e;
1728
1835
  }
1729
1836
  const channels = new Set([...counts.keys(), ...this.channelConfigs.keys()]);
1730
1837
  return [...channels]
@@ -1950,11 +2057,18 @@ export class CotalEndpoint extends EventEmitter {
1950
2057
  watch.consumerName = undefined;
1951
2058
  }
1952
2059
  else {
1953
- // A timeout is deferred only for an epoch that is actually closing/rebuilding; live timeouts stay loud.
1954
2060
  const closedEpoch = err.name === "ClosedConnectionError" || /^closed connection$/i.test(err.message);
1955
- const dyingEpochTimeout = /timeout/i.test(err.message) && (this.reconnecting || !this.nc || this.nc.isClosed());
1956
- if (!closedEpoch && !dyingEpochTimeout)
2061
+ const timeout = err.name === "TimeoutError" || /timeout/i.test(err.message);
2062
+ const dyingEpochTimeout = timeout && (this.reconnecting || !this.nc || this.nc.isClosed());
2063
+ // Cleanup of an ordered consumer: a delete timeout means the broker did not answer in time,
2064
+ // not that the endpoint is unusable. The broker reaps an idle/ephemeral consumer anyway.
2065
+ // Throwing here killed a live observer over a slow VPN (#1047). Catch, surface, continue.
2066
+ if (timeout || closedEpoch || dyingEpochTimeout) {
2067
+ this.emit("error", err);
2068
+ }
2069
+ else {
1957
2070
  throw err;
2071
+ }
1958
2072
  }
1959
2073
  // A terminal close leaves stream/name intact. The endpoint-owned stopped intent is retried
1960
2074
  // through the fresh JetStream manager before its public stop promise may resolve.
@@ -1981,10 +2095,11 @@ export class CotalEndpoint extends EventEmitter {
1981
2095
  }
1982
2096
  }));
1983
2097
  }
1984
- /** Fetch recent messages from a channel's JetStream backlog. */
2098
+ /** Fetch recent messages from a channel's JetStream backlog. `signal` cancels the active pull and
2099
+ * reclaims its ephemeral consumer before the promise rejects. */
1985
2100
  async channelHistory(channel, opts) {
1986
2101
  // history from any sender
1987
- return this.streamHistory(chatStream(this.space), chatSubject(this.space, "*", "*", channel), opts?.limit ?? 100);
2102
+ return this.streamHistory(chatStream(this.space), chatSubject(this.space, "*", "*", channel), opts?.limit ?? 100, undefined, opts?.signal);
1988
2103
  }
1989
2104
  /** Read a channel's recent history THROUGH THE DELIVERY DAEMON instead of through a consumer this
1990
2105
  * connection creates itself — the mediated read of SPEC's "Mediated reads (normative)" rule (no raw
@@ -2025,12 +2140,13 @@ export class CotalEndpoint extends EventEmitter {
2025
2140
  throw new Error("readHistory: the delivery daemon returned a malformed page (expected { items, complete })");
2026
2141
  return { items: data.items, complete: data.complete };
2027
2142
  }
2028
- /** Fetch recent DMs (any sender→any recipient) from the space's DM backlog. God-view only:
2143
+ /** Fetch recent DMs (any sender→any recipient) from the space's DM backlog. `signal` cancels the
2144
+ * active pull and reclaims its ephemeral consumer. God-view only:
2029
2145
  * a normal agent/observer's ACL denies CONSUMER.CREATE on DM_<space>, so this throws-and-
2030
2146
  * skips for them — only an `admin`-profile cred can read it. */
2031
2147
  async dmHistory(opts) {
2032
2148
  // every inst.<recipOwner>.<recipActor>.<sndOwner>.<sndActor> DM — the whole DM subtree (god-view)
2033
- return this.streamHistory(dmStream(this.space), `${spacePrefix(this.space)}.inst.>`, opts?.limit ?? 100);
2149
+ return this.streamHistory(dmStream(this.space), `${spacePrefix(this.space)}.inst.>`, opts?.limit ?? 100, undefined, opts?.signal);
2034
2150
  }
2035
2151
  /**
2036
2152
  * The `limit` MOST RECENT messages matching `subject`, oldest-first within the page.
@@ -2048,14 +2164,17 @@ export class CotalEndpoint extends EventEmitter {
2048
2164
  * A filtered subject's sequences are non-contiguous (other channels interleave in the same
2049
2165
  * stream), so the window cannot be computed arithmetically. A FAILED attempt holds fewer than a
2050
2166
  * page by definition, so wasted transfer stays page-sized and geometric growth keeps the number of
2051
- * attempts logarithmic. The one unbounded case is named in the body: a channel whose matches are
2052
- * all old and sparse walks back to the start of the stream and reads its whole retained set.
2167
+ * attempts logarithmic. A channel with fewer than `limit` matches used to keep widening until
2168
+ * sequence 1 and drain the stream's whole retained set (#840). The walk now stops at the
2169
+ * subject's FIRST matching sequence (the mirror of the last-matching ceiling), probed on the
2170
+ * same CREATE surface, and only after a short drain so a dense page never pays for the floor.
2053
2171
  *
2054
2172
  * `before` pages toward the past: pass the `seq` of the oldest message you already have.
2055
2173
  */
2056
- async streamHistory(stream, subject, limit, before) {
2174
+ async streamHistory(stream, subject, limit, before, signal) {
2057
2175
  if (!this.nc)
2058
2176
  throw new Error("endpoint not started");
2177
+ signal?.throwIfAborted();
2059
2178
  // A LIMIT THAT IS NOT A FINITE NUMBER HAS NO ANSWER, AND THE SEARCH BELOW CANNOT REFUSE IT.
2060
2179
  // Every comparison against NaN is false, so `limit <= 0` does not fire for one, and neither of
2061
2180
  // the widening loop's exits can ever be true either: `page.length >= NaN` is false forever and
@@ -2091,7 +2210,7 @@ export class CotalEndpoint extends EventEmitter {
2091
2210
  // Deliberately NOT `getMessage({ last_by_subj })`, which would be the obvious way to ask: it
2092
2211
  // needs `$JS.API.STREAM.MSG.GET`, which read credentials do not hold. That grant hole already
2093
2212
  // shipped once from this function and turned every non-admin history read into an empty list.
2094
- const ceiling = before !== undefined ? before - 1 : await this.lastMatchingSeq(js, stream, subject);
2213
+ const ceiling = before !== undefined ? before - 1 : await this.lastMatchingSeq(js, stream, subject, signal);
2095
2214
  if (ceiling < 1)
2096
2215
  return [];
2097
2216
  // Widen from the exact ceiling until a window holds a full page, or until the window IS the
@@ -2100,17 +2219,28 @@ export class CotalEndpoint extends EventEmitter {
2100
2219
  // Geometric growth keeps the number of attempts logarithmic, so total wasted transfer is a
2101
2220
  // small multiple of a page.
2102
2221
  //
2103
- // NAMED POLICY for the remaining case: when a channel's matches are all old and sparse, the
2104
- // search walks back to sequence 1 and the final drain transfers that subject's whole retained
2105
- // set. That is chosen deliberately — a FULL page of genuinely recent messages, at the cost of
2106
- // an unbounded read on a channel that has not been used in a long time — over returning a
2107
- // short page while older messages exist. The exact ceiling above means this is now reached
2108
- // only by real sparsity WITHIN a channel, never by the channel simply being quiet lately.
2222
+ // A SHORT PAGE is either "the channel is exhausted" or "the window is still above the
2223
+ // first match". Sequence 1 is the wrong floor for the first of those: three matches at
2224
+ // the high end of a busy stream are exhausted as soon as the window's lower edge passes
2225
+ // the subject's first matching sequence, and walking on to 1 re-reads everyone else's
2226
+ // retained set (#840). Probe that floor only after a short drain so a dense page (one
2227
+ // drain, full) never pays for it. The remaining unbounded-looking case is a subject
2228
+ // whose FIRST match really is near sequence 1; that span is the channel's own, not the
2229
+ // stream's.
2109
2230
  let span = Math.max(limit * 4, 64);
2231
+ let floor = 1;
2232
+ let floorKnown = false;
2110
2233
  for (;;) {
2111
- const start = Math.max(1, ceiling - span + 1);
2112
- const page = await this.drainWindow(js, stream, subject, start, ceiling);
2113
- if (page.length >= limit || start === 1)
2234
+ signal?.throwIfAborted();
2235
+ const start = Math.max(floor, ceiling - span + 1);
2236
+ const page = await this.drainWindow(js, stream, subject, start, ceiling, limit, signal);
2237
+ if (page.length >= limit)
2238
+ return page.slice(-limit);
2239
+ if (!floorKnown) {
2240
+ floor = Math.max(1, await this.firstMatchingSeq(js, stream, subject, signal));
2241
+ floorKnown = true;
2242
+ }
2243
+ if (start <= floor)
2114
2244
  return page.slice(-limit);
2115
2245
  span *= 4;
2116
2246
  }
@@ -2144,7 +2274,51 @@ export class CotalEndpoint extends EventEmitter {
2144
2274
  * (available from the create, before anything is delivered) is 0 for an empty subject, and
2145
2275
  * otherwise one message carries the sequence. Same CREATE/INFO/NEXT/DELETE surface `drainWindow`
2146
2276
  * already uses, so no broker authority is added. */
2147
- async lastMatchingSeq(js, stream, subject) {
2277
+ /** The oldest stream sequence matching `subject`, or 0 when the subject has no messages.
2278
+ * Mirror of {@link lastMatchingSeq}: same CREATE/INFO/NEXT/DELETE surface, `DeliverPolicy.All`
2279
+ * instead of `Last`, first delivered seq instead of last. Read credentials already hold this. */
2280
+ async firstMatchingSeq(js, stream, subject, signal) {
2281
+ signal?.throwIfAborted();
2282
+ const consumer = await js.consumers.get(stream, {
2283
+ filter_subjects: [subject],
2284
+ deliver_policy: DeliverPolicy.All,
2285
+ });
2286
+ try {
2287
+ if ((await consumer.info(true)).num_pending === 0)
2288
+ return 0;
2289
+ signal?.throwIfAborted();
2290
+ const iter = await consumer.fetch({ max_messages: 1 });
2291
+ const stop = () => iter.stop(abortReason(signal));
2292
+ signal?.addEventListener("abort", stop, { once: true });
2293
+ try {
2294
+ signal?.throwIfAborted();
2295
+ for await (const m of iter)
2296
+ return m.seq;
2297
+ }
2298
+ finally {
2299
+ signal?.removeEventListener("abort", stop);
2300
+ iter.stop();
2301
+ }
2302
+ throw new Error(`history: the broker reported messages on ${subject} but delivered none - the read was cut short, not empty`);
2303
+ }
2304
+ finally {
2305
+ try {
2306
+ await consumer.delete();
2307
+ }
2308
+ catch (e) {
2309
+ if (!isJetStreamMissing(e, JetStreamApiCodes.ConsumerNotFound))
2310
+ throw e;
2311
+ }
2312
+ }
2313
+ }
2314
+ /** The newest stream sequence matching `subject`, or 0 when the subject has no messages.
2315
+ *
2316
+ * One ordered consumer at `DeliverPolicy.Last` with this subject's filter: its `num_pending`
2317
+ * (available from the create, before anything is delivered) is 0 for an empty subject, and
2318
+ * otherwise one message carries the sequence. Same CREATE/INFO/NEXT/DELETE surface `drainWindow`
2319
+ * already uses, so no broker authority is added. */
2320
+ async lastMatchingSeq(js, stream, subject, signal) {
2321
+ signal?.throwIfAborted();
2148
2322
  const consumer = await js.consumers.get(stream, {
2149
2323
  filter_subjects: [subject],
2150
2324
  deliver_policy: DeliverPolicy.Last,
@@ -2153,9 +2327,19 @@ export class CotalEndpoint extends EventEmitter {
2153
2327
  // Bind-time zero is the ONLY thing that means "this subject has no messages".
2154
2328
  if ((await consumer.info(true)).num_pending === 0)
2155
2329
  return 0;
2330
+ signal?.throwIfAborted();
2156
2331
  const iter = await consumer.fetch({ max_messages: 1 });
2157
- for await (const m of iter)
2158
- return m.seq;
2332
+ const stop = () => iter.stop(abortReason(signal));
2333
+ signal?.addEventListener("abort", stop, { once: true });
2334
+ try {
2335
+ signal?.throwIfAborted();
2336
+ for await (const m of iter)
2337
+ return m.seq;
2338
+ }
2339
+ finally {
2340
+ signal?.removeEventListener("abort", stop);
2341
+ iter.stop();
2342
+ }
2159
2343
  // Bind said a message was pending and none arrived. The pinned client's pull iterator ends
2160
2344
  // CLEANLY when the connection closes ("we don't propagate the error here"), so this is what a
2161
2345
  // dropped link looks like from here. Returning 0 would make the caller report an empty
@@ -2164,14 +2348,21 @@ export class CotalEndpoint extends EventEmitter {
2164
2348
  throw new Error(`history: the broker reported messages on ${subject} but delivered none - the read was cut short, not empty`);
2165
2349
  }
2166
2350
  finally {
2167
- await consumer.delete().catch(() => { });
2351
+ try {
2352
+ await consumer.delete();
2353
+ }
2354
+ catch (e) {
2355
+ if (!isJetStreamMissing(e, JetStreamApiCodes.ConsumerNotFound))
2356
+ throw e;
2357
+ }
2168
2358
  }
2169
2359
  }
2170
2360
  /** Drain every message matching `subject` with sequence in `[start, ceiling]`, oldest-first.
2171
2361
  * One ephemeral ordered consumer, one batched pull — `AckPolicy.None`, so no per-message ack
2172
2362
  * round trip. Fetches exactly the pending count so it returns as soon as the window is
2173
2363
  * delivered rather than blocking for the pull's full expiry. */
2174
- async drainWindow(js, stream, subject, start, ceiling) {
2364
+ async drainWindow(js, stream, subject, start, ceiling, limit, signal) {
2365
+ signal?.throwIfAborted();
2175
2366
  const out = [];
2176
2367
  const consumer = await js.consumers.get(stream, { filter_subjects: [subject], opt_start_seq: start });
2177
2368
  try {
@@ -2180,36 +2371,51 @@ export class CotalEndpoint extends EventEmitter {
2180
2371
  const pending = (await consumer.info(true)).num_pending;
2181
2372
  if (pending === 0)
2182
2373
  return out;
2183
- const iter = await consumer.fetch({ max_messages: pending });
2374
+ signal?.throwIfAborted();
2375
+ // Keep only a small rolling buffer in flight. Stopping a client iterator cannot unsend bytes the
2376
+ // broker already committed to that pull request, so fetching the whole page let an aborted history
2377
+ // read keep filling a constrained shared connection and starve the next dashboard poll. `consume`
2378
+ // replenishes this bounded buffer as it is read, so large pages still complete without committing
2379
+ // all of their bytes to the connection up front.
2380
+ const iter = await consumer.consume({ max_messages: Math.min(pending, 32) });
2381
+ const stop = () => iter.stop(abortReason(signal));
2382
+ signal?.addEventListener("abort", stop, { once: true });
2184
2383
  // PROVE THE WINDOW COMPLETED. The pull iterator ends cleanly on a dropped connection, so a
2185
2384
  // close after three of ten deliveries would otherwise return a convincing three-message page.
2186
2385
  // The window is done when we have reached its upper bound or consumed everything bind said
2187
2386
  // was pending; anything else is a cut-short read and must say so.
2188
2387
  let delivered = 0;
2189
2388
  let complete = false;
2190
- for await (const m of iter) {
2191
- delivered++;
2192
- if (m.seq >= ceiling) { // reached the page's upper bound
2193
- if (m.seq === ceiling) {
2194
- try {
2195
- out.push(m.json());
2389
+ try {
2390
+ signal?.throwIfAborted();
2391
+ for await (const m of iter) {
2392
+ delivered++;
2393
+ if (m.seq >= ceiling) { // reached the page's upper bound
2394
+ if (m.seq === ceiling) {
2395
+ try {
2396
+ out.push(m.json());
2397
+ }
2398
+ catch { /* skip undecodable */ }
2196
2399
  }
2197
- catch { /* skip undecodable */ }
2400
+ complete = true;
2401
+ break;
2402
+ }
2403
+ try {
2404
+ out.push(m.json());
2405
+ if (out.length > limit)
2406
+ out.shift();
2407
+ }
2408
+ catch { /* skip undecodable */ }
2409
+ if (delivered >= pending) {
2410
+ complete = true;
2411
+ break;
2198
2412
  }
2199
- complete = true;
2200
- break;
2201
- }
2202
- try {
2203
- out.push(m.json());
2204
- }
2205
- catch {
2206
- /* skip undecodable */
2207
- }
2208
- if (delivered >= pending) {
2209
- complete = true;
2210
- break;
2211
2413
  }
2212
2414
  }
2415
+ finally {
2416
+ signal?.removeEventListener("abort", stop);
2417
+ iter.stop();
2418
+ }
2213
2419
  if (!complete)
2214
2420
  throw new Error(`history: read ${delivered} of ${pending} messages on ${subject} before the stream ended early - the window was cut short, not empty`);
2215
2421
  return out;
@@ -2220,7 +2426,13 @@ export class CotalEndpoint extends EventEmitter {
2220
2426
  // up to eight per call, the dashboard makes one call per channel, and a reload repeats it.
2221
2427
  // Left alone that accumulates consumers on the broker until the thresholds expire, and the
2222
2428
  // resulting resource exhaustion would land in streamHistory's catch and read as empty history.
2223
- await consumer.delete().catch(() => { });
2429
+ try {
2430
+ await consumer.delete();
2431
+ }
2432
+ catch (e) {
2433
+ if (!isJetStreamMissing(e, JetStreamApiCodes.ConsumerNotFound))
2434
+ throw e;
2435
+ }
2224
2436
  }
2225
2437
  }
2226
2438
  // ---- internals -----------------------------------------------------------
@@ -2232,10 +2444,27 @@ export class CotalEndpoint extends EventEmitter {
2232
2444
  * denial is never mistaken for absence (which already has a benign cause: MCP reconnect).
2233
2445
  */
2234
2446
  watchStatus() {
2235
- if (!this.nc)
2447
+ const nc = this.nc;
2448
+ if (!nc)
2236
2449
  return;
2237
2450
  void (async () => {
2238
- for await (const s of this.nc.status()) {
2451
+ for await (const s of nc.status()) {
2452
+ // A rebuild can replace `this.nc` before the old iterator finishes. Late disconnect/close
2453
+ // from that old epoch says nothing about the replacement and must not flip its liveness.
2454
+ if (this.nc !== nc)
2455
+ continue;
2456
+ if (s.type === "disconnect") {
2457
+ this.emit("transport", { connected: false, server: s.server });
2458
+ continue;
2459
+ }
2460
+ if (s.type === "reconnect") {
2461
+ this.emit("transport", { connected: true, server: s.server });
2462
+ continue;
2463
+ }
2464
+ if (s.type === "close") {
2465
+ this.emit("transport", { connected: false });
2466
+ continue;
2467
+ }
2239
2468
  if (s.type !== "error")
2240
2469
  continue;
2241
2470
  // Suppress the EXPECTED permission violation from a manager-free join we're confirming: an
@@ -2246,9 +2475,24 @@ export class CotalEndpoint extends EventEmitter {
2246
2475
  this.emit("error", describeStatusError(s.error));
2247
2476
  }
2248
2477
  })().catch((e) => {
2249
- if (!this.stopped)
2478
+ // Defensive symmetry with the reachable in-loop epoch guard above. Measured against five real
2479
+ // broker loss/reconnect/terminal-close cycles on pinned nats.js 3.4.0: status iterators ended
2480
+ // normally and this catch never fired. Keep an old epoch from surfacing an error if a runtime or
2481
+ // future client version can reject here, but do not treat this as a currently reachable edge.
2482
+ if (!this.stopped && this.nc === nc)
2250
2483
  this.emit("error", e);
2251
2484
  });
2485
+ // The transport is already live when connect() returns, while the Cotal bind below is still in
2486
+ // progress. Seed this contract explicitly rather than requiring consumers to combine it with the
2487
+ // later, differently-scoped `connection:true` event.
2488
+ //
2489
+ // The same stopped race as the readiness emit at the end of connectAndBind, and it reaches here
2490
+ // FIRST: connectAndBind calls watchStatus right after the dial, so a stop() landing while the
2491
+ // dial is still pending has this seed fire on an endpoint that is already stopped. Measured
2492
+ // through a real pending dial, a stopped endpoint announced a live transport it never had.
2493
+ if (this.stopped)
2494
+ return;
2495
+ this.emit("transport", { connected: true, server: nc.getServer() });
2252
2496
  }
2253
2497
  /** The error message for a guard that finds the endpoint unbound: "reconnecting" during a
2254
2498
  * rebuild's null window OR an inter-retry backoff (so a concurrent op reports the real
@@ -2309,7 +2553,11 @@ export class CotalEndpoint extends EventEmitter {
2309
2553
  await jsm.consumers.info(stream, name);
2310
2554
  return; // this lifecycle's durable exists — keep its original frontier
2311
2555
  }
2312
- catch { /* absent; create below */ }
2556
+ catch (e) {
2557
+ if (!isJetStreamMissing(e, JetStreamApiCodes.ConsumerNotFound))
2558
+ throw e;
2559
+ // Genuinely absent consumer: create it below. A denial is not absence.
2560
+ }
2313
2561
  const frontier = (await jsm.streams.info(stream)).state.last_seq;
2314
2562
  try {
2315
2563
  await jsm.consumers.add(stream, dmDurableConfig(this.space, owner, actor, lifecycleUid, { ...opts, activationFrontier: frontier }));
@@ -2321,7 +2569,13 @@ export class CotalEndpoint extends EventEmitter {
2321
2569
  await jsm.consumers.info(stream, name);
2322
2570
  return;
2323
2571
  }
2324
- catch { /* not a lost race */ }
2572
+ catch (probeError) {
2573
+ // The probe never creates a successful result: it either proves the concurrent winner above,
2574
+ // or the original create failure stays authoritative. Preserve a denial from the probe because
2575
+ // it names the missing capability more accurately than an unrelated create conflict.
2576
+ if (isPermissionDenied(probeError))
2577
+ throw probeError;
2578
+ }
2325
2579
  throw e;
2326
2580
  }
2327
2581
  }
@@ -2443,7 +2697,10 @@ export class CotalEndpoint extends EventEmitter {
2443
2697
  try {
2444
2698
  await (await this.deliveryRegistry()).delete(leaseKey(shardIndex));
2445
2699
  }
2446
- catch { /* already gone */ }
2700
+ catch {
2701
+ // Intentionally best-effort for EVERY failure: the lease TTL is the crash-safe release authority,
2702
+ // and clean shutdown must continue even when the broker is already gone or draining.
2703
+ }
2447
2704
  }
2448
2705
  /** Read a shard's delivery lease (the daemon-availability signal), or `undefined` if none is live.
2449
2706
  * READ-ONLY surface — drives Component 6's `cotal_channels` delivery-health field (an agent reads it
@@ -2486,6 +2743,9 @@ export class CotalEndpoint extends EventEmitter {
2486
2743
  this.managerLeaseKv = await kvm.create(managerBucket(this.space), { ttl: MANAGER_LEASE_TTL_MS });
2487
2744
  }
2488
2745
  catch {
2746
+ // OPEN mode has no broker ACLs, so a permission denial is impossible here. `create` is only
2747
+ // the ensure-exists attempt; `open` below is the authority after either a pre-existing bucket
2748
+ // or a create race, and it still throws if the bucket cannot actually be used.
2489
2749
  this.managerLeaseKv = await kvm.open(managerBucket(this.space));
2490
2750
  }
2491
2751
  }
@@ -2531,7 +2791,10 @@ export class CotalEndpoint extends EventEmitter {
2531
2791
  else
2532
2792
  await kv.delete(managerLeaseKey(instanceId), { previousSeq: revision });
2533
2793
  }
2534
- catch { /* not ours / already gone */ }
2794
+ catch {
2795
+ // Intentionally best-effort for EVERY failure: a revision mismatch means the lease is no longer
2796
+ // ours, while a broker failure is recovered by the bucket TTL. Shutdown must not claim deletion.
2797
+ }
2535
2798
  }
2536
2799
  /** Read a live manager liveness lease, or undefined if NONE (no manager instance holds the space). A
2537
2800
  * presence/existence check for the CLI's `spawn -f` reuse and `waitLeaseGone`, which only need "is any
@@ -2567,9 +2830,19 @@ export class CotalEndpoint extends EventEmitter {
2567
2830
  // it returns that instance's latest state, so a DEL here retires only its own key. The
2568
2831
  // defect was asking one wildcard for the newest message in the whole subtree, where a
2569
2832
  // stopping peer's tombstone outranks a live sibling's older PUT.
2570
- const m = await jsm.streams.getMessage(stream, { last_by_subj: subject }).catch(() => null);
2833
+ let m;
2834
+ try {
2835
+ m = await jsm.streams.getMessage(stream, { last_by_subj: subject });
2836
+ }
2837
+ catch (e) {
2838
+ if (e.code === JetStreamApiCodes.NoMessageFound)
2839
+ continue;
2840
+ throw e;
2841
+ }
2571
2842
  if (m === null)
2572
2843
  continue; // key vanished between INFO and GET: it is not a live holder
2844
+ // A 10037 above means the key vanished between INFO and GET: it is not a live holder. A
2845
+ // denial or failed read is not absence and is rethrown.
2573
2846
  const op = m.header?.get("KV-Operation");
2574
2847
  if (op === "DEL" || op === "PURGE" || m.data.length === 0)
2575
2848
  continue;
@@ -2752,10 +3025,7 @@ export class CotalEndpoint extends EventEmitter {
2752
3025
  // delete/recreate a predecessor's in-flight catch-up consumer — the uid disambiguates.
2753
3026
  const cuP = parsePrincipalKey(owner);
2754
3027
  const name = `cu_${cuP ? lifecycleNameKey(cuP.owner, cuP.actor, lifecycleUid) : `${token(owner)}-${lifecycleUid}`}_${generation}`;
2755
- try {
2756
- await this.jsm.consumers.delete(chatStream(this.space), name);
2757
- }
2758
- catch { /* none */ }
3028
+ await this.deleteConsumerIfPresent(chatStream(this.space), name);
2759
3029
  await this.jsm.consumers.add(chatStream(this.space), {
2760
3030
  name, filter_subject: subject, ack_policy: AckPolicy.None, mem_storage: true,
2761
3031
  inactive_threshold: nanos(30_000), deliver_policy: DeliverPolicy.StartSequence, opt_start_seq: fromSeqExcl + 1,
@@ -2791,10 +3061,7 @@ export class CotalEndpoint extends EventEmitter {
2791
3061
  }
2792
3062
  }
2793
3063
  finally {
2794
- try {
2795
- await this.jsm.consumers.delete(chatStream(this.space), name);
2796
- }
2797
- catch { /* gone */ }
3064
+ await this.deleteConsumerIfPresent(chatStream(this.space), name);
2798
3065
  }
2799
3066
  return { copied, evicted };
2800
3067
  }
@@ -3155,10 +3422,9 @@ export class CotalEndpoint extends EventEmitter {
3155
3422
  async runFanout() {
3156
3423
  if (!this.js || !this.jsm)
3157
3424
  return;
3158
- try {
3159
- await this.jsm.consumers.add(chatStream(this.space), fanoutDurableConfig(this.space, { ackWaitMs: this.ackWaitMs }));
3160
- }
3161
- catch { /* exists */ }
3425
+ // Named consumer creation is idempotent for the same config. A denial must not be mistaken for
3426
+ // "already exists"; let the broker response decide and propagate every failure.
3427
+ await this.jsm.consumers.add(chatStream(this.space), fanoutDurableConfig(this.space, { ackWaitMs: this.ackWaitMs }));
3162
3428
  const consumer = await this.js.consumers.get(chatStream(this.space), FANOUT_DURABLE);
3163
3429
  const msgs = await consumer.consume();
3164
3430
  this.streamMsgs.push(msgs);
@@ -3244,10 +3510,8 @@ export class CotalEndpoint extends EventEmitter {
3244
3510
  async runReader() {
3245
3511
  if (!this.js || !this.jsm)
3246
3512
  return;
3247
- try {
3248
- await this.jsm.consumers.add(inboxStream(this.space), inboxReaderConfig(this.space, { ackWaitMs: this.ackWaitMs }));
3249
- }
3250
- catch { /* exists */ }
3513
+ // Same fail-loud rule as fan-out: idempotent success is success, denial is not existence.
3514
+ await this.jsm.consumers.add(inboxStream(this.space), inboxReaderConfig(this.space, { ackWaitMs: this.ackWaitMs }));
3251
3515
  const consumer = await this.js.consumers.get(inboxStream(this.space), INBOX_READER_DURABLE);
3252
3516
  const msgs = await consumer.consume();
3253
3517
  this.streamMsgs.push(msgs);
@@ -3340,7 +3604,8 @@ export class CotalEndpoint extends EventEmitter {
3340
3604
  });
3341
3605
  }
3342
3606
  catch {
3343
- // Transfer failed keep the entry pending (redeliver), bounded by the same ceiling so a poison
3607
+ // EVERY transfer failure is deliberately a retry, never a successful result. Keep the entry
3608
+ // pending (redeliver), bounded by the same ceiling so a poison
3344
3609
  // entry can't head-of-line the shared reader forever.
3345
3610
  if (redeliveries >= READER_MAX_REDELIVERIES) {
3346
3611
  m.term();
@@ -3366,9 +3631,11 @@ export class CotalEndpoint extends EventEmitter {
3366
3631
  try {
3367
3632
  consumer = await this.js.consumers.get(dlvStream(this.space), dlvDurable(this.owner, this.actor, this.ownLifecycleUid));
3368
3633
  }
3369
- catch {
3370
- return;
3371
- } // no DLV durable — Plane-3 not active for us
3634
+ catch (e) {
3635
+ if (isJetStreamMissing(e, JetStreamApiCodes.ConsumerNotFound))
3636
+ return;
3637
+ throw e; // a denied bind is not proof Plane-3 is absent
3638
+ }
3372
3639
  const msgs = await consumer.consume();
3373
3640
  this.streamMsgs.push(msgs);
3374
3641
  void (async () => {
@@ -3500,13 +3767,9 @@ export class CotalEndpoint extends EventEmitter {
3500
3767
  for (const channel of this.channels) {
3501
3768
  if (!isConcreteChannel(channel) || this.plane3Channels.has(channel))
3502
3769
  continue;
3503
- let cls;
3504
- try {
3505
- cls = await this.deliveryClassFresh(channel);
3506
- }
3507
- catch {
3508
- continue;
3509
- }
3770
+ // A missing channel row legitimately resolves through the defaults. A failed/denied registry
3771
+ // read does not prove the channel is live-only, so let it fail startup loudly.
3772
+ const cls = await this.deliveryClassFresh(channel);
3510
3773
  if (cls !== "durable")
3511
3774
  continue;
3512
3775
  try {
@@ -3518,7 +3781,7 @@ export class CotalEndpoint extends EventEmitter {
3518
3781
  }
3519
3782
  catch (e) {
3520
3783
  if (!this.isNoResponders(e))
3521
- this.emit("error", e); // no daemon ⇒ retry until it recovers
3784
+ this.emitRecoverable(e); // no daemon ⇒ retry until it recovers
3522
3785
  void this.reconcileBootJoin(channel);
3523
3786
  }
3524
3787
  }
@@ -3551,7 +3814,7 @@ export class CotalEndpoint extends EventEmitter {
3551
3814
  }
3552
3815
  catch (e) {
3553
3816
  if (attempt === 0 && !this.isNoResponders(e))
3554
- this.emit("error", new Error(`channel "${channel}": boot durable self-join not yet established - retrying until the delivery daemon is reachable (${e.message})`));
3817
+ this.emitRecoverable(new Error(`channel "${channel}": boot durable self-join not yet established - retrying until the delivery daemon is reachable (${e.message})`));
3555
3818
  }
3556
3819
  }
3557
3820
  }
@@ -3807,8 +4070,22 @@ export class CotalEndpoint extends EventEmitter {
3807
4070
  try {
3808
4071
  return await this.jsm.consumers.info(stream, durable);
3809
4072
  }
3810
- catch {
3811
- return null; // 404 — fresh durable
4073
+ catch (e) {
4074
+ if (isJetStreamMissing(e, JetStreamApiCodes.ConsumerNotFound))
4075
+ return null; // structured absence — fresh durable
4076
+ throw e;
4077
+ }
4078
+ }
4079
+ /** Delete one named consumer, swallowing ONLY structured consumer/stream absence. */
4080
+ async deleteConsumerIfPresent(stream, durable) {
4081
+ if (!this.jsm)
4082
+ throw new Error("endpoint not started");
4083
+ try {
4084
+ await this.jsm.consumers.delete(stream, durable);
4085
+ }
4086
+ catch (e) {
4087
+ if (!isJetStreamMissing(e, JetStreamApiCodes.ConsumerNotFound, JetStreamApiCodes.StreamNotFound))
4088
+ throw e;
3812
4089
  }
3813
4090
  }
3814
4091
  /** Current frontier (last sequence) of the chat stream — a channel's join watermark, and the
@@ -3884,10 +4161,7 @@ export class CotalEndpoint extends EventEmitter {
3884
4161
  const out = [];
3885
4162
  // Clear any consumer leaked by a crashed prior read before re-creating it with THIS read's
3886
4163
  // single filter (the read ACL is enforced at create — see the doc above).
3887
- try {
3888
- await this.jsm.consumers.delete(stream, name);
3889
- }
3890
- catch { /* none; fine */ }
4164
+ await this.deleteConsumerIfPresent(stream, name);
3891
4165
  await this.jsm.consumers.add(stream, {
3892
4166
  name,
3893
4167
  filter_subject: subject,
@@ -3923,10 +4197,7 @@ export class CotalEndpoint extends EventEmitter {
3923
4197
  }
3924
4198
  }
3925
4199
  finally {
3926
- try {
3927
- await this.jsm.consumers.delete(stream, name);
3928
- }
3929
- catch { /* already gone */ }
4200
+ await this.deleteConsumerIfPresent(stream, name);
3930
4201
  }
3931
4202
  return out;
3932
4203
  }
@@ -3943,6 +4214,8 @@ export class CotalEndpoint extends EventEmitter {
3943
4214
  }
3944
4215
  catch (e) {
3945
4216
  this.emit("error", e);
4217
+ if (isPermissionDenied(e))
4218
+ throw e;
3946
4219
  return 0;
3947
4220
  }
3948
4221
  const noop = { ack: () => { }, nak: () => { }, durable: false };
@@ -3970,14 +4243,18 @@ export class CotalEndpoint extends EventEmitter {
3970
4243
  /**
3971
4244
  * Replay-gated pull of a channel's retained ambient from `sinceSeq` (exclusive) forward — the
3972
4245
  * focus-recall read behind `cotal_inbox`. Returns the messages (NOT emitted — this is a pull,
3973
- * not a push into context) plus `dropped: true` when the channel's earliest *retained* message
3974
- * is already newer than the watermark, i.e. some ambient aged out of the per-subject window and
3975
- * the caller must say so rather than silently short the window.
4246
+ * not a push into context) plus `dropped: true` when the window is not complete: either the
4247
+ * channel's earliest *retained* message is already newer than the watermark (some ambient aged
4248
+ * out of the per-subject window), or replay is off for the channel below. Either way the caller
4249
+ * must say so rather than silently reporting an empty, complete window.
3976
4250
  *
3977
4251
  * Honors the **same** per-channel replay gate as join-backfill ({@link joinPolicyFresh}): a
3978
- * `replay=off` channel returns nothing, so `focus` can't become a history bypass for a channel
3979
- * that denies replay to everyone else (the read ACL bounds *which* channels recall can touch; this
3980
- * app gate bounds *whether* a permitted channel replays).
4252
+ * `replay=off` channel returns no messages, so `focus` can't become a history bypass for a
4253
+ * channel that denies replay to everyone else (the read ACL bounds *which* channels recall can
4254
+ * touch; this app gate bounds *whether* a permitted channel replays). Ingest still ack-drops
4255
+ * focus-mode ambient/mentions on this channel on the promise that they stay recallable (#977) —
4256
+ * the gate means that promise cannot be kept, so it reports `dropped: true` rather than pretend
4257
+ * the window was empty and complete.
3981
4258
  */
3982
4259
  async recallChannel(channel, sinceSeq) {
3983
4260
  if (!this.jsm)
@@ -3986,7 +4263,7 @@ export class CotalEndpoint extends EventEmitter {
3986
4263
  return { messages: [], dropped: false };
3987
4264
  const policy = await this.joinPolicyFresh(channel);
3988
4265
  if (!policy.replay)
3989
- return { messages: [], dropped: false };
4266
+ return { messages: [], dropped: true };
3990
4267
  const subject = chatSubject(this.space, "*", "*", channel);
3991
4268
  let raw;
3992
4269
  try {
@@ -3994,6 +4271,8 @@ export class CotalEndpoint extends EventEmitter {
3994
4271
  }
3995
4272
  catch (e) {
3996
4273
  this.emit("error", e);
4274
+ if (isPermissionDenied(e))
4275
+ throw e;
3997
4276
  raw = [];
3998
4277
  }
3999
4278
  const collected = [];
@@ -4083,6 +4362,7 @@ export class CotalEndpoint extends EventEmitter {
4083
4362
  let hydrated;
4084
4363
  this.presenceSnapshot = new Promise((resolve) => { hydrated = resolve; });
4085
4364
  const iter = await this.kv.watch();
4365
+ this.presenceWatchIter = iter;
4086
4366
  void (async () => {
4087
4367
  let ready = false;
4088
4368
  for await (const e of iter) {
@@ -4103,6 +4383,7 @@ export class CotalEndpoint extends EventEmitter {
4103
4383
  if (!this.channelKv)
4104
4384
  return;
4105
4385
  const iter = await this.channelKv.watch();
4386
+ this.channelWatchIter = iter;
4106
4387
  void (async () => {
4107
4388
  for await (const e of iter)
4108
4389
  this.handleChannelEntry(e);
@@ -4134,6 +4415,7 @@ export class CotalEndpoint extends EventEmitter {
4134
4415
  }
4135
4416
  }
4136
4417
  handleKvEntry(e) {
4418
+ this.lastPresenceWatchAt = Date.now();
4137
4419
  if (e.operation === "DEL" || e.operation === "PURGE") {
4138
4420
  this.markOffline(e.key);
4139
4421
  return;
@@ -4157,6 +4439,19 @@ export class CotalEndpoint extends EventEmitter {
4157
4439
  return;
4158
4440
  const prev = this.roster.get(id);
4159
4441
  const stale = Date.now() - raw.ts > this.ttlMs;
4442
+ // A watch recovering from a stall replays the bucket. Those PUTs still carry the publisher's
4443
+ // pre-stall timestamps, so they look TTL-expired even though the peers kept heartbeating on
4444
+ // the broker. Materializing them as offline is the empty-to-full flicker #1045 named: the
4445
+ // observer's catch-up, not N deaths. Keep last-known rows, stay view-stale, and wait for a
4446
+ // live timestamp (or an explicit offline/delete) before changing a known peer.
4447
+ if (stale && raw.status !== "offline") {
4448
+ if (prev)
4449
+ return;
4450
+ this.roster.set(id, this.toOffline(raw));
4451
+ this.emit("roster", this.getRoster());
4452
+ return;
4453
+ }
4454
+ this.setPresenceViewFresh(true);
4160
4455
  // Any offline materialization (a stale snapshot OR a graceful-leave record) drops the advisory
4161
4456
  // attention fields — an offline peer must not carry a stale `[focus]`/`locally muted` hint.
4162
4457
  const p = stale || raw.status === "offline" ? this.toOffline(raw) : raw;
@@ -4210,8 +4505,22 @@ export class CotalEndpoint extends EventEmitter {
4210
4505
  this.emit("presence", { type: "offline", presence: offline });
4211
4506
  this.emit("roster", this.getRoster());
4212
4507
  }
4508
+ setPresenceViewFresh(fresh) {
4509
+ if (fresh === this.presenceViewFresh)
4510
+ return;
4511
+ this.presenceViewFresh = fresh;
4512
+ this.emit("presence-view", this.presenceView());
4513
+ }
4213
4514
  sweep() {
4214
4515
  const now = Date.now();
4516
+ // Whole-bucket silence past TTL is THIS observer going deaf. Real rosters do not lose every
4517
+ // peer in one window, and treating that silence as N offline verdicts empties an online-only
4518
+ // sidebar with nothing saying the window went blind (#1045). Gate the per-peer age-out on
4519
+ // watch freshness; surface the view as stale instead.
4520
+ if (this.lastPresenceWatchAt !== 0 && now - this.lastPresenceWatchAt > this.ttlMs) {
4521
+ this.setPresenceViewFresh(false);
4522
+ return;
4523
+ }
4215
4524
  let changed = false;
4216
4525
  for (const [id, p] of this.roster) {
4217
4526
  if (p.status !== "offline" && now - p.ts > this.ttlMs) {
@@ -4410,6 +4719,18 @@ export function isPermissionDenied(e) {
4410
4719
  return true;
4411
4720
  return /permissions?\s+violation/i.test(String(e?.message ?? ""));
4412
4721
  }
4722
+ /** True only for the structured JetStream API absence codes named by the caller. A status 404 or
4723
+ * message regex is too broad here: the catch sites use absence to produce a successful empty/fresh
4724
+ * result, so a permission denial, timeout, or protocol failure must never pass as "not found". */
4725
+ function isJetStreamMissing(e, ...codes) {
4726
+ return e instanceof JetStreamApiError && codes.includes(e.code);
4727
+ }
4728
+ /** The signal's exact abort reason, or the platform-standard AbortError when none was supplied. */
4729
+ function abortReason(signal) {
4730
+ return signal?.reason instanceof Error
4731
+ ? signal.reason
4732
+ : new DOMException("This operation was aborted", "AbortError");
4733
+ }
4413
4734
  /** True ONLY for a denial on a **publish** — the single case that proves the message was never
4414
4735
  * ACCEPTED or stored. (Not "never reached the server": the server necessarily received enough of
4415
4736
  * it to reject it. The distinction matters precisely here, because this helper exists to separate