@cotal-ai/core 0.36.0 → 0.38.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-file.d.ts +18 -0
- package/dist/agent-file.d.ts.map +1 -1
- package/dist/agent-file.js +37 -1
- package/dist/agent-file.js.map +1 -1
- package/dist/connector-setup.d.ts +30 -0
- package/dist/connector-setup.d.ts.map +1 -0
- package/dist/connector-setup.js +2 -0
- package/dist/connector-setup.js.map +1 -0
- package/dist/connector.d.ts +31 -3
- package/dist/connector.d.ts.map +1 -1
- package/dist/connector.js +12 -1
- package/dist/connector.js.map +1 -1
- package/dist/endpoint-action.d.ts +6 -1
- package/dist/endpoint-action.d.ts.map +1 -1
- package/dist/endpoint-action.js +6 -2
- package/dist/endpoint-action.js.map +1 -1
- package/dist/endpoint-envelope.d.ts +1 -1
- package/dist/endpoint-envelope.d.ts.map +1 -1
- package/dist/endpoint-envelope.js +1 -1
- package/dist/endpoint-envelope.js.map +1 -1
- package/dist/endpoint-error.d.ts +26 -0
- package/dist/endpoint-error.d.ts.map +1 -1
- package/dist/endpoint-error.js +41 -0
- package/dist/endpoint-error.js.map +1 -1
- package/dist/endpoint-grants.d.ts +8 -7
- package/dist/endpoint-grants.d.ts.map +1 -1
- package/dist/endpoint-grants.js +9 -8
- package/dist/endpoint-grants.js.map +1 -1
- package/dist/endpoint-invoke.d.ts.map +1 -1
- package/dist/endpoint-invoke.js +58 -7
- package/dist/endpoint-invoke.js.map +1 -1
- package/dist/endpoint-serve-kv.d.ts +17 -0
- package/dist/endpoint-serve-kv.d.ts.map +1 -1
- package/dist/endpoint-serve-kv.js +59 -1
- package/dist/endpoint-serve-kv.js.map +1 -1
- package/dist/endpoint-service.d.ts +6 -0
- package/dist/endpoint-service.d.ts.map +1 -1
- package/dist/endpoint-service.js +12 -3
- package/dist/endpoint-service.js.map +1 -1
- package/dist/endpoint.d.ts +78 -14
- package/dist/endpoint.d.ts.map +1 -1
- package/dist/endpoint.js +439 -118
- package/dist/endpoint.js.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/lifecycle-saga.d.ts.map +1 -1
- package/dist/lifecycle-saga.js +24 -5
- package/dist/lifecycle-saga.js.map +1 -1
- package/dist/lifecycle-state.d.ts +15 -0
- package/dist/lifecycle-state.d.ts.map +1 -1
- package/dist/lifecycle-state.js +60 -0
- package/dist/lifecycle-state.js.map +1 -1
- package/dist/provision.d.ts.map +1 -1
- package/dist/provision.js +19 -7
- package/dist/provision.js.map +1 -1
- package/dist/types.d.ts +7 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +1 -1
package/dist/endpoint.js
CHANGED
|
@@ -13,7 +13,7 @@ import { liveKvEntries } from "./kv-scan.js";
|
|
|
13
13
|
import { ARTIFACT_PART_KIND, isArtifactPart } from "./artifact.js";
|
|
14
14
|
import { assertValidName } from "./resolve.js";
|
|
15
15
|
import { createSpaceStreams, dmDurableConfig, dlvDurableConfig, taskDurableConfig, fanoutDurableConfig, inboxReaderConfig, MAX_MSGS_PER_SUBJECT, MANAGER_LEASE_TTL_MS, MANAGER_LEASE_ATTEMPT_MS } from "./streams.js";
|
|
16
|
-
import { jetstream, jetstreamManager, AckPolicy, DeliverPolicy, } from "@nats-io/jetstream";
|
|
16
|
+
import { jetstream, jetstreamManager, AckPolicy, DeliverPolicy, JetStreamApiCodes, JetStreamApiError, } from "@nats-io/jetstream";
|
|
17
17
|
import {} from "@nats-io/jetstream";
|
|
18
18
|
import { Kvm } from "@nats-io/kv";
|
|
19
19
|
import { Bucket, KvWatchInclude } from "@nats-io/kv/internal";
|
|
@@ -31,7 +31,9 @@ export const DEFAULT_SPACE = "main";
|
|
|
31
31
|
* "connection" ({ connected: boolean }) — true on every successful (re)bind (initial start, manual
|
|
32
32
|
* reconnect, AND background self-heal), false the moment the connection drops (rebuild null window /
|
|
33
33
|
* terminal close). Lets an in-process agent track connectedness off the endpoint's own (re)binds
|
|
34
|
-
* instead of an imperative flag the self-heal path can't reach
|
|
34
|
+
* instead of an imperative flag the self-heal path can't reach; "transport" ({ connected, server? })
|
|
35
|
+
* is the lower-level NATS socket edge, true before the full bind finishes and false during an internal
|
|
36
|
+
* nats.js reconnect without changing `connection` readiness.
|
|
35
37
|
*
|
|
36
38
|
* Callers MUST attach an "error" listener before `start()`: async faults (incl. NATS
|
|
37
39
|
* permission denials, surfaced via `watchStatus`) are emitted as "error", and Node throws
|
|
@@ -105,6 +107,12 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
105
107
|
jsm;
|
|
106
108
|
kv;
|
|
107
109
|
channelKv;
|
|
110
|
+
/** The presence/channel-registry watches' own handles. Each is an ORDERED push consumer: its idle
|
|
111
|
+
* heartbeat monitor lives on a JS timer independent of the connection, so a drain that doesn't
|
|
112
|
+
* `.stop()` it first leaves the monitor to fire into a closing connection every 30s, throwing
|
|
113
|
+
* `DrainingConnectionError` out of `reset()` on a timer nothing awaits. */
|
|
114
|
+
presenceWatchIter;
|
|
115
|
+
channelWatchIter;
|
|
108
116
|
/** Plane-3 durable-membership registry KV — lazily opened by the privileged delivery daemon (or a
|
|
109
117
|
* short-lived provisioner). */
|
|
110
118
|
membersKv;
|
|
@@ -177,6 +185,15 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
177
185
|
roster = new Map();
|
|
178
186
|
/** Resolves when the current presence watch has consumed its complete initial KV snapshot. */
|
|
179
187
|
presenceSnapshot = Promise.resolve();
|
|
188
|
+
/**
|
|
189
|
+
* Observer-local age of the last presence-KV delivery (any key, including DEL/PURGE). Distinct
|
|
190
|
+
* from each peer's `ts`: that is the publisher's heartbeat. Whole-bucket silence past TTL is
|
|
191
|
+
* the observer going deaf, not N simultaneous deaths, and sweep must not treat it as the
|
|
192
|
+
* latter (#1045).
|
|
193
|
+
*/
|
|
194
|
+
lastPresenceWatchAt = 0;
|
|
195
|
+
/** Last emitted presence-view freshness. Suppresses duplicate `presence-view` events. */
|
|
196
|
+
presenceViewFresh = true;
|
|
180
197
|
status = "idle";
|
|
181
198
|
activity;
|
|
182
199
|
/** Mirror of the connector's authoritative attention state, published in presence (advisory). The
|
|
@@ -371,10 +388,15 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
371
388
|
}
|
|
372
389
|
async start() {
|
|
373
390
|
await this.connectAndBind();
|
|
374
|
-
//
|
|
375
|
-
//
|
|
376
|
-
//
|
|
377
|
-
//
|
|
391
|
+
// stop() can finish while the INITIAL connectAndBind is still awaiting its broker work. The
|
|
392
|
+
// rebuild path already closes that race; initial start needs the same fence or the late bind
|
|
393
|
+
// leaves a fresh nc, heartbeat, consumers, and presence live on an endpoint already stopped.
|
|
394
|
+
// superviseConnection below: nats.js auto-reconnects transient drops, and when it exhausts its
|
|
395
|
+
// attempts and the connection closes for good we rebuild from scratch, so an in-process agent
|
|
396
|
+
// (e.g. the OpenCode plugin) recovers without a host respawn. Armed only after a successful
|
|
397
|
+
// first connect; a first-connect failure throws to the caller's connect-retry loop instead.
|
|
398
|
+
if (await this.tearDownIfStopped())
|
|
399
|
+
return;
|
|
378
400
|
this.superviseConnection();
|
|
379
401
|
}
|
|
380
402
|
/** How far ahead of the current bearer's `exp` a refresh fires, and how soon a FAILED refresh
|
|
@@ -382,9 +404,17 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
382
404
|
* so whatever `currentBearer` holds is what every attempt presents). */
|
|
383
405
|
static BEARER_REFRESH_MARGIN_MS = 60_000;
|
|
384
406
|
static BEARER_RETRY_MS = 15_000;
|
|
407
|
+
/**
|
|
408
|
+
* A condition the endpoint is already surviving. Node rethrows `error` when no listener is
|
|
409
|
+
* attached, so emitting retry notices on `error` killed hosts that the endpoint intended to
|
|
410
|
+
* keep running (#891). `warning` is observable and never fatal without a listener.
|
|
411
|
+
*/
|
|
412
|
+
emitRecoverable(err) {
|
|
413
|
+
this.emit("warning", err);
|
|
414
|
+
}
|
|
385
415
|
/** Fetch a fresh bearer from the source, pin its principal to ours, arm the next refresh. On a
|
|
386
416
|
* fetch/principal failure: THROWS when `initial` (start() must fail loud before first connect);
|
|
387
|
-
* otherwise emits "
|
|
417
|
+
* otherwise emits "warning" and retries — the live connection keeps working until its current JWT
|
|
388
418
|
* expiry, so a dead auth service is loud without instantly dropping the mesh. */
|
|
389
419
|
async refreshBearer(initial = false) {
|
|
390
420
|
try {
|
|
@@ -398,7 +428,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
398
428
|
catch (e) {
|
|
399
429
|
if (initial)
|
|
400
430
|
throw e;
|
|
401
|
-
this.
|
|
431
|
+
this.emitRecoverable(new Error(`bearer refresh failed (${e instanceof Error ? e.message : String(e)}) - retrying; this connection dies at its current token's expiry if the auth service stays down`));
|
|
402
432
|
this.armBearerRefresh(CotalEndpoint.BEARER_RETRY_MS);
|
|
403
433
|
}
|
|
404
434
|
}
|
|
@@ -466,7 +496,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
466
496
|
catch (e) {
|
|
467
497
|
if (initial)
|
|
468
498
|
throw e;
|
|
469
|
-
this.
|
|
499
|
+
this.emitRecoverable(new Error(`creds refresh failed (${e instanceof Error ? e.message : String(e)}) - retrying; this connection dies at its current JWT's expiry if renewal keeps failing`));
|
|
470
500
|
this.armCredsRefresh(CotalEndpoint.CREDS_RETRY_MS);
|
|
471
501
|
}
|
|
472
502
|
}
|
|
@@ -546,7 +576,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
546
576
|
await this.swapConnectionOntoFreshCreds();
|
|
547
577
|
}
|
|
548
578
|
catch (e) {
|
|
549
|
-
this.
|
|
579
|
+
this.emitRecoverable(new Error(`creds refresh failed (${e instanceof Error ? e.message : String(e)}) - retrying; this connection dies at its current JWT's expiry if renewal keeps failing`));
|
|
550
580
|
this.armCredsRefresh(CotalEndpoint.CREDS_RETRY_MS);
|
|
551
581
|
}
|
|
552
582
|
}
|
|
@@ -633,6 +663,11 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
633
663
|
throw new Error(this.bearerSource
|
|
634
664
|
? "this endpoint's user bearer has expired and renewal through the auth exchange is failing - not presenting the expired token to the broker; retrying with backoff"
|
|
635
665
|
: "this endpoint's user bearer has expired and it holds no bearer source to renew it - re-authenticate and rebuild the endpoint (construct it with a bearer FUNCTION for standing renewal)");
|
|
666
|
+
const credsExp = this.currentCreds && credsClaims(this.currentCreds).exp;
|
|
667
|
+
if (typeof credsExp === "number" && credsExp * 1000 <= Date.now())
|
|
668
|
+
throw new Error(this.credsSource
|
|
669
|
+
? "this endpoint's creds have expired and renewal is failing - not presenting the expired credential to the broker; retrying with backoff"
|
|
670
|
+
: "this endpoint's creds have expired and it holds no creds source to renew them - replace the credential and rebuild the endpoint (pass a creds FUNCTION for standing renewal)");
|
|
636
671
|
this.nc = await dialerFor(this.servers)({
|
|
637
672
|
servers: this.servers,
|
|
638
673
|
// In USER MODE the connection `name` carries the client-chosen inbox nonce (= connId) the callout
|
|
@@ -719,7 +754,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
719
754
|
if (this.doRegister) {
|
|
720
755
|
await this.publishPresence();
|
|
721
756
|
this.heartbeatTimer = setInterval(() => {
|
|
722
|
-
this.publishPresence().catch((e) => this.
|
|
757
|
+
this.publishPresence().catch((e) => this.emitRecoverable(e));
|
|
723
758
|
}, this.heartbeatMs);
|
|
724
759
|
}
|
|
725
760
|
// Caller-owned membership watches are INTENT rather than one-connection iterators. Re-open them
|
|
@@ -731,6 +766,20 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
731
766
|
await this.armPlane3();
|
|
732
767
|
// Bound and live — covers initial start, manual reconnect, AND background self-heal (every
|
|
733
768
|
// path lands here). The single signal an in-process agent's connected flag tracks.
|
|
769
|
+
//
|
|
770
|
+
// The stopped guard: stop() can land in any await above. Both callers tear the fresh
|
|
771
|
+
// connection back down (tearDownIfStopped), but an event has no undo, so a late
|
|
772
|
+
// `connection: true` would be the last edge a listener ever sees on a stopped endpoint and
|
|
773
|
+
// nothing follows it to correct the record. It belongs here rather than in a consumer because
|
|
774
|
+
// every listener reads the same edge; MeshAgent carries its own `stopping` guard and so was
|
|
775
|
+
// never the one exposed, which is the point.
|
|
776
|
+
//
|
|
777
|
+
// Measured for the start() caller by the broker suite's mid-bind cell. doRebuild is
|
|
778
|
+
// also measured, by a separate cell that holds the rebuild's connectAndBind at
|
|
779
|
+
// armPlane3 after a successful first bind and lands stop() in that window (#1028).
|
|
780
|
+
// Shared-line reasoning is no longer the rebuild proof.
|
|
781
|
+
if (this.stopped)
|
|
782
|
+
return;
|
|
734
783
|
this.emit("connection", { connected: true });
|
|
735
784
|
}
|
|
736
785
|
/** Tear down everything {@link connectAndBind} (re)creates, so a rebind can't leak a
|
|
@@ -754,6 +803,20 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
754
803
|
}
|
|
755
804
|
}
|
|
756
805
|
this.streamMsgs.length = 0;
|
|
806
|
+
try {
|
|
807
|
+
this.presenceWatchIter?.stop();
|
|
808
|
+
}
|
|
809
|
+
catch {
|
|
810
|
+
/* already closed with the connection */
|
|
811
|
+
}
|
|
812
|
+
this.presenceWatchIter = undefined;
|
|
813
|
+
try {
|
|
814
|
+
this.channelWatchIter?.stop();
|
|
815
|
+
}
|
|
816
|
+
catch {
|
|
817
|
+
/* already closed with the connection */
|
|
818
|
+
}
|
|
819
|
+
this.channelWatchIter = undefined;
|
|
757
820
|
for (const sub of this.chatSubs.values()) {
|
|
758
821
|
try {
|
|
759
822
|
sub.unsubscribe();
|
|
@@ -766,11 +829,13 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
766
829
|
this.chatSubDenied.clear();
|
|
767
830
|
this.confirmingChatSubs.clear();
|
|
768
831
|
this.roster.clear();
|
|
832
|
+
this.lastPresenceWatchAt = 0;
|
|
833
|
+
this.presenceViewFresh = true;
|
|
769
834
|
this.joinSeq.clear();
|
|
770
835
|
this.channelConfigs.clear();
|
|
771
836
|
this.channelDefaults = {};
|
|
772
837
|
for (const watch of this.membershipFeedWatches)
|
|
773
|
-
watch.arm = watch.arm.catch(() => { }).then(() => this.disarmMembershipWatch(watch));
|
|
838
|
+
watch.arm = watch.arm.catch(() => { }).then(() => this.disarmMembershipWatch(watch)).catch((err) => { this.emit("error", err); });
|
|
774
839
|
}
|
|
775
840
|
/** If stop() ran during a rebuild's `await connectAndBind`, the just-bound connection +
|
|
776
841
|
* heartbeat + supervisor would be left live on a stopped endpoint. Tear that fresh
|
|
@@ -806,7 +871,10 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
806
871
|
return;
|
|
807
872
|
if (this.nc !== nc)
|
|
808
873
|
return; // epoch-stale — a rebuild already swapped this connection
|
|
809
|
-
|
|
874
|
+
// ORDER IS PART OF THE DIAGNOSTIC CONTRACT. MeshAgent retains endpoint errors only while it is
|
|
875
|
+
// not bound, so readiness must turn false before the matching terminal-close error is emitted.
|
|
876
|
+
// Reversing these two lines silently loses the only post-drop reason an agent can report.
|
|
877
|
+
this.emit("connection", { connected: false });
|
|
810
878
|
this.emit("error", new Error(`mesh connection closed${err ? `: ${err.message}` : ""} - re-establishing`));
|
|
811
879
|
void this.reestablishLoop();
|
|
812
880
|
});
|
|
@@ -853,7 +921,11 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
853
921
|
// The manager's liveness-lease handle too: left bound to the old connection, every renew and
|
|
854
922
|
// re-read after a reconnect times out, and the manager reports its lease unknown for good.
|
|
855
923
|
this.managerLeaseKv = undefined;
|
|
856
|
-
|
|
924
|
+
// This is an application-requested epoch teardown, not a transient nats.js blip. The old
|
|
925
|
+
// status iterator is now stale by construction and its close is epoch-dropped, so this line is
|
|
926
|
+
// the authoritative raw-liveness edge for the no-nc window until the new watcher seeds true.
|
|
927
|
+
this.emit("transport", { connected: false });
|
|
928
|
+
this.emit("connection", { connected: false });
|
|
857
929
|
try {
|
|
858
930
|
await oldNc?.drain();
|
|
859
931
|
}
|
|
@@ -864,9 +936,10 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
864
936
|
// stop() may have run during the await — don't leave a live connection + heartbeat +
|
|
865
937
|
// supervisor on a stopped endpoint. (Reads this.nc in its own scope — a bare `this.nc`
|
|
866
938
|
// here in doRebuild narrows to `never` via TS inlining connectAndBind's assignment.)
|
|
939
|
+
// Re-arm on the fresh nc only after this stopped fence accepts it.
|
|
867
940
|
if (await this.tearDownIfStopped())
|
|
868
941
|
return;
|
|
869
|
-
this.superviseConnection();
|
|
942
|
+
this.superviseConnection();
|
|
870
943
|
}
|
|
871
944
|
finally {
|
|
872
945
|
this.reconnecting = false;
|
|
@@ -902,7 +975,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
902
975
|
}
|
|
903
976
|
catch (e) {
|
|
904
977
|
if (!this.stopped)
|
|
905
|
-
this.
|
|
978
|
+
this.emitRecoverable(e);
|
|
906
979
|
const delay = this.nextRetryDelayMs();
|
|
907
980
|
await new Promise((resolve) => {
|
|
908
981
|
this.backoffResolve = resolve;
|
|
@@ -984,6 +1057,20 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
984
1057
|
/* already closed */
|
|
985
1058
|
}
|
|
986
1059
|
}
|
|
1060
|
+
try {
|
|
1061
|
+
this.presenceWatchIter?.stop();
|
|
1062
|
+
}
|
|
1063
|
+
catch {
|
|
1064
|
+
/* already closed */
|
|
1065
|
+
}
|
|
1066
|
+
this.presenceWatchIter = undefined;
|
|
1067
|
+
try {
|
|
1068
|
+
this.channelWatchIter?.stop();
|
|
1069
|
+
}
|
|
1070
|
+
catch {
|
|
1071
|
+
/* already closed */
|
|
1072
|
+
}
|
|
1073
|
+
this.channelWatchIter = undefined;
|
|
987
1074
|
try {
|
|
988
1075
|
if (this.doRegister) {
|
|
989
1076
|
this.status = "offline";
|
|
@@ -1520,6 +1607,22 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
1520
1607
|
getRoster() {
|
|
1521
1608
|
return [...this.roster.values()].sort((a, b) => a.card.name.localeCompare(b.card.name));
|
|
1522
1609
|
}
|
|
1610
|
+
/**
|
|
1611
|
+
* Freshness of THIS observer's presence watch, not of any peer. `fresh: false` means the
|
|
1612
|
+
* whole bucket has been silent past the liveness window — the view is stale as of
|
|
1613
|
+
* `staleSince`, and {@link getRoster} is last-known rather than a current offline verdict.
|
|
1614
|
+
* A watch that has not yet delivered anything is not stale (there is no T to name).
|
|
1615
|
+
*/
|
|
1616
|
+
presenceView() {
|
|
1617
|
+
if (!this.doWatch)
|
|
1618
|
+
return { fresh: true };
|
|
1619
|
+
if (this.lastPresenceWatchAt === 0)
|
|
1620
|
+
return { fresh: true };
|
|
1621
|
+
const staleSince = this.lastPresenceWatchAt + this.ttlMs;
|
|
1622
|
+
if (Date.now() < staleSince)
|
|
1623
|
+
return { fresh: true };
|
|
1624
|
+
return { fresh: false, staleSince };
|
|
1625
|
+
}
|
|
1523
1626
|
/** Wait until the current presence watch has consumed its initial KV snapshot. An empty bucket
|
|
1524
1627
|
* emits no watch entry, so the timeout keeps a genuinely empty mesh bounded. */
|
|
1525
1628
|
async waitForPresenceSnapshot(timeoutMs = 1_000) {
|
|
@@ -1723,8 +1826,12 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
1723
1826
|
}
|
|
1724
1827
|
}
|
|
1725
1828
|
}
|
|
1726
|
-
catch {
|
|
1727
|
-
|
|
1829
|
+
catch (e) {
|
|
1830
|
+
// A genuinely absent CHAT stream means there are no retained message counts yet. Every other
|
|
1831
|
+
// failure, especially a STREAM.INFO permission denial, means the count view could not be read
|
|
1832
|
+
// and must stay loud rather than returning a valid-looking registry-only subset.
|
|
1833
|
+
if (!isJetStreamMissing(e, JetStreamApiCodes.StreamNotFound))
|
|
1834
|
+
throw e;
|
|
1728
1835
|
}
|
|
1729
1836
|
const channels = new Set([...counts.keys(), ...this.channelConfigs.keys()]);
|
|
1730
1837
|
return [...channels]
|
|
@@ -1950,11 +2057,18 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
1950
2057
|
watch.consumerName = undefined;
|
|
1951
2058
|
}
|
|
1952
2059
|
else {
|
|
1953
|
-
// A timeout is deferred only for an epoch that is actually closing/rebuilding; live timeouts stay loud.
|
|
1954
2060
|
const closedEpoch = err.name === "ClosedConnectionError" || /^closed connection$/i.test(err.message);
|
|
1955
|
-
const
|
|
1956
|
-
|
|
2061
|
+
const timeout = err.name === "TimeoutError" || /timeout/i.test(err.message);
|
|
2062
|
+
const dyingEpochTimeout = timeout && (this.reconnecting || !this.nc || this.nc.isClosed());
|
|
2063
|
+
// Cleanup of an ordered consumer: a delete timeout means the broker did not answer in time,
|
|
2064
|
+
// not that the endpoint is unusable. The broker reaps an idle/ephemeral consumer anyway.
|
|
2065
|
+
// Throwing here killed a live observer over a slow VPN (#1047). Catch, surface, continue.
|
|
2066
|
+
if (timeout || closedEpoch || dyingEpochTimeout) {
|
|
2067
|
+
this.emit("error", err);
|
|
2068
|
+
}
|
|
2069
|
+
else {
|
|
1957
2070
|
throw err;
|
|
2071
|
+
}
|
|
1958
2072
|
}
|
|
1959
2073
|
// A terminal close leaves stream/name intact. The endpoint-owned stopped intent is retried
|
|
1960
2074
|
// through the fresh JetStream manager before its public stop promise may resolve.
|
|
@@ -1981,10 +2095,11 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
1981
2095
|
}
|
|
1982
2096
|
}));
|
|
1983
2097
|
}
|
|
1984
|
-
/** Fetch recent messages from a channel's JetStream backlog.
|
|
2098
|
+
/** Fetch recent messages from a channel's JetStream backlog. `signal` cancels the active pull and
|
|
2099
|
+
* reclaims its ephemeral consumer before the promise rejects. */
|
|
1985
2100
|
async channelHistory(channel, opts) {
|
|
1986
2101
|
// history from any sender
|
|
1987
|
-
return this.streamHistory(chatStream(this.space), chatSubject(this.space, "*", "*", channel), opts?.limit ?? 100);
|
|
2102
|
+
return this.streamHistory(chatStream(this.space), chatSubject(this.space, "*", "*", channel), opts?.limit ?? 100, undefined, opts?.signal);
|
|
1988
2103
|
}
|
|
1989
2104
|
/** Read a channel's recent history THROUGH THE DELIVERY DAEMON instead of through a consumer this
|
|
1990
2105
|
* connection creates itself — the mediated read of SPEC's "Mediated reads (normative)" rule (no raw
|
|
@@ -2025,12 +2140,13 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
2025
2140
|
throw new Error("readHistory: the delivery daemon returned a malformed page (expected { items, complete })");
|
|
2026
2141
|
return { items: data.items, complete: data.complete };
|
|
2027
2142
|
}
|
|
2028
|
-
/** Fetch recent DMs (any sender→any recipient) from the space's DM backlog.
|
|
2143
|
+
/** Fetch recent DMs (any sender→any recipient) from the space's DM backlog. `signal` cancels the
|
|
2144
|
+
* active pull and reclaims its ephemeral consumer. God-view only:
|
|
2029
2145
|
* a normal agent/observer's ACL denies CONSUMER.CREATE on DM_<space>, so this throws-and-
|
|
2030
2146
|
* skips for them — only an `admin`-profile cred can read it. */
|
|
2031
2147
|
async dmHistory(opts) {
|
|
2032
2148
|
// every inst.<recipOwner>.<recipActor>.<sndOwner>.<sndActor> DM — the whole DM subtree (god-view)
|
|
2033
|
-
return this.streamHistory(dmStream(this.space), `${spacePrefix(this.space)}.inst.>`, opts?.limit ?? 100);
|
|
2149
|
+
return this.streamHistory(dmStream(this.space), `${spacePrefix(this.space)}.inst.>`, opts?.limit ?? 100, undefined, opts?.signal);
|
|
2034
2150
|
}
|
|
2035
2151
|
/**
|
|
2036
2152
|
* The `limit` MOST RECENT messages matching `subject`, oldest-first within the page.
|
|
@@ -2048,14 +2164,17 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
2048
2164
|
* A filtered subject's sequences are non-contiguous (other channels interleave in the same
|
|
2049
2165
|
* stream), so the window cannot be computed arithmetically. A FAILED attempt holds fewer than a
|
|
2050
2166
|
* page by definition, so wasted transfer stays page-sized and geometric growth keeps the number of
|
|
2051
|
-
* attempts logarithmic.
|
|
2052
|
-
*
|
|
2167
|
+
* attempts logarithmic. A channel with fewer than `limit` matches used to keep widening until
|
|
2168
|
+
* sequence 1 and drain the stream's whole retained set (#840). The walk now stops at the
|
|
2169
|
+
* subject's FIRST matching sequence (the mirror of the last-matching ceiling), probed on the
|
|
2170
|
+
* same CREATE surface, and only after a short drain so a dense page never pays for the floor.
|
|
2053
2171
|
*
|
|
2054
2172
|
* `before` pages toward the past: pass the `seq` of the oldest message you already have.
|
|
2055
2173
|
*/
|
|
2056
|
-
async streamHistory(stream, subject, limit, before) {
|
|
2174
|
+
async streamHistory(stream, subject, limit, before, signal) {
|
|
2057
2175
|
if (!this.nc)
|
|
2058
2176
|
throw new Error("endpoint not started");
|
|
2177
|
+
signal?.throwIfAborted();
|
|
2059
2178
|
// A LIMIT THAT IS NOT A FINITE NUMBER HAS NO ANSWER, AND THE SEARCH BELOW CANNOT REFUSE IT.
|
|
2060
2179
|
// Every comparison against NaN is false, so `limit <= 0` does not fire for one, and neither of
|
|
2061
2180
|
// the widening loop's exits can ever be true either: `page.length >= NaN` is false forever and
|
|
@@ -2091,7 +2210,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
2091
2210
|
// Deliberately NOT `getMessage({ last_by_subj })`, which would be the obvious way to ask: it
|
|
2092
2211
|
// needs `$JS.API.STREAM.MSG.GET`, which read credentials do not hold. That grant hole already
|
|
2093
2212
|
// shipped once from this function and turned every non-admin history read into an empty list.
|
|
2094
|
-
const ceiling = before !== undefined ? before - 1 : await this.lastMatchingSeq(js, stream, subject);
|
|
2213
|
+
const ceiling = before !== undefined ? before - 1 : await this.lastMatchingSeq(js, stream, subject, signal);
|
|
2095
2214
|
if (ceiling < 1)
|
|
2096
2215
|
return [];
|
|
2097
2216
|
// Widen from the exact ceiling until a window holds a full page, or until the window IS the
|
|
@@ -2100,17 +2219,28 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
2100
2219
|
// Geometric growth keeps the number of attempts logarithmic, so total wasted transfer is a
|
|
2101
2220
|
// small multiple of a page.
|
|
2102
2221
|
//
|
|
2103
|
-
//
|
|
2104
|
-
//
|
|
2105
|
-
//
|
|
2106
|
-
//
|
|
2107
|
-
//
|
|
2108
|
-
//
|
|
2222
|
+
// A SHORT PAGE is either "the channel is exhausted" or "the window is still above the
|
|
2223
|
+
// first match". Sequence 1 is the wrong floor for the first of those: three matches at
|
|
2224
|
+
// the high end of a busy stream are exhausted as soon as the window's lower edge passes
|
|
2225
|
+
// the subject's first matching sequence, and walking on to 1 re-reads everyone else's
|
|
2226
|
+
// retained set (#840). Probe that floor only after a short drain so a dense page (one
|
|
2227
|
+
// drain, full) never pays for it. The remaining unbounded-looking case is a subject
|
|
2228
|
+
// whose FIRST match really is near sequence 1; that span is the channel's own, not the
|
|
2229
|
+
// stream's.
|
|
2109
2230
|
let span = Math.max(limit * 4, 64);
|
|
2231
|
+
let floor = 1;
|
|
2232
|
+
let floorKnown = false;
|
|
2110
2233
|
for (;;) {
|
|
2111
|
-
|
|
2112
|
-
const
|
|
2113
|
-
|
|
2234
|
+
signal?.throwIfAborted();
|
|
2235
|
+
const start = Math.max(floor, ceiling - span + 1);
|
|
2236
|
+
const page = await this.drainWindow(js, stream, subject, start, ceiling, limit, signal);
|
|
2237
|
+
if (page.length >= limit)
|
|
2238
|
+
return page.slice(-limit);
|
|
2239
|
+
if (!floorKnown) {
|
|
2240
|
+
floor = Math.max(1, await this.firstMatchingSeq(js, stream, subject, signal));
|
|
2241
|
+
floorKnown = true;
|
|
2242
|
+
}
|
|
2243
|
+
if (start <= floor)
|
|
2114
2244
|
return page.slice(-limit);
|
|
2115
2245
|
span *= 4;
|
|
2116
2246
|
}
|
|
@@ -2144,7 +2274,51 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
2144
2274
|
* (available from the create, before anything is delivered) is 0 for an empty subject, and
|
|
2145
2275
|
* otherwise one message carries the sequence. Same CREATE/INFO/NEXT/DELETE surface `drainWindow`
|
|
2146
2276
|
* already uses, so no broker authority is added. */
|
|
2147
|
-
|
|
2277
|
+
/** The oldest stream sequence matching `subject`, or 0 when the subject has no messages.
|
|
2278
|
+
* Mirror of {@link lastMatchingSeq}: same CREATE/INFO/NEXT/DELETE surface, `DeliverPolicy.All`
|
|
2279
|
+
* instead of `Last`, first delivered seq instead of last. Read credentials already hold this. */
|
|
2280
|
+
async firstMatchingSeq(js, stream, subject, signal) {
|
|
2281
|
+
signal?.throwIfAborted();
|
|
2282
|
+
const consumer = await js.consumers.get(stream, {
|
|
2283
|
+
filter_subjects: [subject],
|
|
2284
|
+
deliver_policy: DeliverPolicy.All,
|
|
2285
|
+
});
|
|
2286
|
+
try {
|
|
2287
|
+
if ((await consumer.info(true)).num_pending === 0)
|
|
2288
|
+
return 0;
|
|
2289
|
+
signal?.throwIfAborted();
|
|
2290
|
+
const iter = await consumer.fetch({ max_messages: 1 });
|
|
2291
|
+
const stop = () => iter.stop(abortReason(signal));
|
|
2292
|
+
signal?.addEventListener("abort", stop, { once: true });
|
|
2293
|
+
try {
|
|
2294
|
+
signal?.throwIfAborted();
|
|
2295
|
+
for await (const m of iter)
|
|
2296
|
+
return m.seq;
|
|
2297
|
+
}
|
|
2298
|
+
finally {
|
|
2299
|
+
signal?.removeEventListener("abort", stop);
|
|
2300
|
+
iter.stop();
|
|
2301
|
+
}
|
|
2302
|
+
throw new Error(`history: the broker reported messages on ${subject} but delivered none - the read was cut short, not empty`);
|
|
2303
|
+
}
|
|
2304
|
+
finally {
|
|
2305
|
+
try {
|
|
2306
|
+
await consumer.delete();
|
|
2307
|
+
}
|
|
2308
|
+
catch (e) {
|
|
2309
|
+
if (!isJetStreamMissing(e, JetStreamApiCodes.ConsumerNotFound))
|
|
2310
|
+
throw e;
|
|
2311
|
+
}
|
|
2312
|
+
}
|
|
2313
|
+
}
|
|
2314
|
+
/** The newest stream sequence matching `subject`, or 0 when the subject has no messages.
|
|
2315
|
+
*
|
|
2316
|
+
* One ordered consumer at `DeliverPolicy.Last` with this subject's filter: its `num_pending`
|
|
2317
|
+
* (available from the create, before anything is delivered) is 0 for an empty subject, and
|
|
2318
|
+
* otherwise one message carries the sequence. Same CREATE/INFO/NEXT/DELETE surface `drainWindow`
|
|
2319
|
+
* already uses, so no broker authority is added. */
|
|
2320
|
+
async lastMatchingSeq(js, stream, subject, signal) {
|
|
2321
|
+
signal?.throwIfAborted();
|
|
2148
2322
|
const consumer = await js.consumers.get(stream, {
|
|
2149
2323
|
filter_subjects: [subject],
|
|
2150
2324
|
deliver_policy: DeliverPolicy.Last,
|
|
@@ -2153,9 +2327,19 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
2153
2327
|
// Bind-time zero is the ONLY thing that means "this subject has no messages".
|
|
2154
2328
|
if ((await consumer.info(true)).num_pending === 0)
|
|
2155
2329
|
return 0;
|
|
2330
|
+
signal?.throwIfAborted();
|
|
2156
2331
|
const iter = await consumer.fetch({ max_messages: 1 });
|
|
2157
|
-
|
|
2158
|
-
|
|
2332
|
+
const stop = () => iter.stop(abortReason(signal));
|
|
2333
|
+
signal?.addEventListener("abort", stop, { once: true });
|
|
2334
|
+
try {
|
|
2335
|
+
signal?.throwIfAborted();
|
|
2336
|
+
for await (const m of iter)
|
|
2337
|
+
return m.seq;
|
|
2338
|
+
}
|
|
2339
|
+
finally {
|
|
2340
|
+
signal?.removeEventListener("abort", stop);
|
|
2341
|
+
iter.stop();
|
|
2342
|
+
}
|
|
2159
2343
|
// Bind said a message was pending and none arrived. The pinned client's pull iterator ends
|
|
2160
2344
|
// CLEANLY when the connection closes ("we don't propagate the error here"), so this is what a
|
|
2161
2345
|
// dropped link looks like from here. Returning 0 would make the caller report an empty
|
|
@@ -2164,14 +2348,21 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
2164
2348
|
throw new Error(`history: the broker reported messages on ${subject} but delivered none - the read was cut short, not empty`);
|
|
2165
2349
|
}
|
|
2166
2350
|
finally {
|
|
2167
|
-
|
|
2351
|
+
try {
|
|
2352
|
+
await consumer.delete();
|
|
2353
|
+
}
|
|
2354
|
+
catch (e) {
|
|
2355
|
+
if (!isJetStreamMissing(e, JetStreamApiCodes.ConsumerNotFound))
|
|
2356
|
+
throw e;
|
|
2357
|
+
}
|
|
2168
2358
|
}
|
|
2169
2359
|
}
|
|
2170
2360
|
/** Drain every message matching `subject` with sequence in `[start, ceiling]`, oldest-first.
|
|
2171
2361
|
* One ephemeral ordered consumer, one batched pull — `AckPolicy.None`, so no per-message ack
|
|
2172
2362
|
* round trip. Fetches exactly the pending count so it returns as soon as the window is
|
|
2173
2363
|
* delivered rather than blocking for the pull's full expiry. */
|
|
2174
|
-
async drainWindow(js, stream, subject, start, ceiling) {
|
|
2364
|
+
async drainWindow(js, stream, subject, start, ceiling, limit, signal) {
|
|
2365
|
+
signal?.throwIfAborted();
|
|
2175
2366
|
const out = [];
|
|
2176
2367
|
const consumer = await js.consumers.get(stream, { filter_subjects: [subject], opt_start_seq: start });
|
|
2177
2368
|
try {
|
|
@@ -2180,36 +2371,51 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
2180
2371
|
const pending = (await consumer.info(true)).num_pending;
|
|
2181
2372
|
if (pending === 0)
|
|
2182
2373
|
return out;
|
|
2183
|
-
|
|
2374
|
+
signal?.throwIfAborted();
|
|
2375
|
+
// Keep only a small rolling buffer in flight. Stopping a client iterator cannot unsend bytes the
|
|
2376
|
+
// broker already committed to that pull request, so fetching the whole page let an aborted history
|
|
2377
|
+
// read keep filling a constrained shared connection and starve the next dashboard poll. `consume`
|
|
2378
|
+
// replenishes this bounded buffer as it is read, so large pages still complete without committing
|
|
2379
|
+
// all of their bytes to the connection up front.
|
|
2380
|
+
const iter = await consumer.consume({ max_messages: Math.min(pending, 32) });
|
|
2381
|
+
const stop = () => iter.stop(abortReason(signal));
|
|
2382
|
+
signal?.addEventListener("abort", stop, { once: true });
|
|
2184
2383
|
// PROVE THE WINDOW COMPLETED. The pull iterator ends cleanly on a dropped connection, so a
|
|
2185
2384
|
// close after three of ten deliveries would otherwise return a convincing three-message page.
|
|
2186
2385
|
// The window is done when we have reached its upper bound or consumed everything bind said
|
|
2187
2386
|
// was pending; anything else is a cut-short read and must say so.
|
|
2188
2387
|
let delivered = 0;
|
|
2189
2388
|
let complete = false;
|
|
2190
|
-
|
|
2191
|
-
|
|
2192
|
-
|
|
2193
|
-
|
|
2194
|
-
|
|
2195
|
-
|
|
2389
|
+
try {
|
|
2390
|
+
signal?.throwIfAborted();
|
|
2391
|
+
for await (const m of iter) {
|
|
2392
|
+
delivered++;
|
|
2393
|
+
if (m.seq >= ceiling) { // reached the page's upper bound
|
|
2394
|
+
if (m.seq === ceiling) {
|
|
2395
|
+
try {
|
|
2396
|
+
out.push(m.json());
|
|
2397
|
+
}
|
|
2398
|
+
catch { /* skip undecodable */ }
|
|
2196
2399
|
}
|
|
2197
|
-
|
|
2400
|
+
complete = true;
|
|
2401
|
+
break;
|
|
2402
|
+
}
|
|
2403
|
+
try {
|
|
2404
|
+
out.push(m.json());
|
|
2405
|
+
if (out.length > limit)
|
|
2406
|
+
out.shift();
|
|
2407
|
+
}
|
|
2408
|
+
catch { /* skip undecodable */ }
|
|
2409
|
+
if (delivered >= pending) {
|
|
2410
|
+
complete = true;
|
|
2411
|
+
break;
|
|
2198
2412
|
}
|
|
2199
|
-
complete = true;
|
|
2200
|
-
break;
|
|
2201
|
-
}
|
|
2202
|
-
try {
|
|
2203
|
-
out.push(m.json());
|
|
2204
|
-
}
|
|
2205
|
-
catch {
|
|
2206
|
-
/* skip undecodable */
|
|
2207
|
-
}
|
|
2208
|
-
if (delivered >= pending) {
|
|
2209
|
-
complete = true;
|
|
2210
|
-
break;
|
|
2211
2413
|
}
|
|
2212
2414
|
}
|
|
2415
|
+
finally {
|
|
2416
|
+
signal?.removeEventListener("abort", stop);
|
|
2417
|
+
iter.stop();
|
|
2418
|
+
}
|
|
2213
2419
|
if (!complete)
|
|
2214
2420
|
throw new Error(`history: read ${delivered} of ${pending} messages on ${subject} before the stream ended early - the window was cut short, not empty`);
|
|
2215
2421
|
return out;
|
|
@@ -2220,7 +2426,13 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
2220
2426
|
// up to eight per call, the dashboard makes one call per channel, and a reload repeats it.
|
|
2221
2427
|
// Left alone that accumulates consumers on the broker until the thresholds expire, and the
|
|
2222
2428
|
// resulting resource exhaustion would land in streamHistory's catch and read as empty history.
|
|
2223
|
-
|
|
2429
|
+
try {
|
|
2430
|
+
await consumer.delete();
|
|
2431
|
+
}
|
|
2432
|
+
catch (e) {
|
|
2433
|
+
if (!isJetStreamMissing(e, JetStreamApiCodes.ConsumerNotFound))
|
|
2434
|
+
throw e;
|
|
2435
|
+
}
|
|
2224
2436
|
}
|
|
2225
2437
|
}
|
|
2226
2438
|
// ---- internals -----------------------------------------------------------
|
|
@@ -2232,10 +2444,27 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
2232
2444
|
* denial is never mistaken for absence (which already has a benign cause: MCP reconnect).
|
|
2233
2445
|
*/
|
|
2234
2446
|
watchStatus() {
|
|
2235
|
-
|
|
2447
|
+
const nc = this.nc;
|
|
2448
|
+
if (!nc)
|
|
2236
2449
|
return;
|
|
2237
2450
|
void (async () => {
|
|
2238
|
-
for await (const s of
|
|
2451
|
+
for await (const s of nc.status()) {
|
|
2452
|
+
// A rebuild can replace `this.nc` before the old iterator finishes. Late disconnect/close
|
|
2453
|
+
// from that old epoch says nothing about the replacement and must not flip its liveness.
|
|
2454
|
+
if (this.nc !== nc)
|
|
2455
|
+
continue;
|
|
2456
|
+
if (s.type === "disconnect") {
|
|
2457
|
+
this.emit("transport", { connected: false, server: s.server });
|
|
2458
|
+
continue;
|
|
2459
|
+
}
|
|
2460
|
+
if (s.type === "reconnect") {
|
|
2461
|
+
this.emit("transport", { connected: true, server: s.server });
|
|
2462
|
+
continue;
|
|
2463
|
+
}
|
|
2464
|
+
if (s.type === "close") {
|
|
2465
|
+
this.emit("transport", { connected: false });
|
|
2466
|
+
continue;
|
|
2467
|
+
}
|
|
2239
2468
|
if (s.type !== "error")
|
|
2240
2469
|
continue;
|
|
2241
2470
|
// Suppress the EXPECTED permission violation from a manager-free join we're confirming: an
|
|
@@ -2246,9 +2475,24 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
2246
2475
|
this.emit("error", describeStatusError(s.error));
|
|
2247
2476
|
}
|
|
2248
2477
|
})().catch((e) => {
|
|
2249
|
-
|
|
2478
|
+
// Defensive symmetry with the reachable in-loop epoch guard above. Measured against five real
|
|
2479
|
+
// broker loss/reconnect/terminal-close cycles on pinned nats.js 3.4.0: status iterators ended
|
|
2480
|
+
// normally and this catch never fired. Keep an old epoch from surfacing an error if a runtime or
|
|
2481
|
+
// future client version can reject here, but do not treat this as a currently reachable edge.
|
|
2482
|
+
if (!this.stopped && this.nc === nc)
|
|
2250
2483
|
this.emit("error", e);
|
|
2251
2484
|
});
|
|
2485
|
+
// The transport is already live when connect() returns, while the Cotal bind below is still in
|
|
2486
|
+
// progress. Seed this contract explicitly rather than requiring consumers to combine it with the
|
|
2487
|
+
// later, differently-scoped `connection:true` event.
|
|
2488
|
+
//
|
|
2489
|
+
// The same stopped race as the readiness emit at the end of connectAndBind, and it reaches here
|
|
2490
|
+
// FIRST: connectAndBind calls watchStatus right after the dial, so a stop() landing while the
|
|
2491
|
+
// dial is still pending has this seed fire on an endpoint that is already stopped. Measured
|
|
2492
|
+
// through a real pending dial, a stopped endpoint announced a live transport it never had.
|
|
2493
|
+
if (this.stopped)
|
|
2494
|
+
return;
|
|
2495
|
+
this.emit("transport", { connected: true, server: nc.getServer() });
|
|
2252
2496
|
}
|
|
2253
2497
|
/** The error message for a guard that finds the endpoint unbound: "reconnecting" during a
|
|
2254
2498
|
* rebuild's null window OR an inter-retry backoff (so a concurrent op reports the real
|
|
@@ -2309,7 +2553,11 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
2309
2553
|
await jsm.consumers.info(stream, name);
|
|
2310
2554
|
return; // this lifecycle's durable exists — keep its original frontier
|
|
2311
2555
|
}
|
|
2312
|
-
catch {
|
|
2556
|
+
catch (e) {
|
|
2557
|
+
if (!isJetStreamMissing(e, JetStreamApiCodes.ConsumerNotFound))
|
|
2558
|
+
throw e;
|
|
2559
|
+
// Genuinely absent consumer: create it below. A denial is not absence.
|
|
2560
|
+
}
|
|
2313
2561
|
const frontier = (await jsm.streams.info(stream)).state.last_seq;
|
|
2314
2562
|
try {
|
|
2315
2563
|
await jsm.consumers.add(stream, dmDurableConfig(this.space, owner, actor, lifecycleUid, { ...opts, activationFrontier: frontier }));
|
|
@@ -2321,7 +2569,13 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
2321
2569
|
await jsm.consumers.info(stream, name);
|
|
2322
2570
|
return;
|
|
2323
2571
|
}
|
|
2324
|
-
catch {
|
|
2572
|
+
catch (probeError) {
|
|
2573
|
+
// The probe never creates a successful result: it either proves the concurrent winner above,
|
|
2574
|
+
// or the original create failure stays authoritative. Preserve a denial from the probe because
|
|
2575
|
+
// it names the missing capability more accurately than an unrelated create conflict.
|
|
2576
|
+
if (isPermissionDenied(probeError))
|
|
2577
|
+
throw probeError;
|
|
2578
|
+
}
|
|
2325
2579
|
throw e;
|
|
2326
2580
|
}
|
|
2327
2581
|
}
|
|
@@ -2443,7 +2697,10 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
2443
2697
|
try {
|
|
2444
2698
|
await (await this.deliveryRegistry()).delete(leaseKey(shardIndex));
|
|
2445
2699
|
}
|
|
2446
|
-
catch {
|
|
2700
|
+
catch {
|
|
2701
|
+
// Intentionally best-effort for EVERY failure: the lease TTL is the crash-safe release authority,
|
|
2702
|
+
// and clean shutdown must continue even when the broker is already gone or draining.
|
|
2703
|
+
}
|
|
2447
2704
|
}
|
|
2448
2705
|
/** Read a shard's delivery lease (the daemon-availability signal), or `undefined` if none is live.
|
|
2449
2706
|
* READ-ONLY surface — drives Component 6's `cotal_channels` delivery-health field (an agent reads it
|
|
@@ -2486,6 +2743,9 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
2486
2743
|
this.managerLeaseKv = await kvm.create(managerBucket(this.space), { ttl: MANAGER_LEASE_TTL_MS });
|
|
2487
2744
|
}
|
|
2488
2745
|
catch {
|
|
2746
|
+
// OPEN mode has no broker ACLs, so a permission denial is impossible here. `create` is only
|
|
2747
|
+
// the ensure-exists attempt; `open` below is the authority after either a pre-existing bucket
|
|
2748
|
+
// or a create race, and it still throws if the bucket cannot actually be used.
|
|
2489
2749
|
this.managerLeaseKv = await kvm.open(managerBucket(this.space));
|
|
2490
2750
|
}
|
|
2491
2751
|
}
|
|
@@ -2531,7 +2791,10 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
2531
2791
|
else
|
|
2532
2792
|
await kv.delete(managerLeaseKey(instanceId), { previousSeq: revision });
|
|
2533
2793
|
}
|
|
2534
|
-
catch {
|
|
2794
|
+
catch {
|
|
2795
|
+
// Intentionally best-effort for EVERY failure: a revision mismatch means the lease is no longer
|
|
2796
|
+
// ours, while a broker failure is recovered by the bucket TTL. Shutdown must not claim deletion.
|
|
2797
|
+
}
|
|
2535
2798
|
}
|
|
2536
2799
|
/** Read a live manager liveness lease, or undefined if NONE (no manager instance holds the space). A
|
|
2537
2800
|
* presence/existence check for the CLI's `spawn -f` reuse and `waitLeaseGone`, which only need "is any
|
|
@@ -2567,9 +2830,19 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
2567
2830
|
// it returns that instance's latest state, so a DEL here retires only its own key. The
|
|
2568
2831
|
// defect was asking one wildcard for the newest message in the whole subtree, where a
|
|
2569
2832
|
// stopping peer's tombstone outranks a live sibling's older PUT.
|
|
2570
|
-
|
|
2833
|
+
let m;
|
|
2834
|
+
try {
|
|
2835
|
+
m = await jsm.streams.getMessage(stream, { last_by_subj: subject });
|
|
2836
|
+
}
|
|
2837
|
+
catch (e) {
|
|
2838
|
+
if (e.code === JetStreamApiCodes.NoMessageFound)
|
|
2839
|
+
continue;
|
|
2840
|
+
throw e;
|
|
2841
|
+
}
|
|
2571
2842
|
if (m === null)
|
|
2572
2843
|
continue; // key vanished between INFO and GET: it is not a live holder
|
|
2844
|
+
// A 10037 above means the key vanished between INFO and GET: it is not a live holder. A
|
|
2845
|
+
// denial or failed read is not absence and is rethrown.
|
|
2573
2846
|
const op = m.header?.get("KV-Operation");
|
|
2574
2847
|
if (op === "DEL" || op === "PURGE" || m.data.length === 0)
|
|
2575
2848
|
continue;
|
|
@@ -2752,10 +3025,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
2752
3025
|
// delete/recreate a predecessor's in-flight catch-up consumer — the uid disambiguates.
|
|
2753
3026
|
const cuP = parsePrincipalKey(owner);
|
|
2754
3027
|
const name = `cu_${cuP ? lifecycleNameKey(cuP.owner, cuP.actor, lifecycleUid) : `${token(owner)}-${lifecycleUid}`}_${generation}`;
|
|
2755
|
-
|
|
2756
|
-
await this.jsm.consumers.delete(chatStream(this.space), name);
|
|
2757
|
-
}
|
|
2758
|
-
catch { /* none */ }
|
|
3028
|
+
await this.deleteConsumerIfPresent(chatStream(this.space), name);
|
|
2759
3029
|
await this.jsm.consumers.add(chatStream(this.space), {
|
|
2760
3030
|
name, filter_subject: subject, ack_policy: AckPolicy.None, mem_storage: true,
|
|
2761
3031
|
inactive_threshold: nanos(30_000), deliver_policy: DeliverPolicy.StartSequence, opt_start_seq: fromSeqExcl + 1,
|
|
@@ -2791,10 +3061,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
2791
3061
|
}
|
|
2792
3062
|
}
|
|
2793
3063
|
finally {
|
|
2794
|
-
|
|
2795
|
-
await this.jsm.consumers.delete(chatStream(this.space), name);
|
|
2796
|
-
}
|
|
2797
|
-
catch { /* gone */ }
|
|
3064
|
+
await this.deleteConsumerIfPresent(chatStream(this.space), name);
|
|
2798
3065
|
}
|
|
2799
3066
|
return { copied, evicted };
|
|
2800
3067
|
}
|
|
@@ -3155,10 +3422,9 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
3155
3422
|
async runFanout() {
|
|
3156
3423
|
if (!this.js || !this.jsm)
|
|
3157
3424
|
return;
|
|
3158
|
-
|
|
3159
|
-
|
|
3160
|
-
}
|
|
3161
|
-
catch { /* exists */ }
|
|
3425
|
+
// Named consumer creation is idempotent for the same config. A denial must not be mistaken for
|
|
3426
|
+
// "already exists"; let the broker response decide and propagate every failure.
|
|
3427
|
+
await this.jsm.consumers.add(chatStream(this.space), fanoutDurableConfig(this.space, { ackWaitMs: this.ackWaitMs }));
|
|
3162
3428
|
const consumer = await this.js.consumers.get(chatStream(this.space), FANOUT_DURABLE);
|
|
3163
3429
|
const msgs = await consumer.consume();
|
|
3164
3430
|
this.streamMsgs.push(msgs);
|
|
@@ -3244,10 +3510,8 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
3244
3510
|
async runReader() {
|
|
3245
3511
|
if (!this.js || !this.jsm)
|
|
3246
3512
|
return;
|
|
3247
|
-
|
|
3248
|
-
|
|
3249
|
-
}
|
|
3250
|
-
catch { /* exists */ }
|
|
3513
|
+
// Same fail-loud rule as fan-out: idempotent success is success, denial is not existence.
|
|
3514
|
+
await this.jsm.consumers.add(inboxStream(this.space), inboxReaderConfig(this.space, { ackWaitMs: this.ackWaitMs }));
|
|
3251
3515
|
const consumer = await this.js.consumers.get(inboxStream(this.space), INBOX_READER_DURABLE);
|
|
3252
3516
|
const msgs = await consumer.consume();
|
|
3253
3517
|
this.streamMsgs.push(msgs);
|
|
@@ -3340,7 +3604,8 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
3340
3604
|
});
|
|
3341
3605
|
}
|
|
3342
3606
|
catch {
|
|
3343
|
-
//
|
|
3607
|
+
// EVERY transfer failure is deliberately a retry, never a successful result. Keep the entry
|
|
3608
|
+
// pending (redeliver), bounded by the same ceiling so a poison
|
|
3344
3609
|
// entry can't head-of-line the shared reader forever.
|
|
3345
3610
|
if (redeliveries >= READER_MAX_REDELIVERIES) {
|
|
3346
3611
|
m.term();
|
|
@@ -3366,9 +3631,11 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
3366
3631
|
try {
|
|
3367
3632
|
consumer = await this.js.consumers.get(dlvStream(this.space), dlvDurable(this.owner, this.actor, this.ownLifecycleUid));
|
|
3368
3633
|
}
|
|
3369
|
-
catch {
|
|
3370
|
-
|
|
3371
|
-
|
|
3634
|
+
catch (e) {
|
|
3635
|
+
if (isJetStreamMissing(e, JetStreamApiCodes.ConsumerNotFound))
|
|
3636
|
+
return;
|
|
3637
|
+
throw e; // a denied bind is not proof Plane-3 is absent
|
|
3638
|
+
}
|
|
3372
3639
|
const msgs = await consumer.consume();
|
|
3373
3640
|
this.streamMsgs.push(msgs);
|
|
3374
3641
|
void (async () => {
|
|
@@ -3500,13 +3767,9 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
3500
3767
|
for (const channel of this.channels) {
|
|
3501
3768
|
if (!isConcreteChannel(channel) || this.plane3Channels.has(channel))
|
|
3502
3769
|
continue;
|
|
3503
|
-
|
|
3504
|
-
|
|
3505
|
-
|
|
3506
|
-
}
|
|
3507
|
-
catch {
|
|
3508
|
-
continue;
|
|
3509
|
-
}
|
|
3770
|
+
// A missing channel row legitimately resolves through the defaults. A failed/denied registry
|
|
3771
|
+
// read does not prove the channel is live-only, so let it fail startup loudly.
|
|
3772
|
+
const cls = await this.deliveryClassFresh(channel);
|
|
3510
3773
|
if (cls !== "durable")
|
|
3511
3774
|
continue;
|
|
3512
3775
|
try {
|
|
@@ -3518,7 +3781,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
3518
3781
|
}
|
|
3519
3782
|
catch (e) {
|
|
3520
3783
|
if (!this.isNoResponders(e))
|
|
3521
|
-
this.
|
|
3784
|
+
this.emitRecoverable(e); // no daemon ⇒ retry until it recovers
|
|
3522
3785
|
void this.reconcileBootJoin(channel);
|
|
3523
3786
|
}
|
|
3524
3787
|
}
|
|
@@ -3551,7 +3814,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
3551
3814
|
}
|
|
3552
3815
|
catch (e) {
|
|
3553
3816
|
if (attempt === 0 && !this.isNoResponders(e))
|
|
3554
|
-
this.
|
|
3817
|
+
this.emitRecoverable(new Error(`channel "${channel}": boot durable self-join not yet established - retrying until the delivery daemon is reachable (${e.message})`));
|
|
3555
3818
|
}
|
|
3556
3819
|
}
|
|
3557
3820
|
}
|
|
@@ -3807,8 +4070,22 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
3807
4070
|
try {
|
|
3808
4071
|
return await this.jsm.consumers.info(stream, durable);
|
|
3809
4072
|
}
|
|
3810
|
-
catch {
|
|
3811
|
-
|
|
4073
|
+
catch (e) {
|
|
4074
|
+
if (isJetStreamMissing(e, JetStreamApiCodes.ConsumerNotFound))
|
|
4075
|
+
return null; // structured absence — fresh durable
|
|
4076
|
+
throw e;
|
|
4077
|
+
}
|
|
4078
|
+
}
|
|
4079
|
+
/** Delete one named consumer, swallowing ONLY structured consumer/stream absence. */
|
|
4080
|
+
async deleteConsumerIfPresent(stream, durable) {
|
|
4081
|
+
if (!this.jsm)
|
|
4082
|
+
throw new Error("endpoint not started");
|
|
4083
|
+
try {
|
|
4084
|
+
await this.jsm.consumers.delete(stream, durable);
|
|
4085
|
+
}
|
|
4086
|
+
catch (e) {
|
|
4087
|
+
if (!isJetStreamMissing(e, JetStreamApiCodes.ConsumerNotFound, JetStreamApiCodes.StreamNotFound))
|
|
4088
|
+
throw e;
|
|
3812
4089
|
}
|
|
3813
4090
|
}
|
|
3814
4091
|
/** Current frontier (last sequence) of the chat stream — a channel's join watermark, and the
|
|
@@ -3884,10 +4161,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
3884
4161
|
const out = [];
|
|
3885
4162
|
// Clear any consumer leaked by a crashed prior read before re-creating it with THIS read's
|
|
3886
4163
|
// single filter (the read ACL is enforced at create — see the doc above).
|
|
3887
|
-
|
|
3888
|
-
await this.jsm.consumers.delete(stream, name);
|
|
3889
|
-
}
|
|
3890
|
-
catch { /* none; fine */ }
|
|
4164
|
+
await this.deleteConsumerIfPresent(stream, name);
|
|
3891
4165
|
await this.jsm.consumers.add(stream, {
|
|
3892
4166
|
name,
|
|
3893
4167
|
filter_subject: subject,
|
|
@@ -3923,10 +4197,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
3923
4197
|
}
|
|
3924
4198
|
}
|
|
3925
4199
|
finally {
|
|
3926
|
-
|
|
3927
|
-
await this.jsm.consumers.delete(stream, name);
|
|
3928
|
-
}
|
|
3929
|
-
catch { /* already gone */ }
|
|
4200
|
+
await this.deleteConsumerIfPresent(stream, name);
|
|
3930
4201
|
}
|
|
3931
4202
|
return out;
|
|
3932
4203
|
}
|
|
@@ -3943,6 +4214,8 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
3943
4214
|
}
|
|
3944
4215
|
catch (e) {
|
|
3945
4216
|
this.emit("error", e);
|
|
4217
|
+
if (isPermissionDenied(e))
|
|
4218
|
+
throw e;
|
|
3946
4219
|
return 0;
|
|
3947
4220
|
}
|
|
3948
4221
|
const noop = { ack: () => { }, nak: () => { }, durable: false };
|
|
@@ -3970,14 +4243,18 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
3970
4243
|
/**
|
|
3971
4244
|
* Replay-gated pull of a channel's retained ambient from `sinceSeq` (exclusive) forward — the
|
|
3972
4245
|
* focus-recall read behind `cotal_inbox`. Returns the messages (NOT emitted — this is a pull,
|
|
3973
|
-
* not a push into context) plus `dropped: true` when the
|
|
3974
|
-
* is already newer than the watermark
|
|
3975
|
-
* the
|
|
4246
|
+
* not a push into context) plus `dropped: true` when the window is not complete: either the
|
|
4247
|
+
* channel's earliest *retained* message is already newer than the watermark (some ambient aged
|
|
4248
|
+
* out of the per-subject window), or replay is off for the channel below. Either way the caller
|
|
4249
|
+
* must say so rather than silently reporting an empty, complete window.
|
|
3976
4250
|
*
|
|
3977
4251
|
* Honors the **same** per-channel replay gate as join-backfill ({@link joinPolicyFresh}): a
|
|
3978
|
-
* `replay=off` channel returns
|
|
3979
|
-
* that denies replay to everyone else (the read ACL bounds *which* channels recall can
|
|
3980
|
-
* app gate bounds *whether* a permitted channel replays).
|
|
4252
|
+
* `replay=off` channel returns no messages, so `focus` can't become a history bypass for a
|
|
4253
|
+
* channel that denies replay to everyone else (the read ACL bounds *which* channels recall can
|
|
4254
|
+
* touch; this app gate bounds *whether* a permitted channel replays). Ingest still ack-drops
|
|
4255
|
+
* focus-mode ambient/mentions on this channel on the promise that they stay recallable (#977) —
|
|
4256
|
+
* the gate means that promise cannot be kept, so it reports `dropped: true` rather than pretend
|
|
4257
|
+
* the window was empty and complete.
|
|
3981
4258
|
*/
|
|
3982
4259
|
async recallChannel(channel, sinceSeq) {
|
|
3983
4260
|
if (!this.jsm)
|
|
@@ -3986,7 +4263,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
3986
4263
|
return { messages: [], dropped: false };
|
|
3987
4264
|
const policy = await this.joinPolicyFresh(channel);
|
|
3988
4265
|
if (!policy.replay)
|
|
3989
|
-
return { messages: [], dropped:
|
|
4266
|
+
return { messages: [], dropped: true };
|
|
3990
4267
|
const subject = chatSubject(this.space, "*", "*", channel);
|
|
3991
4268
|
let raw;
|
|
3992
4269
|
try {
|
|
@@ -3994,6 +4271,8 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
3994
4271
|
}
|
|
3995
4272
|
catch (e) {
|
|
3996
4273
|
this.emit("error", e);
|
|
4274
|
+
if (isPermissionDenied(e))
|
|
4275
|
+
throw e;
|
|
3997
4276
|
raw = [];
|
|
3998
4277
|
}
|
|
3999
4278
|
const collected = [];
|
|
@@ -4083,6 +4362,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
4083
4362
|
let hydrated;
|
|
4084
4363
|
this.presenceSnapshot = new Promise((resolve) => { hydrated = resolve; });
|
|
4085
4364
|
const iter = await this.kv.watch();
|
|
4365
|
+
this.presenceWatchIter = iter;
|
|
4086
4366
|
void (async () => {
|
|
4087
4367
|
let ready = false;
|
|
4088
4368
|
for await (const e of iter) {
|
|
@@ -4103,6 +4383,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
4103
4383
|
if (!this.channelKv)
|
|
4104
4384
|
return;
|
|
4105
4385
|
const iter = await this.channelKv.watch();
|
|
4386
|
+
this.channelWatchIter = iter;
|
|
4106
4387
|
void (async () => {
|
|
4107
4388
|
for await (const e of iter)
|
|
4108
4389
|
this.handleChannelEntry(e);
|
|
@@ -4134,6 +4415,7 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
4134
4415
|
}
|
|
4135
4416
|
}
|
|
4136
4417
|
handleKvEntry(e) {
|
|
4418
|
+
this.lastPresenceWatchAt = Date.now();
|
|
4137
4419
|
if (e.operation === "DEL" || e.operation === "PURGE") {
|
|
4138
4420
|
this.markOffline(e.key);
|
|
4139
4421
|
return;
|
|
@@ -4157,6 +4439,19 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
4157
4439
|
return;
|
|
4158
4440
|
const prev = this.roster.get(id);
|
|
4159
4441
|
const stale = Date.now() - raw.ts > this.ttlMs;
|
|
4442
|
+
// A watch recovering from a stall replays the bucket. Those PUTs still carry the publisher's
|
|
4443
|
+
// pre-stall timestamps, so they look TTL-expired even though the peers kept heartbeating on
|
|
4444
|
+
// the broker. Materializing them as offline is the empty-to-full flicker #1045 named: the
|
|
4445
|
+
// observer's catch-up, not N deaths. Keep last-known rows, stay view-stale, and wait for a
|
|
4446
|
+
// live timestamp (or an explicit offline/delete) before changing a known peer.
|
|
4447
|
+
if (stale && raw.status !== "offline") {
|
|
4448
|
+
if (prev)
|
|
4449
|
+
return;
|
|
4450
|
+
this.roster.set(id, this.toOffline(raw));
|
|
4451
|
+
this.emit("roster", this.getRoster());
|
|
4452
|
+
return;
|
|
4453
|
+
}
|
|
4454
|
+
this.setPresenceViewFresh(true);
|
|
4160
4455
|
// Any offline materialization (a stale snapshot OR a graceful-leave record) drops the advisory
|
|
4161
4456
|
// attention fields — an offline peer must not carry a stale `[focus]`/`locally muted` hint.
|
|
4162
4457
|
const p = stale || raw.status === "offline" ? this.toOffline(raw) : raw;
|
|
@@ -4210,8 +4505,22 @@ export class CotalEndpoint extends EventEmitter {
|
|
|
4210
4505
|
this.emit("presence", { type: "offline", presence: offline });
|
|
4211
4506
|
this.emit("roster", this.getRoster());
|
|
4212
4507
|
}
|
|
4508
|
+
setPresenceViewFresh(fresh) {
|
|
4509
|
+
if (fresh === this.presenceViewFresh)
|
|
4510
|
+
return;
|
|
4511
|
+
this.presenceViewFresh = fresh;
|
|
4512
|
+
this.emit("presence-view", this.presenceView());
|
|
4513
|
+
}
|
|
4213
4514
|
sweep() {
|
|
4214
4515
|
const now = Date.now();
|
|
4516
|
+
// Whole-bucket silence past TTL is THIS observer going deaf. Real rosters do not lose every
|
|
4517
|
+
// peer in one window, and treating that silence as N offline verdicts empties an online-only
|
|
4518
|
+
// sidebar with nothing saying the window went blind (#1045). Gate the per-peer age-out on
|
|
4519
|
+
// watch freshness; surface the view as stale instead.
|
|
4520
|
+
if (this.lastPresenceWatchAt !== 0 && now - this.lastPresenceWatchAt > this.ttlMs) {
|
|
4521
|
+
this.setPresenceViewFresh(false);
|
|
4522
|
+
return;
|
|
4523
|
+
}
|
|
4215
4524
|
let changed = false;
|
|
4216
4525
|
for (const [id, p] of this.roster) {
|
|
4217
4526
|
if (p.status !== "offline" && now - p.ts > this.ttlMs) {
|
|
@@ -4410,6 +4719,18 @@ export function isPermissionDenied(e) {
|
|
|
4410
4719
|
return true;
|
|
4411
4720
|
return /permissions?\s+violation/i.test(String(e?.message ?? ""));
|
|
4412
4721
|
}
|
|
4722
|
+
/** True only for the structured JetStream API absence codes named by the caller. A status 404 or
|
|
4723
|
+
* message regex is too broad here: the catch sites use absence to produce a successful empty/fresh
|
|
4724
|
+
* result, so a permission denial, timeout, or protocol failure must never pass as "not found". */
|
|
4725
|
+
function isJetStreamMissing(e, ...codes) {
|
|
4726
|
+
return e instanceof JetStreamApiError && codes.includes(e.code);
|
|
4727
|
+
}
|
|
4728
|
+
/** The signal's exact abort reason, or the platform-standard AbortError when none was supplied. */
|
|
4729
|
+
function abortReason(signal) {
|
|
4730
|
+
return signal?.reason instanceof Error
|
|
4731
|
+
? signal.reason
|
|
4732
|
+
: new DOMException("This operation was aborted", "AbortError");
|
|
4733
|
+
}
|
|
4413
4734
|
/** True ONLY for a denial on a **publish** — the single case that proves the message was never
|
|
4414
4735
|
* ACCEPTED or stored. (Not "never reached the server": the server necessarily received enough of
|
|
4415
4736
|
* it to reject it. The distinction matters precisely here, because this helper exists to separate
|