@opinionated-machine/sse-fallback 0.1.1 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +258 -4
- package/dist/binding.d.ts +13 -0
- package/dist/binding.d.ts.map +1 -1
- package/dist/binding.js +22 -0
- package/dist/binding.js.map +1 -1
- package/dist/bindingTypes.d.ts +57 -2
- package/dist/bindingTypes.d.ts.map +1 -1
- package/dist/bindingTypes.js +4 -1
- package/dist/bindingTypes.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/subscription.d.ts +144 -3
- package/dist/subscription.d.ts.map +1 -1
- package/dist/subscription.js +421 -42
- package/dist/subscription.js.map +1 -1
- package/dist/transport.d.ts +5 -1
- package/dist/transport.d.ts.map +1 -1
- package/dist/transport.js +5 -2
- package/dist/transport.js.map +1 -1
- package/package.json +3 -3
package/dist/subscription.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { createSSEStreamParser } from '@opinionated-machine/sse-parser';
|
|
2
|
+
import { validateSnapshotSource } from "./binding.js";
|
|
2
3
|
import { DEFAULT_POLICY } from "./bindingTypes.js";
|
|
3
4
|
import { Reconciler } from "./reconciler.js";
|
|
4
5
|
import { backoffDelay, ResettableTimer, sleep } from "./scheduler.js";
|
|
@@ -13,6 +14,7 @@ export class SubscriptionStoppedError extends Error {
|
|
|
13
14
|
status;
|
|
14
15
|
limit;
|
|
15
16
|
channel;
|
|
17
|
+
streamWasLive;
|
|
16
18
|
constructor(detail) {
|
|
17
19
|
super(`Subscription stopped (${detail.reason}) before the awaited event arrived`);
|
|
18
20
|
this.name = 'SubscriptionStoppedError';
|
|
@@ -20,8 +22,83 @@ export class SubscriptionStoppedError extends Error {
|
|
|
20
22
|
this.status = detail.status;
|
|
21
23
|
this.limit = detail.limit;
|
|
22
24
|
this.channel = detail.channel;
|
|
25
|
+
this.streamWasLive = detail.streamWasLive;
|
|
23
26
|
}
|
|
24
27
|
}
|
|
28
|
+
/**
|
|
29
|
+
* A request answered with a status the channel cannot use. Handed to
|
|
30
|
+
* `onStreamError` / `onPollError`, and as the `cause` of a
|
|
31
|
+
* {@link FallbackDegradedError}.
|
|
32
|
+
*/
|
|
33
|
+
export class FallbackHttpError extends Error {
|
|
34
|
+
channel;
|
|
35
|
+
status;
|
|
36
|
+
request;
|
|
37
|
+
constructor(message, detail) {
|
|
38
|
+
super(message);
|
|
39
|
+
this.name = 'FallbackHttpError';
|
|
40
|
+
this.channel = detail.channel;
|
|
41
|
+
this.status = detail.status;
|
|
42
|
+
this.request = detail.request;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* The stream stopped working for a reason the client can name, and the subscription is running
|
|
47
|
+
* without it: on its poll for an endpoint snapshot, or on nothing at all for a synthesized one.
|
|
48
|
+
*
|
|
49
|
+
* Delivered to `diagnostics.onDegraded` when the subscription degrades, and again every
|
|
50
|
+
* `degradationReportIntervalMs` for as long as it stays degraded, so a tab that lives on the
|
|
51
|
+
* fallback keeps showing up in an error tracker instead of reporting once and going quiet.
|
|
52
|
+
* Hand it to one as it is: the message names the route, the status and the channel left.
|
|
53
|
+
*/
|
|
54
|
+
export class FallbackDegradedError extends Error {
|
|
55
|
+
kind;
|
|
56
|
+
/** The refusing or rejecting status, for `'stream-refused'` and `'stream-rejected'`. */
|
|
57
|
+
status;
|
|
58
|
+
/** The stream request that failed. */
|
|
59
|
+
request;
|
|
60
|
+
/** Whether any stream of this subscription had carried bytes before it broke. */
|
|
61
|
+
streamWasLive;
|
|
62
|
+
/** Whether a poll keeps delivering while the stream is down. */
|
|
63
|
+
pollCarriesDelivery;
|
|
64
|
+
/** How many times this degradation has been reported, starting at 1. */
|
|
65
|
+
reportCount;
|
|
66
|
+
/** Time since the subscription degraded, 0 on the first report. */
|
|
67
|
+
degradedForMs;
|
|
68
|
+
constructor(detail) {
|
|
69
|
+
super(describeDegradation(detail), { cause: detail.cause });
|
|
70
|
+
this.name = 'FallbackDegradedError';
|
|
71
|
+
this.kind = detail.kind;
|
|
72
|
+
this.status = detail.status;
|
|
73
|
+
this.request = detail.request;
|
|
74
|
+
this.streamWasLive = detail.streamWasLive;
|
|
75
|
+
this.pollCarriesDelivery = detail.pollCarriesDelivery;
|
|
76
|
+
this.reportCount = detail.reportCount;
|
|
77
|
+
this.degradedForMs = detail.degradedForMs;
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
function describeDegradation(detail) {
|
|
81
|
+
const route = `${detail.request.method.toUpperCase()} ${detail.request.path}`;
|
|
82
|
+
const history = detail.streamWasLive ? 'it had been live' : 'it was never live';
|
|
83
|
+
const fault = {
|
|
84
|
+
'stream-refused': `was refused with ${detail.status}`,
|
|
85
|
+
'stream-rejected': detail.status === 200
|
|
86
|
+
? `answered 200 with content-type ${detail.contentType || '(none)'}`
|
|
87
|
+
: `was rejected with ${detail.status}`,
|
|
88
|
+
'stream-silent': 'was accepted but closed without carrying any bytes',
|
|
89
|
+
'stream-unreachable': 'could not be reached',
|
|
90
|
+
}[detail.kind];
|
|
91
|
+
const fallback = detail.pollCarriesDelivery
|
|
92
|
+
? 'updates arrive late, on the fallback poll'
|
|
93
|
+
: 'no poll covers this binding, so nothing is delivered until it recovers';
|
|
94
|
+
return `SSE stream ${route} ${fault} (${history}); ${fallback}`;
|
|
95
|
+
}
|
|
96
|
+
/**
|
|
97
|
+
* Consecutive versionless polls dropped for racing a pushed event, after
|
|
98
|
+
* which the next one is delivered anyway: a busy stream must not starve the
|
|
99
|
+
* poll that repairs an outage.
|
|
100
|
+
*/
|
|
101
|
+
const MAX_SUPERSEDED_POLLS = 3;
|
|
25
102
|
class ResilientSubscriptionImpl {
|
|
26
103
|
binding;
|
|
27
104
|
transport;
|
|
@@ -33,8 +110,18 @@ class ResilientSubscriptionImpl {
|
|
|
33
110
|
reconciler;
|
|
34
111
|
pollGate;
|
|
35
112
|
onAuthChallenge;
|
|
113
|
+
/**
|
|
114
|
+
* Whether a poll can deliver anything. False for a synthesized snapshot,
|
|
115
|
+
* which answers without asking the server: every poll would be a request
|
|
116
|
+
* for news that only the stream carries.
|
|
117
|
+
*/
|
|
118
|
+
pollCanDeliver;
|
|
119
|
+
/** {@link FallbackPolicy.streamRefusal}, with `'auto'` resolved. */
|
|
120
|
+
streamRefusal;
|
|
36
121
|
abortController = new AbortController();
|
|
37
122
|
currentStreamAbort;
|
|
123
|
+
/** Wakes the stream loop out of its reconnect backoff, while it sleeps one. */
|
|
124
|
+
reconnectBackoffWake;
|
|
38
125
|
statusValue = 'connecting';
|
|
39
126
|
stopped = false;
|
|
40
127
|
stopDetail;
|
|
@@ -50,27 +137,51 @@ class ResilientSubscriptionImpl {
|
|
|
50
137
|
degraded = false;
|
|
51
138
|
/** Whether the current stream has produced any bytes at all. */
|
|
52
139
|
streamProducedBytes = false;
|
|
140
|
+
/** Whether ANY stream of this subscription ever produced bytes. */
|
|
141
|
+
streamEverProducedBytes = false;
|
|
142
|
+
/** Whether the stream has been given up for good, see `abandonStream`. */
|
|
143
|
+
streamAbandonedValue = false;
|
|
144
|
+
/** The degradation last reported through `onDegraded`, until the stream recovers. */
|
|
145
|
+
reportedDegradation;
|
|
146
|
+
/** Events the stream delivered, so a versionless poll can tell it was overtaken. */
|
|
147
|
+
streamDeliveries = 0;
|
|
148
|
+
supersededPolls = 0;
|
|
53
149
|
pollInFlight = false;
|
|
54
150
|
pollQueued = false;
|
|
55
151
|
pollFailures = 0;
|
|
56
152
|
idlePolls = 0;
|
|
57
153
|
/** Polls attempted, for `subscriptionBudget.maxPolls`. */
|
|
58
154
|
pollsAttempted = 0;
|
|
59
|
-
/**
|
|
60
|
-
|
|
155
|
+
/**
|
|
156
|
+
* The one auth retry per failure streak: the channel that spent it once
|
|
157
|
+
* `onAuthChallenge` ran, `'declined'` once it said it could not recover.
|
|
158
|
+
* Both hold until a request succeeds.
|
|
159
|
+
*/
|
|
160
|
+
authCredit = 'available';
|
|
61
161
|
deadman = new ResettableTimer(() => this.schedulePoll());
|
|
62
162
|
staleConnection = new ResettableTimer(() => this.onStaleConnection());
|
|
63
163
|
budgetTimer = new ResettableTimer(() => this.stopWith({ reason: 'budget-exhausted', limit: 'maxDurationMs' }));
|
|
164
|
+
degradationReminder = new ResettableTimer(() => this.emitDegradation());
|
|
64
165
|
eventListeners = new Set();
|
|
65
166
|
stateListeners = new Set();
|
|
66
167
|
statusListeners = new Set();
|
|
67
168
|
stopListeners = new Set();
|
|
169
|
+
establishedListeners = new Set();
|
|
68
170
|
iteratorFeeds = new Set();
|
|
69
171
|
constructor(binding, options) {
|
|
70
172
|
this.binding = binding;
|
|
71
173
|
this.transport = options.transport;
|
|
72
174
|
this.params = options.params ?? {};
|
|
73
175
|
this.policy = { ...DEFAULT_POLICY, ...binding.config.policy, ...options.policy };
|
|
176
|
+
validateSnapshotSource(binding.config);
|
|
177
|
+
assertPolicySuitsBinding(binding.config.snapshotSource, this.policy);
|
|
178
|
+
this.pollCanDeliver = binding.config.snapshotSource === 'endpoint';
|
|
179
|
+
this.streamRefusal =
|
|
180
|
+
this.policy.streamRefusal === 'auto'
|
|
181
|
+
? this.pollCanDeliver
|
|
182
|
+
? 'keep-polling'
|
|
183
|
+
: 'stop'
|
|
184
|
+
: this.policy.streamRefusal;
|
|
74
185
|
this.diagnostics = options.diagnostics ?? {};
|
|
75
186
|
this.random = options.random ?? Math.random;
|
|
76
187
|
this.parseEventData = options.parseEventData ?? JSON.parse;
|
|
@@ -101,14 +212,7 @@ class ResilientSubscriptionImpl {
|
|
|
101
212
|
if (this.policy.mode === 'poll-only') {
|
|
102
213
|
// No stream is ever opened, so the machine starts where a dual-mode
|
|
103
214
|
// subscription only lands after repeated connect failures.
|
|
104
|
-
this.
|
|
105
|
-
this.setStatus('polling');
|
|
106
|
-
if (this.policy.initialPoll === 'eager') {
|
|
107
|
-
this.schedulePoll();
|
|
108
|
-
}
|
|
109
|
-
else {
|
|
110
|
-
this.armDeadman();
|
|
111
|
-
}
|
|
215
|
+
this.degrade({ forcePoll: this.policy.initialPoll === 'eager' });
|
|
112
216
|
return;
|
|
113
217
|
}
|
|
114
218
|
void this.runStreamLoop();
|
|
@@ -123,6 +227,9 @@ class ResilientSubscriptionImpl {
|
|
|
123
227
|
get status() {
|
|
124
228
|
return this.statusValue;
|
|
125
229
|
}
|
|
230
|
+
get streamAbandoned() {
|
|
231
|
+
return this.streamAbandonedValue;
|
|
232
|
+
}
|
|
126
233
|
get result() {
|
|
127
234
|
return this.stopDetail;
|
|
128
235
|
}
|
|
@@ -141,6 +248,10 @@ class ResilientSubscriptionImpl {
|
|
|
141
248
|
this.statusListeners.add(listener);
|
|
142
249
|
return () => this.statusListeners.delete(listener);
|
|
143
250
|
}
|
|
251
|
+
onStreamEstablished(listener) {
|
|
252
|
+
this.establishedListeners.add(listener);
|
|
253
|
+
return () => this.establishedListeners.delete(listener);
|
|
254
|
+
}
|
|
144
255
|
onStop(listener) {
|
|
145
256
|
if (this.stopDetail !== undefined) {
|
|
146
257
|
this.runListener(listener, this.stopDetail);
|
|
@@ -202,12 +313,16 @@ class ResilientSubscriptionImpl {
|
|
|
202
313
|
nudge() {
|
|
203
314
|
if (this.stopped)
|
|
204
315
|
return;
|
|
205
|
-
this.
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
// Byte-activity watchdog stays authoritative; nothing else to do here.
|
|
316
|
+
if (this.pollCanDeliver) {
|
|
317
|
+
this.schedulePoll();
|
|
318
|
+
// The byte-activity watchdog stays authoritative for the stream.
|
|
209
319
|
return;
|
|
210
320
|
}
|
|
321
|
+
// No poll to force: a fresh stream is the only repair there is, so
|
|
322
|
+
// reconnect now rather than wait out the backoff. A connect in flight or
|
|
323
|
+
// an open stream is not cut: aborting either would count as a connect
|
|
324
|
+
// failure and add a backoff, slowing down the recovery it was asked for.
|
|
325
|
+
this.reconnectBackoffWake?.abort();
|
|
211
326
|
}
|
|
212
327
|
stop() {
|
|
213
328
|
this.stopWith({ reason: 'manual' });
|
|
@@ -220,6 +335,7 @@ class ResilientSubscriptionImpl {
|
|
|
220
335
|
this.deadman.clear();
|
|
221
336
|
this.staleConnection.clear();
|
|
222
337
|
this.budgetTimer.clear();
|
|
338
|
+
this.degradationReminder.clear();
|
|
223
339
|
this.currentStreamAbort?.abort();
|
|
224
340
|
this.abortController.abort();
|
|
225
341
|
this.setStatus('stopped', detail);
|
|
@@ -229,6 +345,7 @@ class ResilientSubscriptionImpl {
|
|
|
229
345
|
for (const feed of this.iteratorFeeds)
|
|
230
346
|
feed.finish();
|
|
231
347
|
this.iteratorFeeds.clear();
|
|
348
|
+
this.establishedListeners.clear();
|
|
232
349
|
}
|
|
233
350
|
// --------------------------------------------------------------------
|
|
234
351
|
// Stream loop
|
|
@@ -236,13 +353,16 @@ class ResilientSubscriptionImpl {
|
|
|
236
353
|
// biome-ignore lint/complexity/noExcessiveCognitiveComplexity: the connect/consume/reconnect loop is a single state machine — splitting it would obscure the transitions
|
|
237
354
|
async runStreamLoop() {
|
|
238
355
|
let firstConnect = true;
|
|
239
|
-
while (!this.stopped) {
|
|
356
|
+
while (!this.stopped && !this.streamAbandoned) {
|
|
240
357
|
const streamAbort = new AbortController();
|
|
241
358
|
this.currentStreamAbort = streamAbort;
|
|
242
359
|
const onMasterAbort = () => streamAbort.abort();
|
|
243
360
|
this.abortController.signal.addEventListener('abort', onMasterAbort, { once: true });
|
|
244
361
|
let connectFailed = false;
|
|
362
|
+
let failure;
|
|
363
|
+
let credentialsRefreshed = false;
|
|
245
364
|
this.streamProducedBytes = false;
|
|
365
|
+
const request = this.binding.buildStreamRequest(this.params);
|
|
246
366
|
// A connect that never produces headers must not park the subscription:
|
|
247
367
|
// bound it here so it fails like any other connect failure (backoff,
|
|
248
368
|
// degradation, fallback polling) instead of hanging forever.
|
|
@@ -251,7 +371,10 @@ class ResilientSubscriptionImpl {
|
|
|
251
371
|
connectTimeout.arm(this.policy.connectTimeoutMs);
|
|
252
372
|
}
|
|
253
373
|
try {
|
|
254
|
-
const response = await this.transport.openStream(
|
|
374
|
+
const response = await this.transport.openStream(request, {
|
|
375
|
+
signal: streamAbort.signal,
|
|
376
|
+
lastEventId: this.lastEventId,
|
|
377
|
+
});
|
|
255
378
|
connectTimeout.clear();
|
|
256
379
|
if (this.stopped)
|
|
257
380
|
return;
|
|
@@ -263,7 +386,17 @@ class ResilientSubscriptionImpl {
|
|
|
263
386
|
// chunks of a refused connect are never consumed, and without this
|
|
264
387
|
// every retry would leak a socket.
|
|
265
388
|
streamAbort.abort();
|
|
266
|
-
|
|
389
|
+
const error = new FallbackHttpError(response.status === 200
|
|
390
|
+
? `SSE connect answered 200 with content-type ${contentType}`
|
|
391
|
+
: `SSE connect failed with status ${response.status}`, { channel: 'stream', status: response.status, request });
|
|
392
|
+
failure = {
|
|
393
|
+
kind: 'stream-rejected',
|
|
394
|
+
status: response.status,
|
|
395
|
+
contentType,
|
|
396
|
+
request,
|
|
397
|
+
cause: error,
|
|
398
|
+
};
|
|
399
|
+
this.diagnostics.onStreamError?.(error);
|
|
267
400
|
if (this.policy.unretryableStatuses.includes(response.status)) {
|
|
268
401
|
// An expired token is the common case behind a 401 in a SPA, and
|
|
269
402
|
// recovering without a page reload is the point of this package.
|
|
@@ -273,21 +406,39 @@ class ResilientSubscriptionImpl {
|
|
|
273
406
|
if (this.stopped)
|
|
274
407
|
return;
|
|
275
408
|
if (!recovered) {
|
|
409
|
+
if (this.streamRefusal === 'keep-polling') {
|
|
410
|
+
this.abandonStream({
|
|
411
|
+
kind: 'stream-refused',
|
|
412
|
+
status: response.status,
|
|
413
|
+
request,
|
|
414
|
+
cause: error,
|
|
415
|
+
});
|
|
416
|
+
return;
|
|
417
|
+
}
|
|
276
418
|
this.stopWith({
|
|
277
419
|
reason: 'unretryable-status',
|
|
278
420
|
status: response.status,
|
|
279
421
|
channel: 'stream',
|
|
422
|
+
streamWasLive: this.streamEverProducedBytes,
|
|
280
423
|
});
|
|
281
424
|
return;
|
|
282
425
|
}
|
|
426
|
+
// A refreshed token says nothing about the stream's health, so
|
|
427
|
+
// the retry skips the failure count and the backoff, as a poll does.
|
|
428
|
+
credentialsRefreshed = true;
|
|
283
429
|
}
|
|
284
430
|
}
|
|
285
431
|
else {
|
|
286
432
|
this.serverRetryHintMs = undefined;
|
|
287
433
|
if (firstConnect) {
|
|
288
|
-
|
|
434
|
+
// A synthesized snapshot has nothing to hydrate from, so it takes
|
|
435
|
+
// the second branch: quiet is the normal state for a stream whose
|
|
436
|
+
// events are rare, and holding it out of 'live' until a byte
|
|
437
|
+
// arrives would mark a healthy surface as broken for hours. A
|
|
438
|
+
// connection that is open and dead is the stale watchdog's job.
|
|
439
|
+
if (this.pollCanDeliver && this.policy.initialPoll === 'eager') {
|
|
289
440
|
// Subscribe-first hydration: buffer live events until the
|
|
290
|
-
// snapshot lands
|
|
441
|
+
// snapshot lands, a zero missed-event window.
|
|
291
442
|
this.reconciler.beginHydration();
|
|
292
443
|
this.schedulePoll();
|
|
293
444
|
}
|
|
@@ -297,8 +448,11 @@ class ResilientSubscriptionImpl {
|
|
|
297
448
|
}
|
|
298
449
|
else {
|
|
299
450
|
// While degraded, an accepted connect is not evidence of anything:
|
|
300
|
-
// only bytes
|
|
301
|
-
|
|
451
|
+
// only bytes move the status back out of 'polling'. A synthesized
|
|
452
|
+
// snapshot has no 'polling' to hold, and holding it in
|
|
453
|
+
// 'reconnecting' would mark a quiet, healthy stream as broken for
|
|
454
|
+
// as long as its events are rare, the same case as a first connect.
|
|
455
|
+
if (!this.degraded || !this.pollCanDeliver) {
|
|
302
456
|
this.setStatus('live');
|
|
303
457
|
}
|
|
304
458
|
if ((this.binding.config.replay ?? 'untrusted') === 'untrusted') {
|
|
@@ -321,6 +475,11 @@ class ResilientSubscriptionImpl {
|
|
|
321
475
|
if (this.stopped)
|
|
322
476
|
return;
|
|
323
477
|
connectFailed = !this.streamConnected;
|
|
478
|
+
failure = {
|
|
479
|
+
kind: connectFailed ? 'stream-unreachable' : 'stream-silent',
|
|
480
|
+
request,
|
|
481
|
+
cause: error,
|
|
482
|
+
};
|
|
324
483
|
this.diagnostics.onStreamError?.(error);
|
|
325
484
|
}
|
|
326
485
|
finally {
|
|
@@ -329,25 +488,38 @@ class ResilientSubscriptionImpl {
|
|
|
329
488
|
this.streamConnected = false;
|
|
330
489
|
this.abortController.signal.removeEventListener('abort', onMasterAbort);
|
|
331
490
|
}
|
|
332
|
-
if (this.stopped || this.reconciler.isTerminated)
|
|
491
|
+
if (this.stopped || this.streamAbandoned || this.reconciler.isTerminated)
|
|
333
492
|
return;
|
|
493
|
+
if (credentialsRefreshed)
|
|
494
|
+
continue;
|
|
495
|
+
// Armed before any status is published below, so a listener that
|
|
496
|
+
// nudges on 'reconnecting' cuts this backoff short.
|
|
497
|
+
const backoffWake = new AbortController();
|
|
498
|
+
this.reconnectBackoffWake = backoffWake;
|
|
334
499
|
// A connection only counts as successful once it has actually carried
|
|
335
500
|
// bytes. A stream that is accepted and then closes immediately would
|
|
336
501
|
// otherwise reset the backoff on every attempt and never degrade,
|
|
337
502
|
// turning a broken upstream into a reconnect-and-poll storm.
|
|
503
|
+
let countedFailure;
|
|
338
504
|
if (connectFailed || !this.streamProducedBytes) {
|
|
339
505
|
this.consecutiveConnectFailures += 1;
|
|
506
|
+
countedFailure = failure ?? { kind: 'stream-silent', request };
|
|
507
|
+
// Reminders name the cause as it stands now, not the one that degraded it.
|
|
508
|
+
if (this.reportedDegradation)
|
|
509
|
+
this.reportedDegradation.failure = countedFailure;
|
|
340
510
|
}
|
|
341
|
-
if (this.consecutiveConnectFailures >= this.policy.degradedAfterFailures) {
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
this.
|
|
347
|
-
}
|
|
511
|
+
if (!this.degraded && this.consecutiveConnectFailures >= this.policy.degradedAfterFailures) {
|
|
512
|
+
// The poll this drop needs is scheduled below; the deadman carries
|
|
513
|
+
// the degraded cadence from its next arm.
|
|
514
|
+
this.degrade({ forcePoll: false });
|
|
515
|
+
if (countedFailure)
|
|
516
|
+
this.reportDegradation(countedFailure);
|
|
348
517
|
}
|
|
349
518
|
else {
|
|
350
|
-
|
|
519
|
+
// Already degraded, a drop keeps an endpoint binding on 'polling'. A
|
|
520
|
+
// synthesized one may have reported 'live' off a quiet reconnect, and
|
|
521
|
+
// is back to having nothing that covers the outage.
|
|
522
|
+
this.setStatus(this.degraded && this.pollCanDeliver ? 'polling' : 'reconnecting');
|
|
351
523
|
}
|
|
352
524
|
// The stream just dropped — data may have been lost; poll now.
|
|
353
525
|
this.schedulePoll();
|
|
@@ -357,10 +529,116 @@ class ResilientSubscriptionImpl {
|
|
|
357
529
|
const delay = this.serverRetryHintMs !== undefined && !connectFailed
|
|
358
530
|
? this.serverRetryHintMs
|
|
359
531
|
: backoffDelay(backoffConfig, this.consecutiveConnectFailures, this.random);
|
|
360
|
-
|
|
361
|
-
if (
|
|
532
|
+
await this.waitOutReconnectBackoff(delay, backoffWake);
|
|
533
|
+
if (this.stopped)
|
|
534
|
+
return;
|
|
535
|
+
}
|
|
536
|
+
}
|
|
537
|
+
/**
|
|
538
|
+
* Sleep out the reconnect backoff, cut short by `stop()` or by `nudge()`.
|
|
539
|
+
* A nudge ends the wait without stopping anything, so the caller tells the
|
|
540
|
+
* two apart by `stopped`.
|
|
541
|
+
*/
|
|
542
|
+
async waitOutReconnectBackoff(delayMs, wake) {
|
|
543
|
+
const onMasterAbort = () => wake.abort();
|
|
544
|
+
this.abortController.signal.addEventListener('abort', onMasterAbort, { once: true });
|
|
545
|
+
try {
|
|
546
|
+
// A status listener run on the way here may already have stopped it.
|
|
547
|
+
if (this.stopped)
|
|
362
548
|
return;
|
|
549
|
+
await sleep(delayMs, wake.signal);
|
|
363
550
|
}
|
|
551
|
+
finally {
|
|
552
|
+
this.reconnectBackoffWake = undefined;
|
|
553
|
+
this.abortController.signal.removeEventListener('abort', onMasterAbort);
|
|
554
|
+
}
|
|
555
|
+
}
|
|
556
|
+
/**
|
|
557
|
+
* Give the stream up for the life of the subscription and let the poll
|
|
558
|
+
* carry it, after a refusal no reconnect can get past.
|
|
559
|
+
*
|
|
560
|
+
* One-way, and held in `streamAbandonedValue` rather than in the caller's
|
|
561
|
+
* `return`: the connect loop tests it, so neither a later edit nor a
|
|
562
|
+
* throwing hook can put the subscription back on a request the server has
|
|
563
|
+
* already refused. Re-probing on a timer would win the latency back, at the
|
|
564
|
+
* cost of turning one refusal into an unbounded retry.
|
|
565
|
+
*
|
|
566
|
+
* The poll is forced rather than left to the deadman: under
|
|
567
|
+
* `initialPoll: 'eager'` the hydration poll is scheduled by an accepted
|
|
568
|
+
* connect, so a first connect that is refused leaves nothing armed.
|
|
569
|
+
*/
|
|
570
|
+
abandonStream(refusal) {
|
|
571
|
+
const { status } = refusal;
|
|
572
|
+
this.streamAbandonedValue = true;
|
|
573
|
+
// A retry the STREAM spent on a refresh that worked, only to be refused
|
|
574
|
+
// anyway, went to a channel that no longer exists. The poll now carries
|
|
575
|
+
// the subscription alone, so it gets the credit back instead of dying on
|
|
576
|
+
// a refusal the application was never asked about. A hook that declined
|
|
577
|
+
// keeps its answer: the poll carries the same credentials, and asking
|
|
578
|
+
// again would mean a second refresh or login prompt for one outage. A
|
|
579
|
+
// credit the poll spent stays spent: its own retry is what it bought.
|
|
580
|
+
if (this.authCredit === 'spent-by-stream')
|
|
581
|
+
this.authCredit = 'available';
|
|
582
|
+
const streamWasLive = this.streamEverProducedBytes;
|
|
583
|
+
this.runListener(() => this.diagnostics.onStreamRefused?.({ status, streamWasLive }), undefined);
|
|
584
|
+
// The hook may have stopped the subscription; a stopped one must not be
|
|
585
|
+
// put back into 'polling'.
|
|
586
|
+
if (this.stopped || this.reconciler.isTerminated)
|
|
587
|
+
return;
|
|
588
|
+
this.degrade({ forcePoll: true });
|
|
589
|
+
this.reportDegradation(refusal);
|
|
590
|
+
}
|
|
591
|
+
/**
|
|
592
|
+
* Report that the stream broke and the subscription runs without it, then
|
|
593
|
+
* keep reporting on `degradationReportIntervalMs` until it recovers.
|
|
594
|
+
*
|
|
595
|
+
* One report per degradation would not be enough: a tab that degraded an
|
|
596
|
+
* hour ago looks, in an error tracker, like a fault that went away. A new
|
|
597
|
+
* cause (a refusal after a run of silent connects) is reported at once.
|
|
598
|
+
*/
|
|
599
|
+
reportDegradation(failure) {
|
|
600
|
+
if (this.stopped || this.reconciler.isTerminated)
|
|
601
|
+
return;
|
|
602
|
+
const current = this.reportedDegradation;
|
|
603
|
+
this.reportedDegradation = {
|
|
604
|
+
failure,
|
|
605
|
+
streamWasLive: this.streamEverProducedBytes,
|
|
606
|
+
since: current?.since ?? Date.now(),
|
|
607
|
+
reports: current?.reports ?? 0,
|
|
608
|
+
};
|
|
609
|
+
this.emitDegradation();
|
|
610
|
+
}
|
|
611
|
+
emitDegradation() {
|
|
612
|
+
const degradation = this.reportedDegradation;
|
|
613
|
+
if (!degradation || this.stopped || this.reconciler.isTerminated)
|
|
614
|
+
return;
|
|
615
|
+
degradation.reports += 1;
|
|
616
|
+
const { failure } = degradation;
|
|
617
|
+
const error = new FallbackDegradedError({
|
|
618
|
+
...failure,
|
|
619
|
+
streamWasLive: degradation.streamWasLive,
|
|
620
|
+
pollCarriesDelivery: this.pollCanDeliver,
|
|
621
|
+
reportCount: degradation.reports,
|
|
622
|
+
degradedForMs: Date.now() - degradation.since,
|
|
623
|
+
});
|
|
624
|
+
this.runListener(() => this.diagnostics.onDegraded?.(error), undefined);
|
|
625
|
+
if (this.stopped)
|
|
626
|
+
return;
|
|
627
|
+
const interval = this.policy.degradationReportIntervalMs;
|
|
628
|
+
if (interval !== 'off')
|
|
629
|
+
this.degradationReminder.arm(interval);
|
|
630
|
+
}
|
|
631
|
+
endDegradation() {
|
|
632
|
+
const degradation = this.reportedDegradation;
|
|
633
|
+
if (!degradation)
|
|
634
|
+
return;
|
|
635
|
+
this.reportedDegradation = undefined;
|
|
636
|
+
this.degradationReminder.clear();
|
|
637
|
+
const recovery = {
|
|
638
|
+
kind: degradation.failure.kind,
|
|
639
|
+
degradedForMs: Date.now() - degradation.since,
|
|
640
|
+
};
|
|
641
|
+
this.runListener(() => this.diagnostics.onRecovered?.(recovery), undefined);
|
|
364
642
|
}
|
|
365
643
|
consumeStream(response) {
|
|
366
644
|
this.streamConnected = true;
|
|
@@ -426,12 +704,17 @@ class ResilientSubscriptionImpl {
|
|
|
426
704
|
if (this.streamProducedBytes)
|
|
427
705
|
return;
|
|
428
706
|
this.streamProducedBytes = true;
|
|
707
|
+
this.streamEverProducedBytes = true;
|
|
429
708
|
// The stream is demonstrably working — clear the failure history and
|
|
430
709
|
// leave degraded mode. Doing this here rather than at connect keeps a
|
|
431
710
|
// connect-then-close loop counted as the failure it is.
|
|
432
711
|
this.consecutiveConnectFailures = 0;
|
|
433
|
-
this.
|
|
712
|
+
this.authCredit = 'available';
|
|
434
713
|
this.degraded = false;
|
|
714
|
+
this.endDegradation();
|
|
715
|
+
// `onRecovered` may have stopped the subscription, which must stay 'stopped'.
|
|
716
|
+
if (this.stopped)
|
|
717
|
+
return;
|
|
435
718
|
// Bytes on the wire are the only evidence that delivery works, so they are
|
|
436
719
|
// what promotes the subscription to 'live' — from degraded polling, and
|
|
437
720
|
// equally from the 'connecting' a byte-less stream is parked in after
|
|
@@ -441,6 +724,11 @@ class ResilientSubscriptionImpl {
|
|
|
441
724
|
if (!this.reconciler.isHydrating) {
|
|
442
725
|
this.setStatus('live');
|
|
443
726
|
}
|
|
727
|
+
for (const listener of this.establishedListeners) {
|
|
728
|
+
if (this.stopped)
|
|
729
|
+
return;
|
|
730
|
+
this.runListener(listener, undefined);
|
|
731
|
+
}
|
|
444
732
|
}
|
|
445
733
|
/**
|
|
446
734
|
* Move the reconnect cursor to where this frame leaves it.
|
|
@@ -499,8 +787,10 @@ class ResilientSubscriptionImpl {
|
|
|
499
787
|
// every 15s polled between nearly every pair of events, forever, while a
|
|
500
788
|
// fully idle one backed off to `deadmanIdleBackoff.maxMs`. A poll that
|
|
501
789
|
// does find news still resets it, in `executePoll`.
|
|
502
|
-
if (outcome.deliveries.length > 0)
|
|
790
|
+
if (outcome.deliveries.length > 0) {
|
|
791
|
+
this.streamDeliveries += 1;
|
|
503
792
|
this.armDeadman();
|
|
793
|
+
}
|
|
504
794
|
if (outcome.duplicate) {
|
|
505
795
|
this.diagnostics.onDuplicate?.(parsed.event ?? 'message');
|
|
506
796
|
}
|
|
@@ -555,7 +845,39 @@ class ResilientSubscriptionImpl {
|
|
|
555
845
|
// --------------------------------------------------------------------
|
|
556
846
|
// Polling
|
|
557
847
|
// --------------------------------------------------------------------
|
|
848
|
+
/**
|
|
849
|
+
* Enter the degraded state, from wherever the subscription came:
|
|
850
|
+
* `mode: 'poll-only'` at construction, enough consecutive connect failures,
|
|
851
|
+
* or a stream given up on a refusal. It caps the reconnect backoff and
|
|
852
|
+
* stops an accepted-but-silent connect from claiming `'live'`.
|
|
853
|
+
*
|
|
854
|
+
* With a synthesized snapshot it stops there. `'polling'` would name a
|
|
855
|
+
* channel that delivers nothing, so the status stays `'reconnecting'`: the
|
|
856
|
+
* stream being down IS the outage, and nothing covers it.
|
|
857
|
+
*
|
|
858
|
+
* `forcePoll` polls now instead of waiting for the deadman, which the
|
|
859
|
+
* callers with nothing else armed depend on.
|
|
860
|
+
*/
|
|
861
|
+
degrade(opts) {
|
|
862
|
+
this.degraded = true;
|
|
863
|
+
if (!this.pollCanDeliver) {
|
|
864
|
+
this.setStatus('reconnecting');
|
|
865
|
+
return;
|
|
866
|
+
}
|
|
867
|
+
this.setStatus('polling');
|
|
868
|
+
if (opts.forcePoll) {
|
|
869
|
+
this.schedulePoll();
|
|
870
|
+
}
|
|
871
|
+
else {
|
|
872
|
+
this.armDeadman();
|
|
873
|
+
}
|
|
874
|
+
}
|
|
558
875
|
schedulePoll() {
|
|
876
|
+
// A synthesized snapshot has no news in it: every caller here (hydration,
|
|
877
|
+
// the deadman, a gap repair, a dropped stream, `nudge`) is asking for
|
|
878
|
+
// reconciliation that only the stream can provide.
|
|
879
|
+
if (!this.pollCanDeliver)
|
|
880
|
+
return;
|
|
559
881
|
if (this.stopped || this.reconciler.isTerminated)
|
|
560
882
|
return;
|
|
561
883
|
if (this.pollInFlight) {
|
|
@@ -594,11 +916,17 @@ class ResilientSubscriptionImpl {
|
|
|
594
916
|
if (this.stopped || this.reconciler.isTerminated)
|
|
595
917
|
return;
|
|
596
918
|
}
|
|
597
|
-
const
|
|
919
|
+
const request = this.binding.buildSnapshotRequest(this.params);
|
|
920
|
+
const streamDeliveriesAtRequest = this.streamDeliveries;
|
|
921
|
+
const response = await this.transport.fetchSnapshot(request, { signal: pollAbort.signal });
|
|
598
922
|
if (this.stopped || this.reconciler.isTerminated)
|
|
599
923
|
return;
|
|
600
924
|
if (response.status < 200 || response.status >= 300) {
|
|
601
|
-
this.diagnostics.onPollError?.(new
|
|
925
|
+
this.diagnostics.onPollError?.(new FallbackHttpError(`Snapshot poll failed with status ${response.status}`, {
|
|
926
|
+
channel: 'poll',
|
|
927
|
+
status: response.status,
|
|
928
|
+
request,
|
|
929
|
+
}));
|
|
602
930
|
if (this.policy.unretryableStatuses.includes(response.status)) {
|
|
603
931
|
const recovered = await this.tryAuthChallenge(response.status, 'poll');
|
|
604
932
|
if (this.stopped || this.reconciler.isTerminated)
|
|
@@ -620,7 +948,14 @@ class ResilientSubscriptionImpl {
|
|
|
620
948
|
return;
|
|
621
949
|
}
|
|
622
950
|
this.pollFailures = 0;
|
|
623
|
-
this.
|
|
951
|
+
this.authCredit = 'available';
|
|
952
|
+
if (this.isSupersededByStream(streamDeliveriesAtRequest)) {
|
|
953
|
+
// Asked again rather than dropped: this may be the poll repairing an
|
|
954
|
+
// outage, and a fresh one reads a state at least as new as the event.
|
|
955
|
+
this.diagnostics.onStaleSnapshot?.();
|
|
956
|
+
this.pollQueued = true;
|
|
957
|
+
return;
|
|
958
|
+
}
|
|
624
959
|
const outcome = this.reconciler.handleSnapshot(response.body);
|
|
625
960
|
if (outcome.stale) {
|
|
626
961
|
this.diagnostics.onStaleSnapshot?.();
|
|
@@ -671,6 +1006,25 @@ class ResilientSubscriptionImpl {
|
|
|
671
1006
|
}
|
|
672
1007
|
}
|
|
673
1008
|
}
|
|
1009
|
+
/**
|
|
1010
|
+
* Whether a versionless snapshot was overtaken by the stream while it was
|
|
1011
|
+
* in flight. Its body may predate the pushed event, and with no version to
|
|
1012
|
+
* compare, delivering it would overwrite newer news with older.
|
|
1013
|
+
*/
|
|
1014
|
+
isSupersededByStream(streamDeliveriesAtRequest) {
|
|
1015
|
+
if (this.binding.config.version !== 'none')
|
|
1016
|
+
return false;
|
|
1017
|
+
if (this.streamDeliveries === streamDeliveriesAtRequest) {
|
|
1018
|
+
this.supersededPolls = 0;
|
|
1019
|
+
return false;
|
|
1020
|
+
}
|
|
1021
|
+
if (this.supersededPolls >= MAX_SUPERSEDED_POLLS) {
|
|
1022
|
+
this.supersededPolls = 0;
|
|
1023
|
+
return false;
|
|
1024
|
+
}
|
|
1025
|
+
this.supersededPolls += 1;
|
|
1026
|
+
return true;
|
|
1027
|
+
}
|
|
674
1028
|
/**
|
|
675
1029
|
* Offer an auth refusal to the application once, so an expired token can be
|
|
676
1030
|
* refreshed instead of killing the subscription.
|
|
@@ -693,17 +1047,22 @@ class ResilientSubscriptionImpl {
|
|
|
693
1047
|
const inFlight = this.authRefresh;
|
|
694
1048
|
if (inFlight)
|
|
695
1049
|
return await inFlight;
|
|
696
|
-
if (this.
|
|
1050
|
+
if (this.authCredit !== 'available')
|
|
697
1051
|
return false;
|
|
698
|
-
|
|
1052
|
+
const spent = channel === 'stream' ? 'spent-by-stream' : 'spent-by-poll';
|
|
1053
|
+
this.authCredit = spent;
|
|
699
1054
|
const refresh = (async () => {
|
|
1055
|
+
let recovered = false;
|
|
700
1056
|
try {
|
|
701
|
-
|
|
1057
|
+
recovered = (await onAuthChallenge({ status, channel })) === true;
|
|
702
1058
|
}
|
|
703
1059
|
catch (error) {
|
|
704
1060
|
this.diagnostics.onListenerError?.(error);
|
|
705
|
-
return false;
|
|
706
1061
|
}
|
|
1062
|
+
// A request that succeeded meanwhile already started a new streak.
|
|
1063
|
+
if (!recovered && this.authCredit === spent)
|
|
1064
|
+
this.authCredit = 'declined';
|
|
1065
|
+
return recovered;
|
|
707
1066
|
})();
|
|
708
1067
|
this.authRefresh = refresh;
|
|
709
1068
|
try {
|
|
@@ -711,7 +1070,7 @@ class ResilientSubscriptionImpl {
|
|
|
711
1070
|
}
|
|
712
1071
|
finally {
|
|
713
1072
|
// Only a refusal AFTER this refresh completed counts as the second
|
|
714
|
-
// failure, which is what `
|
|
1073
|
+
// failure, which is what `authCredit` now gates.
|
|
715
1074
|
if (this.authRefresh === refresh)
|
|
716
1075
|
this.authRefresh = undefined;
|
|
717
1076
|
}
|
|
@@ -758,6 +1117,8 @@ class ResilientSubscriptionImpl {
|
|
|
758
1117
|
return this.streamConnected ? 'connecting' : 'reconnecting';
|
|
759
1118
|
}
|
|
760
1119
|
armDeadman() {
|
|
1120
|
+
if (!this.pollCanDeliver)
|
|
1121
|
+
return;
|
|
761
1122
|
if (this.stopped || this.reconciler.isTerminated)
|
|
762
1123
|
return;
|
|
763
1124
|
if (this.degraded) {
|
|
@@ -849,6 +1210,20 @@ class ResilientSubscriptionImpl {
|
|
|
849
1210
|
});
|
|
850
1211
|
}
|
|
851
1212
|
}
|
|
1213
|
+
/**
|
|
1214
|
+
* Reject the combinations that could only ever deliver nothing, at
|
|
1215
|
+
* construction rather than as a subscription that looks alive.
|
|
1216
|
+
*/
|
|
1217
|
+
function assertPolicySuitsBinding(snapshotSource, policy) {
|
|
1218
|
+
if (snapshotSource === 'endpoint')
|
|
1219
|
+
return;
|
|
1220
|
+
if (policy.mode === 'poll-only') {
|
|
1221
|
+
throw new Error("mode: 'poll-only' requires snapshotSource: 'endpoint'. A synthesized snapshot is answered without asking the server, so a subscription that never opens a stream could not deliver anything.");
|
|
1222
|
+
}
|
|
1223
|
+
if (policy.streamRefusal === 'keep-polling') {
|
|
1224
|
+
throw new Error("streamRefusal: 'keep-polling' requires snapshotSource: 'endpoint'. A synthesized snapshot is answered without asking the server, so polling on after a refused stream would report a healthy subscription that delivers nothing.");
|
|
1225
|
+
}
|
|
1226
|
+
}
|
|
852
1227
|
/**
|
|
853
1228
|
* Create a resilient subscription: SSE as the low-latency channel, short
|
|
854
1229
|
* polls as the correctness backbone. See the package README for the state
|
|
@@ -864,7 +1239,11 @@ export function createResilientSubscription(binding, options) {
|
|
|
864
1239
|
get status() {
|
|
865
1240
|
return impl.status;
|
|
866
1241
|
},
|
|
1242
|
+
get streamAbandoned() {
|
|
1243
|
+
return impl.streamAbandoned;
|
|
1244
|
+
},
|
|
867
1245
|
onStatusChange: (listener) => impl.onStatusChange(listener),
|
|
1246
|
+
onStreamEstablished: (listener) => impl.onStreamEstablished(listener),
|
|
868
1247
|
get result() {
|
|
869
1248
|
return impl.result;
|
|
870
1249
|
},
|