@cratis/chronicle 6.62.0 → 6.63.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ChronicleClient.ts +86 -3
- package/connection/ConnectionLifecycle.ts +42 -1
- package/dist/ChronicleClient.d.ts +28 -1
- package/dist/ChronicleClient.d.ts.map +1 -1
- package/dist/ChronicleClient.js +71 -3
- package/dist/ChronicleClient.js.map +1 -1
- package/dist/connection/ConnectionLifecycle.d.ts +24 -0
- package/dist/connection/ConnectionLifecycle.d.ts.map +1 -1
- package/dist/connection/ConnectionLifecycle.js +40 -1
- package/dist/connection/ConnectionLifecycle.js.map +1 -1
- package/dist/observation/ObservationReconnectBackoff.d.ts +64 -0
- package/dist/observation/ObservationReconnectBackoff.d.ts.map +1 -0
- package/dist/observation/ObservationReconnectBackoff.js +75 -0
- package/dist/observation/ObservationReconnectBackoff.js.map +1 -0
- package/dist/observation/ObservationStreamOutcome.d.ts +10 -0
- package/dist/observation/ObservationStreamOutcome.d.ts.map +1 -0
- package/dist/observation/ObservationStreamOutcome.js +4 -0
- package/dist/observation/ObservationStreamOutcome.js.map +1 -0
- package/dist/observation/classifyStreamError.d.ts +25 -0
- package/dist/observation/classifyStreamError.d.ts.map +1 -0
- package/dist/observation/classifyStreamError.js +55 -0
- package/dist/observation/classifyStreamError.js.map +1 -0
- package/dist/observation/for_ObservationReconnectBackoff/given/a_backoff.d.ts +13 -0
- package/dist/observation/for_ObservationReconnectBackoff/given/a_backoff.d.ts.map +1 -0
- package/dist/observation/for_ObservationReconnectBackoff/given/a_backoff.js +25 -0
- package/dist/observation/for_ObservationReconnectBackoff/given/a_backoff.js.map +1 -0
- package/dist/observation/for_ObservationReconnectBackoff/given/a_scripted_stream.d.ts +27 -0
- package/dist/observation/for_ObservationReconnectBackoff/given/a_scripted_stream.d.ts.map +1 -0
- package/dist/observation/for_ObservationReconnectBackoff/given/a_scripted_stream.js +36 -0
- package/dist/observation/for_ObservationReconnectBackoff/given/a_scripted_stream.js.map +1 -0
- package/dist/reactors/Reactors.d.ts +15 -5
- package/dist/reactors/Reactors.d.ts.map +1 -1
- package/dist/reactors/Reactors.js +95 -18
- package/dist/reactors/Reactors.js.map +1 -1
- package/dist/reactors/for_Reactors/when_reconnecting_the_observation_stream/given/a_reactor_whose_stream_ends.d.ts +26 -0
- package/dist/reactors/for_Reactors/when_reconnecting_the_observation_stream/given/a_reactor_whose_stream_ends.d.ts.map +1 -0
- package/dist/reactors/for_Reactors/when_reconnecting_the_observation_stream/given/a_reactor_whose_stream_ends.js +64 -0
- package/dist/reactors/for_Reactors/when_reconnecting_the_observation_stream/given/a_reactor_whose_stream_ends.js.map +1 -0
- package/dist/reducers/Reducers.d.ts +11 -5
- package/dist/reducers/Reducers.d.ts.map +1 -1
- package/dist/reducers/Reducers.js +93 -19
- package/dist/reducers/Reducers.js.map +1 -1
- package/dist/reducers/for_Reducers/when_reconnecting_the_observation_stream/given/a_reducer_whose_stream_ends.d.ts +25 -0
- package/dist/reducers/for_Reducers/when_reconnecting_the_observation_stream/given/a_reducer_whose_stream_ends.d.ts.map +1 -0
- package/dist/reducers/for_Reducers/when_reconnecting_the_observation_stream/given/a_reducer_whose_stream_ends.js +70 -0
- package/dist/reducers/for_Reducers/when_reconnecting_the_observation_stream/given/a_reducer_whose_stream_ends.js.map +1 -0
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/for_ChronicleClient/when_reconnecting/and_an_observation_stream_lost_its_session.spec.ts +212 -0
- package/observation/ObservationReconnectBackoff.ts +109 -0
- package/observation/ObservationStreamOutcome.ts +12 -0
- package/observation/classifyStreamError.ts +67 -0
- package/observation/for_ObservationReconnectBackoff/given/a_backoff.ts +29 -0
- package/observation/for_ObservationReconnectBackoff/given/a_scripted_stream.ts +52 -0
- package/observation/for_ObservationReconnectBackoff/when_a_stream_delivered_before_it_ended.spec.ts +23 -0
- package/observation/for_ObservationReconnectBackoff/when_a_stream_stayed_open_before_it_ended.spec.ts +25 -0
- package/observation/for_ObservationReconnectBackoff/when_jittering_the_delay.spec.ts +24 -0
- package/observation/for_ObservationReconnectBackoff/when_streams_keep_ending_without_working.spec.ts +24 -0
- package/observation/for_ObservationReconnectBackoff/when_the_base_delay_is_not_positive.spec.ts +18 -0
- package/observation/for_ObservationReconnectBackoff/when_the_settings_are_not_finite.spec.ts +32 -0
- package/observation/for_ObservationReconnectBackoff/when_using_the_defaults.spec.ts +29 -0
- package/observation/for_classifyStreamError/when_classifying_a_stream_error.spec.ts +36 -0
- package/package.json +1 -1
- package/reactors/Reactors.ts +101 -18
- package/reactors/for_Reactors/when_reconnecting_the_observation_stream/and_a_handler_failed_with_a_lost_session.spec.ts +32 -0
- package/reactors/for_Reactors/when_reconnecting_the_observation_stream/and_a_replaced_stream_fails_late.spec.ts +28 -0
- package/reactors/for_Reactors/when_reconnecting_the_observation_stream/and_a_stream_delivered_before_it_ended.spec.ts +24 -0
- package/reactors/for_Reactors/when_reconnecting_the_observation_stream/and_a_stream_stayed_open_before_it_ended.spec.ts +23 -0
- package/reactors/for_Reactors/when_reconnecting_the_observation_stream/and_it_is_disposed_while_waiting_to_reopen.spec.ts +27 -0
- package/reactors/for_Reactors/when_reconnecting_the_observation_stream/and_streams_keep_ending_without_working.spec.ts +24 -0
- package/reactors/for_Reactors/when_reconnecting_the_observation_stream/and_the_client_reconnected_after_streams_kept_ending.spec.ts +24 -0
- package/reactors/for_Reactors/when_reconnecting_the_observation_stream/and_the_client_reconnected_while_waiting_to_reopen.spec.ts +25 -0
- package/reactors/for_Reactors/when_reconnecting_the_observation_stream/and_the_kernel_lost_the_session.spec.ts +24 -0
- package/reactors/for_Reactors/when_reconnecting_the_observation_stream/given/a_reactor_whose_stream_ends.ts +69 -0
- package/reducers/Reducers.ts +99 -19
- package/reducers/for_Reducers/when_reconnecting_the_observation_stream/and_a_reducer_failed_with_a_lost_session.spec.ts +33 -0
- package/reducers/for_Reducers/when_reconnecting_the_observation_stream/and_a_replaced_stream_fails_late.spec.ts +28 -0
- package/reducers/for_Reducers/when_reconnecting_the_observation_stream/and_a_stream_delivered_before_it_ended.spec.ts +24 -0
- package/reducers/for_Reducers/when_reconnecting_the_observation_stream/and_a_stream_stayed_open_before_it_ended.spec.ts +23 -0
- package/reducers/for_Reducers/when_reconnecting_the_observation_stream/and_it_is_disposed_while_waiting_to_reopen.spec.ts +27 -0
- package/reducers/for_Reducers/when_reconnecting_the_observation_stream/and_streams_keep_ending_without_working.spec.ts +24 -0
- package/reducers/for_Reducers/when_reconnecting_the_observation_stream/and_the_client_reconnected_after_streams_kept_ending.spec.ts +24 -0
- package/reducers/for_Reducers/when_reconnecting_the_observation_stream/and_the_client_reconnected_while_waiting_to_reopen.spec.ts +25 -0
- package/reducers/for_Reducers/when_reconnecting_the_observation_stream/and_the_kernel_lost_the_session.spec.ts +24 -0
- package/reducers/for_Reducers/when_reconnecting_the_observation_stream/given/a_reducer_whose_stream_ends.ts +69 -0
|
@@ -8,6 +8,7 @@ export class ConnectionLifecycle {
|
|
|
8
8
|
_onConnected = new Set();
|
|
9
9
|
_onDisconnected = new Set();
|
|
10
10
|
_onFailed = new Set();
|
|
11
|
+
_onReconnectRequested = new Set();
|
|
11
12
|
_failure;
|
|
12
13
|
_isConnected = false;
|
|
13
14
|
_connectionId = Guid.create().toString();
|
|
@@ -45,6 +46,44 @@ export class ConnectionLifecycle {
|
|
|
45
46
|
this._onDisconnected.add(handler);
|
|
46
47
|
return () => this._onDisconnected.delete(handler);
|
|
47
48
|
}
|
|
49
|
+
/**
|
|
50
|
+
* Registers a callback for when something asks the whole client to reconnect.
|
|
51
|
+
* @param handler - Callback to invoke with the reason and the error behind the request.
|
|
52
|
+
* @returns A function that unsubscribes the callback.
|
|
53
|
+
*/
|
|
54
|
+
onReconnectRequested(handler) {
|
|
55
|
+
this._onReconnectRequested.add(handler);
|
|
56
|
+
return () => this._onReconnectRequested.delete(handler);
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Asks the whole client to drop its session and reconnect with a new connection id, for instance because
|
|
60
|
+
* the kernel no longer knows the current one.
|
|
61
|
+
*
|
|
62
|
+
* A request about a connection id that has already been replaced is ignored: the session it reports on is
|
|
63
|
+
* gone, and the observer that reported it is opened again by the reconnect that replaced it.
|
|
64
|
+
* @param connectionId - The connection id of the session the request is about.
|
|
65
|
+
* @param reason - Why the reconnect is requested.
|
|
66
|
+
* @param error - The error behind the request.
|
|
67
|
+
* @param onError - Called with the error when a handler throws.
|
|
68
|
+
*/
|
|
69
|
+
requestReconnect(connectionId, reason, error, onError) {
|
|
70
|
+
if (this._failure || connectionId !== this._connectionId)
|
|
71
|
+
return;
|
|
72
|
+
for (const handler of this._onReconnectRequested) {
|
|
73
|
+
try {
|
|
74
|
+
handler(reason, error);
|
|
75
|
+
}
|
|
76
|
+
catch (handlerError) {
|
|
77
|
+
onError(handlerError);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Replaces the connection id, so the next session is registered with one the kernel has not seen.
|
|
83
|
+
*/
|
|
84
|
+
renewConnectionId() {
|
|
85
|
+
this._connectionId = Guid.create().toString();
|
|
86
|
+
}
|
|
48
87
|
/** Registers a callback for a terminal connection failure. */
|
|
49
88
|
onFailed(handler) {
|
|
50
89
|
this._onFailed.add(handler);
|
|
@@ -73,7 +112,7 @@ export class ConnectionLifecycle {
|
|
|
73
112
|
async disconnected(onError) {
|
|
74
113
|
this._isConnected = false;
|
|
75
114
|
await Promise.all([...this._onDisconnected].map(handler => this.invokeHandler(handler, onError)));
|
|
76
|
-
this.
|
|
115
|
+
this.renewConnectionId();
|
|
77
116
|
}
|
|
78
117
|
async invokeHandler(handler, onError) {
|
|
79
118
|
try {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ConnectionLifecycle.js","sourceRoot":"","sources":["../../connection/ConnectionLifecycle.ts"],"names":[],"mappings":"AAAA,6CAA6C;AAC7C,qGAAqG;AAErG,OAAO,EAAE,IAAI,EAAE,MAAM,sBAAsB,CAAC;
|
|
1
|
+
{"version":3,"file":"ConnectionLifecycle.js","sourceRoot":"","sources":["../../connection/ConnectionLifecycle.ts"],"names":[],"mappings":"AAAA,6CAA6C;AAC7C,qGAAqG;AAErG,OAAO,EAAE,IAAI,EAAE,MAAM,sBAAsB,CAAC;AAM5C;;GAEG;AACH,MAAM,OAAO,mBAAmB;IACX,YAAY,GAAG,IAAI,GAAG,EAAoB,CAAC;IAC3C,eAAe,GAAG,IAAI,GAAG,EAAuB,CAAC;IACjD,SAAS,GAAG,IAAI,GAAG,EAAmC,CAAC;IACvD,qBAAqB,GAAG,IAAI,GAAG,EAA6B,CAAC;IACtE,QAAQ,CAAS;IAEjB,YAAY,GAAG,KAAK,CAAC;IACrB,aAAa,GAAW,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC;IAEzD;;OAEG;IACH,IAAI,WAAW;QACX,OAAO,IAAI,CAAC,YAAY,CAAC;IAC7B,CAAC;IAED,qEAAqE;IACrE,IAAI,OAAO;QACP,OAAO,IAAI,CAAC,QAAQ,CAAC;IACzB,CAAC;IAED;;OAEG;IACH,IAAI,YAAY;QACZ,OAAO,IAAI,CAAC,aAAa,CAAC;IAC9B,CAAC;IAED;;;;OAIG;IACH,WAAW,CAAC,OAAyB;QACjC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAC/B,OAAO,GAAG,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACnD,CAAC;IAED;;;;OAIG;IACH,cAAc,CAAC,OAA4B;QACvC,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAClC,OAAO,GAAG,EAAE,CAAC,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACtD,CAAC;IAED;;;;OAIG;IACH,oBAAoB,CAAC,OAAkC;QACnD,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACxC,OAAO,GAAG,EAAE,CAAC,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAC5D,CAAC;IAED;;;;;;;;;;OAUG;IACH,gBAAgB,CAAC,YAAoB,EAAE,MAAc,EAAE,KAAc,EAAE,OAAiC;QACpG,IAAI,IAAI,CAAC,QAAQ,IAAI,YAAY,KAAK,IAAI,CAAC,aAAa;YAAE,OAAO;QACjE,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,qBAAqB,EAAE,CAAC;YAC/C,IAAI,CAAC;gBACD,OAAO,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;YAC3B,CAAC;YAAC,OAAO,YAAY,EAAE,CAAC;gBACpB,OAAO,CAAC,YAAY,CAAC,CAAC;YAC1B,CAAC;QACL,CAAC;IACL,CAAC;IAED;;OAEG;IACH,iBAAiB;QACb,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC;IAClD,CAAC;IAED,8DAA8D;IAC9D,QAAQ,CAAC,OAAwC;QAC7C,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAC5B,OAAO,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAChD,CAAC;IAED,gFAAgF;IAChF,KAAK,CAAC,MAAM,CAAC,KAAY,EAAE,OAAiC;QACxD,IAAI,IAAI,CAAC,QAAQ;YAAE,OAAO;QAC1B,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;QACtB,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;QAC1B,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;IAC7G,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,SAAS,CAAC,OAAiC;QAC7C,IAAI,IAAI,CAAC,QAAQ;YAAE,OAAO;QAC1B,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QACzB,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;IACnG,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,YAAY,CAAC,OAAiC;QAChD,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;QAC1B,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,eAAe,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;QAClG,IAAI,CAAC,iBAAiB,EAAE,CAAC;IAC7B,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,OAA4B,EAAE,OAAiC;QACvF,IAAI,CAAC;YACD,MAAM,OAAO,EAAE,CAAC;QACpB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,OAAO,CAAC,KAAK,CAAC,CAAC;QACnB,CAAC;IACL,CAAC;CACJ"}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Settings for how a reactor or reducer waits before opening its observation stream again.
|
|
3
|
+
* Every setting is optional; the defaults match the .NET client.
|
|
4
|
+
*/
|
|
5
|
+
export interface ObservationReconnectBackoffOptions {
|
|
6
|
+
/** The wait in milliseconds after a stream that had worked. Defaults to 2 seconds. */
|
|
7
|
+
baseDelayMs?: number;
|
|
8
|
+
/** The longest wait in milliseconds, however often streams end. Defaults to 60 seconds. */
|
|
9
|
+
maxDelayMs?: number;
|
|
10
|
+
/** How long in milliseconds a stream must stay open to count as having worked. Defaults to 3 minutes. */
|
|
11
|
+
healthyStreamDurationMs?: number;
|
|
12
|
+
/** Returns the current time in milliseconds, measuring how long a stream stayed open. Defaults to `performance.now`. */
|
|
13
|
+
now?: () => number;
|
|
14
|
+
/** Returns a value in [0, 1) choosing where in the upper half of the wait this one lands. Defaults to `Math.random`. */
|
|
15
|
+
nextJitter?: () => number;
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* Decides how long one reactor or reducer waits before opening its observation stream again after it ended.
|
|
19
|
+
*
|
|
20
|
+
* A fixed delay is right for one observer and wrong for many: seventy reactors whose streams fail together
|
|
21
|
+
* come back together, every two seconds, against a kernel that is still busy. So each stream that ends without
|
|
22
|
+
* having worked doubles the wait, up to a ceiling, and every wait lands at random in its upper half so a herd
|
|
23
|
+
* spreads out.
|
|
24
|
+
*
|
|
25
|
+
* The wait starts over once a stream has worked: it delivered something, or it stayed open long enough to have
|
|
26
|
+
* been established. A base delay of zero or less is used as it is.
|
|
27
|
+
*/
|
|
28
|
+
export declare class ObservationReconnectBackoff {
|
|
29
|
+
/** The default wait after a stream that had worked. */
|
|
30
|
+
static readonly defaultBaseDelayMs = 2000;
|
|
31
|
+
/** The default longest wait between attempts. */
|
|
32
|
+
static readonly defaultMaxDelayMs = 60000;
|
|
33
|
+
/**
|
|
34
|
+
* The default time a stream must stay open to count as having worked.
|
|
35
|
+
*
|
|
36
|
+
* It is longer than the kernel takes to give up establishing a stream: the kernel asks a busy observer again
|
|
37
|
+
* on the same stream, up to four attempts of a thirty-second grain call each, before it fails the stream. A
|
|
38
|
+
* shorter threshold would count every such stream as having worked and start the wait over at the base delay.
|
|
39
|
+
*/
|
|
40
|
+
static readonly defaultHealthyStreamDurationMs = 180000;
|
|
41
|
+
private static readonly _maxExponent;
|
|
42
|
+
private readonly _baseDelayMs;
|
|
43
|
+
private readonly _maxDelayMs;
|
|
44
|
+
private readonly _healthyStreamDurationMs;
|
|
45
|
+
private readonly _now;
|
|
46
|
+
private readonly _nextJitter;
|
|
47
|
+
private _streamsEndedWithoutWorking;
|
|
48
|
+
private _streamOpenedAt;
|
|
49
|
+
/**
|
|
50
|
+
* Creates a new {@link ObservationReconnectBackoff}.
|
|
51
|
+
* @param options - The settings; anything left out uses its default.
|
|
52
|
+
*/
|
|
53
|
+
constructor(options?: ObservationReconnectBackoffOptions);
|
|
54
|
+
/** Records that a new observation stream was opened. */
|
|
55
|
+
streamOpened(): void;
|
|
56
|
+
/** Records that the open stream delivered something, which proves it was established. */
|
|
57
|
+
streamDelivered(): void;
|
|
58
|
+
/**
|
|
59
|
+
* Gets how long to wait before opening the stream again, now that the open one has ended.
|
|
60
|
+
* @returns The wait in milliseconds.
|
|
61
|
+
*/
|
|
62
|
+
nextDelayMs(): number;
|
|
63
|
+
}
|
|
64
|
+
//# sourceMappingURL=ObservationReconnectBackoff.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ObservationReconnectBackoff.d.ts","sourceRoot":"","sources":["../../observation/ObservationReconnectBackoff.ts"],"names":[],"mappings":"AAGA;;;GAGG;AACH,MAAM,WAAW,kCAAkC;IAC/C,sFAAsF;IACtF,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB,2FAA2F;IAC3F,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB,yGAAyG;IACzG,uBAAuB,CAAC,EAAE,MAAM,CAAC;IAEjC,wHAAwH;IACxH,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IAEnB,wHAAwH;IACxH,UAAU,CAAC,EAAE,MAAM,MAAM,CAAC;CAC7B;AAKD;;;;;;;;;;GAUG;AACH,qBAAa,2BAA2B;IACpC,uDAAuD;IACvD,MAAM,CAAC,QAAQ,CAAC,kBAAkB,QAAS;IAE3C,iDAAiD;IACjD,MAAM,CAAC,QAAQ,CAAC,iBAAiB,SAAU;IAE3C;;;;;;OAMG;IACH,MAAM,CAAC,QAAQ,CAAC,8BAA8B,UAAW;IAEzD,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAM;IAE1C,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IACtC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAAS;IAClD,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAe;IACpC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAe;IAC3C,OAAO,CAAC,2BAA2B,CAAK;IACxC,OAAO,CAAC,eAAe,CAAS;IAEhC;;;OAGG;IACH,YAAY,OAAO,GAAE,kCAAuC,EAS3D;IAED,wDAAwD;IACxD,YAAY,IAAI,IAAI,CAEnB;IAED,yFAAyF;IACzF,eAAe,IAAI,IAAI,CAEtB;IAED;;;OAGG;IACH,WAAW,IAAI,MAAM,CAcpB;CACJ"}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
// Copyright (c) Cratis. All rights reserved.
|
|
2
|
+
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
|
|
3
|
+
const finiteOr = (value, fallback, isAllowed) => value !== undefined && Number.isFinite(value) && isAllowed(value) ? value : fallback;
|
|
4
|
+
/**
|
|
5
|
+
* Decides how long one reactor or reducer waits before opening its observation stream again after it ended.
|
|
6
|
+
*
|
|
7
|
+
* A fixed delay is right for one observer and wrong for many: seventy reactors whose streams fail together
|
|
8
|
+
* come back together, every two seconds, against a kernel that is still busy. So each stream that ends without
|
|
9
|
+
* having worked doubles the wait, up to a ceiling, and every wait lands at random in its upper half so a herd
|
|
10
|
+
* spreads out.
|
|
11
|
+
*
|
|
12
|
+
* The wait starts over once a stream has worked: it delivered something, or it stayed open long enough to have
|
|
13
|
+
* been established. A base delay of zero or less is used as it is.
|
|
14
|
+
*/
|
|
15
|
+
export class ObservationReconnectBackoff {
|
|
16
|
+
/** The default wait after a stream that had worked. */
|
|
17
|
+
static defaultBaseDelayMs = 2_000;
|
|
18
|
+
/** The default longest wait between attempts. */
|
|
19
|
+
static defaultMaxDelayMs = 60_000;
|
|
20
|
+
/**
|
|
21
|
+
* The default time a stream must stay open to count as having worked.
|
|
22
|
+
*
|
|
23
|
+
* It is longer than the kernel takes to give up establishing a stream: the kernel asks a busy observer again
|
|
24
|
+
* on the same stream, up to four attempts of a thirty-second grain call each, before it fails the stream. A
|
|
25
|
+
* shorter threshold would count every such stream as having worked and start the wait over at the base delay.
|
|
26
|
+
*/
|
|
27
|
+
static defaultHealthyStreamDurationMs = 180_000;
|
|
28
|
+
static _maxExponent = 30;
|
|
29
|
+
_baseDelayMs;
|
|
30
|
+
_maxDelayMs;
|
|
31
|
+
_healthyStreamDurationMs;
|
|
32
|
+
_now;
|
|
33
|
+
_nextJitter;
|
|
34
|
+
_streamsEndedWithoutWorking = 0;
|
|
35
|
+
_streamOpenedAt;
|
|
36
|
+
/**
|
|
37
|
+
* Creates a new {@link ObservationReconnectBackoff}.
|
|
38
|
+
* @param options - The settings; anything left out uses its default.
|
|
39
|
+
*/
|
|
40
|
+
constructor(options = {}) {
|
|
41
|
+
// A wait that is not a finite number would reach setTimeout as about one millisecond: a hot loop.
|
|
42
|
+
this._baseDelayMs = finiteOr(options.baseDelayMs, ObservationReconnectBackoff.defaultBaseDelayMs, () => true);
|
|
43
|
+
this._maxDelayMs = finiteOr(options.maxDelayMs, ObservationReconnectBackoff.defaultMaxDelayMs, value => value > 0);
|
|
44
|
+
this._healthyStreamDurationMs = finiteOr(options.healthyStreamDurationMs, ObservationReconnectBackoff.defaultHealthyStreamDurationMs, value => value >= 0);
|
|
45
|
+
this._now = options.now ?? (() => performance.now());
|
|
46
|
+
this._nextJitter = options.nextJitter ?? Math.random;
|
|
47
|
+
this._streamOpenedAt = this._now();
|
|
48
|
+
}
|
|
49
|
+
/** Records that a new observation stream was opened. */
|
|
50
|
+
streamOpened() {
|
|
51
|
+
this._streamOpenedAt = this._now();
|
|
52
|
+
}
|
|
53
|
+
/** Records that the open stream delivered something, which proves it was established. */
|
|
54
|
+
streamDelivered() {
|
|
55
|
+
this._streamsEndedWithoutWorking = 0;
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* Gets how long to wait before opening the stream again, now that the open one has ended.
|
|
59
|
+
* @returns The wait in milliseconds.
|
|
60
|
+
*/
|
|
61
|
+
nextDelayMs() {
|
|
62
|
+
if (this._baseDelayMs <= 0) {
|
|
63
|
+
return this._baseDelayMs;
|
|
64
|
+
}
|
|
65
|
+
if (this._now() - this._streamOpenedAt >= this._healthyStreamDurationMs) {
|
|
66
|
+
this._streamsEndedWithoutWorking = 0;
|
|
67
|
+
}
|
|
68
|
+
const exponent = Math.min(this._streamsEndedWithoutWorking, ObservationReconnectBackoff._maxExponent);
|
|
69
|
+
this._streamsEndedWithoutWorking++;
|
|
70
|
+
const ceiling = Math.min(this._maxDelayMs, this._baseDelayMs * Math.pow(2, exponent));
|
|
71
|
+
const jitter = 0.5 + 0.5 * Math.min(Math.max(this._nextJitter(), 0), 1);
|
|
72
|
+
return Math.floor(ceiling * jitter);
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
//# sourceMappingURL=ObservationReconnectBackoff.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ObservationReconnectBackoff.js","sourceRoot":"","sources":["../../observation/ObservationReconnectBackoff.ts"],"names":[],"mappings":"AAAA,6CAA6C;AAC7C,qGAAqG;AAuBrG,MAAM,QAAQ,GAAG,CAAC,KAAyB,EAAE,QAAgB,EAAE,SAAqC,EAAU,EAAE,CAC5G,KAAK,KAAK,SAAS,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC;AAEzF;;;;;;;;;;GAUG;AACH,MAAM,OAAO,2BAA2B;IACpC,uDAAuD;IACvD,MAAM,CAAU,kBAAkB,GAAG,KAAK,CAAC;IAE3C,iDAAiD;IACjD,MAAM,CAAU,iBAAiB,GAAG,MAAM,CAAC;IAE3C;;;;;;OAMG;IACH,MAAM,CAAU,8BAA8B,GAAG,OAAO,CAAC;IAEjD,MAAM,CAAU,YAAY,GAAG,EAAE,CAAC;IAEzB,YAAY,CAAS;IACrB,WAAW,CAAS;IACpB,wBAAwB,CAAS;IACjC,IAAI,CAAe;IACnB,WAAW,CAAe;IACnC,2BAA2B,GAAG,CAAC,CAAC;IAChC,eAAe,CAAS;IAEhC;;;OAGG;IACH,YAAY,OAAO,GAAuC,EAAE;QACxD,kGAAkG;QAClG,IAAI,CAAC,YAAY,GAAG,QAAQ,CAAC,OAAO,CAAC,WAAW,EAAE,2BAA2B,CAAC,kBAAkB,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;QAC9G,IAAI,CAAC,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,UAAU,EAAE,2BAA2B,CAAC,iBAAiB,EAAE,KAAK,CAAC,EAAE,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;QACnH,IAAI,CAAC,wBAAwB,GAAG,QAAQ,CAAC,OAAO,CAAC,uBAAuB,EACpE,2BAA2B,CAAC,8BAA8B,EAAE,KAAK,CAAC,EAAE,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;QACrF,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC;QACrD,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,UAAU,IAAI,IAAI,CAAC,MAAM,CAAC;QACrD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IACvC,CAAC;IAED,wDAAwD;IACxD,YAAY;QACR,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IACvC,CAAC;IAED,yFAAyF;IACzF,eAAe;QACX,IAAI,CAAC,2BAA2B,GAAG,CAAC,CAAC;IACzC,CAAC;IAED;;;OAGG;IACH,WAAW;QACP,IAAI,IAAI,CAAC,YAAY,IAAI,CAAC,EAAE,CAAC;YACzB,OAAO,IAAI,CAAC,YAAY,CAAC;QAC7B,CAAC;QAED,IAAI,IAAI,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC,eAAe,IAAI,IAAI,CAAC,wBAAwB,EAAE,CAAC;YACtE,IAAI,CAAC,2BAA2B,GAAG,CAAC,CAAC;QACzC,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,2BAA2B,EAAE,2BAA2B,CAAC,YAAY,CAAC,CAAC;QACtG,IAAI,CAAC,2BAA2B,EAAE,CAAC;QACnC,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC;QACtF,MAAM,MAAM,GAAG,GAAG,GAAG,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACxE,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,CAAC;IACxC,CAAC;CACJ"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* How one reactor or reducer observation stream ended, which decides what happens next.
|
|
3
|
+
*
|
|
4
|
+
* - `ended`: the stream ended or failed while it was still the current one; open it again after a backoff.
|
|
5
|
+
* - `sessionLost`: the kernel no longer knows the connection; the whole client reconnects and re-registers it.
|
|
6
|
+
* - `superseded`: a disconnect, a dispose or a newer stream replaced it; nothing to do.
|
|
7
|
+
* - `stopped`: the client cancelled it on purpose; nothing to do.
|
|
8
|
+
*/
|
|
9
|
+
export type ObservationStreamOutcome = 'ended' | 'sessionLost' | 'superseded' | 'stopped';
|
|
10
|
+
//# sourceMappingURL=ObservationStreamOutcome.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ObservationStreamOutcome.d.ts","sourceRoot":"","sources":["../../observation/ObservationStreamOutcome.ts"],"names":[],"mappings":"AAGA;;;;;;;GAOG;AACH,MAAM,MAAM,wBAAwB,GAAG,OAAO,GAAG,aAAa,GAAG,YAAY,GAAG,SAAS,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ObservationStreamOutcome.js","sourceRoot":"","sources":["../../observation/ObservationStreamOutcome.ts"],"names":[],"mappings":"AAAA,6CAA6C;AAC7C,qGAAqG"}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/** What ended a kernel observation stream, deciding how the client recovers. */
|
|
2
|
+
export declare enum StreamErrorKind {
|
|
3
|
+
/** The stream failed for a reason of its own; it is opened again after a backoff. */
|
|
4
|
+
Failure = "Failure",
|
|
5
|
+
/** The client cancelled the stream itself; it is not opened again. */
|
|
6
|
+
Cancelled = "Cancelled",
|
|
7
|
+
/** The kernel is stopping or unreachable; the stream is opened again after a backoff. */
|
|
8
|
+
ConnectionLost = "ConnectionLost",
|
|
9
|
+
/**
|
|
10
|
+
* The kernel no longer knows this client's connection, for instance after it restarted. Opening the stream again
|
|
11
|
+
* with the same connection id can never work, so the whole client reconnects with a new one.
|
|
12
|
+
*/
|
|
13
|
+
SessionLost = "SessionLost"
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Classifies an error that ended a kernel observation stream, looking through wrapping errors.
|
|
17
|
+
*
|
|
18
|
+
* An {@link AggregateError} takes the most serious kind among the errors it wraps, in this order:
|
|
19
|
+
* {@link StreamErrorKind.Failure}, then {@link StreamErrorKind.SessionLost}, then {@link StreamErrorKind.ConnectionLost},
|
|
20
|
+
* then {@link StreamErrorKind.Cancelled}. This matches the .NET client.
|
|
21
|
+
* @param error - The error the stream ended with.
|
|
22
|
+
* @returns The {@link StreamErrorKind} for the error.
|
|
23
|
+
*/
|
|
24
|
+
export declare function classifyStreamError(error: unknown): StreamErrorKind;
|
|
25
|
+
//# sourceMappingURL=classifyStreamError.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"classifyStreamError.d.ts","sourceRoot":"","sources":["../../observation/classifyStreamError.ts"],"names":[],"mappings":"AAGA,gFAAgF;AAChF,oBAAY,eAAe;IACvB,qFAAqF;IACrF,OAAO,YAAY;IAEnB,sEAAsE;IACtE,SAAS,cAAc;IAEvB,yFAAyF;IACzF,cAAc,mBAAmB;IAEjC;;;OAGG;IACH,WAAW,gBAAgB;CAC9B;AAiBD;;;;;;;;GAQG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,OAAO,GAAG,eAAe,CAqBnE"}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
// Copyright (c) Cratis. All rights reserved.
|
|
2
|
+
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
|
|
3
|
+
/** What ended a kernel observation stream, deciding how the client recovers. */
|
|
4
|
+
export var StreamErrorKind;
|
|
5
|
+
(function (StreamErrorKind) {
|
|
6
|
+
/** The stream failed for a reason of its own; it is opened again after a backoff. */
|
|
7
|
+
StreamErrorKind["Failure"] = "Failure";
|
|
8
|
+
/** The client cancelled the stream itself; it is not opened again. */
|
|
9
|
+
StreamErrorKind["Cancelled"] = "Cancelled";
|
|
10
|
+
/** The kernel is stopping or unreachable; the stream is opened again after a backoff. */
|
|
11
|
+
StreamErrorKind["ConnectionLost"] = "ConnectionLost";
|
|
12
|
+
/**
|
|
13
|
+
* The kernel no longer knows this client's connection, for instance after it restarted. Opening the stream again
|
|
14
|
+
* with the same connection id can never work, so the whole client reconnects with a new one.
|
|
15
|
+
*/
|
|
16
|
+
StreamErrorKind["SessionLost"] = "SessionLost";
|
|
17
|
+
})(StreamErrorKind || (StreamErrorKind = {}));
|
|
18
|
+
const CANCELLED = 1;
|
|
19
|
+
const UNAVAILABLE = 14;
|
|
20
|
+
const CLIENT_IS_NOT_CONNECTED = 'ClientIsNotConnected';
|
|
21
|
+
const mentionsClientIsNotConnected = (error) => [error.details, error.message].some(text => typeof text === 'string' && text.includes(CLIENT_IS_NOT_CONNECTED));
|
|
22
|
+
/**
|
|
23
|
+
* Classifies an error that ended a kernel observation stream, looking through wrapping errors.
|
|
24
|
+
*
|
|
25
|
+
* An {@link AggregateError} takes the most serious kind among the errors it wraps, in this order:
|
|
26
|
+
* {@link StreamErrorKind.Failure}, then {@link StreamErrorKind.SessionLost}, then {@link StreamErrorKind.ConnectionLost},
|
|
27
|
+
* then {@link StreamErrorKind.Cancelled}. This matches the .NET client.
|
|
28
|
+
* @param error - The error the stream ended with.
|
|
29
|
+
* @returns The {@link StreamErrorKind} for the error.
|
|
30
|
+
*/
|
|
31
|
+
export function classifyStreamError(error) {
|
|
32
|
+
if (error === null || typeof error !== 'object') {
|
|
33
|
+
return StreamErrorKind.Failure;
|
|
34
|
+
}
|
|
35
|
+
if (error instanceof AggregateError && error.errors.length > 0) {
|
|
36
|
+
const kinds = error.errors.map(classifyStreamError);
|
|
37
|
+
if (kinds.includes(StreamErrorKind.Failure))
|
|
38
|
+
return StreamErrorKind.Failure;
|
|
39
|
+
if (kinds.includes(StreamErrorKind.SessionLost))
|
|
40
|
+
return StreamErrorKind.SessionLost;
|
|
41
|
+
return kinds.includes(StreamErrorKind.ConnectionLost) ? StreamErrorKind.ConnectionLost : StreamErrorKind.Cancelled;
|
|
42
|
+
}
|
|
43
|
+
const grpcError = error;
|
|
44
|
+
if (grpcError.name === 'AbortError')
|
|
45
|
+
return StreamErrorKind.Cancelled;
|
|
46
|
+
if (typeof grpcError.code === 'number') {
|
|
47
|
+
if (grpcError.code === CANCELLED)
|
|
48
|
+
return StreamErrorKind.Cancelled;
|
|
49
|
+
if (grpcError.code === UNAVAILABLE)
|
|
50
|
+
return StreamErrorKind.ConnectionLost;
|
|
51
|
+
return mentionsClientIsNotConnected(grpcError) ? StreamErrorKind.SessionLost : StreamErrorKind.Failure;
|
|
52
|
+
}
|
|
53
|
+
return grpcError.cause !== undefined ? classifyStreamError(grpcError.cause) : StreamErrorKind.Failure;
|
|
54
|
+
}
|
|
55
|
+
//# sourceMappingURL=classifyStreamError.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"classifyStreamError.js","sourceRoot":"","sources":["../../observation/classifyStreamError.ts"],"names":[],"mappings":"AAAA,6CAA6C;AAC7C,qGAAqG;AAErG,gFAAgF;AAChF,MAAM,CAAN,IAAY,eAeX;AAfD,WAAY,eAAe;IACvB,qFAAqF;IACrF,sCAAmB,CAAA;IAEnB,sEAAsE;IACtE,0CAAuB,CAAA;IAEvB,yFAAyF;IACzF,oDAAiC,CAAA;IAEjC;;;OAGG;IACH,8CAA2B,CAAA;AAC/B,CAAC,EAfW,eAAe,KAAf,eAAe,QAe1B;AAED,MAAM,SAAS,GAAG,CAAC,CAAC;AACpB,MAAM,WAAW,GAAG,EAAE,CAAC;AACvB,MAAM,uBAAuB,GAAG,sBAAsB,CAAC;AAUvD,MAAM,4BAA4B,GAAG,CAAC,KAAoB,EAAW,EAAE,CACnE,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,QAAQ,CAAC,uBAAuB,CAAC,CAAC,CAAC;AAEpH;;;;;;;;GAQG;AACH,MAAM,UAAU,mBAAmB,CAAC,KAAc;IAC9C,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9C,OAAO,eAAe,CAAC,OAAO,CAAC;IACnC,CAAC;IAED,IAAI,KAAK,YAAY,cAAc,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7D,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;QACpD,IAAI,KAAK,CAAC,QAAQ,CAAC,eAAe,CAAC,OAAO,CAAC;YAAE,OAAO,eAAe,CAAC,OAAO,CAAC;QAC5E,IAAI,KAAK,CAAC,QAAQ,CAAC,eAAe,CAAC,WAAW,CAAC;YAAE,OAAO,eAAe,CAAC,WAAW,CAAC;QACpF,OAAO,KAAK,CAAC,QAAQ,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,cAAc,CAAC,CAAC,CAAC,eAAe,CAAC,SAAS,CAAC;IACvH,CAAC;IAED,MAAM,SAAS,GAAG,KAAsB,CAAC;IACzC,IAAI,SAAS,CAAC,IAAI,KAAK,YAAY;QAAE,OAAO,eAAe,CAAC,SAAS,CAAC;IACtE,IAAI,OAAO,SAAS,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QACrC,IAAI,SAAS,CAAC,IAAI,KAAK,SAAS;YAAE,OAAO,eAAe,CAAC,SAAS,CAAC;QACnE,IAAI,SAAS,CAAC,IAAI,KAAK,WAAW;YAAE,OAAO,eAAe,CAAC,cAAc,CAAC;QAC1E,OAAO,4BAA4B,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC,CAAC,eAAe,CAAC,OAAO,CAAC;IAC3G,CAAC;IAED,OAAO,SAAS,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,mBAAmB,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,OAAO,CAAC;AAC1G,CAAC"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { ObservationReconnectBackoff } from '../../ObservationReconnectBackoff.js';
|
|
2
|
+
/** A backoff with a 2 second base, a 60 second ceiling and a 3 minute healthy threshold, on a clock and jitter the spec controls. */
|
|
3
|
+
export declare class a_backoff {
|
|
4
|
+
static readonly baseDelayMs = 2000;
|
|
5
|
+
static readonly maxDelayMs = 60000;
|
|
6
|
+
static readonly healthyStreamDurationMs = 180000;
|
|
7
|
+
now: number;
|
|
8
|
+
jitter: number;
|
|
9
|
+
readonly backoff: ObservationReconnectBackoff;
|
|
10
|
+
/** Opens a stream, lets it stay open for a while and ends it, returning the wait before the next one. */
|
|
11
|
+
streamEndsAfter(openForMs: number): number;
|
|
12
|
+
}
|
|
13
|
+
//# sourceMappingURL=a_backoff.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"a_backoff.d.ts","sourceRoot":"","sources":["../../../../observation/for_ObservationReconnectBackoff/given/a_backoff.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,2BAA2B,EAAE,MAAM,sCAAsC,CAAC;AAEnF,qIAAqI;AACrI,qBAAa,SAAS;IAClB,MAAM,CAAC,QAAQ,CAAC,WAAW,QAAS;IACpC,MAAM,CAAC,QAAQ,CAAC,UAAU,SAAU;IACpC,MAAM,CAAC,QAAQ,CAAC,uBAAuB,UAAW;IAElD,GAAG,SAAK;IACR,MAAM,SAAK;IAEX,QAAQ,CAAC,OAAO,8BAMb;IAEH,yGAAyG;IACzG,eAAe,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAIzC;CACJ"}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
// Copyright (c) Cratis. All rights reserved.
|
|
2
|
+
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
|
|
3
|
+
import { ObservationReconnectBackoff } from '../../ObservationReconnectBackoff.js';
|
|
4
|
+
/** A backoff with a 2 second base, a 60 second ceiling and a 3 minute healthy threshold, on a clock and jitter the spec controls. */
|
|
5
|
+
export class a_backoff {
|
|
6
|
+
static baseDelayMs = 2_000;
|
|
7
|
+
static maxDelayMs = 60_000;
|
|
8
|
+
static healthyStreamDurationMs = 180_000;
|
|
9
|
+
now = 0;
|
|
10
|
+
jitter = 0;
|
|
11
|
+
backoff = new ObservationReconnectBackoff({
|
|
12
|
+
baseDelayMs: a_backoff.baseDelayMs,
|
|
13
|
+
maxDelayMs: a_backoff.maxDelayMs,
|
|
14
|
+
healthyStreamDurationMs: a_backoff.healthyStreamDurationMs,
|
|
15
|
+
now: () => this.now,
|
|
16
|
+
nextJitter: () => this.jitter
|
|
17
|
+
});
|
|
18
|
+
/** Opens a stream, lets it stay open for a while and ends it, returning the wait before the next one. */
|
|
19
|
+
streamEndsAfter(openForMs) {
|
|
20
|
+
this.backoff.streamOpened();
|
|
21
|
+
this.now += openForMs;
|
|
22
|
+
return this.backoff.nextDelayMs();
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
//# sourceMappingURL=a_backoff.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"a_backoff.js","sourceRoot":"","sources":["../../../../observation/for_ObservationReconnectBackoff/given/a_backoff.ts"],"names":[],"mappings":"AAAA,6CAA6C;AAC7C,qGAAqG;AAErG,OAAO,EAAE,2BAA2B,EAAE,MAAM,sCAAsC,CAAC;AAEnF,qIAAqI;AACrI,MAAM,OAAO,SAAS;IAClB,MAAM,CAAU,WAAW,GAAG,KAAK,CAAC;IACpC,MAAM,CAAU,UAAU,GAAG,MAAM,CAAC;IACpC,MAAM,CAAU,uBAAuB,GAAG,OAAO,CAAC;IAElD,GAAG,GAAG,CAAC,CAAC;IACR,MAAM,GAAG,CAAC,CAAC;IAEF,OAAO,GAAG,IAAI,2BAA2B,CAAC;QAC/C,WAAW,EAAE,SAAS,CAAC,WAAW;QAClC,UAAU,EAAE,SAAS,CAAC,UAAU;QAChC,uBAAuB,EAAE,SAAS,CAAC,uBAAuB;QAC1D,GAAG,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG;QACnB,UAAU,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM;KAChC,CAAC,CAAC;IAEH,yGAAyG;IACzG,eAAe,CAAC,SAAiB;QAC7B,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC;QAC5B,IAAI,CAAC,GAAG,IAAI,SAAS,CAAC;QACtB,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;IACtC,CAAC;CACJ"}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
/** How one observation stream behaves: what it delivers, how long it stays open and the error it ends with, if any. */
|
|
2
|
+
export interface StreamScript<T> {
|
|
3
|
+
/** What the stream delivers before it ends. */
|
|
4
|
+
deliver?: T[];
|
|
5
|
+
/** How long, in milliseconds, the stream stays open after delivering before it ends. */
|
|
6
|
+
openForMs?: number;
|
|
7
|
+
/** Whether the stream stays open until the client aborts it. */
|
|
8
|
+
untilAborted?: boolean;
|
|
9
|
+
/** How long, in milliseconds, an aborted stream takes to end, as a slow transport would. */
|
|
10
|
+
endsLateByMs?: number;
|
|
11
|
+
/** The error the stream ends with. */
|
|
12
|
+
error?: unknown;
|
|
13
|
+
}
|
|
14
|
+
/** Records the observation streams a reactor or reducer opens and plays them by their scripts. */
|
|
15
|
+
export declare class a_scripted_stream<T> {
|
|
16
|
+
readonly openedAt: number[];
|
|
17
|
+
readonly scripts: StreamScript<T>[];
|
|
18
|
+
open: number;
|
|
19
|
+
/** The observe method of the fake kernel connection. */
|
|
20
|
+
readonly observe: (_messages: unknown, options: {
|
|
21
|
+
signal: AbortSignal;
|
|
22
|
+
}) => AsyncGenerator<T>;
|
|
23
|
+
/** Gets the wait before each stream after the first was opened. */
|
|
24
|
+
get waits(): number[];
|
|
25
|
+
private play;
|
|
26
|
+
}
|
|
27
|
+
//# sourceMappingURL=a_scripted_stream.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"a_scripted_stream.d.ts","sourceRoot":"","sources":["../../../../observation/for_ObservationReconnectBackoff/given/a_scripted_stream.ts"],"names":[],"mappings":"AAGA,uHAAuH;AACvH,MAAM,WAAW,YAAY,CAAC,CAAC;IAC3B,+CAA+C;IAC/C,OAAO,CAAC,EAAE,CAAC,EAAE,CAAC;IAEd,wFAAwF;IACxF,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,gEAAgE;IAChE,YAAY,CAAC,EAAE,OAAO,CAAC;IAEvB,4FAA4F;IAC5F,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB,sCAAsC;IACtC,KAAK,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,kGAAkG;AAClG,qBAAa,iBAAiB,CAAC,CAAC;IAC5B,QAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,CAAM;IACjC,QAAQ,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC,CAAC,EAAE,CAAM;IACzC,IAAI,SAAK;IAET,wDAAwD;IACxD,QAAQ,CAAC,OAAO,cAAe,OAAO,WAAW;QAAE,MAAM,EAAE,WAAW,CAAA;KAAE,KAAG,cAAc,CAAC,CAAC,CAAC,CAA8B;IAE1H,mEAAmE;IACnE,IAAI,KAAK,IAAI,MAAM,EAAE,CAEpB;YAEc,IAAI;CAgBtB"}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
// Copyright (c) Cratis. All rights reserved.
|
|
2
|
+
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
|
|
3
|
+
/** Records the observation streams a reactor or reducer opens and plays them by their scripts. */
|
|
4
|
+
export class a_scripted_stream {
|
|
5
|
+
openedAt = [];
|
|
6
|
+
scripts = [];
|
|
7
|
+
open = 0;
|
|
8
|
+
/** The observe method of the fake kernel connection. */
|
|
9
|
+
observe = (_messages, options) => this.play(options.signal);
|
|
10
|
+
/** Gets the wait before each stream after the first was opened. */
|
|
11
|
+
get waits() {
|
|
12
|
+
return this.openedAt.slice(1).map((at, index) => at - this.openedAt[index]);
|
|
13
|
+
}
|
|
14
|
+
async *play(signal) {
|
|
15
|
+
this.openedAt.push(Date.now());
|
|
16
|
+
this.open++;
|
|
17
|
+
try {
|
|
18
|
+
const script = this.scripts.shift() ?? {};
|
|
19
|
+
for (const delivery of script.deliver ?? [])
|
|
20
|
+
yield delivery;
|
|
21
|
+
if (script.openForMs)
|
|
22
|
+
await new Promise(resolve => setTimeout(resolve, script.openForMs));
|
|
23
|
+
if (script.untilAborted) {
|
|
24
|
+
await new Promise(resolve => signal.aborted ? resolve() : signal.addEventListener('abort', () => resolve(), { once: true }));
|
|
25
|
+
if (script.endsLateByMs)
|
|
26
|
+
await new Promise(resolve => setTimeout(resolve, script.endsLateByMs));
|
|
27
|
+
}
|
|
28
|
+
if (script.error)
|
|
29
|
+
throw script.error;
|
|
30
|
+
}
|
|
31
|
+
finally {
|
|
32
|
+
this.open--;
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
//# sourceMappingURL=a_scripted_stream.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"a_scripted_stream.js","sourceRoot":"","sources":["../../../../observation/for_ObservationReconnectBackoff/given/a_scripted_stream.ts"],"names":[],"mappings":"AAAA,6CAA6C;AAC7C,qGAAqG;AAoBrG,kGAAkG;AAClG,MAAM,OAAO,iBAAiB;IACjB,QAAQ,GAAa,EAAE,CAAC;IACxB,OAAO,GAAsB,EAAE,CAAC;IACzC,IAAI,GAAG,CAAC,CAAC;IAET,wDAAwD;IAC/C,OAAO,GAAG,CAAC,SAAkB,EAAE,OAAgC,EAAqB,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAE1H,mEAAmE;IACnE,IAAI,KAAK;QACL,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,EAAE,CAAC,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;IAChF,CAAC;IAEO,KAAK,CAAC,CAAC,IAAI,CAAC,MAAmB;QACnC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QAC/B,IAAI,CAAC,IAAI,EAAE,CAAC;QACZ,IAAI,CAAC;YACD,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC;YAC1C,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,OAAO,IAAI,EAAE;gBAAE,MAAM,QAAQ,CAAC;YAC5D,IAAI,MAAM,CAAC,SAAS;gBAAE,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC;YAC1F,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;gBACtB,MAAM,IAAI,OAAO,CAAO,OAAO,CAAC,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;gBACnI,IAAI,MAAM,CAAC,YAAY;oBAAE,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC;YACpG,CAAC;YACD,IAAI,MAAM,CAAC,KAAK;gBAAE,MAAM,MAAM,CAAC,KAAK,CAAC;QACzC,CAAC;gBAAS,CAAC;YACP,IAAI,CAAC,IAAI,EAAE,CAAC;QAChB,CAAC;IACL,CAAC;CACJ"}
|
|
@@ -7,6 +7,7 @@ import { IReactors } from './IReactors.js';
|
|
|
7
7
|
import type { ReactorResultHandler } from './ReactorResultHandler.js';
|
|
8
8
|
import type { IEventStore } from '../IEventStore.js';
|
|
9
9
|
import type { ClientArtifactsActivator } from '../artifacts/ClientArtifactsActivator.js';
|
|
10
|
+
import { type ObservationReconnectBackoffOptions } from '../observation/ObservationReconnectBackoff.js';
|
|
10
11
|
/**
|
|
11
12
|
* Implements {@link IReactors}, managing discovery and registration of reactors
|
|
12
13
|
* with the Chronicle Kernel via bidirectional gRPC streaming.
|
|
@@ -21,14 +22,15 @@ export declare class Reactors implements IReactors {
|
|
|
21
22
|
private readonly _eventStore?;
|
|
22
23
|
private readonly _artifactActivator?;
|
|
23
24
|
private readonly _skipKeepAlive;
|
|
24
|
-
|
|
25
|
-
private static readonly _reobserveDelayMs;
|
|
25
|
+
private readonly _reconnectBackoff;
|
|
26
26
|
private readonly _logger;
|
|
27
27
|
private readonly _lifecycle;
|
|
28
28
|
private readonly _unsubscribeDisconnected;
|
|
29
29
|
private readonly _reactors;
|
|
30
30
|
private readonly _queues;
|
|
31
31
|
private readonly _observations;
|
|
32
|
+
private readonly _reconnectBackoffs;
|
|
33
|
+
private readonly _reobserveTimers;
|
|
32
34
|
private _registered;
|
|
33
35
|
private _disposed;
|
|
34
36
|
/**
|
|
@@ -39,8 +41,13 @@ export declare class Reactors implements IReactors {
|
|
|
39
41
|
* @param _namespace - The namespace within the event store.
|
|
40
42
|
* @param lifecycle - The connection lifecycle used to react to disconnect events.
|
|
41
43
|
* @param _eventLog - The event log used to append side-effect events a reactor handler returns.
|
|
44
|
+
* @param _resultHandler - Optional handler for values a reactor handler returns.
|
|
45
|
+
* @param _eventStore - Optional event store handed to reactor handlers.
|
|
46
|
+
* @param _artifactActivator - Optional activator creating reactor instances.
|
|
47
|
+
* @param _skipKeepAlive - Whether the client runs without a keep-alive, which refuses client-hosted reactors.
|
|
48
|
+
* @param _reconnectBackoff - How a reactor waits before opening its observation stream again after it ended.
|
|
42
49
|
*/
|
|
43
|
-
constructor(_clientArtifacts: IClientArtifactsProvider, _connection: ChronicleConnection, _eventStoreName: string, _namespace: string, lifecycle: ConnectionLifecycle, _eventLog: IEventLog, _resultHandler?: ReactorResultHandler | undefined, _eventStore?: IEventStore | undefined, _artifactActivator?: ClientArtifactsActivator | undefined, _skipKeepAlive?: boolean);
|
|
50
|
+
constructor(_clientArtifacts: IClientArtifactsProvider, _connection: ChronicleConnection, _eventStoreName: string, _namespace: string, lifecycle: ConnectionLifecycle, _eventLog: IEventLog, _resultHandler?: ReactorResultHandler | undefined, _eventStore?: IEventStore | undefined, _artifactActivator?: ClientArtifactsActivator | undefined, _skipKeepAlive?: boolean, _reconnectBackoff?: ObservationReconnectBackoffOptions);
|
|
44
51
|
/** Stops observations permanently when the owning client is disposed. */
|
|
45
52
|
dispose(): void;
|
|
46
53
|
/** @inheritdoc */
|
|
@@ -49,17 +56,20 @@ export declare class Reactors implements IReactors {
|
|
|
49
56
|
register(): Promise<void>;
|
|
50
57
|
private startObservation;
|
|
51
58
|
private runObservation;
|
|
59
|
+
private reconnectBackoffFor;
|
|
52
60
|
/**
|
|
53
61
|
* Re-establishes an observation whose stream ended.
|
|
54
62
|
*
|
|
55
63
|
* The stream ending without an error is not proof that all is well — the kernel
|
|
56
64
|
* closes a cross-store (inbox) reactor's stream rather than tailing it forever,
|
|
57
65
|
* so a reactor that does not re-subscribe there silently stops observing until
|
|
58
|
-
* the whole client reconnects. Both endings are therefore retried
|
|
59
|
-
*
|
|
66
|
+
* the whole client reconnects. Both endings are therefore retried. The delay keeps
|
|
67
|
+
* a stream that keeps ending from becoming a hot loop, and grows with jitter while
|
|
68
|
+
* it does (see {@link ObservationReconnectBackoff}).
|
|
60
69
|
*/
|
|
61
70
|
private scheduleReobserve;
|
|
62
71
|
private observeReactor;
|
|
72
|
+
private streamFailed;
|
|
63
73
|
private disconnectAll;
|
|
64
74
|
}
|
|
65
75
|
//# sourceMappingURL=Reactors.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Reactors.d.ts","sourceRoot":"","sources":["../../reactors/Reactors.ts"],"names":[],"mappings":"AAGA,OAAO,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"Reactors.d.ts","sourceRoot":"","sources":["../../reactors/Reactors.ts"],"names":[],"mappings":"AAGA,OAAO,kBAAkB,CAAC;AAM1B,OAAO,EAAE,wBAAwB,EAAE,MAAM,uBAAuB,CAAC;AACjE,OAAO,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAC;AAC7D,OAAO,EAAE,mBAAmB,EAAE,MAAM,sCAAsC,CAAC;AAM3E,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gCAAgC,CAAC;AAEhE,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAQ3C,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAC;AAEtE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,KAAK,EAAE,wBAAwB,EAAE,MAAM,0CAA0C,CAAC;AAEzF,OAAO,EAA+B,KAAK,kCAAkC,EAAE,MAAM,+CAA+C,CAAC;AAiErI;;;GAGG;AACH,qBAAa,QAAS,YAAW,SAAS;IA2BlC,OAAO,CAAC,QAAQ,CAAC,gBAAgB;IACjC,OAAO,CAAC,QAAQ,CAAC,WAAW;IAC5B,OAAO,CAAC,QAAQ,CAAC,eAAe;IAChC,OAAO,CAAC,QAAQ,CAAC,UAAU;IAE3B,OAAO,CAAC,QAAQ,CAAC,SAAS;IAC1B,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC;IAChC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC;IAC7B,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAC;IACpC,OAAO,CAAC,QAAQ,CAAC,cAAc;IAC/B,OAAO,CAAC,QAAQ,CAAC,iBAAiB;IApCtC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAkC;IAC1D,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAsB;IACjD,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAAa;IACtD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAkC;IAC5D,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAiD;IACzE,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAsC;IACpE,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAkD;IACrF,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAoD;IACrF,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,SAAS,CAAS;IAE1B;;;;;;;;;;;;;OAaG;IACH,YACqB,gBAAgB,EAAE,wBAAwB,EAC1C,WAAW,EAAE,mBAAmB,EAChC,eAAe,EAAE,MAAM,EACvB,UAAU,EAAE,MAAM,EACnC,SAAS,EAAE,mBAAmB,EACb,SAAS,EAAE,SAAS,EACpB,cAAc,CAAC,EAAE,oBAAoB,YAAA,EACrC,WAAW,CAAC,EAAE,WAAW,YAAA,EACzB,kBAAkB,CAAC,EAAE,wBAAwB,YAAA,EAC7C,cAAc,GAAE,OAAe,EAC/B,iBAAiB,GAAE,kCAAuC,EAS9E;IAED,yEAAyE;IACzE,OAAO,IAAI,IAAI,CAMd;IAED,kBAAkB;IACZ,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC,CAS9B;IAED,kBAAkB;IACZ,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC,CAwB9B;IAED,OAAO,CAAC,gBAAgB;YAkBV,cAAc;IAiB5B,OAAO,CAAC,mBAAmB;IAS3B;;;;;;;;;OASG;IACH,OAAO,CAAC,iBAAiB;YAyBX,cAAc;IA+G5B,OAAO,CAAC,YAAY;IAsBpB,OAAO,CAAC,aAAa;CAQxB"}
|