@cratis/chronicle 6.62.0 → 6.63.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ChronicleClient.ts +86 -3
- package/connection/ConnectionLifecycle.ts +42 -1
- package/dist/ChronicleClient.d.ts +28 -1
- package/dist/ChronicleClient.d.ts.map +1 -1
- package/dist/ChronicleClient.js +71 -3
- package/dist/ChronicleClient.js.map +1 -1
- package/dist/connection/ConnectionLifecycle.d.ts +24 -0
- package/dist/connection/ConnectionLifecycle.d.ts.map +1 -1
- package/dist/connection/ConnectionLifecycle.js +40 -1
- package/dist/connection/ConnectionLifecycle.js.map +1 -1
- package/dist/observation/ObservationReconnectBackoff.d.ts +64 -0
- package/dist/observation/ObservationReconnectBackoff.d.ts.map +1 -0
- package/dist/observation/ObservationReconnectBackoff.js +75 -0
- package/dist/observation/ObservationReconnectBackoff.js.map +1 -0
- package/dist/observation/ObservationStreamOutcome.d.ts +10 -0
- package/dist/observation/ObservationStreamOutcome.d.ts.map +1 -0
- package/dist/observation/ObservationStreamOutcome.js +4 -0
- package/dist/observation/ObservationStreamOutcome.js.map +1 -0
- package/dist/observation/classifyStreamError.d.ts +25 -0
- package/dist/observation/classifyStreamError.d.ts.map +1 -0
- package/dist/observation/classifyStreamError.js +55 -0
- package/dist/observation/classifyStreamError.js.map +1 -0
- package/dist/observation/for_ObservationReconnectBackoff/given/a_backoff.d.ts +13 -0
- package/dist/observation/for_ObservationReconnectBackoff/given/a_backoff.d.ts.map +1 -0
- package/dist/observation/for_ObservationReconnectBackoff/given/a_backoff.js +25 -0
- package/dist/observation/for_ObservationReconnectBackoff/given/a_backoff.js.map +1 -0
- package/dist/observation/for_ObservationReconnectBackoff/given/a_scripted_stream.d.ts +27 -0
- package/dist/observation/for_ObservationReconnectBackoff/given/a_scripted_stream.d.ts.map +1 -0
- package/dist/observation/for_ObservationReconnectBackoff/given/a_scripted_stream.js +36 -0
- package/dist/observation/for_ObservationReconnectBackoff/given/a_scripted_stream.js.map +1 -0
- package/dist/reactors/Reactors.d.ts +15 -5
- package/dist/reactors/Reactors.d.ts.map +1 -1
- package/dist/reactors/Reactors.js +95 -18
- package/dist/reactors/Reactors.js.map +1 -1
- package/dist/reactors/for_Reactors/when_reconnecting_the_observation_stream/given/a_reactor_whose_stream_ends.d.ts +26 -0
- package/dist/reactors/for_Reactors/when_reconnecting_the_observation_stream/given/a_reactor_whose_stream_ends.d.ts.map +1 -0
- package/dist/reactors/for_Reactors/when_reconnecting_the_observation_stream/given/a_reactor_whose_stream_ends.js +64 -0
- package/dist/reactors/for_Reactors/when_reconnecting_the_observation_stream/given/a_reactor_whose_stream_ends.js.map +1 -0
- package/dist/reducers/Reducers.d.ts +11 -5
- package/dist/reducers/Reducers.d.ts.map +1 -1
- package/dist/reducers/Reducers.js +93 -19
- package/dist/reducers/Reducers.js.map +1 -1
- package/dist/reducers/for_Reducers/when_reconnecting_the_observation_stream/given/a_reducer_whose_stream_ends.d.ts +25 -0
- package/dist/reducers/for_Reducers/when_reconnecting_the_observation_stream/given/a_reducer_whose_stream_ends.d.ts.map +1 -0
- package/dist/reducers/for_Reducers/when_reconnecting_the_observation_stream/given/a_reducer_whose_stream_ends.js +70 -0
- package/dist/reducers/for_Reducers/when_reconnecting_the_observation_stream/given/a_reducer_whose_stream_ends.js.map +1 -0
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/for_ChronicleClient/when_reconnecting/and_an_observation_stream_lost_its_session.spec.ts +212 -0
- package/observation/ObservationReconnectBackoff.ts +109 -0
- package/observation/ObservationStreamOutcome.ts +12 -0
- package/observation/classifyStreamError.ts +67 -0
- package/observation/for_ObservationReconnectBackoff/given/a_backoff.ts +29 -0
- package/observation/for_ObservationReconnectBackoff/given/a_scripted_stream.ts +52 -0
- package/observation/for_ObservationReconnectBackoff/when_a_stream_delivered_before_it_ended.spec.ts +23 -0
- package/observation/for_ObservationReconnectBackoff/when_a_stream_stayed_open_before_it_ended.spec.ts +25 -0
- package/observation/for_ObservationReconnectBackoff/when_jittering_the_delay.spec.ts +24 -0
- package/observation/for_ObservationReconnectBackoff/when_streams_keep_ending_without_working.spec.ts +24 -0
- package/observation/for_ObservationReconnectBackoff/when_the_base_delay_is_not_positive.spec.ts +18 -0
- package/observation/for_ObservationReconnectBackoff/when_the_settings_are_not_finite.spec.ts +32 -0
- package/observation/for_ObservationReconnectBackoff/when_using_the_defaults.spec.ts +29 -0
- package/observation/for_classifyStreamError/when_classifying_a_stream_error.spec.ts +36 -0
- package/package.json +1 -1
- package/reactors/Reactors.ts +101 -18
- package/reactors/for_Reactors/when_reconnecting_the_observation_stream/and_a_handler_failed_with_a_lost_session.spec.ts +32 -0
- package/reactors/for_Reactors/when_reconnecting_the_observation_stream/and_a_replaced_stream_fails_late.spec.ts +28 -0
- package/reactors/for_Reactors/when_reconnecting_the_observation_stream/and_a_stream_delivered_before_it_ended.spec.ts +24 -0
- package/reactors/for_Reactors/when_reconnecting_the_observation_stream/and_a_stream_stayed_open_before_it_ended.spec.ts +23 -0
- package/reactors/for_Reactors/when_reconnecting_the_observation_stream/and_it_is_disposed_while_waiting_to_reopen.spec.ts +27 -0
- package/reactors/for_Reactors/when_reconnecting_the_observation_stream/and_streams_keep_ending_without_working.spec.ts +24 -0
- package/reactors/for_Reactors/when_reconnecting_the_observation_stream/and_the_client_reconnected_after_streams_kept_ending.spec.ts +24 -0
- package/reactors/for_Reactors/when_reconnecting_the_observation_stream/and_the_client_reconnected_while_waiting_to_reopen.spec.ts +25 -0
- package/reactors/for_Reactors/when_reconnecting_the_observation_stream/and_the_kernel_lost_the_session.spec.ts +24 -0
- package/reactors/for_Reactors/when_reconnecting_the_observation_stream/given/a_reactor_whose_stream_ends.ts +69 -0
- package/reducers/Reducers.ts +99 -19
- package/reducers/for_Reducers/when_reconnecting_the_observation_stream/and_a_reducer_failed_with_a_lost_session.spec.ts +33 -0
- package/reducers/for_Reducers/when_reconnecting_the_observation_stream/and_a_replaced_stream_fails_late.spec.ts +28 -0
- package/reducers/for_Reducers/when_reconnecting_the_observation_stream/and_a_stream_delivered_before_it_ended.spec.ts +24 -0
- package/reducers/for_Reducers/when_reconnecting_the_observation_stream/and_a_stream_stayed_open_before_it_ended.spec.ts +23 -0
- package/reducers/for_Reducers/when_reconnecting_the_observation_stream/and_it_is_disposed_while_waiting_to_reopen.spec.ts +27 -0
- package/reducers/for_Reducers/when_reconnecting_the_observation_stream/and_streams_keep_ending_without_working.spec.ts +24 -0
- package/reducers/for_Reducers/when_reconnecting_the_observation_stream/and_the_client_reconnected_after_streams_kept_ending.spec.ts +24 -0
- package/reducers/for_Reducers/when_reconnecting_the_observation_stream/and_the_client_reconnected_while_waiting_to_reopen.spec.ts +25 -0
- package/reducers/for_Reducers/when_reconnecting_the_observation_stream/and_the_kernel_lost_the_session.spec.ts +24 -0
- package/reducers/for_Reducers/when_reconnecting_the_observation_stream/given/a_reducer_whose_stream_ends.ts +69 -0
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
// Copyright (c) Cratis. All rights reserved.
|
|
2
|
+
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
|
|
3
|
+
|
|
4
|
+
/** What ended a kernel observation stream, deciding how the client recovers. */
|
|
5
|
+
export enum StreamErrorKind {
|
|
6
|
+
/** The stream failed for a reason of its own; it is opened again after a backoff. */
|
|
7
|
+
Failure = 'Failure',
|
|
8
|
+
|
|
9
|
+
/** The client cancelled the stream itself; it is not opened again. */
|
|
10
|
+
Cancelled = 'Cancelled',
|
|
11
|
+
|
|
12
|
+
/** The kernel is stopping or unreachable; the stream is opened again after a backoff. */
|
|
13
|
+
ConnectionLost = 'ConnectionLost',
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* The kernel no longer knows this client's connection, for instance after it restarted. Opening the stream again
|
|
17
|
+
* with the same connection id can never work, so the whole client reconnects with a new one.
|
|
18
|
+
*/
|
|
19
|
+
SessionLost = 'SessionLost'
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
const CANCELLED = 1;
|
|
23
|
+
const UNAVAILABLE = 14;
|
|
24
|
+
const CLIENT_IS_NOT_CONNECTED = 'ClientIsNotConnected';
|
|
25
|
+
|
|
26
|
+
interface GrpcLikeError {
|
|
27
|
+
code?: unknown;
|
|
28
|
+
details?: unknown;
|
|
29
|
+
message?: unknown;
|
|
30
|
+
name?: unknown;
|
|
31
|
+
cause?: unknown;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
const mentionsClientIsNotConnected = (error: GrpcLikeError): boolean =>
|
|
35
|
+
[error.details, error.message].some(text => typeof text === 'string' && text.includes(CLIENT_IS_NOT_CONNECTED));
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Classifies an error that ended a kernel observation stream, looking through wrapping errors.
|
|
39
|
+
*
|
|
40
|
+
* An {@link AggregateError} takes the most serious kind among the errors it wraps, in this order:
|
|
41
|
+
* {@link StreamErrorKind.Failure}, then {@link StreamErrorKind.SessionLost}, then {@link StreamErrorKind.ConnectionLost},
|
|
42
|
+
* then {@link StreamErrorKind.Cancelled}. This matches the .NET client.
|
|
43
|
+
* @param error - The error the stream ended with.
|
|
44
|
+
* @returns The {@link StreamErrorKind} for the error.
|
|
45
|
+
*/
|
|
46
|
+
export function classifyStreamError(error: unknown): StreamErrorKind {
|
|
47
|
+
if (error === null || typeof error !== 'object') {
|
|
48
|
+
return StreamErrorKind.Failure;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
if (error instanceof AggregateError && error.errors.length > 0) {
|
|
52
|
+
const kinds = error.errors.map(classifyStreamError);
|
|
53
|
+
if (kinds.includes(StreamErrorKind.Failure)) return StreamErrorKind.Failure;
|
|
54
|
+
if (kinds.includes(StreamErrorKind.SessionLost)) return StreamErrorKind.SessionLost;
|
|
55
|
+
return kinds.includes(StreamErrorKind.ConnectionLost) ? StreamErrorKind.ConnectionLost : StreamErrorKind.Cancelled;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
const grpcError = error as GrpcLikeError;
|
|
59
|
+
if (grpcError.name === 'AbortError') return StreamErrorKind.Cancelled;
|
|
60
|
+
if (typeof grpcError.code === 'number') {
|
|
61
|
+
if (grpcError.code === CANCELLED) return StreamErrorKind.Cancelled;
|
|
62
|
+
if (grpcError.code === UNAVAILABLE) return StreamErrorKind.ConnectionLost;
|
|
63
|
+
return mentionsClientIsNotConnected(grpcError) ? StreamErrorKind.SessionLost : StreamErrorKind.Failure;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
return grpcError.cause !== undefined ? classifyStreamError(grpcError.cause) : StreamErrorKind.Failure;
|
|
67
|
+
}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
// Copyright (c) Cratis. All rights reserved.
|
|
2
|
+
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
|
|
3
|
+
|
|
4
|
+
import { ObservationReconnectBackoff } from '../../ObservationReconnectBackoff.js';
|
|
5
|
+
|
|
6
|
+
/** A backoff with a 2 second base, a 60 second ceiling and a 3 minute healthy threshold, on a clock and jitter the spec controls. */
|
|
7
|
+
export class a_backoff {
|
|
8
|
+
static readonly baseDelayMs = 2_000;
|
|
9
|
+
static readonly maxDelayMs = 60_000;
|
|
10
|
+
static readonly healthyStreamDurationMs = 180_000;
|
|
11
|
+
|
|
12
|
+
now = 0;
|
|
13
|
+
jitter = 0;
|
|
14
|
+
|
|
15
|
+
readonly backoff = new ObservationReconnectBackoff({
|
|
16
|
+
baseDelayMs: a_backoff.baseDelayMs,
|
|
17
|
+
maxDelayMs: a_backoff.maxDelayMs,
|
|
18
|
+
healthyStreamDurationMs: a_backoff.healthyStreamDurationMs,
|
|
19
|
+
now: () => this.now,
|
|
20
|
+
nextJitter: () => this.jitter
|
|
21
|
+
});
|
|
22
|
+
|
|
23
|
+
/** Opens a stream, lets it stay open for a while and ends it, returning the wait before the next one. */
|
|
24
|
+
streamEndsAfter(openForMs: number): number {
|
|
25
|
+
this.backoff.streamOpened();
|
|
26
|
+
this.now += openForMs;
|
|
27
|
+
return this.backoff.nextDelayMs();
|
|
28
|
+
}
|
|
29
|
+
}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
// Copyright (c) Cratis. All rights reserved.
|
|
2
|
+
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
|
|
3
|
+
|
|
4
|
+
/** How one observation stream behaves: what it delivers, how long it stays open and the error it ends with, if any. */
|
|
5
|
+
export interface StreamScript<T> {
|
|
6
|
+
/** What the stream delivers before it ends. */
|
|
7
|
+
deliver?: T[];
|
|
8
|
+
|
|
9
|
+
/** How long, in milliseconds, the stream stays open after delivering before it ends. */
|
|
10
|
+
openForMs?: number;
|
|
11
|
+
|
|
12
|
+
/** Whether the stream stays open until the client aborts it. */
|
|
13
|
+
untilAborted?: boolean;
|
|
14
|
+
|
|
15
|
+
/** How long, in milliseconds, an aborted stream takes to end, as a slow transport would. */
|
|
16
|
+
endsLateByMs?: number;
|
|
17
|
+
|
|
18
|
+
/** The error the stream ends with. */
|
|
19
|
+
error?: unknown;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/** Records the observation streams a reactor or reducer opens and plays them by their scripts. */
|
|
23
|
+
export class a_scripted_stream<T> {
|
|
24
|
+
readonly openedAt: number[] = [];
|
|
25
|
+
readonly scripts: StreamScript<T>[] = [];
|
|
26
|
+
open = 0;
|
|
27
|
+
|
|
28
|
+
/** The observe method of the fake kernel connection. */
|
|
29
|
+
readonly observe = (_messages: unknown, options: { signal: AbortSignal }): AsyncGenerator<T> => this.play(options.signal);
|
|
30
|
+
|
|
31
|
+
/** Gets the wait before each stream after the first was opened. */
|
|
32
|
+
get waits(): number[] {
|
|
33
|
+
return this.openedAt.slice(1).map((at, index) => at - this.openedAt[index]);
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
private async *play(signal: AbortSignal): AsyncGenerator<T> {
|
|
37
|
+
this.openedAt.push(Date.now());
|
|
38
|
+
this.open++;
|
|
39
|
+
try {
|
|
40
|
+
const script = this.scripts.shift() ?? {};
|
|
41
|
+
for (const delivery of script.deliver ?? []) yield delivery;
|
|
42
|
+
if (script.openForMs) await new Promise(resolve => setTimeout(resolve, script.openForMs));
|
|
43
|
+
if (script.untilAborted) {
|
|
44
|
+
await new Promise<void>(resolve => signal.aborted ? resolve() : signal.addEventListener('abort', () => resolve(), { once: true }));
|
|
45
|
+
if (script.endsLateByMs) await new Promise(resolve => setTimeout(resolve, script.endsLateByMs));
|
|
46
|
+
}
|
|
47
|
+
if (script.error) throw script.error;
|
|
48
|
+
} finally {
|
|
49
|
+
this.open--;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
}
|
package/observation/for_ObservationReconnectBackoff/when_a_stream_delivered_before_it_ended.spec.ts
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
// Copyright (c) Cratis. All rights reserved.
|
|
2
|
+
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
|
|
3
|
+
|
|
4
|
+
import { beforeEach, chai, describe, it } from 'vitest';
|
|
5
|
+
import { a_backoff } from './given/a_backoff.js';
|
|
6
|
+
|
|
7
|
+
chai.should();
|
|
8
|
+
|
|
9
|
+
describe('when a stream delivered before it ended', () => {
|
|
10
|
+
let context: a_backoff;
|
|
11
|
+
let delay: number;
|
|
12
|
+
|
|
13
|
+
beforeEach(() => {
|
|
14
|
+
context = new a_backoff();
|
|
15
|
+
for (let attempt = 0; attempt < 4; attempt++) context.streamEndsAfter(1_000);
|
|
16
|
+
context.backoff.streamOpened();
|
|
17
|
+
context.backoff.streamDelivered();
|
|
18
|
+
context.now += 1_000;
|
|
19
|
+
delay = context.backoff.nextDelayMs();
|
|
20
|
+
});
|
|
21
|
+
|
|
22
|
+
it('should start the wait over at the base delay', () => delay.should.equal(a_backoff.baseDelayMs / 2));
|
|
23
|
+
});
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
// Copyright (c) Cratis. All rights reserved.
|
|
2
|
+
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
|
|
3
|
+
|
|
4
|
+
import { beforeEach, chai, describe, it } from 'vitest';
|
|
5
|
+
import { a_backoff } from './given/a_backoff.js';
|
|
6
|
+
|
|
7
|
+
chai.should();
|
|
8
|
+
|
|
9
|
+
describe('when a stream stayed open before it ended', () => {
|
|
10
|
+
let longEnough: number;
|
|
11
|
+
let tooShort: number;
|
|
12
|
+
|
|
13
|
+
beforeEach(() => {
|
|
14
|
+
const healthy = new a_backoff();
|
|
15
|
+
for (let attempt = 0; attempt < 3; attempt++) healthy.streamEndsAfter(1_000);
|
|
16
|
+
longEnough = healthy.streamEndsAfter(a_backoff.healthyStreamDurationMs);
|
|
17
|
+
|
|
18
|
+
const unhealthy = new a_backoff();
|
|
19
|
+
for (let attempt = 0; attempt < 3; attempt++) unhealthy.streamEndsAfter(1_000);
|
|
20
|
+
tooShort = unhealthy.streamEndsAfter(a_backoff.healthyStreamDurationMs - 1);
|
|
21
|
+
});
|
|
22
|
+
|
|
23
|
+
it('should start the wait over at the base delay once it stayed open long enough', () => longEnough.should.equal(a_backoff.baseDelayMs / 2));
|
|
24
|
+
it('should keep backing off when it closed just before counting as having worked', () => tooShort.should.equal(8_000));
|
|
25
|
+
});
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
// Copyright (c) Cratis. All rights reserved.
|
|
2
|
+
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
|
|
3
|
+
|
|
4
|
+
import { beforeEach, chai, describe, it } from 'vitest';
|
|
5
|
+
import { a_backoff } from './given/a_backoff.js';
|
|
6
|
+
|
|
7
|
+
chai.should();
|
|
8
|
+
|
|
9
|
+
describe('when jittering the delay', () => {
|
|
10
|
+
let delays: number[];
|
|
11
|
+
|
|
12
|
+
beforeEach(() => {
|
|
13
|
+
delays = [0, 0.5, -1, 2].map(jitter => {
|
|
14
|
+
const context = new a_backoff();
|
|
15
|
+
context.jitter = jitter;
|
|
16
|
+
return context.streamEndsAfter(0);
|
|
17
|
+
});
|
|
18
|
+
});
|
|
19
|
+
|
|
20
|
+
it('should land at half the wait for the lowest jitter', () => delays[0].should.equal(1_000));
|
|
21
|
+
it('should land in the upper half of the wait', () => delays[1].should.equal(1_500));
|
|
22
|
+
it('should clamp a jitter below zero to the lower bound', () => delays[2].should.equal(1_000));
|
|
23
|
+
it('should clamp a jitter above one to the full wait', () => delays[3].should.equal(2_000));
|
|
24
|
+
});
|
package/observation/for_ObservationReconnectBackoff/when_streams_keep_ending_without_working.spec.ts
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
// Copyright (c) Cratis. All rights reserved.
|
|
2
|
+
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
|
|
3
|
+
|
|
4
|
+
import { beforeEach, chai, describe, it } from 'vitest';
|
|
5
|
+
import { a_backoff } from './given/a_backoff.js';
|
|
6
|
+
|
|
7
|
+
chai.should();
|
|
8
|
+
|
|
9
|
+
describe('when streams keep ending without working', () => {
|
|
10
|
+
let context: a_backoff;
|
|
11
|
+
let delays: number[];
|
|
12
|
+
|
|
13
|
+
beforeEach(() => {
|
|
14
|
+
context = new a_backoff();
|
|
15
|
+
context.jitter = 0.999999;
|
|
16
|
+
delays = Array.from({ length: 8 }, () => context.streamEndsAfter(1_000));
|
|
17
|
+
});
|
|
18
|
+
|
|
19
|
+
it('should double the wait each time, starting from the base delay', () =>
|
|
20
|
+
delays.slice(0, 5).map(delay => Math.round(delay / 1_000)).should.deep.equal([2, 4, 8, 16, 32]));
|
|
21
|
+
|
|
22
|
+
it('should never wait longer than the ceiling', () => delays.forEach(delay => delay.should.be.at.most(a_backoff.maxDelayMs)));
|
|
23
|
+
it('should keep waiting close to the ceiling once it is reached', () => delays.slice(5).forEach(delay => delay.should.be.at.least(59_999)));
|
|
24
|
+
});
|
package/observation/for_ObservationReconnectBackoff/when_the_base_delay_is_not_positive.spec.ts
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
// Copyright (c) Cratis. All rights reserved.
|
|
2
|
+
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
|
|
3
|
+
|
|
4
|
+
import { beforeEach, chai, describe, it } from 'vitest';
|
|
5
|
+
import { ObservationReconnectBackoff } from '../ObservationReconnectBackoff.js';
|
|
6
|
+
|
|
7
|
+
chai.should();
|
|
8
|
+
|
|
9
|
+
describe('when the base delay is not positive', () => {
|
|
10
|
+
let delays: number[];
|
|
11
|
+
|
|
12
|
+
beforeEach(() => {
|
|
13
|
+
const backoff = new ObservationReconnectBackoff({ baseDelayMs: 0, nextJitter: () => 0.5 });
|
|
14
|
+
delays = [backoff.nextDelayMs(), backoff.nextDelayMs(), backoff.nextDelayMs()];
|
|
15
|
+
});
|
|
16
|
+
|
|
17
|
+
it('should use the base delay as it is, without backing off', () => delays.should.deep.equal([0, 0, 0]));
|
|
18
|
+
});
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
// Copyright (c) Cratis. All rights reserved.
|
|
2
|
+
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
|
|
3
|
+
|
|
4
|
+
import { beforeEach, chai, describe, it } from 'vitest';
|
|
5
|
+
import { ObservationReconnectBackoff } from '../ObservationReconnectBackoff.js';
|
|
6
|
+
|
|
7
|
+
chai.should();
|
|
8
|
+
|
|
9
|
+
describe('when the settings are not finite', () => {
|
|
10
|
+
let delays: number[];
|
|
11
|
+
|
|
12
|
+
beforeEach(() => {
|
|
13
|
+
let now = 0;
|
|
14
|
+
const backoff = new ObservationReconnectBackoff({
|
|
15
|
+
baseDelayMs: Number.NaN,
|
|
16
|
+
maxDelayMs: -1,
|
|
17
|
+
healthyStreamDurationMs: Number.POSITIVE_INFINITY,
|
|
18
|
+
now: () => now,
|
|
19
|
+
nextJitter: () => 1
|
|
20
|
+
});
|
|
21
|
+
delays = [];
|
|
22
|
+
for (let attempt = 0; attempt < 7; attempt++) {
|
|
23
|
+
backoff.streamOpened();
|
|
24
|
+
delays.push(backoff.nextDelayMs());
|
|
25
|
+
}
|
|
26
|
+
now = ObservationReconnectBackoff.defaultHealthyStreamDurationMs;
|
|
27
|
+
delays.push(backoff.nextDelayMs());
|
|
28
|
+
});
|
|
29
|
+
|
|
30
|
+
it('should fall back to the default delays', () =>
|
|
31
|
+
delays.should.deep.equal([2_000, 4_000, 8_000, 16_000, 32_000, 60_000, 60_000, 2_000]));
|
|
32
|
+
});
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
// Copyright (c) Cratis. All rights reserved.
|
|
2
|
+
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
|
|
3
|
+
|
|
4
|
+
import { beforeEach, chai, describe, it } from 'vitest';
|
|
5
|
+
import { ObservationReconnectBackoff } from '../ObservationReconnectBackoff.js';
|
|
6
|
+
|
|
7
|
+
chai.should();
|
|
8
|
+
|
|
9
|
+
describe('when using the defaults', () => {
|
|
10
|
+
let delays: number[];
|
|
11
|
+
|
|
12
|
+
beforeEach(() => {
|
|
13
|
+
let now = 0;
|
|
14
|
+
const backoff = new ObservationReconnectBackoff({ now: () => now, nextJitter: () => 0.999999 });
|
|
15
|
+
delays = Array.from({ length: 7 }, () => {
|
|
16
|
+
backoff.streamOpened();
|
|
17
|
+
now += 1_000;
|
|
18
|
+
return backoff.nextDelayMs();
|
|
19
|
+
});
|
|
20
|
+
backoff.streamOpened();
|
|
21
|
+
now += ObservationReconnectBackoff.defaultHealthyStreamDurationMs;
|
|
22
|
+
delays.push(backoff.nextDelayMs());
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
it('should start at two seconds', () => Math.round(delays[0] / 1_000).should.equal(2));
|
|
26
|
+
it('should never wait longer than a minute', () => delays.forEach(delay => delay.should.be.at.most(60_000)));
|
|
27
|
+
it('should reach a minute while streams keep ending', () => Math.round(delays[6] / 1_000).should.equal(60));
|
|
28
|
+
it('should start over after a stream stayed open for three minutes', () => Math.round(delays[7] / 1_000).should.equal(2));
|
|
29
|
+
});
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
// Copyright (c) Cratis. All rights reserved.
|
|
2
|
+
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
|
|
3
|
+
|
|
4
|
+
import { ClientError, Status } from 'nice-grpc';
|
|
5
|
+
import { chai, describe, it } from 'vitest';
|
|
6
|
+
import { classifyStreamError, StreamErrorKind } from '../classifyStreamError.js';
|
|
7
|
+
|
|
8
|
+
chai.should();
|
|
9
|
+
|
|
10
|
+
const sessionLost = () => new ClientError('/Reactors/Observe', Status.UNKNOWN, 'ClientIsNotConnected: connection 42 is not connected');
|
|
11
|
+
const abort = () => Object.assign(new Error('The operation has been aborted'), { name: 'AbortError' });
|
|
12
|
+
|
|
13
|
+
describe('when classifying a stream error', () => {
|
|
14
|
+
const cases: [string, unknown, StreamErrorKind][] = [
|
|
15
|
+
['the kernel no longer knows the connection', sessionLost(), StreamErrorKind.SessionLost],
|
|
16
|
+
['the message names the lost connection', Object.assign(new Error('ClientIsNotConnected'), { code: Status.INTERNAL }), StreamErrorKind.SessionLost],
|
|
17
|
+
['the lost connection is wrapped', new Error('stream failed', { cause: sessionLost() }), StreamErrorKind.SessionLost],
|
|
18
|
+
['every wrapped error lost the session or connection',
|
|
19
|
+
new AggregateError([sessionLost(), new ClientError('/x', Status.UNAVAILABLE, '')]), StreamErrorKind.SessionLost],
|
|
20
|
+
['a wrapped error is a failure of its own',
|
|
21
|
+
new AggregateError([sessionLost(), new ClientError('/x', Status.INTERNAL, 'boom')]), StreamErrorKind.Failure],
|
|
22
|
+
['a wrapped error lost the connection and another was cancelled',
|
|
23
|
+
new AggregateError([new ClientError('/x', Status.CANCELLED, ''), new ClientError('/x', Status.UNAVAILABLE, '')]),
|
|
24
|
+
StreamErrorKind.ConnectionLost],
|
|
25
|
+
['every wrapped error was cancelled', new AggregateError([abort(), new ClientError('/x', Status.CANCELLED, '')]), StreamErrorKind.Cancelled],
|
|
26
|
+
['the kernel is unreachable', new ClientError('/x', Status.UNAVAILABLE, 'ClientIsNotConnected'), StreamErrorKind.ConnectionLost],
|
|
27
|
+
['the call was cancelled', new ClientError('/x', Status.CANCELLED, ''), StreamErrorKind.Cancelled],
|
|
28
|
+
['the client aborted the stream', abort(), StreamErrorKind.Cancelled],
|
|
29
|
+
['the stream failed for another reason', new ClientError('/x', Status.INTERNAL, 'boom'), StreamErrorKind.Failure],
|
|
30
|
+
['the error is not an object', 'boom', StreamErrorKind.Failure]
|
|
31
|
+
];
|
|
32
|
+
|
|
33
|
+
for (const [scenario, error, kind] of cases) {
|
|
34
|
+
it(`should classify as ${kind} when ${scenario}`, () => classifyStreamError(error).should.equal(kind));
|
|
35
|
+
}
|
|
36
|
+
});
|
package/package.json
CHANGED
package/reactors/Reactors.ts
CHANGED
|
@@ -6,6 +6,7 @@ import { createLogger } from '../logging/createLogger.js';
|
|
|
6
6
|
import { runInBackgroundContext } from '../telemetry/runInBackgroundContext.js';
|
|
7
7
|
import { Constructor } from '@cratis/fundamentals';
|
|
8
8
|
import { EventGenerationDelivery, ReactorMessage } from '@cratis/chronicle.contracts';
|
|
9
|
+
import type { EventsToObserve } from '@cratis/chronicle.contracts';
|
|
9
10
|
import { IClientArtifactsProvider } from '../artifacts/index.js';
|
|
10
11
|
import { ChronicleConnection } from '../connection/index.js';
|
|
11
12
|
import { ConnectionLifecycle } from '../connection/ConnectionLifecycle.js';
|
|
@@ -29,6 +30,9 @@ import type { ReactorServices } from './ReactorServices.js';
|
|
|
29
30
|
import type { IEventStore } from '../IEventStore.js';
|
|
30
31
|
import type { ClientArtifactsActivator } from '../artifacts/ClientArtifactsActivator.js';
|
|
31
32
|
import { refuseClientHostedObservers } from '../artifacts/refuseClientHostedObservers.js';
|
|
33
|
+
import { ObservationReconnectBackoff, type ObservationReconnectBackoffOptions } from '../observation/ObservationReconnectBackoff.js';
|
|
34
|
+
import { classifyStreamError, StreamErrorKind } from '../observation/classifyStreamError.js';
|
|
35
|
+
import type { ObservationStreamOutcome } from '../observation/ObservationStreamOutcome.js';
|
|
32
36
|
|
|
33
37
|
/** Expression used to partition reactor observations by event source ID. */
|
|
34
38
|
const EVENT_SOURCE_ID_KEY = '$eventSourceId';
|
|
@@ -96,15 +100,14 @@ class AsyncQueue<T> {
|
|
|
96
100
|
* with the Chronicle Kernel via bidirectional gRPC streaming.
|
|
97
101
|
*/
|
|
98
102
|
export class Reactors implements IReactors {
|
|
99
|
-
/** How long to wait before re-establishing an observation whose stream ended. */
|
|
100
|
-
private static readonly _reobserveDelayMs = 2000;
|
|
101
|
-
|
|
102
103
|
private readonly _logger: ReturnType<typeof createLogger>;
|
|
103
104
|
private readonly _lifecycle: ConnectionLifecycle;
|
|
104
105
|
private readonly _unsubscribeDisconnected: () => void;
|
|
105
106
|
private readonly _reactors = new Map<string, Constructor>();
|
|
106
107
|
private readonly _queues = new Map<string, AsyncQueue<ReactorMessage>>();
|
|
107
108
|
private readonly _observations = new Map<string, AbortController>();
|
|
109
|
+
private readonly _reconnectBackoffs = new Map<string, ObservationReconnectBackoff>();
|
|
110
|
+
private readonly _reobserveTimers = new Map<string, ReturnType<typeof setTimeout>>();
|
|
108
111
|
private _registered = false;
|
|
109
112
|
private _disposed = false;
|
|
110
113
|
|
|
@@ -116,6 +119,11 @@ export class Reactors implements IReactors {
|
|
|
116
119
|
* @param _namespace - The namespace within the event store.
|
|
117
120
|
* @param lifecycle - The connection lifecycle used to react to disconnect events.
|
|
118
121
|
* @param _eventLog - The event log used to append side-effect events a reactor handler returns.
|
|
122
|
+
* @param _resultHandler - Optional handler for values a reactor handler returns.
|
|
123
|
+
* @param _eventStore - Optional event store handed to reactor handlers.
|
|
124
|
+
* @param _artifactActivator - Optional activator creating reactor instances.
|
|
125
|
+
* @param _skipKeepAlive - Whether the client runs without a keep-alive, which refuses client-hosted reactors.
|
|
126
|
+
* @param _reconnectBackoff - How a reactor waits before opening its observation stream again after it ended.
|
|
119
127
|
*/
|
|
120
128
|
constructor(
|
|
121
129
|
private readonly _clientArtifacts: IClientArtifactsProvider,
|
|
@@ -127,7 +135,8 @@ export class Reactors implements IReactors {
|
|
|
127
135
|
private readonly _resultHandler?: ReactorResultHandler,
|
|
128
136
|
private readonly _eventStore?: IEventStore,
|
|
129
137
|
private readonly _artifactActivator?: ClientArtifactsActivator,
|
|
130
|
-
private readonly _skipKeepAlive: boolean = false
|
|
138
|
+
private readonly _skipKeepAlive: boolean = false,
|
|
139
|
+
private readonly _reconnectBackoff: ObservationReconnectBackoffOptions = {}
|
|
131
140
|
) {
|
|
132
141
|
this._logger = createLogger('@cratis/chronicle/reactors', _connection.logger);
|
|
133
142
|
this._lifecycle = lifecycle;
|
|
@@ -144,6 +153,7 @@ export class Reactors implements IReactors {
|
|
|
144
153
|
this._disposed = true;
|
|
145
154
|
this._registered = false;
|
|
146
155
|
this.disconnectAll();
|
|
156
|
+
this._reconnectBackoffs.clear();
|
|
147
157
|
}
|
|
148
158
|
|
|
149
159
|
/** @inheritdoc */
|
|
@@ -210,13 +220,23 @@ export class Reactors implements IReactors {
|
|
|
210
220
|
eventTypes: ReactorEventTypeEntry[],
|
|
211
221
|
validateInstance: (instance: object) => void
|
|
212
222
|
): Promise<void> {
|
|
223
|
+
let outcome: ObservationStreamOutcome = 'ended';
|
|
213
224
|
try {
|
|
214
|
-
await this.observeReactor(id, reactorType, eventSequenceId, eventTypes, validateInstance);
|
|
225
|
+
outcome = await this.observeReactor(id, reactorType, eventSequenceId, eventTypes, validateInstance);
|
|
215
226
|
} catch (error) {
|
|
216
227
|
this._logger.error('Reactor observation loop exited with error', { reactorId: id, error });
|
|
217
228
|
}
|
|
218
229
|
|
|
219
|
-
this.scheduleReobserve(id, reactorType, eventTypes, validateInstance);
|
|
230
|
+
if (outcome === 'ended') this.scheduleReobserve(id, reactorType, eventTypes, validateInstance);
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
private reconnectBackoffFor(id: string): ObservationReconnectBackoff {
|
|
234
|
+
let backoff = this._reconnectBackoffs.get(id);
|
|
235
|
+
if (!backoff) {
|
|
236
|
+
backoff = new ObservationReconnectBackoff(this._reconnectBackoff);
|
|
237
|
+
this._reconnectBackoffs.set(id, backoff);
|
|
238
|
+
}
|
|
239
|
+
return backoff;
|
|
220
240
|
}
|
|
221
241
|
|
|
222
242
|
/**
|
|
@@ -225,8 +245,9 @@ export class Reactors implements IReactors {
|
|
|
225
245
|
* The stream ending without an error is not proof that all is well — the kernel
|
|
226
246
|
* closes a cross-store (inbox) reactor's stream rather than tailing it forever,
|
|
227
247
|
* so a reactor that does not re-subscribe there silently stops observing until
|
|
228
|
-
* the whole client reconnects. Both endings are therefore retried
|
|
229
|
-
*
|
|
248
|
+
* the whole client reconnects. Both endings are therefore retried. The delay keeps
|
|
249
|
+
* a stream that keeps ending from becoming a hot loop, and grows with jitter while
|
|
250
|
+
* it does (see {@link ObservationReconnectBackoff}).
|
|
230
251
|
*/
|
|
231
252
|
private scheduleReobserve(id: string, reactorType: Constructor, eventTypes: ReactorEventTypeEntry[], validateInstance: (instance: object) => void): void {
|
|
232
253
|
// A disconnect clears the registration; the reconnect re-registers every
|
|
@@ -235,16 +256,22 @@ export class Reactors implements IReactors {
|
|
|
235
256
|
return;
|
|
236
257
|
}
|
|
237
258
|
|
|
259
|
+
clearTimeout(this._reobserveTimers.get(id));
|
|
260
|
+
const delayMs = this.reconnectBackoffFor(id).nextDelayMs();
|
|
261
|
+
this._logger.debug('Waiting before re-establishing reactor observation', { reactorId: id, delayMs });
|
|
238
262
|
const handle = runInBackgroundContext(() => setTimeout(() => {
|
|
239
|
-
|
|
263
|
+
this._reobserveTimers.delete(id);
|
|
264
|
+
// A reconnect in the meantime has already opened a new stream; opening another would leave one untracked.
|
|
265
|
+
if (!this._registered || this._disposed || this._observations.has(id)) {
|
|
240
266
|
return;
|
|
241
267
|
}
|
|
242
268
|
|
|
243
269
|
this._logger.info('Re-establishing reactor observation', { reactorId: id });
|
|
244
270
|
this.startObservation(id, reactorType, eventTypes, validateInstance);
|
|
245
|
-
},
|
|
271
|
+
}, delayMs));
|
|
246
272
|
|
|
247
273
|
handle.unref?.();
|
|
274
|
+
this._reobserveTimers.set(id, handle);
|
|
248
275
|
}
|
|
249
276
|
|
|
250
277
|
private async observeReactor(
|
|
@@ -253,17 +280,20 @@ export class Reactors implements IReactors {
|
|
|
253
280
|
eventSequenceId: string,
|
|
254
281
|
eventTypes: ReactorEventTypeEntry[],
|
|
255
282
|
validateInstance: (instance: object) => void
|
|
256
|
-
): Promise<
|
|
283
|
+
): Promise<ObservationStreamOutcome> {
|
|
257
284
|
const batchMethods = getReactorBatchMethods(reactorType, this._clientArtifacts.eventTypes);
|
|
258
285
|
const queue = new AsyncQueue<ReactorMessage>();
|
|
259
286
|
const controller = new AbortController();
|
|
260
287
|
this._queues.set(id, queue);
|
|
261
288
|
this._observations.set(id, controller);
|
|
289
|
+
const connectionId = this._lifecycle.connectionId;
|
|
290
|
+
const backoff = this.reconnectBackoffFor(id);
|
|
291
|
+
backoff.streamOpened();
|
|
262
292
|
|
|
263
293
|
queue.send({
|
|
264
294
|
Content: {
|
|
265
295
|
Value0: {
|
|
266
|
-
ConnectionId:
|
|
296
|
+
ConnectionId: connectionId,
|
|
267
297
|
EventStore: this._eventStoreName,
|
|
268
298
|
Namespace: this._namespace,
|
|
269
299
|
Reactor: {
|
|
@@ -290,6 +320,8 @@ export class Reactors implements IReactors {
|
|
|
290
320
|
}
|
|
291
321
|
});
|
|
292
322
|
|
|
323
|
+
let iterator: AsyncIterator<EventsToObserve> | undefined;
|
|
324
|
+
let streamFinished = false;
|
|
293
325
|
try {
|
|
294
326
|
const reactorInstance = this._artifactActivator ? undefined : new (reactorType as new () => Record<string, Function>)();
|
|
295
327
|
if (reactorInstance) validateInstance(reactorInstance);
|
|
@@ -304,17 +336,44 @@ export class Reactors implements IReactors {
|
|
|
304
336
|
logger: this._logger, connectionLogger: this._connection.logger
|
|
305
337
|
});
|
|
306
338
|
|
|
307
|
-
|
|
308
|
-
|
|
339
|
+
// Pull each message by hand so only a failure of the stream itself is classified: an error from a
|
|
340
|
+
// handler is the handler's failure, and must never make the whole client reconnect.
|
|
341
|
+
iterator = this._connection.reactors.observe(queue, { signal: controller.signal })[Symbol.asyncIterator]();
|
|
342
|
+
for (;;) {
|
|
343
|
+
let next: IteratorResult<EventsToObserve>;
|
|
344
|
+
try {
|
|
345
|
+
next = await iterator.next();
|
|
346
|
+
} catch (streamError) {
|
|
347
|
+
streamFinished = true;
|
|
348
|
+
return this.streamFailed(id, queue, controller.signal, connectionId, streamError);
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
if (next.done) {
|
|
352
|
+
streamFinished = true;
|
|
353
|
+
break;
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
backoff.streamDelivered();
|
|
357
|
+
const result = await dispatcher.dispatch(next.value);
|
|
309
358
|
queue.send({ Content: { Value0: undefined, Value1: result } });
|
|
310
359
|
}
|
|
311
|
-
|
|
312
|
-
if (
|
|
360
|
+
|
|
361
|
+
if (this._queues.get(id) !== queue) {
|
|
313
362
|
this._logger.debug('Reactor observation stream closed cleanly', { reactorId: id });
|
|
314
|
-
|
|
315
|
-
this._logger.error('Reactor observation stream ended unexpectedly', { reactorId: id, error: err });
|
|
363
|
+
return 'superseded';
|
|
316
364
|
}
|
|
365
|
+
|
|
366
|
+
return 'ended';
|
|
367
|
+
} catch (handlerError) {
|
|
368
|
+
if (this._queues.get(id) !== queue) return 'superseded';
|
|
369
|
+
this._logger.error('Reactor observation failed while handling a delivery', { reactorId: id, error: handlerError });
|
|
370
|
+
return 'ended';
|
|
317
371
|
} finally {
|
|
372
|
+
if (!streamFinished) {
|
|
373
|
+
controller.abort();
|
|
374
|
+
void iterator?.return?.()?.catch(() => undefined);
|
|
375
|
+
}
|
|
376
|
+
|
|
318
377
|
// Only retire our own queue: a reconnect can already have replaced it,
|
|
319
378
|
// and deleting the new one would leave that observation untracked and
|
|
320
379
|
// leak a duplicate stream on every reconnect.
|
|
@@ -326,7 +385,31 @@ export class Reactors implements IReactors {
|
|
|
326
385
|
}
|
|
327
386
|
}
|
|
328
387
|
|
|
388
|
+
private streamFailed(id: string, queue: AsyncQueue<ReactorMessage>, signal: AbortSignal, connectionId: string, error: unknown): ObservationStreamOutcome {
|
|
389
|
+
// A late error from a stream that a disconnect, a dispose or a newer stream replaced says nothing about now.
|
|
390
|
+
if (this._queues.get(id) !== queue) {
|
|
391
|
+
this._logger.debug('Reactor observation stream closed cleanly', { reactorId: id });
|
|
392
|
+
return 'superseded';
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
const kind = classifyStreamError(error);
|
|
396
|
+
if (kind === StreamErrorKind.Cancelled && signal.aborted) return 'stopped';
|
|
397
|
+
if (kind === StreamErrorKind.SessionLost) {
|
|
398
|
+
// The kernel lost this client's registration. Opening the same stream with the same connection id
|
|
399
|
+
// can never work, so reconnect the whole client, which re-registers every reactor.
|
|
400
|
+
this._logger.warn('Reactor observation stream lost its session; reconnecting the client', { reactorId: id, error });
|
|
401
|
+
this._lifecycle.requestReconnect(connectionId, 'reactor-session-lost', error,
|
|
402
|
+
handlerError => this._logger.error('Reconnect request handler failed', { reactorId: id, error: handlerError }));
|
|
403
|
+
return 'sessionLost';
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
this._logger.error('Reactor observation stream ended unexpectedly', { reactorId: id, error });
|
|
407
|
+
return 'ended';
|
|
408
|
+
}
|
|
409
|
+
|
|
329
410
|
private disconnectAll(): void {
|
|
411
|
+
for (const timer of this._reobserveTimers.values()) clearTimeout(timer);
|
|
412
|
+
this._reobserveTimers.clear();
|
|
330
413
|
for (const controller of this._observations.values()) controller.abort();
|
|
331
414
|
for (const queue of this._queues.values()) queue.complete();
|
|
332
415
|
this._observations.clear();
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
// Copyright (c) Cratis. All rights reserved.
|
|
2
|
+
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
|
|
3
|
+
|
|
4
|
+
import { afterEach, beforeEach, chai, describe, it, vi } from 'vitest';
|
|
5
|
+
import { ClientError, Status } from 'nice-grpc';
|
|
6
|
+
import { a_reactor_whose_stream_ends } from './given/a_reactor_whose_stream_ends.js';
|
|
7
|
+
import { page } from '../given/observe_reactor_messages.js';
|
|
8
|
+
import { ReactorPageDispatcher } from '../../ReactorPageDispatcher.js';
|
|
9
|
+
|
|
10
|
+
chai.should();
|
|
11
|
+
|
|
12
|
+
describe('when reconnecting a reactor observation stream and a handler failed with a lost session', () => {
|
|
13
|
+
let context: a_reactor_whose_stream_ends;
|
|
14
|
+
|
|
15
|
+
beforeEach(async () => {
|
|
16
|
+
context = new a_reactor_whose_stream_ends();
|
|
17
|
+
vi.spyOn(ReactorPageDispatcher.prototype, 'dispatch')
|
|
18
|
+
.mockRejectedValueOnce(new ClientError('/EventSequences/Append', Status.UNKNOWN, 'ClientIsNotConnected: connection is not connected'));
|
|
19
|
+
context.scripts.push({ deliver: [page(1n, {})], untilAborted: true }, { untilAborted: true });
|
|
20
|
+
await context.reactors.register();
|
|
21
|
+
await vi.advanceTimersByTimeAsync(120_000);
|
|
22
|
+
});
|
|
23
|
+
|
|
24
|
+
afterEach(() => {
|
|
25
|
+
context.dispose();
|
|
26
|
+
vi.restoreAllMocks();
|
|
27
|
+
});
|
|
28
|
+
|
|
29
|
+
it('should not ask the whole client to reconnect', () => context.reconnectRequests.should.be.empty);
|
|
30
|
+
it('should open the stream again after the base delay', () => context.waits.should.deep.equal([1_000]));
|
|
31
|
+
it('should close the stream the handler failed on', () => context.stream.open.should.equal(1));
|
|
32
|
+
});
|