@cratis/chronicle 6.62.0 → 6.63.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. package/ChronicleClient.ts +86 -3
  2. package/connection/ConnectionLifecycle.ts +42 -1
  3. package/dist/ChronicleClient.d.ts +28 -1
  4. package/dist/ChronicleClient.d.ts.map +1 -1
  5. package/dist/ChronicleClient.js +71 -3
  6. package/dist/ChronicleClient.js.map +1 -1
  7. package/dist/connection/ConnectionLifecycle.d.ts +24 -0
  8. package/dist/connection/ConnectionLifecycle.d.ts.map +1 -1
  9. package/dist/connection/ConnectionLifecycle.js +40 -1
  10. package/dist/connection/ConnectionLifecycle.js.map +1 -1
  11. package/dist/observation/ObservationReconnectBackoff.d.ts +64 -0
  12. package/dist/observation/ObservationReconnectBackoff.d.ts.map +1 -0
  13. package/dist/observation/ObservationReconnectBackoff.js +75 -0
  14. package/dist/observation/ObservationReconnectBackoff.js.map +1 -0
  15. package/dist/observation/ObservationStreamOutcome.d.ts +10 -0
  16. package/dist/observation/ObservationStreamOutcome.d.ts.map +1 -0
  17. package/dist/observation/ObservationStreamOutcome.js +4 -0
  18. package/dist/observation/ObservationStreamOutcome.js.map +1 -0
  19. package/dist/observation/classifyStreamError.d.ts +25 -0
  20. package/dist/observation/classifyStreamError.d.ts.map +1 -0
  21. package/dist/observation/classifyStreamError.js +55 -0
  22. package/dist/observation/classifyStreamError.js.map +1 -0
  23. package/dist/observation/for_ObservationReconnectBackoff/given/a_backoff.d.ts +13 -0
  24. package/dist/observation/for_ObservationReconnectBackoff/given/a_backoff.d.ts.map +1 -0
  25. package/dist/observation/for_ObservationReconnectBackoff/given/a_backoff.js +25 -0
  26. package/dist/observation/for_ObservationReconnectBackoff/given/a_backoff.js.map +1 -0
  27. package/dist/observation/for_ObservationReconnectBackoff/given/a_scripted_stream.d.ts +27 -0
  28. package/dist/observation/for_ObservationReconnectBackoff/given/a_scripted_stream.d.ts.map +1 -0
  29. package/dist/observation/for_ObservationReconnectBackoff/given/a_scripted_stream.js +36 -0
  30. package/dist/observation/for_ObservationReconnectBackoff/given/a_scripted_stream.js.map +1 -0
  31. package/dist/reactors/Reactors.d.ts +15 -5
  32. package/dist/reactors/Reactors.d.ts.map +1 -1
  33. package/dist/reactors/Reactors.js +95 -18
  34. package/dist/reactors/Reactors.js.map +1 -1
  35. package/dist/reactors/for_Reactors/when_reconnecting_the_observation_stream/given/a_reactor_whose_stream_ends.d.ts +26 -0
  36. package/dist/reactors/for_Reactors/when_reconnecting_the_observation_stream/given/a_reactor_whose_stream_ends.d.ts.map +1 -0
  37. package/dist/reactors/for_Reactors/when_reconnecting_the_observation_stream/given/a_reactor_whose_stream_ends.js +64 -0
  38. package/dist/reactors/for_Reactors/when_reconnecting_the_observation_stream/given/a_reactor_whose_stream_ends.js.map +1 -0
  39. package/dist/reducers/Reducers.d.ts +11 -5
  40. package/dist/reducers/Reducers.d.ts.map +1 -1
  41. package/dist/reducers/Reducers.js +93 -19
  42. package/dist/reducers/Reducers.js.map +1 -1
  43. package/dist/reducers/for_Reducers/when_reconnecting_the_observation_stream/given/a_reducer_whose_stream_ends.d.ts +25 -0
  44. package/dist/reducers/for_Reducers/when_reconnecting_the_observation_stream/given/a_reducer_whose_stream_ends.d.ts.map +1 -0
  45. package/dist/reducers/for_Reducers/when_reconnecting_the_observation_stream/given/a_reducer_whose_stream_ends.js +70 -0
  46. package/dist/reducers/for_Reducers/when_reconnecting_the_observation_stream/given/a_reducer_whose_stream_ends.js.map +1 -0
  47. package/dist/tsconfig.tsbuildinfo +1 -1
  48. package/for_ChronicleClient/when_reconnecting/and_an_observation_stream_lost_its_session.spec.ts +212 -0
  49. package/observation/ObservationReconnectBackoff.ts +109 -0
  50. package/observation/ObservationStreamOutcome.ts +12 -0
  51. package/observation/classifyStreamError.ts +67 -0
  52. package/observation/for_ObservationReconnectBackoff/given/a_backoff.ts +29 -0
  53. package/observation/for_ObservationReconnectBackoff/given/a_scripted_stream.ts +52 -0
  54. package/observation/for_ObservationReconnectBackoff/when_a_stream_delivered_before_it_ended.spec.ts +23 -0
  55. package/observation/for_ObservationReconnectBackoff/when_a_stream_stayed_open_before_it_ended.spec.ts +25 -0
  56. package/observation/for_ObservationReconnectBackoff/when_jittering_the_delay.spec.ts +24 -0
  57. package/observation/for_ObservationReconnectBackoff/when_streams_keep_ending_without_working.spec.ts +24 -0
  58. package/observation/for_ObservationReconnectBackoff/when_the_base_delay_is_not_positive.spec.ts +18 -0
  59. package/observation/for_ObservationReconnectBackoff/when_the_settings_are_not_finite.spec.ts +32 -0
  60. package/observation/for_ObservationReconnectBackoff/when_using_the_defaults.spec.ts +29 -0
  61. package/observation/for_classifyStreamError/when_classifying_a_stream_error.spec.ts +36 -0
  62. package/package.json +1 -1
  63. package/reactors/Reactors.ts +101 -18
  64. package/reactors/for_Reactors/when_reconnecting_the_observation_stream/and_a_handler_failed_with_a_lost_session.spec.ts +32 -0
  65. package/reactors/for_Reactors/when_reconnecting_the_observation_stream/and_a_replaced_stream_fails_late.spec.ts +28 -0
  66. package/reactors/for_Reactors/when_reconnecting_the_observation_stream/and_a_stream_delivered_before_it_ended.spec.ts +24 -0
  67. package/reactors/for_Reactors/when_reconnecting_the_observation_stream/and_a_stream_stayed_open_before_it_ended.spec.ts +23 -0
  68. package/reactors/for_Reactors/when_reconnecting_the_observation_stream/and_it_is_disposed_while_waiting_to_reopen.spec.ts +27 -0
  69. package/reactors/for_Reactors/when_reconnecting_the_observation_stream/and_streams_keep_ending_without_working.spec.ts +24 -0
  70. package/reactors/for_Reactors/when_reconnecting_the_observation_stream/and_the_client_reconnected_after_streams_kept_ending.spec.ts +24 -0
  71. package/reactors/for_Reactors/when_reconnecting_the_observation_stream/and_the_client_reconnected_while_waiting_to_reopen.spec.ts +25 -0
  72. package/reactors/for_Reactors/when_reconnecting_the_observation_stream/and_the_kernel_lost_the_session.spec.ts +24 -0
  73. package/reactors/for_Reactors/when_reconnecting_the_observation_stream/given/a_reactor_whose_stream_ends.ts +69 -0
  74. package/reducers/Reducers.ts +99 -19
  75. package/reducers/for_Reducers/when_reconnecting_the_observation_stream/and_a_reducer_failed_with_a_lost_session.spec.ts +33 -0
  76. package/reducers/for_Reducers/when_reconnecting_the_observation_stream/and_a_replaced_stream_fails_late.spec.ts +28 -0
  77. package/reducers/for_Reducers/when_reconnecting_the_observation_stream/and_a_stream_delivered_before_it_ended.spec.ts +24 -0
  78. package/reducers/for_Reducers/when_reconnecting_the_observation_stream/and_a_stream_stayed_open_before_it_ended.spec.ts +23 -0
  79. package/reducers/for_Reducers/when_reconnecting_the_observation_stream/and_it_is_disposed_while_waiting_to_reopen.spec.ts +27 -0
  80. package/reducers/for_Reducers/when_reconnecting_the_observation_stream/and_streams_keep_ending_without_working.spec.ts +24 -0
  81. package/reducers/for_Reducers/when_reconnecting_the_observation_stream/and_the_client_reconnected_after_streams_kept_ending.spec.ts +24 -0
  82. package/reducers/for_Reducers/when_reconnecting_the_observation_stream/and_the_client_reconnected_while_waiting_to_reopen.spec.ts +25 -0
  83. package/reducers/for_Reducers/when_reconnecting_the_observation_stream/and_the_kernel_lost_the_session.spec.ts +24 -0
  84. package/reducers/for_Reducers/when_reconnecting_the_observation_stream/given/a_reducer_whose_stream_ends.ts +69 -0
@@ -0,0 +1,67 @@
1
+ // Copyright (c) Cratis. All rights reserved.
2
+ // Licensed under the MIT license. See LICENSE file in the project root for full license information.
3
+
4
+ /** What ended a kernel observation stream, deciding how the client recovers. */
5
+ export enum StreamErrorKind {
6
+ /** The stream failed for a reason of its own; it is opened again after a backoff. */
7
+ Failure = 'Failure',
8
+
9
+ /** The client cancelled the stream itself; it is not opened again. */
10
+ Cancelled = 'Cancelled',
11
+
12
+ /** The kernel is stopping or unreachable; the stream is opened again after a backoff. */
13
+ ConnectionLost = 'ConnectionLost',
14
+
15
+ /**
16
+ * The kernel no longer knows this client's connection, for instance after it restarted. Opening the stream again
17
+ * with the same connection id can never work, so the whole client reconnects with a new one.
18
+ */
19
+ SessionLost = 'SessionLost'
20
+ }
21
+
22
+ const CANCELLED = 1;
23
+ const UNAVAILABLE = 14;
24
+ const CLIENT_IS_NOT_CONNECTED = 'ClientIsNotConnected';
25
+
26
+ interface GrpcLikeError {
27
+ code?: unknown;
28
+ details?: unknown;
29
+ message?: unknown;
30
+ name?: unknown;
31
+ cause?: unknown;
32
+ }
33
+
34
+ const mentionsClientIsNotConnected = (error: GrpcLikeError): boolean =>
35
+ [error.details, error.message].some(text => typeof text === 'string' && text.includes(CLIENT_IS_NOT_CONNECTED));
36
+
37
+ /**
38
+ * Classifies an error that ended a kernel observation stream, looking through wrapping errors.
39
+ *
40
+ * An {@link AggregateError} takes the most serious kind among the errors it wraps, in this order:
41
+ * {@link StreamErrorKind.Failure}, then {@link StreamErrorKind.SessionLost}, then {@link StreamErrorKind.ConnectionLost},
42
+ * then {@link StreamErrorKind.Cancelled}. This matches the .NET client.
43
+ * @param error - The error the stream ended with.
44
+ * @returns The {@link StreamErrorKind} for the error.
45
+ */
46
+ export function classifyStreamError(error: unknown): StreamErrorKind {
47
+ if (error === null || typeof error !== 'object') {
48
+ return StreamErrorKind.Failure;
49
+ }
50
+
51
+ if (error instanceof AggregateError && error.errors.length > 0) {
52
+ const kinds = error.errors.map(classifyStreamError);
53
+ if (kinds.includes(StreamErrorKind.Failure)) return StreamErrorKind.Failure;
54
+ if (kinds.includes(StreamErrorKind.SessionLost)) return StreamErrorKind.SessionLost;
55
+ return kinds.includes(StreamErrorKind.ConnectionLost) ? StreamErrorKind.ConnectionLost : StreamErrorKind.Cancelled;
56
+ }
57
+
58
+ const grpcError = error as GrpcLikeError;
59
+ if (grpcError.name === 'AbortError') return StreamErrorKind.Cancelled;
60
+ if (typeof grpcError.code === 'number') {
61
+ if (grpcError.code === CANCELLED) return StreamErrorKind.Cancelled;
62
+ if (grpcError.code === UNAVAILABLE) return StreamErrorKind.ConnectionLost;
63
+ return mentionsClientIsNotConnected(grpcError) ? StreamErrorKind.SessionLost : StreamErrorKind.Failure;
64
+ }
65
+
66
+ return grpcError.cause !== undefined ? classifyStreamError(grpcError.cause) : StreamErrorKind.Failure;
67
+ }
@@ -0,0 +1,29 @@
1
+ // Copyright (c) Cratis. All rights reserved.
2
+ // Licensed under the MIT license. See LICENSE file in the project root for full license information.
3
+
4
+ import { ObservationReconnectBackoff } from '../../ObservationReconnectBackoff.js';
5
+
6
+ /** A backoff with a 2 second base, a 60 second ceiling and a 3 minute healthy threshold, on a clock and jitter the spec controls. */
7
+ export class a_backoff {
8
+ static readonly baseDelayMs = 2_000;
9
+ static readonly maxDelayMs = 60_000;
10
+ static readonly healthyStreamDurationMs = 180_000;
11
+
12
+ now = 0;
13
+ jitter = 0;
14
+
15
+ readonly backoff = new ObservationReconnectBackoff({
16
+ baseDelayMs: a_backoff.baseDelayMs,
17
+ maxDelayMs: a_backoff.maxDelayMs,
18
+ healthyStreamDurationMs: a_backoff.healthyStreamDurationMs,
19
+ now: () => this.now,
20
+ nextJitter: () => this.jitter
21
+ });
22
+
23
+ /** Opens a stream, lets it stay open for a while and ends it, returning the wait before the next one. */
24
+ streamEndsAfter(openForMs: number): number {
25
+ this.backoff.streamOpened();
26
+ this.now += openForMs;
27
+ return this.backoff.nextDelayMs();
28
+ }
29
+ }
@@ -0,0 +1,52 @@
1
+ // Copyright (c) Cratis. All rights reserved.
2
+ // Licensed under the MIT license. See LICENSE file in the project root for full license information.
3
+
4
+ /** How one observation stream behaves: what it delivers, how long it stays open and the error it ends with, if any. */
5
+ export interface StreamScript<T> {
6
+ /** What the stream delivers before it ends. */
7
+ deliver?: T[];
8
+
9
+ /** How long, in milliseconds, the stream stays open after delivering before it ends. */
10
+ openForMs?: number;
11
+
12
+ /** Whether the stream stays open until the client aborts it. */
13
+ untilAborted?: boolean;
14
+
15
+ /** How long, in milliseconds, an aborted stream takes to end, as a slow transport would. */
16
+ endsLateByMs?: number;
17
+
18
+ /** The error the stream ends with. */
19
+ error?: unknown;
20
+ }
21
+
22
+ /** Records the observation streams a reactor or reducer opens and plays them by their scripts. */
23
+ export class a_scripted_stream<T> {
24
+ readonly openedAt: number[] = [];
25
+ readonly scripts: StreamScript<T>[] = [];
26
+ open = 0;
27
+
28
+ /** The observe method of the fake kernel connection. */
29
+ readonly observe = (_messages: unknown, options: { signal: AbortSignal }): AsyncGenerator<T> => this.play(options.signal);
30
+
31
+ /** Gets the wait before each stream after the first was opened. */
32
+ get waits(): number[] {
33
+ return this.openedAt.slice(1).map((at, index) => at - this.openedAt[index]);
34
+ }
35
+
36
+ private async *play(signal: AbortSignal): AsyncGenerator<T> {
37
+ this.openedAt.push(Date.now());
38
+ this.open++;
39
+ try {
40
+ const script = this.scripts.shift() ?? {};
41
+ for (const delivery of script.deliver ?? []) yield delivery;
42
+ if (script.openForMs) await new Promise(resolve => setTimeout(resolve, script.openForMs));
43
+ if (script.untilAborted) {
44
+ await new Promise<void>(resolve => signal.aborted ? resolve() : signal.addEventListener('abort', () => resolve(), { once: true }));
45
+ if (script.endsLateByMs) await new Promise(resolve => setTimeout(resolve, script.endsLateByMs));
46
+ }
47
+ if (script.error) throw script.error;
48
+ } finally {
49
+ this.open--;
50
+ }
51
+ }
52
+ }
@@ -0,0 +1,23 @@
1
+ // Copyright (c) Cratis. All rights reserved.
2
+ // Licensed under the MIT license. See LICENSE file in the project root for full license information.
3
+
4
+ import { beforeEach, chai, describe, it } from 'vitest';
5
+ import { a_backoff } from './given/a_backoff.js';
6
+
7
+ chai.should();
8
+
9
+ describe('when a stream delivered before it ended', () => {
10
+ let context: a_backoff;
11
+ let delay: number;
12
+
13
+ beforeEach(() => {
14
+ context = new a_backoff();
15
+ for (let attempt = 0; attempt < 4; attempt++) context.streamEndsAfter(1_000);
16
+ context.backoff.streamOpened();
17
+ context.backoff.streamDelivered();
18
+ context.now += 1_000;
19
+ delay = context.backoff.nextDelayMs();
20
+ });
21
+
22
+ it('should start the wait over at the base delay', () => delay.should.equal(a_backoff.baseDelayMs / 2));
23
+ });
@@ -0,0 +1,25 @@
1
+ // Copyright (c) Cratis. All rights reserved.
2
+ // Licensed under the MIT license. See LICENSE file in the project root for full license information.
3
+
4
+ import { beforeEach, chai, describe, it } from 'vitest';
5
+ import { a_backoff } from './given/a_backoff.js';
6
+
7
+ chai.should();
8
+
9
+ describe('when a stream stayed open before it ended', () => {
10
+ let longEnough: number;
11
+ let tooShort: number;
12
+
13
+ beforeEach(() => {
14
+ const healthy = new a_backoff();
15
+ for (let attempt = 0; attempt < 3; attempt++) healthy.streamEndsAfter(1_000);
16
+ longEnough = healthy.streamEndsAfter(a_backoff.healthyStreamDurationMs);
17
+
18
+ const unhealthy = new a_backoff();
19
+ for (let attempt = 0; attempt < 3; attempt++) unhealthy.streamEndsAfter(1_000);
20
+ tooShort = unhealthy.streamEndsAfter(a_backoff.healthyStreamDurationMs - 1);
21
+ });
22
+
23
+ it('should start the wait over at the base delay once it stayed open long enough', () => longEnough.should.equal(a_backoff.baseDelayMs / 2));
24
+ it('should keep backing off when it closed just before counting as having worked', () => tooShort.should.equal(8_000));
25
+ });
@@ -0,0 +1,24 @@
1
+ // Copyright (c) Cratis. All rights reserved.
2
+ // Licensed under the MIT license. See LICENSE file in the project root for full license information.
3
+
4
+ import { beforeEach, chai, describe, it } from 'vitest';
5
+ import { a_backoff } from './given/a_backoff.js';
6
+
7
+ chai.should();
8
+
9
+ describe('when jittering the delay', () => {
10
+ let delays: number[];
11
+
12
+ beforeEach(() => {
13
+ delays = [0, 0.5, -1, 2].map(jitter => {
14
+ const context = new a_backoff();
15
+ context.jitter = jitter;
16
+ return context.streamEndsAfter(0);
17
+ });
18
+ });
19
+
20
+ it('should land at half the wait for the lowest jitter', () => delays[0].should.equal(1_000));
21
+ it('should land in the upper half of the wait', () => delays[1].should.equal(1_500));
22
+ it('should clamp a jitter below zero to the lower bound', () => delays[2].should.equal(1_000));
23
+ it('should clamp a jitter above one to the full wait', () => delays[3].should.equal(2_000));
24
+ });
@@ -0,0 +1,24 @@
1
+ // Copyright (c) Cratis. All rights reserved.
2
+ // Licensed under the MIT license. See LICENSE file in the project root for full license information.
3
+
4
+ import { beforeEach, chai, describe, it } from 'vitest';
5
+ import { a_backoff } from './given/a_backoff.js';
6
+
7
+ chai.should();
8
+
9
+ describe('when streams keep ending without working', () => {
10
+ let context: a_backoff;
11
+ let delays: number[];
12
+
13
+ beforeEach(() => {
14
+ context = new a_backoff();
15
+ context.jitter = 0.999999;
16
+ delays = Array.from({ length: 8 }, () => context.streamEndsAfter(1_000));
17
+ });
18
+
19
+ it('should double the wait each time, starting from the base delay', () =>
20
+ delays.slice(0, 5).map(delay => Math.round(delay / 1_000)).should.deep.equal([2, 4, 8, 16, 32]));
21
+
22
+ it('should never wait longer than the ceiling', () => delays.forEach(delay => delay.should.be.at.most(a_backoff.maxDelayMs)));
23
+ it('should keep waiting close to the ceiling once it is reached', () => delays.slice(5).forEach(delay => delay.should.be.at.least(59_999)));
24
+ });
@@ -0,0 +1,18 @@
1
+ // Copyright (c) Cratis. All rights reserved.
2
+ // Licensed under the MIT license. See LICENSE file in the project root for full license information.
3
+
4
+ import { beforeEach, chai, describe, it } from 'vitest';
5
+ import { ObservationReconnectBackoff } from '../ObservationReconnectBackoff.js';
6
+
7
+ chai.should();
8
+
9
+ describe('when the base delay is not positive', () => {
10
+ let delays: number[];
11
+
12
+ beforeEach(() => {
13
+ const backoff = new ObservationReconnectBackoff({ baseDelayMs: 0, nextJitter: () => 0.5 });
14
+ delays = [backoff.nextDelayMs(), backoff.nextDelayMs(), backoff.nextDelayMs()];
15
+ });
16
+
17
+ it('should use the base delay as it is, without backing off', () => delays.should.deep.equal([0, 0, 0]));
18
+ });
@@ -0,0 +1,32 @@
1
+ // Copyright (c) Cratis. All rights reserved.
2
+ // Licensed under the MIT license. See LICENSE file in the project root for full license information.
3
+
4
+ import { beforeEach, chai, describe, it } from 'vitest';
5
+ import { ObservationReconnectBackoff } from '../ObservationReconnectBackoff.js';
6
+
7
+ chai.should();
8
+
9
+ describe('when the settings are not finite', () => {
10
+ let delays: number[];
11
+
12
+ beforeEach(() => {
13
+ let now = 0;
14
+ const backoff = new ObservationReconnectBackoff({
15
+ baseDelayMs: Number.NaN,
16
+ maxDelayMs: -1,
17
+ healthyStreamDurationMs: Number.POSITIVE_INFINITY,
18
+ now: () => now,
19
+ nextJitter: () => 1
20
+ });
21
+ delays = [];
22
+ for (let attempt = 0; attempt < 7; attempt++) {
23
+ backoff.streamOpened();
24
+ delays.push(backoff.nextDelayMs());
25
+ }
26
+ now = ObservationReconnectBackoff.defaultHealthyStreamDurationMs;
27
+ delays.push(backoff.nextDelayMs());
28
+ });
29
+
30
+ it('should fall back to the default delays', () =>
31
+ delays.should.deep.equal([2_000, 4_000, 8_000, 16_000, 32_000, 60_000, 60_000, 2_000]));
32
+ });
@@ -0,0 +1,29 @@
1
+ // Copyright (c) Cratis. All rights reserved.
2
+ // Licensed under the MIT license. See LICENSE file in the project root for full license information.
3
+
4
+ import { beforeEach, chai, describe, it } from 'vitest';
5
+ import { ObservationReconnectBackoff } from '../ObservationReconnectBackoff.js';
6
+
7
+ chai.should();
8
+
9
+ describe('when using the defaults', () => {
10
+ let delays: number[];
11
+
12
+ beforeEach(() => {
13
+ let now = 0;
14
+ const backoff = new ObservationReconnectBackoff({ now: () => now, nextJitter: () => 0.999999 });
15
+ delays = Array.from({ length: 7 }, () => {
16
+ backoff.streamOpened();
17
+ now += 1_000;
18
+ return backoff.nextDelayMs();
19
+ });
20
+ backoff.streamOpened();
21
+ now += ObservationReconnectBackoff.defaultHealthyStreamDurationMs;
22
+ delays.push(backoff.nextDelayMs());
23
+ });
24
+
25
+ it('should start at two seconds', () => Math.round(delays[0] / 1_000).should.equal(2));
26
+ it('should never wait longer than a minute', () => delays.forEach(delay => delay.should.be.at.most(60_000)));
27
+ it('should reach a minute while streams keep ending', () => Math.round(delays[6] / 1_000).should.equal(60));
28
+ it('should start over after a stream stayed open for three minutes', () => Math.round(delays[7] / 1_000).should.equal(2));
29
+ });
@@ -0,0 +1,36 @@
1
+ // Copyright (c) Cratis. All rights reserved.
2
+ // Licensed under the MIT license. See LICENSE file in the project root for full license information.
3
+
4
+ import { ClientError, Status } from 'nice-grpc';
5
+ import { chai, describe, it } from 'vitest';
6
+ import { classifyStreamError, StreamErrorKind } from '../classifyStreamError.js';
7
+
8
+ chai.should();
9
+
10
+ const sessionLost = () => new ClientError('/Reactors/Observe', Status.UNKNOWN, 'ClientIsNotConnected: connection 42 is not connected');
11
+ const abort = () => Object.assign(new Error('The operation has been aborted'), { name: 'AbortError' });
12
+
13
+ describe('when classifying a stream error', () => {
14
+ const cases: [string, unknown, StreamErrorKind][] = [
15
+ ['the kernel no longer knows the connection', sessionLost(), StreamErrorKind.SessionLost],
16
+ ['the message names the lost connection', Object.assign(new Error('ClientIsNotConnected'), { code: Status.INTERNAL }), StreamErrorKind.SessionLost],
17
+ ['the lost connection is wrapped', new Error('stream failed', { cause: sessionLost() }), StreamErrorKind.SessionLost],
18
+ ['every wrapped error lost the session or connection',
19
+ new AggregateError([sessionLost(), new ClientError('/x', Status.UNAVAILABLE, '')]), StreamErrorKind.SessionLost],
20
+ ['a wrapped error is a failure of its own',
21
+ new AggregateError([sessionLost(), new ClientError('/x', Status.INTERNAL, 'boom')]), StreamErrorKind.Failure],
22
+ ['a wrapped error lost the connection and another was cancelled',
23
+ new AggregateError([new ClientError('/x', Status.CANCELLED, ''), new ClientError('/x', Status.UNAVAILABLE, '')]),
24
+ StreamErrorKind.ConnectionLost],
25
+ ['every wrapped error was cancelled', new AggregateError([abort(), new ClientError('/x', Status.CANCELLED, '')]), StreamErrorKind.Cancelled],
26
+ ['the kernel is unreachable', new ClientError('/x', Status.UNAVAILABLE, 'ClientIsNotConnected'), StreamErrorKind.ConnectionLost],
27
+ ['the call was cancelled', new ClientError('/x', Status.CANCELLED, ''), StreamErrorKind.Cancelled],
28
+ ['the client aborted the stream', abort(), StreamErrorKind.Cancelled],
29
+ ['the stream failed for another reason', new ClientError('/x', Status.INTERNAL, 'boom'), StreamErrorKind.Failure],
30
+ ['the error is not an object', 'boom', StreamErrorKind.Failure]
31
+ ];
32
+
33
+ for (const [scenario, error, kind] of cases) {
34
+ it(`should classify as ${kind} when ${scenario}`, () => classifyStreamError(error).should.equal(kind));
35
+ }
36
+ });
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@cratis/chronicle",
3
- "version": "6.62.0",
3
+ "version": "6.63.0",
4
4
  "description": "Event sourcing for TypeScript and Node.js — the idiomatic client for Cratis Chronicle, the open-source event-sourcing database",
5
5
  "author": "Cratis",
6
6
  "license": "MIT",
@@ -6,6 +6,7 @@ import { createLogger } from '../logging/createLogger.js';
6
6
  import { runInBackgroundContext } from '../telemetry/runInBackgroundContext.js';
7
7
  import { Constructor } from '@cratis/fundamentals';
8
8
  import { EventGenerationDelivery, ReactorMessage } from '@cratis/chronicle.contracts';
9
+ import type { EventsToObserve } from '@cratis/chronicle.contracts';
9
10
  import { IClientArtifactsProvider } from '../artifacts/index.js';
10
11
  import { ChronicleConnection } from '../connection/index.js';
11
12
  import { ConnectionLifecycle } from '../connection/ConnectionLifecycle.js';
@@ -29,6 +30,9 @@ import type { ReactorServices } from './ReactorServices.js';
29
30
  import type { IEventStore } from '../IEventStore.js';
30
31
  import type { ClientArtifactsActivator } from '../artifacts/ClientArtifactsActivator.js';
31
32
  import { refuseClientHostedObservers } from '../artifacts/refuseClientHostedObservers.js';
33
+ import { ObservationReconnectBackoff, type ObservationReconnectBackoffOptions } from '../observation/ObservationReconnectBackoff.js';
34
+ import { classifyStreamError, StreamErrorKind } from '../observation/classifyStreamError.js';
35
+ import type { ObservationStreamOutcome } from '../observation/ObservationStreamOutcome.js';
32
36
 
33
37
  /** Expression used to partition reactor observations by event source ID. */
34
38
  const EVENT_SOURCE_ID_KEY = '$eventSourceId';
@@ -96,15 +100,14 @@ class AsyncQueue<T> {
96
100
  * with the Chronicle Kernel via bidirectional gRPC streaming.
97
101
  */
98
102
  export class Reactors implements IReactors {
99
- /** How long to wait before re-establishing an observation whose stream ended. */
100
- private static readonly _reobserveDelayMs = 2000;
101
-
102
103
  private readonly _logger: ReturnType<typeof createLogger>;
103
104
  private readonly _lifecycle: ConnectionLifecycle;
104
105
  private readonly _unsubscribeDisconnected: () => void;
105
106
  private readonly _reactors = new Map<string, Constructor>();
106
107
  private readonly _queues = new Map<string, AsyncQueue<ReactorMessage>>();
107
108
  private readonly _observations = new Map<string, AbortController>();
109
+ private readonly _reconnectBackoffs = new Map<string, ObservationReconnectBackoff>();
110
+ private readonly _reobserveTimers = new Map<string, ReturnType<typeof setTimeout>>();
108
111
  private _registered = false;
109
112
  private _disposed = false;
110
113
 
@@ -116,6 +119,11 @@ export class Reactors implements IReactors {
116
119
  * @param _namespace - The namespace within the event store.
117
120
  * @param lifecycle - The connection lifecycle used to react to disconnect events.
118
121
  * @param _eventLog - The event log used to append side-effect events a reactor handler returns.
122
+ * @param _resultHandler - Optional handler for values a reactor handler returns.
123
+ * @param _eventStore - Optional event store handed to reactor handlers.
124
+ * @param _artifactActivator - Optional activator creating reactor instances.
125
+ * @param _skipKeepAlive - Whether the client runs without a keep-alive, which refuses client-hosted reactors.
126
+ * @param _reconnectBackoff - How a reactor waits before opening its observation stream again after it ended.
119
127
  */
120
128
  constructor(
121
129
  private readonly _clientArtifacts: IClientArtifactsProvider,
@@ -127,7 +135,8 @@ export class Reactors implements IReactors {
127
135
  private readonly _resultHandler?: ReactorResultHandler,
128
136
  private readonly _eventStore?: IEventStore,
129
137
  private readonly _artifactActivator?: ClientArtifactsActivator,
130
- private readonly _skipKeepAlive: boolean = false
138
+ private readonly _skipKeepAlive: boolean = false,
139
+ private readonly _reconnectBackoff: ObservationReconnectBackoffOptions = {}
131
140
  ) {
132
141
  this._logger = createLogger('@cratis/chronicle/reactors', _connection.logger);
133
142
  this._lifecycle = lifecycle;
@@ -144,6 +153,7 @@ export class Reactors implements IReactors {
144
153
  this._disposed = true;
145
154
  this._registered = false;
146
155
  this.disconnectAll();
156
+ this._reconnectBackoffs.clear();
147
157
  }
148
158
 
149
159
  /** @inheritdoc */
@@ -210,13 +220,23 @@ export class Reactors implements IReactors {
210
220
  eventTypes: ReactorEventTypeEntry[],
211
221
  validateInstance: (instance: object) => void
212
222
  ): Promise<void> {
223
+ let outcome: ObservationStreamOutcome = 'ended';
213
224
  try {
214
- await this.observeReactor(id, reactorType, eventSequenceId, eventTypes, validateInstance);
225
+ outcome = await this.observeReactor(id, reactorType, eventSequenceId, eventTypes, validateInstance);
215
226
  } catch (error) {
216
227
  this._logger.error('Reactor observation loop exited with error', { reactorId: id, error });
217
228
  }
218
229
 
219
- this.scheduleReobserve(id, reactorType, eventTypes, validateInstance);
230
+ if (outcome === 'ended') this.scheduleReobserve(id, reactorType, eventTypes, validateInstance);
231
+ }
232
+
233
+ private reconnectBackoffFor(id: string): ObservationReconnectBackoff {
234
+ let backoff = this._reconnectBackoffs.get(id);
235
+ if (!backoff) {
236
+ backoff = new ObservationReconnectBackoff(this._reconnectBackoff);
237
+ this._reconnectBackoffs.set(id, backoff);
238
+ }
239
+ return backoff;
220
240
  }
221
241
 
222
242
  /**
@@ -225,8 +245,9 @@ export class Reactors implements IReactors {
225
245
  * The stream ending without an error is not proof that all is well — the kernel
226
246
  * closes a cross-store (inbox) reactor's stream rather than tailing it forever,
227
247
  * so a reactor that does not re-subscribe there silently stops observing until
228
- * the whole client reconnects. Both endings are therefore retried, and the delay
229
- * keeps a stream that keeps ending from becoming a hot loop.
248
+ * the whole client reconnects. Both endings are therefore retried. The delay keeps
249
+ * a stream that keeps ending from becoming a hot loop, and grows with jitter while
250
+ * it does (see {@link ObservationReconnectBackoff}).
230
251
  */
231
252
  private scheduleReobserve(id: string, reactorType: Constructor, eventTypes: ReactorEventTypeEntry[], validateInstance: (instance: object) => void): void {
232
253
  // A disconnect clears the registration; the reconnect re-registers every
@@ -235,16 +256,22 @@ export class Reactors implements IReactors {
235
256
  return;
236
257
  }
237
258
 
259
+ clearTimeout(this._reobserveTimers.get(id));
260
+ const delayMs = this.reconnectBackoffFor(id).nextDelayMs();
261
+ this._logger.debug('Waiting before re-establishing reactor observation', { reactorId: id, delayMs });
238
262
  const handle = runInBackgroundContext(() => setTimeout(() => {
239
- if (!this._registered || this._disposed) {
263
+ this._reobserveTimers.delete(id);
264
+ // A reconnect in the meantime has already opened a new stream; opening another would leave one untracked.
265
+ if (!this._registered || this._disposed || this._observations.has(id)) {
240
266
  return;
241
267
  }
242
268
 
243
269
  this._logger.info('Re-establishing reactor observation', { reactorId: id });
244
270
  this.startObservation(id, reactorType, eventTypes, validateInstance);
245
- }, Reactors._reobserveDelayMs));
271
+ }, delayMs));
246
272
 
247
273
  handle.unref?.();
274
+ this._reobserveTimers.set(id, handle);
248
275
  }
249
276
 
250
277
  private async observeReactor(
@@ -253,17 +280,20 @@ export class Reactors implements IReactors {
253
280
  eventSequenceId: string,
254
281
  eventTypes: ReactorEventTypeEntry[],
255
282
  validateInstance: (instance: object) => void
256
- ): Promise<void> {
283
+ ): Promise<ObservationStreamOutcome> {
257
284
  const batchMethods = getReactorBatchMethods(reactorType, this._clientArtifacts.eventTypes);
258
285
  const queue = new AsyncQueue<ReactorMessage>();
259
286
  const controller = new AbortController();
260
287
  this._queues.set(id, queue);
261
288
  this._observations.set(id, controller);
289
+ const connectionId = this._lifecycle.connectionId;
290
+ const backoff = this.reconnectBackoffFor(id);
291
+ backoff.streamOpened();
262
292
 
263
293
  queue.send({
264
294
  Content: {
265
295
  Value0: {
266
- ConnectionId: this._lifecycle.connectionId,
296
+ ConnectionId: connectionId,
267
297
  EventStore: this._eventStoreName,
268
298
  Namespace: this._namespace,
269
299
  Reactor: {
@@ -290,6 +320,8 @@ export class Reactors implements IReactors {
290
320
  }
291
321
  });
292
322
 
323
+ let iterator: AsyncIterator<EventsToObserve> | undefined;
324
+ let streamFinished = false;
293
325
  try {
294
326
  const reactorInstance = this._artifactActivator ? undefined : new (reactorType as new () => Record<string, Function>)();
295
327
  if (reactorInstance) validateInstance(reactorInstance);
@@ -304,17 +336,44 @@ export class Reactors implements IReactors {
304
336
  logger: this._logger, connectionLogger: this._connection.logger
305
337
  });
306
338
 
307
- for await (const eventsToObserve of this._connection.reactors.observe(queue, { signal: controller.signal })) {
308
- const result = await dispatcher.dispatch(eventsToObserve);
339
+ // Pull each message by hand so only a failure of the stream itself is classified: an error from a
340
+ // handler is the handler's failure, and must never make the whole client reconnect.
341
+ iterator = this._connection.reactors.observe(queue, { signal: controller.signal })[Symbol.asyncIterator]();
342
+ for (;;) {
343
+ let next: IteratorResult<EventsToObserve>;
344
+ try {
345
+ next = await iterator.next();
346
+ } catch (streamError) {
347
+ streamFinished = true;
348
+ return this.streamFailed(id, queue, controller.signal, connectionId, streamError);
349
+ }
350
+
351
+ if (next.done) {
352
+ streamFinished = true;
353
+ break;
354
+ }
355
+
356
+ backoff.streamDelivered();
357
+ const result = await dispatcher.dispatch(next.value);
309
358
  queue.send({ Content: { Value0: undefined, Value1: result } });
310
359
  }
311
- } catch (err) {
312
- if (!this._queues.has(id)) {
360
+
361
+ if (this._queues.get(id) !== queue) {
313
362
  this._logger.debug('Reactor observation stream closed cleanly', { reactorId: id });
314
- } else {
315
- this._logger.error('Reactor observation stream ended unexpectedly', { reactorId: id, error: err });
363
+ return 'superseded';
316
364
  }
365
+
366
+ return 'ended';
367
+ } catch (handlerError) {
368
+ if (this._queues.get(id) !== queue) return 'superseded';
369
+ this._logger.error('Reactor observation failed while handling a delivery', { reactorId: id, error: handlerError });
370
+ return 'ended';
317
371
  } finally {
372
+ if (!streamFinished) {
373
+ controller.abort();
374
+ void iterator?.return?.()?.catch(() => undefined);
375
+ }
376
+
318
377
  // Only retire our own queue: a reconnect can already have replaced it,
319
378
  // and deleting the new one would leave that observation untracked and
320
379
  // leak a duplicate stream on every reconnect.
@@ -326,7 +385,31 @@ export class Reactors implements IReactors {
326
385
  }
327
386
  }
328
387
 
388
+ private streamFailed(id: string, queue: AsyncQueue<ReactorMessage>, signal: AbortSignal, connectionId: string, error: unknown): ObservationStreamOutcome {
389
+ // A late error from a stream that a disconnect, a dispose or a newer stream replaced says nothing about now.
390
+ if (this._queues.get(id) !== queue) {
391
+ this._logger.debug('Reactor observation stream closed cleanly', { reactorId: id });
392
+ return 'superseded';
393
+ }
394
+
395
+ const kind = classifyStreamError(error);
396
+ if (kind === StreamErrorKind.Cancelled && signal.aborted) return 'stopped';
397
+ if (kind === StreamErrorKind.SessionLost) {
398
+ // The kernel lost this client's registration. Opening the same stream with the same connection id
399
+ // can never work, so reconnect the whole client, which re-registers every reactor.
400
+ this._logger.warn('Reactor observation stream lost its session; reconnecting the client', { reactorId: id, error });
401
+ this._lifecycle.requestReconnect(connectionId, 'reactor-session-lost', error,
402
+ handlerError => this._logger.error('Reconnect request handler failed', { reactorId: id, error: handlerError }));
403
+ return 'sessionLost';
404
+ }
405
+
406
+ this._logger.error('Reactor observation stream ended unexpectedly', { reactorId: id, error });
407
+ return 'ended';
408
+ }
409
+
329
410
  private disconnectAll(): void {
411
+ for (const timer of this._reobserveTimers.values()) clearTimeout(timer);
412
+ this._reobserveTimers.clear();
330
413
  for (const controller of this._observations.values()) controller.abort();
331
414
  for (const queue of this._queues.values()) queue.complete();
332
415
  this._observations.clear();
@@ -0,0 +1,32 @@
1
+ // Copyright (c) Cratis. All rights reserved.
2
+ // Licensed under the MIT license. See LICENSE file in the project root for full license information.
3
+
4
+ import { afterEach, beforeEach, chai, describe, it, vi } from 'vitest';
5
+ import { ClientError, Status } from 'nice-grpc';
6
+ import { a_reactor_whose_stream_ends } from './given/a_reactor_whose_stream_ends.js';
7
+ import { page } from '../given/observe_reactor_messages.js';
8
+ import { ReactorPageDispatcher } from '../../ReactorPageDispatcher.js';
9
+
10
+ chai.should();
11
+
12
+ describe('when reconnecting a reactor observation stream and a handler failed with a lost session', () => {
13
+ let context: a_reactor_whose_stream_ends;
14
+
15
+ beforeEach(async () => {
16
+ context = new a_reactor_whose_stream_ends();
17
+ vi.spyOn(ReactorPageDispatcher.prototype, 'dispatch')
18
+ .mockRejectedValueOnce(new ClientError('/EventSequences/Append', Status.UNKNOWN, 'ClientIsNotConnected: connection is not connected'));
19
+ context.scripts.push({ deliver: [page(1n, {})], untilAborted: true }, { untilAborted: true });
20
+ await context.reactors.register();
21
+ await vi.advanceTimersByTimeAsync(120_000);
22
+ });
23
+
24
+ afterEach(() => {
25
+ context.dispose();
26
+ vi.restoreAllMocks();
27
+ });
28
+
29
+ it('should not ask the whole client to reconnect', () => context.reconnectRequests.should.be.empty);
30
+ it('should open the stream again after the base delay', () => context.waits.should.deep.equal([1_000]));
31
+ it('should close the stream the handler failed on', () => context.stream.open.should.equal(1));
32
+ });