@cratis/chronicle 6.62.0 → 6.63.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. package/ChronicleClient.ts +86 -3
  2. package/connection/ConnectionLifecycle.ts +42 -1
  3. package/dist/ChronicleClient.d.ts +28 -1
  4. package/dist/ChronicleClient.d.ts.map +1 -1
  5. package/dist/ChronicleClient.js +71 -3
  6. package/dist/ChronicleClient.js.map +1 -1
  7. package/dist/connection/ConnectionLifecycle.d.ts +24 -0
  8. package/dist/connection/ConnectionLifecycle.d.ts.map +1 -1
  9. package/dist/connection/ConnectionLifecycle.js +40 -1
  10. package/dist/connection/ConnectionLifecycle.js.map +1 -1
  11. package/dist/observation/ObservationReconnectBackoff.d.ts +64 -0
  12. package/dist/observation/ObservationReconnectBackoff.d.ts.map +1 -0
  13. package/dist/observation/ObservationReconnectBackoff.js +75 -0
  14. package/dist/observation/ObservationReconnectBackoff.js.map +1 -0
  15. package/dist/observation/ObservationStreamOutcome.d.ts +10 -0
  16. package/dist/observation/ObservationStreamOutcome.d.ts.map +1 -0
  17. package/dist/observation/ObservationStreamOutcome.js +4 -0
  18. package/dist/observation/ObservationStreamOutcome.js.map +1 -0
  19. package/dist/observation/classifyStreamError.d.ts +25 -0
  20. package/dist/observation/classifyStreamError.d.ts.map +1 -0
  21. package/dist/observation/classifyStreamError.js +55 -0
  22. package/dist/observation/classifyStreamError.js.map +1 -0
  23. package/dist/observation/for_ObservationReconnectBackoff/given/a_backoff.d.ts +13 -0
  24. package/dist/observation/for_ObservationReconnectBackoff/given/a_backoff.d.ts.map +1 -0
  25. package/dist/observation/for_ObservationReconnectBackoff/given/a_backoff.js +25 -0
  26. package/dist/observation/for_ObservationReconnectBackoff/given/a_backoff.js.map +1 -0
  27. package/dist/observation/for_ObservationReconnectBackoff/given/a_scripted_stream.d.ts +27 -0
  28. package/dist/observation/for_ObservationReconnectBackoff/given/a_scripted_stream.d.ts.map +1 -0
  29. package/dist/observation/for_ObservationReconnectBackoff/given/a_scripted_stream.js +36 -0
  30. package/dist/observation/for_ObservationReconnectBackoff/given/a_scripted_stream.js.map +1 -0
  31. package/dist/reactors/Reactors.d.ts +15 -5
  32. package/dist/reactors/Reactors.d.ts.map +1 -1
  33. package/dist/reactors/Reactors.js +95 -18
  34. package/dist/reactors/Reactors.js.map +1 -1
  35. package/dist/reactors/for_Reactors/when_reconnecting_the_observation_stream/given/a_reactor_whose_stream_ends.d.ts +26 -0
  36. package/dist/reactors/for_Reactors/when_reconnecting_the_observation_stream/given/a_reactor_whose_stream_ends.d.ts.map +1 -0
  37. package/dist/reactors/for_Reactors/when_reconnecting_the_observation_stream/given/a_reactor_whose_stream_ends.js +64 -0
  38. package/dist/reactors/for_Reactors/when_reconnecting_the_observation_stream/given/a_reactor_whose_stream_ends.js.map +1 -0
  39. package/dist/reducers/Reducers.d.ts +11 -5
  40. package/dist/reducers/Reducers.d.ts.map +1 -1
  41. package/dist/reducers/Reducers.js +93 -19
  42. package/dist/reducers/Reducers.js.map +1 -1
  43. package/dist/reducers/for_Reducers/when_reconnecting_the_observation_stream/given/a_reducer_whose_stream_ends.d.ts +25 -0
  44. package/dist/reducers/for_Reducers/when_reconnecting_the_observation_stream/given/a_reducer_whose_stream_ends.d.ts.map +1 -0
  45. package/dist/reducers/for_Reducers/when_reconnecting_the_observation_stream/given/a_reducer_whose_stream_ends.js +70 -0
  46. package/dist/reducers/for_Reducers/when_reconnecting_the_observation_stream/given/a_reducer_whose_stream_ends.js.map +1 -0
  47. package/dist/tsconfig.tsbuildinfo +1 -1
  48. package/for_ChronicleClient/when_reconnecting/and_an_observation_stream_lost_its_session.spec.ts +212 -0
  49. package/observation/ObservationReconnectBackoff.ts +109 -0
  50. package/observation/ObservationStreamOutcome.ts +12 -0
  51. package/observation/classifyStreamError.ts +67 -0
  52. package/observation/for_ObservationReconnectBackoff/given/a_backoff.ts +29 -0
  53. package/observation/for_ObservationReconnectBackoff/given/a_scripted_stream.ts +52 -0
  54. package/observation/for_ObservationReconnectBackoff/when_a_stream_delivered_before_it_ended.spec.ts +23 -0
  55. package/observation/for_ObservationReconnectBackoff/when_a_stream_stayed_open_before_it_ended.spec.ts +25 -0
  56. package/observation/for_ObservationReconnectBackoff/when_jittering_the_delay.spec.ts +24 -0
  57. package/observation/for_ObservationReconnectBackoff/when_streams_keep_ending_without_working.spec.ts +24 -0
  58. package/observation/for_ObservationReconnectBackoff/when_the_base_delay_is_not_positive.spec.ts +18 -0
  59. package/observation/for_ObservationReconnectBackoff/when_the_settings_are_not_finite.spec.ts +32 -0
  60. package/observation/for_ObservationReconnectBackoff/when_using_the_defaults.spec.ts +29 -0
  61. package/observation/for_classifyStreamError/when_classifying_a_stream_error.spec.ts +36 -0
  62. package/package.json +1 -1
  63. package/reactors/Reactors.ts +101 -18
  64. package/reactors/for_Reactors/when_reconnecting_the_observation_stream/and_a_handler_failed_with_a_lost_session.spec.ts +32 -0
  65. package/reactors/for_Reactors/when_reconnecting_the_observation_stream/and_a_replaced_stream_fails_late.spec.ts +28 -0
  66. package/reactors/for_Reactors/when_reconnecting_the_observation_stream/and_a_stream_delivered_before_it_ended.spec.ts +24 -0
  67. package/reactors/for_Reactors/when_reconnecting_the_observation_stream/and_a_stream_stayed_open_before_it_ended.spec.ts +23 -0
  68. package/reactors/for_Reactors/when_reconnecting_the_observation_stream/and_it_is_disposed_while_waiting_to_reopen.spec.ts +27 -0
  69. package/reactors/for_Reactors/when_reconnecting_the_observation_stream/and_streams_keep_ending_without_working.spec.ts +24 -0
  70. package/reactors/for_Reactors/when_reconnecting_the_observation_stream/and_the_client_reconnected_after_streams_kept_ending.spec.ts +24 -0
  71. package/reactors/for_Reactors/when_reconnecting_the_observation_stream/and_the_client_reconnected_while_waiting_to_reopen.spec.ts +25 -0
  72. package/reactors/for_Reactors/when_reconnecting_the_observation_stream/and_the_kernel_lost_the_session.spec.ts +24 -0
  73. package/reactors/for_Reactors/when_reconnecting_the_observation_stream/given/a_reactor_whose_stream_ends.ts +69 -0
  74. package/reducers/Reducers.ts +99 -19
  75. package/reducers/for_Reducers/when_reconnecting_the_observation_stream/and_a_reducer_failed_with_a_lost_session.spec.ts +33 -0
  76. package/reducers/for_Reducers/when_reconnecting_the_observation_stream/and_a_replaced_stream_fails_late.spec.ts +28 -0
  77. package/reducers/for_Reducers/when_reconnecting_the_observation_stream/and_a_stream_delivered_before_it_ended.spec.ts +24 -0
  78. package/reducers/for_Reducers/when_reconnecting_the_observation_stream/and_a_stream_stayed_open_before_it_ended.spec.ts +23 -0
  79. package/reducers/for_Reducers/when_reconnecting_the_observation_stream/and_it_is_disposed_while_waiting_to_reopen.spec.ts +27 -0
  80. package/reducers/for_Reducers/when_reconnecting_the_observation_stream/and_streams_keep_ending_without_working.spec.ts +24 -0
  81. package/reducers/for_Reducers/when_reconnecting_the_observation_stream/and_the_client_reconnected_after_streams_kept_ending.spec.ts +24 -0
  82. package/reducers/for_Reducers/when_reconnecting_the_observation_stream/and_the_client_reconnected_while_waiting_to_reopen.spec.ts +25 -0
  83. package/reducers/for_Reducers/when_reconnecting_the_observation_stream/and_the_kernel_lost_the_session.spec.ts +24 -0
  84. package/reducers/for_Reducers/when_reconnecting_the_observation_stream/given/a_reducer_whose_stream_ends.ts +69 -0
@@ -0,0 +1,28 @@
1
+ // Copyright (c) Cratis. All rights reserved.
2
+ // Licensed under the MIT license. See LICENSE file in the project root for full license information.
3
+
4
+ import { afterEach, beforeEach, chai, describe, it, vi } from 'vitest';
5
+ import { ClientError, Status } from 'nice-grpc';
6
+ import { a_reactor_whose_stream_ends } from './given/a_reactor_whose_stream_ends.js';
7
+
8
+ chai.should();
9
+
10
+ describe('when reconnecting a reactor observation stream and a replaced stream fails late', () => {
11
+ let context: a_reactor_whose_stream_ends;
12
+
13
+ beforeEach(async () => {
14
+ context = new a_reactor_whose_stream_ends();
15
+ context.scripts.push(
16
+ { untilAborted: true, endsLateByMs: 10, error: new ClientError('/Reactors/Observe', Status.INTERNAL, 'boom') },
17
+ { untilAborted: true });
18
+ await context.reactors.register();
19
+ await vi.advanceTimersByTimeAsync(0);
20
+ await context.reconnect();
21
+ await vi.advanceTimersByTimeAsync(120_000);
22
+ });
23
+
24
+ afterEach(() => context.dispose());
25
+
26
+ it('should not open the stream again', () => context.openedAt.should.have.lengthOf(2));
27
+ it('should leave exactly one stream open', () => context.stream.open.should.equal(1));
28
+ });
@@ -0,0 +1,24 @@
1
+ // Copyright (c) Cratis. All rights reserved.
2
+ // Licensed under the MIT license. See LICENSE file in the project root for full license information.
3
+
4
+ import { afterEach, beforeEach, chai, describe, it, vi } from 'vitest';
5
+ import { a_reactor_whose_stream_ends } from './given/a_reactor_whose_stream_ends.js';
6
+ import { page } from '../given/observe_reactor_messages.js';
7
+
8
+ chai.should();
9
+
10
+ describe('when reconnecting a reactor observation stream and a stream delivered before it ended', () => {
11
+ let context: a_reactor_whose_stream_ends;
12
+
13
+ beforeEach(async () => {
14
+ context = new a_reactor_whose_stream_ends();
15
+ context.scripts.push({}, {}, { deliver: [page(1n, {})] }, {});
16
+ await context.reactors.register();
17
+ await vi.advanceTimersByTimeAsync(6_500);
18
+ });
19
+
20
+ afterEach(() => context.dispose());
21
+
22
+ it('should start the wait over at the base delay after the stream that delivered', () =>
23
+ context.waits.slice(0, 4).should.deep.equal([1_000, 2_000, 1_000, 2_000]));
24
+ });
@@ -0,0 +1,23 @@
1
+ // Copyright (c) Cratis. All rights reserved.
2
+ // Licensed under the MIT license. See LICENSE file in the project root for full license information.
3
+
4
+ import { afterEach, beforeEach, chai, describe, it, vi } from 'vitest';
5
+ import { a_reactor_whose_stream_ends } from './given/a_reactor_whose_stream_ends.js';
6
+
7
+ chai.should();
8
+
9
+ describe('when reconnecting a reactor observation stream and a stream stayed open before it ended', () => {
10
+ let context: a_reactor_whose_stream_ends;
11
+
12
+ beforeEach(async () => {
13
+ context = new a_reactor_whose_stream_ends();
14
+ context.scripts.push({}, {}, { openForMs: 180_000 }, { untilAborted: true });
15
+ await context.reactors.register();
16
+ await vi.advanceTimersByTimeAsync(185_000);
17
+ });
18
+
19
+ afterEach(() => context.dispose());
20
+
21
+ it('should start the wait over at the base delay after the stream that stayed open', () =>
22
+ context.waits.should.deep.equal([1_000, 2_000, 181_000]));
23
+ });
@@ -0,0 +1,27 @@
1
+ // Copyright (c) Cratis. All rights reserved.
2
+ // Licensed under the MIT license. See LICENSE file in the project root for full license information.
3
+
4
+ import { afterEach, beforeEach, chai, describe, it, vi } from 'vitest';
5
+ import { a_reactor_whose_stream_ends } from './given/a_reactor_whose_stream_ends.js';
6
+
7
+ chai.should();
8
+
9
+ describe('when reconnecting a reactor observation stream and it is disposed while it waits to reopen', () => {
10
+ let context: a_reactor_whose_stream_ends;
11
+ let timersAfterDispose: number;
12
+
13
+ beforeEach(async () => {
14
+ context = new a_reactor_whose_stream_ends();
15
+ context.scripts.push({});
16
+ await context.reactors.register();
17
+ await vi.advanceTimersByTimeAsync(0);
18
+ context.reactors.dispose();
19
+ timersAfterDispose = vi.getTimerCount();
20
+ await vi.advanceTimersByTimeAsync(120_000);
21
+ });
22
+
23
+ afterEach(() => context.dispose());
24
+
25
+ it('should not open the stream again', () => context.openedAt.should.have.lengthOf(1));
26
+ it('should leave no timer behind', () => timersAfterDispose.should.equal(0));
27
+ });
@@ -0,0 +1,24 @@
1
+ // Copyright (c) Cratis. All rights reserved.
2
+ // Licensed under the MIT license. See LICENSE file in the project root for full license information.
3
+
4
+ import { afterEach, beforeEach, chai, describe, it, vi } from 'vitest';
5
+ import { ClientError, Status } from 'nice-grpc';
6
+ import { a_reactor_whose_stream_ends } from './given/a_reactor_whose_stream_ends.js';
7
+
8
+ chai.should();
9
+
10
+ describe('when reconnecting a reactor observation stream and streams keep ending without working', () => {
11
+ let context: a_reactor_whose_stream_ends;
12
+
13
+ beforeEach(async () => {
14
+ context = new a_reactor_whose_stream_ends();
15
+ context.scripts.push({}, { error: new ClientError('/Reactors/Observe', Status.INTERNAL, 'boom') }, {}, {}, {});
16
+ await context.reactors.register();
17
+ await vi.advanceTimersByTimeAsync(60_000);
18
+ });
19
+
20
+ afterEach(() => context.dispose());
21
+
22
+ it('should double the wait each time a stream ends', () => context.waits.slice(0, 5).should.deep.equal([1_000, 2_000, 4_000, 8_000, 16_000]));
23
+ it('should not reconnect the client', () => context.reconnectRequests.should.be.empty);
24
+ });
@@ -0,0 +1,24 @@
1
+ // Copyright (c) Cratis. All rights reserved.
2
+ // Licensed under the MIT license. See LICENSE file in the project root for full license information.
3
+
4
+ import { afterEach, beforeEach, chai, describe, it, vi } from 'vitest';
5
+ import { a_reactor_whose_stream_ends } from './given/a_reactor_whose_stream_ends.js';
6
+
7
+ chai.should();
8
+
9
+ describe('when reconnecting a reactor observation stream and the client reconnected after streams kept ending', () => {
10
+ let context: a_reactor_whose_stream_ends;
11
+
12
+ beforeEach(async () => {
13
+ context = new a_reactor_whose_stream_ends();
14
+ context.scripts.push({}, {}, {}, { untilAborted: true });
15
+ await context.reactors.register();
16
+ await vi.advanceTimersByTimeAsync(1_000);
17
+ await context.reconnect();
18
+ await vi.advanceTimersByTimeAsync(4_000);
19
+ });
20
+
21
+ afterEach(() => context.dispose());
22
+
23
+ it('should keep the wait it had grown to', () => context.waits.should.deep.equal([1_000, 0, 4_000]));
24
+ });
@@ -0,0 +1,25 @@
1
+ // Copyright (c) Cratis. All rights reserved.
2
+ // Licensed under the MIT license. See LICENSE file in the project root for full license information.
3
+
4
+ import { afterEach, beforeEach, chai, describe, it, vi } from 'vitest';
5
+ import { a_reactor_whose_stream_ends } from './given/a_reactor_whose_stream_ends.js';
6
+
7
+ chai.should();
8
+
9
+ describe('when reconnecting a reactor observation stream and the client reconnected while it waited to reopen', () => {
10
+ let context: a_reactor_whose_stream_ends;
11
+
12
+ beforeEach(async () => {
13
+ context = new a_reactor_whose_stream_ends();
14
+ context.scripts.push({}, { untilAborted: true });
15
+ await context.reactors.register();
16
+ await vi.advanceTimersByTimeAsync(0);
17
+ await context.reconnect();
18
+ await vi.advanceTimersByTimeAsync(120_000);
19
+ });
20
+
21
+ afterEach(() => context.dispose());
22
+
23
+ it('should only open the stream the reconnect registered', () => context.openedAt.should.have.lengthOf(2));
24
+ it('should leave exactly one stream open', () => context.stream.open.should.equal(1));
25
+ });
@@ -0,0 +1,24 @@
1
+ // Copyright (c) Cratis. All rights reserved.
2
+ // Licensed under the MIT license. See LICENSE file in the project root for full license information.
3
+
4
+ import { afterEach, beforeEach, chai, describe, it, vi } from 'vitest';
5
+ import { ClientError, Status } from 'nice-grpc';
6
+ import { a_reactor_whose_stream_ends } from './given/a_reactor_whose_stream_ends.js';
7
+
8
+ chai.should();
9
+
10
+ describe('when reconnecting a reactor observation stream and the kernel lost the session', () => {
11
+ let context: a_reactor_whose_stream_ends;
12
+
13
+ beforeEach(async () => {
14
+ context = new a_reactor_whose_stream_ends();
15
+ context.scripts.push({ error: new ClientError('/Reactors/Observe', Status.UNKNOWN, 'ClientIsNotConnected: connection is not connected') });
16
+ await context.reactors.register();
17
+ await vi.advanceTimersByTimeAsync(120_000);
18
+ });
19
+
20
+ afterEach(() => context.dispose());
21
+
22
+ it('should ask the whole client to reconnect', () => context.reconnectRequests.should.deep.equal(['reactor-session-lost']));
23
+ it('should not open the stream again with the same connection', () => context.openedAt.should.have.lengthOf(1));
24
+ });
@@ -0,0 +1,69 @@
1
+ // Copyright (c) Cratis. All rights reserved.
2
+ // Licensed under the MIT license. See LICENSE file in the project root for full license information.
3
+
4
+ import { vi } from 'vitest';
5
+ import type { EventsToObserve } from '@cratis/chronicle.contracts';
6
+ import { Reactors } from '../../../Reactors.js';
7
+ import { reactor } from '../../../reactor.js';
8
+ import { eventType } from '../../../../events/eventTypeDecorator.js';
9
+ import { ConnectionLifecycle } from '../../../../connection/ConnectionLifecycle.js';
10
+ import type { ChronicleConnection } from '../../../../connection/ChronicleConnection.js';
11
+ import type { IClientArtifactsProvider } from '../../../../artifacts/IClientArtifactsProvider.js';
12
+ import type { IEventLog } from '../../../../eventSequences/IEventLog.js';
13
+ import type { IEventStore } from '../../../../IEventStore.js';
14
+ import { a_scripted_stream, type StreamScript } from '../../../../observation/for_ObservationReconnectBackoff/given/a_scripted_stream.js';
15
+
16
+ @eventType('reconnecting-reactor-item')
17
+ class Item {}
18
+
19
+ @reactor('reconnecting-reactor')
20
+ class Observer { item() {} }
21
+
22
+ export type { StreamScript };
23
+
24
+ /**
25
+ * A reactor whose observation streams follow a script, on fake timers and a jitter of zero, so every wait is
26
+ * exactly half of its ceiling.
27
+ */
28
+ export class a_reactor_whose_stream_ends {
29
+ readonly lifecycle = new ConnectionLifecycle();
30
+ readonly reconnectRequests: string[] = [];
31
+ readonly stream = new a_scripted_stream<EventsToObserve>();
32
+ readonly reactors: Reactors;
33
+
34
+ constructor() {
35
+ vi.useFakeTimers();
36
+ this.lifecycle.onReconnectRequested(reason => this.reconnectRequests.push(reason));
37
+ const artifacts = { eventTypes: [Item], reactors: [Observer], reducers: [], readModels: [], seeders: [], constraints: [],
38
+ projections: [], webhooks: [], eventTypeMigrations: [], globalForHandlers: [] } as IClientArtifactsProvider;
39
+ this.reactors = new Reactors(artifacts, { reactors: { observe: this.stream.observe } } as unknown as ChronicleConnection,
40
+ 'store', 'Default', this.lifecycle, {} as IEventLog, undefined, { readModels: {} } as IEventStore, undefined, false,
41
+ { now: () => Date.now(), nextJitter: () => 0 });
42
+ }
43
+
44
+ /** Gets the scripts the next streams follow. */
45
+ get scripts(): StreamScript<EventsToObserve>[] {
46
+ return this.stream.scripts;
47
+ }
48
+
49
+ /** Gets when each stream was opened. */
50
+ get openedAt(): number[] {
51
+ return this.stream.openedAt;
52
+ }
53
+
54
+ /** Gets the wait before each stream after the first was opened. */
55
+ get waits(): number[] {
56
+ return this.stream.waits;
57
+ }
58
+
59
+ /** Drops the session and registers again, as a client reconnect does. */
60
+ async reconnect(): Promise<void> {
61
+ await this.lifecycle.disconnected(() => undefined);
62
+ await this.reactors.register();
63
+ }
64
+
65
+ dispose(): void {
66
+ this.reactors.dispose();
67
+ vi.useRealTimers();
68
+ }
69
+ }
@@ -45,6 +45,9 @@ import { EventBatch } from '../observation/batches/EventBatch.js';
45
45
  import { isPage, partitionsOfOperation } from '../observation/batches/PageMessages.js';
46
46
  import { PartitionFailedInBatch } from '../observation/batches/PartitionFailedInBatch.js';
47
47
  import { runConcurrently } from '../observation/batches/runConcurrently.js';
48
+ import { ObservationReconnectBackoff, type ObservationReconnectBackoffOptions } from '../observation/ObservationReconnectBackoff.js';
49
+ import { classifyStreamError, StreamErrorKind } from '../observation/classifyStreamError.js';
50
+ import type { ObservationStreamOutcome } from '../observation/ObservationStreamOutcome.js';
48
51
 
49
52
  /** Expression used to partition reducer observations by event source ID. */
50
53
  const EVENT_SOURCE_ID_KEY = '$eventSourceId';
@@ -112,15 +115,14 @@ class AsyncQueue<T> {
112
115
  * with the Chronicle Kernel via bidirectional gRPC streaming.
113
116
  */
114
117
  export class Reducers implements IReducers {
115
- /** How long to wait before re-establishing an observation whose stream ended. */
116
- private static readonly _reobserveDelayMs = 2000;
117
-
118
118
  private readonly _logger: ReturnType<typeof createLogger>;
119
119
  private readonly _lifecycle: ConnectionLifecycle;
120
120
  private readonly _unsubscribeDisconnected: () => void;
121
121
  private readonly _reducers = new Map<string, Constructor>();
122
122
  private readonly _queues = new Map<string, AsyncQueue<ReducerMessage>>();
123
123
  private readonly _observations = new Map<string, AbortController>();
124
+ private readonly _reconnectBackoffs = new Map<string, ObservationReconnectBackoff>();
125
+ private readonly _reobserveTimers = new Map<string, ReturnType<typeof setTimeout>>();
124
126
  private _registered = false;
125
127
  private _disposed = false;
126
128
 
@@ -135,6 +137,8 @@ export class Reducers implements IReducers {
135
137
  * @param _eventStore - The event store handed to reducers that request it.
136
138
  * @param _artifactActivator - Optional per-delivery activator.
137
139
  * @param _readModelNamingPolicy - Optional policy naming the container of each read model.
140
+ * @param _skipKeepAlive - Whether the client runs without a keep-alive, which refuses client-hosted reducers.
141
+ * @param _reconnectBackoff - How a reducer waits before opening its observation stream again after it ended.
138
142
  */
139
143
  constructor(
140
144
  private readonly _clientArtifacts: IClientArtifactsProvider,
@@ -146,7 +150,8 @@ export class Reducers implements IReducers {
146
150
  private readonly _eventStore?: IEventStore,
147
151
  private readonly _artifactActivator?: ClientArtifactsActivator,
148
152
  private readonly _readModelNamingPolicy?: ReadModelNamingPolicy,
149
- private readonly _skipKeepAlive: boolean = false
153
+ private readonly _skipKeepAlive: boolean = false,
154
+ private readonly _reconnectBackoff: ObservationReconnectBackoffOptions = {}
150
155
  ) {
151
156
  this._logger = createLogger('@cratis/chronicle/reducers', _connection.logger);
152
157
  this._lifecycle = lifecycle;
@@ -163,6 +168,7 @@ export class Reducers implements IReducers {
163
168
  this._disposed = true;
164
169
  this._registered = false;
165
170
  this.disconnectAll();
171
+ this._reconnectBackoffs.clear();
166
172
  }
167
173
 
168
174
  /** @inheritdoc */
@@ -276,13 +282,23 @@ export class Reducers implements IReducers {
276
282
  }
277
283
 
278
284
  private async runObservation(observation: ReducerObservation): Promise<void> {
285
+ let outcome: ObservationStreamOutcome = 'ended';
279
286
  try {
280
- await this.observeReducer(observation);
287
+ outcome = await this.observeReducer(observation);
281
288
  } catch (error) {
282
289
  this._logger.error('Reducer observation loop exited with error', { reducerId: observation.id, error });
283
290
  }
284
291
 
285
- this.scheduleReobserve(observation.id, observation.reducerType, observation.dispatcher);
292
+ if (outcome === 'ended') this.scheduleReobserve(observation.id, observation.reducerType, observation.dispatcher);
293
+ }
294
+
295
+ private reconnectBackoffFor(id: string): ObservationReconnectBackoff {
296
+ let backoff = this._reconnectBackoffs.get(id);
297
+ if (!backoff) {
298
+ backoff = new ObservationReconnectBackoff(this._reconnectBackoff);
299
+ this._reconnectBackoffs.set(id, backoff);
300
+ }
301
+ return backoff;
286
302
  }
287
303
 
288
304
  /**
@@ -292,8 +308,8 @@ export class Reducers implements IReducers {
292
308
  * closes a cross-store (inbox) observation's stream rather than tailing it
293
309
  * forever, so a reducer that does not re-subscribe there silently stops
294
310
  * observing until the whole client reconnects. Both endings are therefore
295
- * retried, and the delay keeps a stream that keeps ending from becoming a hot
296
- * loop.
311
+ * retried. The delay keeps a stream that keeps ending from becoming a hot loop,
312
+ * and grows with jitter while it does (see {@link ObservationReconnectBackoff}).
297
313
  */
298
314
  private scheduleReobserve(id: string, reducerType: Constructor, dispatcher: ReducerEventDispatcher): void {
299
315
  // A disconnect clears the registration; the reconnect re-registers every
@@ -302,16 +318,22 @@ export class Reducers implements IReducers {
302
318
  return;
303
319
  }
304
320
 
321
+ clearTimeout(this._reobserveTimers.get(id));
322
+ const delayMs = this.reconnectBackoffFor(id).nextDelayMs();
323
+ this._logger.debug('Waiting before re-establishing reducer observation', { reducerId: id, delayMs });
305
324
  const handle = runInBackgroundContext(() => setTimeout(() => {
306
- if (!this._registered || this._disposed) {
325
+ this._reobserveTimers.delete(id);
326
+ // A reconnect in the meantime has already opened a new stream; opening another would leave one untracked.
327
+ if (!this._registered || this._disposed || this._observations.has(id)) {
307
328
  return;
308
329
  }
309
330
 
310
331
  this._logger.info('Re-establishing reducer observation', { reducerId: id });
311
332
  this.startObservation(id, reducerType, dispatcher);
312
- }, Reducers._reobserveDelayMs));
333
+ }, delayMs));
313
334
 
314
335
  handle.unref?.();
336
+ this._reobserveTimers.set(id, handle);
315
337
  }
316
338
 
317
339
  private getReducerReadModelIdentifier(reducerType: Constructor): string {
@@ -323,21 +345,44 @@ export class Reducers implements IReducers {
323
345
  return (reducerType as Function).name;
324
346
  }
325
347
 
326
- private async observeReducer(observation: ReducerObservation): Promise<void> {
348
+ private async observeReducer(observation: ReducerObservation): Promise<ObservationStreamOutcome> {
327
349
  const { id, reducerType } = observation;
328
350
  const queue = new AsyncQueue<ReducerMessage>();
329
351
  const controller = new AbortController();
330
352
  this._queues.set(id, queue);
331
353
  this._observations.set(id, controller);
354
+ const connectionId = this._lifecycle.connectionId;
355
+ const backoff = this.reconnectBackoffFor(id);
356
+ backoff.streamOpened();
332
357
 
333
- queue.send(this.createRegistrationMessage(observation));
358
+ queue.send(this.createRegistrationMessage(observation, connectionId));
334
359
 
360
+ let iterator: AsyncIterator<ReduceOperationMessage> | undefined;
361
+ let streamFinished = false;
335
362
  try {
336
363
  const reducerInstance = this._artifactActivator ? undefined : new (reducerType as new () => Record<string, Function>)();
337
364
  if (reducerInstance) observation.dispatcher.validateInstance(reducerInstance);
338
365
  const session: ReducerObservationSession = { ...observation, signal: controller.signal, reducerInstance };
339
366
 
340
- for await (const operation of this._connection.reducers.observe(queue, { signal: controller.signal })) {
367
+ // Pull each message by hand so only a failure of the stream itself is classified: an error from a
368
+ // reducer is the reducer's failure, and must never make the whole client reconnect.
369
+ iterator = this._connection.reducers.observe(queue, { signal: controller.signal })[Symbol.asyncIterator]();
370
+ for (;;) {
371
+ let next: IteratorResult<ReduceOperationMessage>;
372
+ try {
373
+ next = await iterator.next();
374
+ } catch (streamError) {
375
+ streamFinished = true;
376
+ return this.streamFailed(id, queue, controller.signal, connectionId, streamError);
377
+ }
378
+
379
+ if (next.done) {
380
+ streamFinished = true;
381
+ break;
382
+ }
383
+
384
+ const operation = next.value;
385
+ backoff.streamDelivered();
341
386
  const partitions = partitionsOfOperation(operation);
342
387
  const notification = this.beginReduction(session, operation, partitions);
343
388
  await this.notifyReplayTransition(session, operation, notification);
@@ -347,13 +392,23 @@ export class Reducers implements IReducers {
347
392
  partition => this.reducePartition(session, partition, notification, context));
348
393
  queue.send(toResultMessage(operation, partitions, outcomes));
349
394
  }
350
- } catch (err) {
351
- if (!this._queues.has(id)) {
395
+
396
+ if (this._queues.get(id) !== queue) {
352
397
  this._logger.debug('Reducer observation stream closed cleanly', { reducerId: id });
353
- } else {
354
- this._logger.error('Reducer observation stream ended unexpectedly', { reducerId: id, error: err });
398
+ return 'superseded';
355
399
  }
400
+
401
+ return 'ended';
402
+ } catch (handlerError) {
403
+ if (this._queues.get(id) !== queue) return 'superseded';
404
+ this._logger.error('Reducer observation failed while handling a delivery', { reducerId: id, error: handlerError });
405
+ return 'ended';
356
406
  } finally {
407
+ if (!streamFinished) {
408
+ controller.abort();
409
+ void iterator?.return?.()?.catch(() => undefined);
410
+ }
411
+
357
412
  // Only retire our own queue: a reconnect can already have replaced it,
358
413
  // and deleting the new one would leave that observation untracked and
359
414
  // leak a duplicate stream on every reconnect.
@@ -365,11 +420,34 @@ export class Reducers implements IReducers {
365
420
  }
366
421
  }
367
422
 
368
- private createRegistrationMessage({ id, reducerType, eventSequenceId, readModelName, dispatcher }: ReducerObservation): ReducerMessage {
423
+ private streamFailed(id: string, queue: AsyncQueue<ReducerMessage>, signal: AbortSignal, connectionId: string, error: unknown): ObservationStreamOutcome {
424
+ // A late error from a stream that a disconnect, a dispose or a newer stream replaced says nothing about now.
425
+ if (this._queues.get(id) !== queue) {
426
+ this._logger.debug('Reducer observation stream closed cleanly', { reducerId: id });
427
+ return 'superseded';
428
+ }
429
+
430
+ const kind = classifyStreamError(error);
431
+ if (kind === StreamErrorKind.Cancelled && signal.aborted) return 'stopped';
432
+ if (kind === StreamErrorKind.SessionLost) {
433
+ // The kernel lost this client's registration. Opening the same stream with the same connection id
434
+ // can never work, so reconnect the whole client, which re-registers every reducer.
435
+ this._logger.warn('Reducer observation stream lost its session; reconnecting the client', { reducerId: id, error });
436
+ this._lifecycle.requestReconnect(connectionId, 'reducer-session-lost', error,
437
+ handlerError => this._logger.error('Reconnect request handler failed', { reducerId: id, error: handlerError }));
438
+ return 'sessionLost';
439
+ }
440
+
441
+ this._logger.error('Reducer observation stream ended unexpectedly', { reducerId: id, error });
442
+ return 'ended';
443
+ }
444
+
445
+ private createRegistrationMessage({ id, reducerType, eventSequenceId, readModelName, dispatcher }: ReducerObservation,
446
+ connectionId: string): ReducerMessage {
369
447
  return {
370
448
  Content: {
371
449
  Value0: {
372
- ConnectionId: this._lifecycle.connectionId,
450
+ ConnectionId: connectionId,
373
451
  EventStore: this._eventStoreName,
374
452
  Namespace: this._namespace,
375
453
  Reducer: {
@@ -596,6 +674,8 @@ export class Reducers implements IReducers {
596
674
  }
597
675
 
598
676
  private disconnectAll(): void {
677
+ for (const timer of this._reobserveTimers.values()) clearTimeout(timer);
678
+ this._reobserveTimers.clear();
599
679
  for (const controller of this._observations.values()) controller.abort();
600
680
  for (const queue of this._queues.values()) queue.complete();
601
681
  this._observations.clear();
@@ -0,0 +1,33 @@
1
+ // Copyright (c) Cratis. All rights reserved.
2
+ // Licensed under the MIT license. See LICENSE file in the project root for full license information.
3
+
4
+ import { afterEach, beforeEach, chai, describe, it, vi } from 'vitest';
5
+ import { ClientError, Status } from 'nice-grpc';
6
+ import { a_reducer_whose_stream_ends } from './given/a_reducer_whose_stream_ends.js';
7
+ import { page } from '../given/observe_reducer_messages.js';
8
+ import { Reducers } from '../../Reducers.js';
9
+
10
+ chai.should();
11
+
12
+ describe('when reconnecting a reducer observation stream and reducing failed with a lost session', () => {
13
+ let context: a_reducer_whose_stream_ends;
14
+
15
+ beforeEach(async () => {
16
+ context = new a_reducer_whose_stream_ends();
17
+ vi.spyOn(Reducers.prototype as unknown as { beginReduction: () => unknown }, 'beginReduction').mockImplementationOnce(() => {
18
+ throw new ClientError('/ReadModels/Get', Status.UNKNOWN, 'ClientIsNotConnected: connection is not connected');
19
+ });
20
+ context.scripts.push({ deliver: [page(1n, {})], untilAborted: true }, { untilAborted: true });
21
+ await context.reducers.register();
22
+ await vi.advanceTimersByTimeAsync(120_000);
23
+ });
24
+
25
+ afterEach(() => {
26
+ context.dispose();
27
+ vi.restoreAllMocks();
28
+ });
29
+
30
+ it('should not ask the whole client to reconnect', () => context.reconnectRequests.should.be.empty);
31
+ it('should open the stream again after the base delay', () => context.waits.should.deep.equal([1_000]));
32
+ it('should close the stream the reducer failed on', () => context.stream.open.should.equal(1));
33
+ });
@@ -0,0 +1,28 @@
1
+ // Copyright (c) Cratis. All rights reserved.
2
+ // Licensed under the MIT license. See LICENSE file in the project root for full license information.
3
+
4
+ import { afterEach, beforeEach, chai, describe, it, vi } from 'vitest';
5
+ import { ClientError, Status } from 'nice-grpc';
6
+ import { a_reducer_whose_stream_ends } from './given/a_reducer_whose_stream_ends.js';
7
+
8
+ chai.should();
9
+
10
+ describe('when reconnecting a reducer observation stream and a replaced stream fails late', () => {
11
+ let context: a_reducer_whose_stream_ends;
12
+
13
+ beforeEach(async () => {
14
+ context = new a_reducer_whose_stream_ends();
15
+ context.scripts.push(
16
+ { untilAborted: true, endsLateByMs: 10, error: new ClientError('/Reducers/Observe', Status.INTERNAL, 'boom') },
17
+ { untilAborted: true });
18
+ await context.reducers.register();
19
+ await vi.advanceTimersByTimeAsync(0);
20
+ await context.reconnect();
21
+ await vi.advanceTimersByTimeAsync(120_000);
22
+ });
23
+
24
+ afterEach(() => context.dispose());
25
+
26
+ it('should not open the stream again', () => context.openedAt.should.have.lengthOf(2));
27
+ it('should leave exactly one stream open', () => context.stream.open.should.equal(1));
28
+ });
@@ -0,0 +1,24 @@
1
+ // Copyright (c) Cratis. All rights reserved.
2
+ // Licensed under the MIT license. See LICENSE file in the project root for full license information.
3
+
4
+ import { afterEach, beforeEach, chai, describe, it, vi } from 'vitest';
5
+ import { a_reducer_whose_stream_ends } from './given/a_reducer_whose_stream_ends.js';
6
+ import { page } from '../given/observe_reducer_messages.js';
7
+
8
+ chai.should();
9
+
10
+ describe('when reconnecting a reducer observation stream and a stream delivered before it ended', () => {
11
+ let context: a_reducer_whose_stream_ends;
12
+
13
+ beforeEach(async () => {
14
+ context = new a_reducer_whose_stream_ends();
15
+ context.scripts.push({}, {}, { deliver: [page(1n, {})] }, {});
16
+ await context.reducers.register();
17
+ await vi.advanceTimersByTimeAsync(6_500);
18
+ });
19
+
20
+ afterEach(() => context.dispose());
21
+
22
+ it('should start the wait over at the base delay after the stream that delivered', () =>
23
+ context.waits.slice(0, 4).should.deep.equal([1_000, 2_000, 1_000, 2_000]));
24
+ });
@@ -0,0 +1,23 @@
1
+ // Copyright (c) Cratis. All rights reserved.
2
+ // Licensed under the MIT license. See LICENSE file in the project root for full license information.
3
+
4
+ import { afterEach, beforeEach, chai, describe, it, vi } from 'vitest';
5
+ import { a_reducer_whose_stream_ends } from './given/a_reducer_whose_stream_ends.js';
6
+
7
+ chai.should();
8
+
9
+ describe('when reconnecting a reducer observation stream and a stream stayed open before it ended', () => {
10
+ let context: a_reducer_whose_stream_ends;
11
+
12
+ beforeEach(async () => {
13
+ context = new a_reducer_whose_stream_ends();
14
+ context.scripts.push({}, {}, { openForMs: 180_000 }, { untilAborted: true });
15
+ await context.reducers.register();
16
+ await vi.advanceTimersByTimeAsync(185_000);
17
+ });
18
+
19
+ afterEach(() => context.dispose());
20
+
21
+ it('should start the wait over at the base delay after the stream that stayed open', () =>
22
+ context.waits.should.deep.equal([1_000, 2_000, 181_000]));
23
+ });
@@ -0,0 +1,27 @@
1
+ // Copyright (c) Cratis. All rights reserved.
2
+ // Licensed under the MIT license. See LICENSE file in the project root for full license information.
3
+
4
+ import { afterEach, beforeEach, chai, describe, it, vi } from 'vitest';
5
+ import { a_reducer_whose_stream_ends } from './given/a_reducer_whose_stream_ends.js';
6
+
7
+ chai.should();
8
+
9
+ describe('when reconnecting a reducer observation stream and it is disposed while it waits to reopen', () => {
10
+ let context: a_reducer_whose_stream_ends;
11
+ let timersAfterDispose: number;
12
+
13
+ beforeEach(async () => {
14
+ context = new a_reducer_whose_stream_ends();
15
+ context.scripts.push({});
16
+ await context.reducers.register();
17
+ await vi.advanceTimersByTimeAsync(0);
18
+ context.reducers.dispose();
19
+ timersAfterDispose = vi.getTimerCount();
20
+ await vi.advanceTimersByTimeAsync(120_000);
21
+ });
22
+
23
+ afterEach(() => context.dispose());
24
+
25
+ it('should not open the stream again', () => context.openedAt.should.have.lengthOf(1));
26
+ it('should leave no timer behind', () => timersAfterDispose.should.equal(0));
27
+ });