@cratis/chronicle 6.61.0 → 6.63.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ChronicleClient.ts +86 -3
- package/connection/ChronicleConnection.ts +7 -1
- package/connection/ChronicleServices.ts +2 -0
- package/connection/ConnectionLifecycle.ts +42 -1
- package/dist/ChronicleClient.d.ts +28 -1
- package/dist/ChronicleClient.d.ts.map +1 -1
- package/dist/ChronicleClient.js +71 -3
- package/dist/ChronicleClient.js.map +1 -1
- package/dist/connection/ChronicleConnection.d.ts +1 -0
- package/dist/connection/ChronicleConnection.d.ts.map +1 -1
- package/dist/connection/ChronicleConnection.js +6 -2
- package/dist/connection/ChronicleConnection.js.map +1 -1
- package/dist/connection/ChronicleServices.d.ts +2 -1
- package/dist/connection/ChronicleServices.d.ts.map +1 -1
- package/dist/connection/ConnectionLifecycle.d.ts +24 -0
- package/dist/connection/ConnectionLifecycle.d.ts.map +1 -1
- package/dist/connection/ConnectionLifecycle.js +40 -1
- package/dist/connection/ConnectionLifecycle.js.map +1 -1
- package/dist/eventSequences/EventForEventSourceId.d.ts +21 -1
- package/dist/eventSequences/EventForEventSourceId.d.ts.map +1 -1
- package/dist/eventSequences/EventSequence.d.ts +9 -0
- package/dist/eventSequences/EventSequence.d.ts.map +1 -1
- package/dist/eventSequences/EventSequence.js +18 -0
- package/dist/eventSequences/EventSequence.js.map +1 -1
- package/dist/eventSequences/IEventSequence.d.ts +30 -0
- package/dist/eventSequences/IEventSequence.d.ts.map +1 -1
- package/dist/imports/EventImport.d.ts +29 -0
- package/dist/imports/EventImport.d.ts.map +1 -0
- package/dist/imports/EventImport.js +248 -0
- package/dist/imports/EventImport.js.map +1 -0
- package/dist/imports/EventImportSettings.d.ts +32 -0
- package/dist/imports/EventImportSettings.d.ts.map +1 -0
- package/dist/imports/EventImportSettings.js +4 -0
- package/dist/imports/EventImportSettings.js.map +1 -0
- package/dist/imports/IEventImport.d.ts +50 -0
- package/dist/imports/IEventImport.d.ts.map +1 -0
- package/dist/imports/IEventImport.js +4 -0
- package/dist/imports/IEventImport.js.map +1 -0
- package/dist/imports/ImportChunkRefused.d.ts +12 -0
- package/dist/imports/ImportChunkRefused.d.ts.map +1 -0
- package/dist/imports/ImportChunkRefused.js +19 -0
- package/dist/imports/ImportChunkRefused.js.map +1 -0
- package/dist/imports/ImportCouldNotBegin.d.ts +10 -0
- package/dist/imports/ImportCouldNotBegin.d.ts.map +1 -0
- package/dist/imports/ImportCouldNotBegin.js +16 -0
- package/dist/imports/ImportCouldNotBegin.js.map +1 -0
- package/dist/imports/ImportOptions.d.ts +26 -0
- package/dist/imports/ImportOptions.d.ts.map +1 -0
- package/dist/imports/ImportOptions.js +4 -0
- package/dist/imports/ImportOptions.js.map +1 -0
- package/dist/imports/ImportProgress.d.ts +20 -0
- package/dist/imports/ImportProgress.d.ts.map +1 -0
- package/dist/imports/ImportProgress.js +4 -0
- package/dist/imports/ImportProgress.js.map +1 -0
- package/dist/imports/ImportRejection.d.ts +10 -0
- package/dist/imports/ImportRejection.d.ts.map +1 -0
- package/dist/imports/ImportRejection.js +4 -0
- package/dist/imports/ImportRejection.js.map +1 -0
- package/dist/imports/ImportResult.d.ts +30 -0
- package/dist/imports/ImportResult.d.ts.map +1 -0
- package/dist/imports/ImportResult.js +4 -0
- package/dist/imports/ImportResult.js.map +1 -0
- package/dist/imports/ImportState.d.ts +18 -0
- package/dist/imports/ImportState.d.ts.map +1 -0
- package/dist/imports/ImportState.js +21 -0
- package/dist/imports/ImportState.js.map +1 -0
- package/dist/imports/ImportTarget.d.ts +16 -0
- package/dist/imports/ImportTarget.d.ts.map +1 -0
- package/dist/imports/ImportTarget.js +4 -0
- package/dist/imports/ImportTarget.js.map +1 -0
- package/dist/imports/ImportWasAbandoned.d.ts +11 -0
- package/dist/imports/ImportWasAbandoned.d.ts.map +1 -0
- package/dist/imports/ImportWasAbandoned.js +16 -0
- package/dist/imports/ImportWasAbandoned.js.map +1 -0
- package/dist/imports/ImportsNotSupported.d.ts +5 -0
- package/dist/imports/ImportsNotSupported.d.ts.map +1 -0
- package/dist/imports/ImportsNotSupported.js +10 -0
- package/dist/imports/ImportsNotSupported.js.map +1 -0
- package/dist/imports/OnInvalidEvent.d.ts +8 -0
- package/dist/imports/OnInvalidEvent.d.ts.map +1 -0
- package/dist/imports/OnInvalidEvent.js +11 -0
- package/dist/imports/OnInvalidEvent.js.map +1 -0
- package/dist/imports/abortable.d.ts +23 -0
- package/dist/imports/abortable.d.ts.map +1 -0
- package/dist/imports/abortable.js +60 -0
- package/dist/imports/abortable.js.map +1 -0
- package/dist/imports/beginEventImport.d.ts +16 -0
- package/dist/imports/beginEventImport.d.ts.map +1 -0
- package/dist/imports/beginEventImport.js +74 -0
- package/dist/imports/beginEventImport.js.map +1 -0
- package/dist/imports/for_EventImport/given/a_kernel_import_session.d.ts +144 -0
- package/dist/imports/for_EventImport/given/a_kernel_import_session.d.ts.map +1 -0
- package/dist/imports/for_EventImport/given/a_kernel_import_session.js +132 -0
- package/dist/imports/for_EventImport/given/a_kernel_import_session.js.map +1 -0
- package/dist/imports/importEvents.d.ts +13 -0
- package/dist/imports/importEvents.d.ts.map +1 -0
- package/dist/imports/importEvents.js +35 -0
- package/dist/imports/importEvents.js.map +1 -0
- package/dist/imports/index.d.ts +12 -0
- package/dist/imports/index.d.ts.map +1 -0
- package/dist/imports/index.js +9 -0
- package/dist/imports/index.js.map +1 -0
- package/dist/imports/isTransientImportFailure.d.ts +7 -0
- package/dist/imports/isTransientImportFailure.d.ts.map +1 -0
- package/dist/imports/isTransientImportFailure.js +13 -0
- package/dist/imports/isTransientImportFailure.js.map +1 -0
- package/dist/imports/sizeOfImportEvent.d.ts +9 -0
- package/dist/imports/sizeOfImportEvent.d.ts.map +1 -0
- package/dist/imports/sizeOfImportEvent.js +25 -0
- package/dist/imports/sizeOfImportEvent.js.map +1 -0
- package/dist/imports/toImportEvent.d.ts +14 -0
- package/dist/imports/toImportEvent.d.ts.map +1 -0
- package/dist/imports/toImportEvent.js +54 -0
- package/dist/imports/toImportEvent.js.map +1 -0
- package/dist/imports/toImportProgress.d.ts +9 -0
- package/dist/imports/toImportProgress.d.ts.map +1 -0
- package/dist/imports/toImportProgress.js +20 -0
- package/dist/imports/toImportProgress.js.map +1 -0
- package/dist/imports/toImportResult.d.ts +9 -0
- package/dist/imports/toImportResult.d.ts.map +1 -0
- package/dist/imports/toImportResult.js +33 -0
- package/dist/imports/toImportResult.js.map +1 -0
- package/dist/index.d.ts +3 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -1
- package/dist/index.js.map +1 -1
- package/dist/observation/ObservationReconnectBackoff.d.ts +64 -0
- package/dist/observation/ObservationReconnectBackoff.d.ts.map +1 -0
- package/dist/observation/ObservationReconnectBackoff.js +75 -0
- package/dist/observation/ObservationReconnectBackoff.js.map +1 -0
- package/dist/observation/ObservationStreamOutcome.d.ts +10 -0
- package/dist/observation/ObservationStreamOutcome.d.ts.map +1 -0
- package/dist/observation/ObservationStreamOutcome.js +4 -0
- package/dist/observation/ObservationStreamOutcome.js.map +1 -0
- package/dist/observation/classifyStreamError.d.ts +25 -0
- package/dist/observation/classifyStreamError.d.ts.map +1 -0
- package/dist/observation/classifyStreamError.js +55 -0
- package/dist/observation/classifyStreamError.js.map +1 -0
- package/dist/observation/for_ObservationReconnectBackoff/given/a_backoff.d.ts +13 -0
- package/dist/observation/for_ObservationReconnectBackoff/given/a_backoff.d.ts.map +1 -0
- package/dist/observation/for_ObservationReconnectBackoff/given/a_backoff.js +25 -0
- package/dist/observation/for_ObservationReconnectBackoff/given/a_backoff.js.map +1 -0
- package/dist/observation/for_ObservationReconnectBackoff/given/a_scripted_stream.d.ts +27 -0
- package/dist/observation/for_ObservationReconnectBackoff/given/a_scripted_stream.d.ts.map +1 -0
- package/dist/observation/for_ObservationReconnectBackoff/given/a_scripted_stream.js +36 -0
- package/dist/observation/for_ObservationReconnectBackoff/given/a_scripted_stream.js.map +1 -0
- package/dist/observation/index.d.ts +2 -0
- package/dist/observation/index.d.ts.map +1 -1
- package/dist/observation/index.js +1 -0
- package/dist/observation/index.js.map +1 -1
- package/dist/reactors/Reactors.d.ts +15 -5
- package/dist/reactors/Reactors.d.ts.map +1 -1
- package/dist/reactors/Reactors.js +95 -18
- package/dist/reactors/Reactors.js.map +1 -1
- package/dist/reactors/for_Reactors/when_reconnecting_the_observation_stream/given/a_reactor_whose_stream_ends.d.ts +26 -0
- package/dist/reactors/for_Reactors/when_reconnecting_the_observation_stream/given/a_reactor_whose_stream_ends.d.ts.map +1 -0
- package/dist/reactors/for_Reactors/when_reconnecting_the_observation_stream/given/a_reactor_whose_stream_ends.js +64 -0
- package/dist/reactors/for_Reactors/when_reconnecting_the_observation_stream/given/a_reactor_whose_stream_ends.js.map +1 -0
- package/dist/reducers/Reducers.d.ts +11 -5
- package/dist/reducers/Reducers.d.ts.map +1 -1
- package/dist/reducers/Reducers.js +93 -19
- package/dist/reducers/Reducers.js.map +1 -1
- package/dist/reducers/for_Reducers/when_reconnecting_the_observation_stream/given/a_reducer_whose_stream_ends.d.ts +25 -0
- package/dist/reducers/for_Reducers/when_reconnecting_the_observation_stream/given/a_reducer_whose_stream_ends.d.ts.map +1 -0
- package/dist/reducers/for_Reducers/when_reconnecting_the_observation_stream/given/a_reducer_whose_stream_ends.js +70 -0
- package/dist/reducers/for_Reducers/when_reconnecting_the_observation_stream/given/a_reducer_whose_stream_ends.js.map +1 -0
- package/dist/testing/BatchPartitionOutcome.d.ts +7 -0
- package/dist/testing/BatchPartitionOutcome.d.ts.map +1 -0
- package/dist/testing/BatchPartitionOutcome.js +4 -0
- package/dist/testing/BatchPartitionOutcome.js.map +1 -0
- package/dist/testing/InProcessEventSequence.d.ts +4 -0
- package/dist/testing/InProcessEventSequence.d.ts.map +1 -1
- package/dist/testing/InProcessEventSequence.js +2 -0
- package/dist/testing/InProcessEventSequence.js.map +1 -1
- package/dist/testing/ReactorDeliveryResult.d.ts +2 -0
- package/dist/testing/ReactorDeliveryResult.d.ts.map +1 -1
- package/dist/testing/ReactorScenario.d.ts +46 -10
- package/dist/testing/ReactorScenario.d.ts.map +1 -1
- package/dist/testing/ReactorScenario.js +120 -29
- package/dist/testing/ReactorScenario.js.map +1 -1
- package/dist/testing/deliverToBatchMethods.d.ts +35 -0
- package/dist/testing/deliverToBatchMethods.d.ts.map +1 -0
- package/dist/testing/deliverToBatchMethods.js +55 -0
- package/dist/testing/deliverToBatchMethods.js.map +1 -0
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/eventSequences/EventForEventSourceId.ts +25 -1
- package/eventSequences/EventSequence.ts +28 -0
- package/eventSequences/IEventSequence.ts +33 -0
- package/for_ChronicleClient/when_reconnecting/and_an_observation_stream_lost_its_session.spec.ts +212 -0
- package/imports/EventImport.ts +263 -0
- package/imports/EventImportSettings.ts +46 -0
- package/imports/IEventImport.ts +58 -0
- package/imports/ImportChunkRefused.ts +15 -0
- package/imports/ImportCouldNotBegin.ts +14 -0
- package/imports/ImportOptions.ts +36 -0
- package/imports/ImportProgress.ts +29 -0
- package/imports/ImportRejection.ts +14 -0
- package/imports/ImportResult.ts +43 -0
- package/imports/ImportState.ts +26 -0
- package/imports/ImportTarget.ts +23 -0
- package/imports/ImportWasAbandoned.ts +16 -0
- package/imports/ImportsNotSupported.ts +10 -0
- package/imports/OnInvalidEvent.ts +11 -0
- package/imports/abortable.ts +57 -0
- package/imports/beginEventImport.ts +85 -0
- package/imports/for_EventImport/given/a_kernel_import_session.ts +160 -0
- package/imports/for_EventImport/when_a_chunk_fails.spec.ts +65 -0
- package/imports/for_EventImport/when_abandoning.spec.ts +52 -0
- package/imports/for_EventImport/when_cancelling_with_an_abort_signal.spec.ts +59 -0
- package/imports/for_EventImport/when_completing.spec.ts +71 -0
- package/imports/for_EventImport/when_cutting_events_into_chunks.spec.ts +70 -0
- package/imports/for_EventImport/when_keeping_chunks_in_flight.spec.ts +49 -0
- package/imports/for_beginEventImport/when_reattaching.spec.ts +82 -0
- package/imports/for_beginEventImport/when_the_kernel_does_not_serve_imports.spec.ts +55 -0
- package/imports/for_importEvents/when_importing_in_one_call.spec.ts +99 -0
- package/imports/for_toImportEvent/when_converting_an_event.spec.ts +74 -0
- package/imports/importEvents.ts +44 -0
- package/imports/index.ts +14 -0
- package/imports/isTransientImportFailure.ts +15 -0
- package/imports/sizeOfImportEvent.ts +25 -0
- package/imports/toImportEvent.ts +60 -0
- package/imports/toImportProgress.ts +24 -0
- package/imports/toImportResult.ts +36 -0
- package/index.ts +3 -0
- package/observation/ObservationReconnectBackoff.ts +109 -0
- package/observation/ObservationStreamOutcome.ts +12 -0
- package/observation/classifyStreamError.ts +67 -0
- package/observation/for_ObservationReconnectBackoff/given/a_backoff.ts +29 -0
- package/observation/for_ObservationReconnectBackoff/given/a_scripted_stream.ts +52 -0
- package/observation/for_ObservationReconnectBackoff/when_a_stream_delivered_before_it_ended.spec.ts +23 -0
- package/observation/for_ObservationReconnectBackoff/when_a_stream_stayed_open_before_it_ended.spec.ts +25 -0
- package/observation/for_ObservationReconnectBackoff/when_jittering_the_delay.spec.ts +24 -0
- package/observation/for_ObservationReconnectBackoff/when_streams_keep_ending_without_working.spec.ts +24 -0
- package/observation/for_ObservationReconnectBackoff/when_the_base_delay_is_not_positive.spec.ts +18 -0
- package/observation/for_ObservationReconnectBackoff/when_the_settings_are_not_finite.spec.ts +32 -0
- package/observation/for_ObservationReconnectBackoff/when_using_the_defaults.spec.ts +29 -0
- package/observation/for_classifyStreamError/when_classifying_a_stream_error.spec.ts +36 -0
- package/observation/index.ts +2 -0
- package/package.json +7 -2
- package/projections/for_childrenOrderingSupport/when_detecting_support_in_the_pinned_contracts.spec.ts +1 -1
- package/reactors/Reactors.ts +101 -18
- package/reactors/for_Reactors/when_reconnecting_the_observation_stream/and_a_handler_failed_with_a_lost_session.spec.ts +32 -0
- package/reactors/for_Reactors/when_reconnecting_the_observation_stream/and_a_replaced_stream_fails_late.spec.ts +28 -0
- package/reactors/for_Reactors/when_reconnecting_the_observation_stream/and_a_stream_delivered_before_it_ended.spec.ts +24 -0
- package/reactors/for_Reactors/when_reconnecting_the_observation_stream/and_a_stream_stayed_open_before_it_ended.spec.ts +23 -0
- package/reactors/for_Reactors/when_reconnecting_the_observation_stream/and_it_is_disposed_while_waiting_to_reopen.spec.ts +27 -0
- package/reactors/for_Reactors/when_reconnecting_the_observation_stream/and_streams_keep_ending_without_working.spec.ts +24 -0
- package/reactors/for_Reactors/when_reconnecting_the_observation_stream/and_the_client_reconnected_after_streams_kept_ending.spec.ts +24 -0
- package/reactors/for_Reactors/when_reconnecting_the_observation_stream/and_the_client_reconnected_while_waiting_to_reopen.spec.ts +25 -0
- package/reactors/for_Reactors/when_reconnecting_the_observation_stream/and_the_kernel_lost_the_session.spec.ts +24 -0
- package/reactors/for_Reactors/when_reconnecting_the_observation_stream/given/a_reactor_whose_stream_ends.ts +69 -0
- package/reducers/Reducers.ts +99 -19
- package/reducers/for_Reducers/when_reconnecting_the_observation_stream/and_a_reducer_failed_with_a_lost_session.spec.ts +33 -0
- package/reducers/for_Reducers/when_reconnecting_the_observation_stream/and_a_replaced_stream_fails_late.spec.ts +28 -0
- package/reducers/for_Reducers/when_reconnecting_the_observation_stream/and_a_stream_delivered_before_it_ended.spec.ts +24 -0
- package/reducers/for_Reducers/when_reconnecting_the_observation_stream/and_a_stream_stayed_open_before_it_ended.spec.ts +23 -0
- package/reducers/for_Reducers/when_reconnecting_the_observation_stream/and_it_is_disposed_while_waiting_to_reopen.spec.ts +27 -0
- package/reducers/for_Reducers/when_reconnecting_the_observation_stream/and_streams_keep_ending_without_working.spec.ts +24 -0
- package/reducers/for_Reducers/when_reconnecting_the_observation_stream/and_the_client_reconnected_after_streams_kept_ending.spec.ts +24 -0
- package/reducers/for_Reducers/when_reconnecting_the_observation_stream/and_the_client_reconnected_while_waiting_to_reopen.spec.ts +25 -0
- package/reducers/for_Reducers/when_reconnecting_the_observation_stream/and_the_kernel_lost_the_session.spec.ts +24 -0
- package/reducers/for_Reducers/when_reconnecting_the_observation_stream/given/a_reducer_whose_stream_ends.ts +69 -0
- package/testing/BatchPartitionOutcome.ts +10 -0
- package/testing/InProcessEventSequence.ts +4 -0
- package/testing/ReactorDeliveryResult.ts +2 -0
- package/testing/ReactorScenario.ts +147 -39
- package/testing/deliverToBatchMethods.ts +90 -0
- package/testing/for_ReactorScenario/when_the_reactor_has_a_batch_method.spec.ts +143 -9
package/reducers/Reducers.ts
CHANGED
|
@@ -45,6 +45,9 @@ import { EventBatch } from '../observation/batches/EventBatch.js';
|
|
|
45
45
|
import { isPage, partitionsOfOperation } from '../observation/batches/PageMessages.js';
|
|
46
46
|
import { PartitionFailedInBatch } from '../observation/batches/PartitionFailedInBatch.js';
|
|
47
47
|
import { runConcurrently } from '../observation/batches/runConcurrently.js';
|
|
48
|
+
import { ObservationReconnectBackoff, type ObservationReconnectBackoffOptions } from '../observation/ObservationReconnectBackoff.js';
|
|
49
|
+
import { classifyStreamError, StreamErrorKind } from '../observation/classifyStreamError.js';
|
|
50
|
+
import type { ObservationStreamOutcome } from '../observation/ObservationStreamOutcome.js';
|
|
48
51
|
|
|
49
52
|
/** Expression used to partition reducer observations by event source ID. */
|
|
50
53
|
const EVENT_SOURCE_ID_KEY = '$eventSourceId';
|
|
@@ -112,15 +115,14 @@ class AsyncQueue<T> {
|
|
|
112
115
|
* with the Chronicle Kernel via bidirectional gRPC streaming.
|
|
113
116
|
*/
|
|
114
117
|
export class Reducers implements IReducers {
|
|
115
|
-
/** How long to wait before re-establishing an observation whose stream ended. */
|
|
116
|
-
private static readonly _reobserveDelayMs = 2000;
|
|
117
|
-
|
|
118
118
|
private readonly _logger: ReturnType<typeof createLogger>;
|
|
119
119
|
private readonly _lifecycle: ConnectionLifecycle;
|
|
120
120
|
private readonly _unsubscribeDisconnected: () => void;
|
|
121
121
|
private readonly _reducers = new Map<string, Constructor>();
|
|
122
122
|
private readonly _queues = new Map<string, AsyncQueue<ReducerMessage>>();
|
|
123
123
|
private readonly _observations = new Map<string, AbortController>();
|
|
124
|
+
private readonly _reconnectBackoffs = new Map<string, ObservationReconnectBackoff>();
|
|
125
|
+
private readonly _reobserveTimers = new Map<string, ReturnType<typeof setTimeout>>();
|
|
124
126
|
private _registered = false;
|
|
125
127
|
private _disposed = false;
|
|
126
128
|
|
|
@@ -135,6 +137,8 @@ export class Reducers implements IReducers {
|
|
|
135
137
|
* @param _eventStore - The event store handed to reducers that request it.
|
|
136
138
|
* @param _artifactActivator - Optional per-delivery activator.
|
|
137
139
|
* @param _readModelNamingPolicy - Optional policy naming the container of each read model.
|
|
140
|
+
* @param _skipKeepAlive - Whether the client runs without a keep-alive, which refuses client-hosted reducers.
|
|
141
|
+
* @param _reconnectBackoff - How a reducer waits before opening its observation stream again after it ended.
|
|
138
142
|
*/
|
|
139
143
|
constructor(
|
|
140
144
|
private readonly _clientArtifacts: IClientArtifactsProvider,
|
|
@@ -146,7 +150,8 @@ export class Reducers implements IReducers {
|
|
|
146
150
|
private readonly _eventStore?: IEventStore,
|
|
147
151
|
private readonly _artifactActivator?: ClientArtifactsActivator,
|
|
148
152
|
private readonly _readModelNamingPolicy?: ReadModelNamingPolicy,
|
|
149
|
-
private readonly _skipKeepAlive: boolean = false
|
|
153
|
+
private readonly _skipKeepAlive: boolean = false,
|
|
154
|
+
private readonly _reconnectBackoff: ObservationReconnectBackoffOptions = {}
|
|
150
155
|
) {
|
|
151
156
|
this._logger = createLogger('@cratis/chronicle/reducers', _connection.logger);
|
|
152
157
|
this._lifecycle = lifecycle;
|
|
@@ -163,6 +168,7 @@ export class Reducers implements IReducers {
|
|
|
163
168
|
this._disposed = true;
|
|
164
169
|
this._registered = false;
|
|
165
170
|
this.disconnectAll();
|
|
171
|
+
this._reconnectBackoffs.clear();
|
|
166
172
|
}
|
|
167
173
|
|
|
168
174
|
/** @inheritdoc */
|
|
@@ -276,13 +282,23 @@ export class Reducers implements IReducers {
|
|
|
276
282
|
}
|
|
277
283
|
|
|
278
284
|
private async runObservation(observation: ReducerObservation): Promise<void> {
|
|
285
|
+
let outcome: ObservationStreamOutcome = 'ended';
|
|
279
286
|
try {
|
|
280
|
-
await this.observeReducer(observation);
|
|
287
|
+
outcome = await this.observeReducer(observation);
|
|
281
288
|
} catch (error) {
|
|
282
289
|
this._logger.error('Reducer observation loop exited with error', { reducerId: observation.id, error });
|
|
283
290
|
}
|
|
284
291
|
|
|
285
|
-
this.scheduleReobserve(observation.id, observation.reducerType, observation.dispatcher);
|
|
292
|
+
if (outcome === 'ended') this.scheduleReobserve(observation.id, observation.reducerType, observation.dispatcher);
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
private reconnectBackoffFor(id: string): ObservationReconnectBackoff {
|
|
296
|
+
let backoff = this._reconnectBackoffs.get(id);
|
|
297
|
+
if (!backoff) {
|
|
298
|
+
backoff = new ObservationReconnectBackoff(this._reconnectBackoff);
|
|
299
|
+
this._reconnectBackoffs.set(id, backoff);
|
|
300
|
+
}
|
|
301
|
+
return backoff;
|
|
286
302
|
}
|
|
287
303
|
|
|
288
304
|
/**
|
|
@@ -292,8 +308,8 @@ export class Reducers implements IReducers {
|
|
|
292
308
|
* closes a cross-store (inbox) observation's stream rather than tailing it
|
|
293
309
|
* forever, so a reducer that does not re-subscribe there silently stops
|
|
294
310
|
* observing until the whole client reconnects. Both endings are therefore
|
|
295
|
-
* retried
|
|
296
|
-
*
|
|
311
|
+
* retried. The delay keeps a stream that keeps ending from becoming a hot loop,
|
|
312
|
+
* and grows with jitter while it does (see {@link ObservationReconnectBackoff}).
|
|
297
313
|
*/
|
|
298
314
|
private scheduleReobserve(id: string, reducerType: Constructor, dispatcher: ReducerEventDispatcher): void {
|
|
299
315
|
// A disconnect clears the registration; the reconnect re-registers every
|
|
@@ -302,16 +318,22 @@ export class Reducers implements IReducers {
|
|
|
302
318
|
return;
|
|
303
319
|
}
|
|
304
320
|
|
|
321
|
+
clearTimeout(this._reobserveTimers.get(id));
|
|
322
|
+
const delayMs = this.reconnectBackoffFor(id).nextDelayMs();
|
|
323
|
+
this._logger.debug('Waiting before re-establishing reducer observation', { reducerId: id, delayMs });
|
|
305
324
|
const handle = runInBackgroundContext(() => setTimeout(() => {
|
|
306
|
-
|
|
325
|
+
this._reobserveTimers.delete(id);
|
|
326
|
+
// A reconnect in the meantime has already opened a new stream; opening another would leave one untracked.
|
|
327
|
+
if (!this._registered || this._disposed || this._observations.has(id)) {
|
|
307
328
|
return;
|
|
308
329
|
}
|
|
309
330
|
|
|
310
331
|
this._logger.info('Re-establishing reducer observation', { reducerId: id });
|
|
311
332
|
this.startObservation(id, reducerType, dispatcher);
|
|
312
|
-
},
|
|
333
|
+
}, delayMs));
|
|
313
334
|
|
|
314
335
|
handle.unref?.();
|
|
336
|
+
this._reobserveTimers.set(id, handle);
|
|
315
337
|
}
|
|
316
338
|
|
|
317
339
|
private getReducerReadModelIdentifier(reducerType: Constructor): string {
|
|
@@ -323,21 +345,44 @@ export class Reducers implements IReducers {
|
|
|
323
345
|
return (reducerType as Function).name;
|
|
324
346
|
}
|
|
325
347
|
|
|
326
|
-
private async observeReducer(observation: ReducerObservation): Promise<
|
|
348
|
+
private async observeReducer(observation: ReducerObservation): Promise<ObservationStreamOutcome> {
|
|
327
349
|
const { id, reducerType } = observation;
|
|
328
350
|
const queue = new AsyncQueue<ReducerMessage>();
|
|
329
351
|
const controller = new AbortController();
|
|
330
352
|
this._queues.set(id, queue);
|
|
331
353
|
this._observations.set(id, controller);
|
|
354
|
+
const connectionId = this._lifecycle.connectionId;
|
|
355
|
+
const backoff = this.reconnectBackoffFor(id);
|
|
356
|
+
backoff.streamOpened();
|
|
332
357
|
|
|
333
|
-
queue.send(this.createRegistrationMessage(observation));
|
|
358
|
+
queue.send(this.createRegistrationMessage(observation, connectionId));
|
|
334
359
|
|
|
360
|
+
let iterator: AsyncIterator<ReduceOperationMessage> | undefined;
|
|
361
|
+
let streamFinished = false;
|
|
335
362
|
try {
|
|
336
363
|
const reducerInstance = this._artifactActivator ? undefined : new (reducerType as new () => Record<string, Function>)();
|
|
337
364
|
if (reducerInstance) observation.dispatcher.validateInstance(reducerInstance);
|
|
338
365
|
const session: ReducerObservationSession = { ...observation, signal: controller.signal, reducerInstance };
|
|
339
366
|
|
|
340
|
-
|
|
367
|
+
// Pull each message by hand so only a failure of the stream itself is classified: an error from a
|
|
368
|
+
// reducer is the reducer's failure, and must never make the whole client reconnect.
|
|
369
|
+
iterator = this._connection.reducers.observe(queue, { signal: controller.signal })[Symbol.asyncIterator]();
|
|
370
|
+
for (;;) {
|
|
371
|
+
let next: IteratorResult<ReduceOperationMessage>;
|
|
372
|
+
try {
|
|
373
|
+
next = await iterator.next();
|
|
374
|
+
} catch (streamError) {
|
|
375
|
+
streamFinished = true;
|
|
376
|
+
return this.streamFailed(id, queue, controller.signal, connectionId, streamError);
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
if (next.done) {
|
|
380
|
+
streamFinished = true;
|
|
381
|
+
break;
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
const operation = next.value;
|
|
385
|
+
backoff.streamDelivered();
|
|
341
386
|
const partitions = partitionsOfOperation(operation);
|
|
342
387
|
const notification = this.beginReduction(session, operation, partitions);
|
|
343
388
|
await this.notifyReplayTransition(session, operation, notification);
|
|
@@ -347,13 +392,23 @@ export class Reducers implements IReducers {
|
|
|
347
392
|
partition => this.reducePartition(session, partition, notification, context));
|
|
348
393
|
queue.send(toResultMessage(operation, partitions, outcomes));
|
|
349
394
|
}
|
|
350
|
-
|
|
351
|
-
if (
|
|
395
|
+
|
|
396
|
+
if (this._queues.get(id) !== queue) {
|
|
352
397
|
this._logger.debug('Reducer observation stream closed cleanly', { reducerId: id });
|
|
353
|
-
|
|
354
|
-
this._logger.error('Reducer observation stream ended unexpectedly', { reducerId: id, error: err });
|
|
398
|
+
return 'superseded';
|
|
355
399
|
}
|
|
400
|
+
|
|
401
|
+
return 'ended';
|
|
402
|
+
} catch (handlerError) {
|
|
403
|
+
if (this._queues.get(id) !== queue) return 'superseded';
|
|
404
|
+
this._logger.error('Reducer observation failed while handling a delivery', { reducerId: id, error: handlerError });
|
|
405
|
+
return 'ended';
|
|
356
406
|
} finally {
|
|
407
|
+
if (!streamFinished) {
|
|
408
|
+
controller.abort();
|
|
409
|
+
void iterator?.return?.()?.catch(() => undefined);
|
|
410
|
+
}
|
|
411
|
+
|
|
357
412
|
// Only retire our own queue: a reconnect can already have replaced it,
|
|
358
413
|
// and deleting the new one would leave that observation untracked and
|
|
359
414
|
// leak a duplicate stream on every reconnect.
|
|
@@ -365,11 +420,34 @@ export class Reducers implements IReducers {
|
|
|
365
420
|
}
|
|
366
421
|
}
|
|
367
422
|
|
|
368
|
-
private
|
|
423
|
+
private streamFailed(id: string, queue: AsyncQueue<ReducerMessage>, signal: AbortSignal, connectionId: string, error: unknown): ObservationStreamOutcome {
|
|
424
|
+
// A late error from a stream that a disconnect, a dispose or a newer stream replaced says nothing about now.
|
|
425
|
+
if (this._queues.get(id) !== queue) {
|
|
426
|
+
this._logger.debug('Reducer observation stream closed cleanly', { reducerId: id });
|
|
427
|
+
return 'superseded';
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
const kind = classifyStreamError(error);
|
|
431
|
+
if (kind === StreamErrorKind.Cancelled && signal.aborted) return 'stopped';
|
|
432
|
+
if (kind === StreamErrorKind.SessionLost) {
|
|
433
|
+
// The kernel lost this client's registration. Opening the same stream with the same connection id
|
|
434
|
+
// can never work, so reconnect the whole client, which re-registers every reducer.
|
|
435
|
+
this._logger.warn('Reducer observation stream lost its session; reconnecting the client', { reducerId: id, error });
|
|
436
|
+
this._lifecycle.requestReconnect(connectionId, 'reducer-session-lost', error,
|
|
437
|
+
handlerError => this._logger.error('Reconnect request handler failed', { reducerId: id, error: handlerError }));
|
|
438
|
+
return 'sessionLost';
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
this._logger.error('Reducer observation stream ended unexpectedly', { reducerId: id, error });
|
|
442
|
+
return 'ended';
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
private createRegistrationMessage({ id, reducerType, eventSequenceId, readModelName, dispatcher }: ReducerObservation,
|
|
446
|
+
connectionId: string): ReducerMessage {
|
|
369
447
|
return {
|
|
370
448
|
Content: {
|
|
371
449
|
Value0: {
|
|
372
|
-
ConnectionId:
|
|
450
|
+
ConnectionId: connectionId,
|
|
373
451
|
EventStore: this._eventStoreName,
|
|
374
452
|
Namespace: this._namespace,
|
|
375
453
|
Reducer: {
|
|
@@ -596,6 +674,8 @@ export class Reducers implements IReducers {
|
|
|
596
674
|
}
|
|
597
675
|
|
|
598
676
|
private disconnectAll(): void {
|
|
677
|
+
for (const timer of this._reobserveTimers.values()) clearTimeout(timer);
|
|
678
|
+
this._reobserveTimers.clear();
|
|
599
679
|
for (const controller of this._observations.values()) controller.abort();
|
|
600
680
|
for (const queue of this._queues.values()) queue.complete();
|
|
601
681
|
this._observations.clear();
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
// Copyright (c) Cratis. All rights reserved.
|
|
2
|
+
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
|
|
3
|
+
|
|
4
|
+
import { afterEach, beforeEach, chai, describe, it, vi } from 'vitest';
|
|
5
|
+
import { ClientError, Status } from 'nice-grpc';
|
|
6
|
+
import { a_reducer_whose_stream_ends } from './given/a_reducer_whose_stream_ends.js';
|
|
7
|
+
import { page } from '../given/observe_reducer_messages.js';
|
|
8
|
+
import { Reducers } from '../../Reducers.js';
|
|
9
|
+
|
|
10
|
+
chai.should();
|
|
11
|
+
|
|
12
|
+
describe('when reconnecting a reducer observation stream and reducing failed with a lost session', () => {
|
|
13
|
+
let context: a_reducer_whose_stream_ends;
|
|
14
|
+
|
|
15
|
+
beforeEach(async () => {
|
|
16
|
+
context = new a_reducer_whose_stream_ends();
|
|
17
|
+
vi.spyOn(Reducers.prototype as unknown as { beginReduction: () => unknown }, 'beginReduction').mockImplementationOnce(() => {
|
|
18
|
+
throw new ClientError('/ReadModels/Get', Status.UNKNOWN, 'ClientIsNotConnected: connection is not connected');
|
|
19
|
+
});
|
|
20
|
+
context.scripts.push({ deliver: [page(1n, {})], untilAborted: true }, { untilAborted: true });
|
|
21
|
+
await context.reducers.register();
|
|
22
|
+
await vi.advanceTimersByTimeAsync(120_000);
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
afterEach(() => {
|
|
26
|
+
context.dispose();
|
|
27
|
+
vi.restoreAllMocks();
|
|
28
|
+
});
|
|
29
|
+
|
|
30
|
+
it('should not ask the whole client to reconnect', () => context.reconnectRequests.should.be.empty);
|
|
31
|
+
it('should open the stream again after the base delay', () => context.waits.should.deep.equal([1_000]));
|
|
32
|
+
it('should close the stream the reducer failed on', () => context.stream.open.should.equal(1));
|
|
33
|
+
});
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
// Copyright (c) Cratis. All rights reserved.
|
|
2
|
+
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
|
|
3
|
+
|
|
4
|
+
import { afterEach, beforeEach, chai, describe, it, vi } from 'vitest';
|
|
5
|
+
import { ClientError, Status } from 'nice-grpc';
|
|
6
|
+
import { a_reducer_whose_stream_ends } from './given/a_reducer_whose_stream_ends.js';
|
|
7
|
+
|
|
8
|
+
chai.should();
|
|
9
|
+
|
|
10
|
+
describe('when reconnecting a reducer observation stream and a replaced stream fails late', () => {
|
|
11
|
+
let context: a_reducer_whose_stream_ends;
|
|
12
|
+
|
|
13
|
+
beforeEach(async () => {
|
|
14
|
+
context = new a_reducer_whose_stream_ends();
|
|
15
|
+
context.scripts.push(
|
|
16
|
+
{ untilAborted: true, endsLateByMs: 10, error: new ClientError('/Reducers/Observe', Status.INTERNAL, 'boom') },
|
|
17
|
+
{ untilAborted: true });
|
|
18
|
+
await context.reducers.register();
|
|
19
|
+
await vi.advanceTimersByTimeAsync(0);
|
|
20
|
+
await context.reconnect();
|
|
21
|
+
await vi.advanceTimersByTimeAsync(120_000);
|
|
22
|
+
});
|
|
23
|
+
|
|
24
|
+
afterEach(() => context.dispose());
|
|
25
|
+
|
|
26
|
+
it('should not open the stream again', () => context.openedAt.should.have.lengthOf(2));
|
|
27
|
+
it('should leave exactly one stream open', () => context.stream.open.should.equal(1));
|
|
28
|
+
});
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
// Copyright (c) Cratis. All rights reserved.
|
|
2
|
+
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
|
|
3
|
+
|
|
4
|
+
import { afterEach, beforeEach, chai, describe, it, vi } from 'vitest';
|
|
5
|
+
import { a_reducer_whose_stream_ends } from './given/a_reducer_whose_stream_ends.js';
|
|
6
|
+
import { page } from '../given/observe_reducer_messages.js';
|
|
7
|
+
|
|
8
|
+
chai.should();
|
|
9
|
+
|
|
10
|
+
describe('when reconnecting a reducer observation stream and a stream delivered before it ended', () => {
|
|
11
|
+
let context: a_reducer_whose_stream_ends;
|
|
12
|
+
|
|
13
|
+
beforeEach(async () => {
|
|
14
|
+
context = new a_reducer_whose_stream_ends();
|
|
15
|
+
context.scripts.push({}, {}, { deliver: [page(1n, {})] }, {});
|
|
16
|
+
await context.reducers.register();
|
|
17
|
+
await vi.advanceTimersByTimeAsync(6_500);
|
|
18
|
+
});
|
|
19
|
+
|
|
20
|
+
afterEach(() => context.dispose());
|
|
21
|
+
|
|
22
|
+
it('should start the wait over at the base delay after the stream that delivered', () =>
|
|
23
|
+
context.waits.slice(0, 4).should.deep.equal([1_000, 2_000, 1_000, 2_000]));
|
|
24
|
+
});
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
// Copyright (c) Cratis. All rights reserved.
|
|
2
|
+
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
|
|
3
|
+
|
|
4
|
+
import { afterEach, beforeEach, chai, describe, it, vi } from 'vitest';
|
|
5
|
+
import { a_reducer_whose_stream_ends } from './given/a_reducer_whose_stream_ends.js';
|
|
6
|
+
|
|
7
|
+
chai.should();
|
|
8
|
+
|
|
9
|
+
describe('when reconnecting a reducer observation stream and a stream stayed open before it ended', () => {
|
|
10
|
+
let context: a_reducer_whose_stream_ends;
|
|
11
|
+
|
|
12
|
+
beforeEach(async () => {
|
|
13
|
+
context = new a_reducer_whose_stream_ends();
|
|
14
|
+
context.scripts.push({}, {}, { openForMs: 180_000 }, { untilAborted: true });
|
|
15
|
+
await context.reducers.register();
|
|
16
|
+
await vi.advanceTimersByTimeAsync(185_000);
|
|
17
|
+
});
|
|
18
|
+
|
|
19
|
+
afterEach(() => context.dispose());
|
|
20
|
+
|
|
21
|
+
it('should start the wait over at the base delay after the stream that stayed open', () =>
|
|
22
|
+
context.waits.should.deep.equal([1_000, 2_000, 181_000]));
|
|
23
|
+
});
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
// Copyright (c) Cratis. All rights reserved.
|
|
2
|
+
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
|
|
3
|
+
|
|
4
|
+
import { afterEach, beforeEach, chai, describe, it, vi } from 'vitest';
|
|
5
|
+
import { a_reducer_whose_stream_ends } from './given/a_reducer_whose_stream_ends.js';
|
|
6
|
+
|
|
7
|
+
chai.should();
|
|
8
|
+
|
|
9
|
+
describe('when reconnecting a reducer observation stream and it is disposed while it waits to reopen', () => {
|
|
10
|
+
let context: a_reducer_whose_stream_ends;
|
|
11
|
+
let timersAfterDispose: number;
|
|
12
|
+
|
|
13
|
+
beforeEach(async () => {
|
|
14
|
+
context = new a_reducer_whose_stream_ends();
|
|
15
|
+
context.scripts.push({});
|
|
16
|
+
await context.reducers.register();
|
|
17
|
+
await vi.advanceTimersByTimeAsync(0);
|
|
18
|
+
context.reducers.dispose();
|
|
19
|
+
timersAfterDispose = vi.getTimerCount();
|
|
20
|
+
await vi.advanceTimersByTimeAsync(120_000);
|
|
21
|
+
});
|
|
22
|
+
|
|
23
|
+
afterEach(() => context.dispose());
|
|
24
|
+
|
|
25
|
+
it('should not open the stream again', () => context.openedAt.should.have.lengthOf(1));
|
|
26
|
+
it('should leave no timer behind', () => timersAfterDispose.should.equal(0));
|
|
27
|
+
});
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
// Copyright (c) Cratis. All rights reserved.
|
|
2
|
+
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
|
|
3
|
+
|
|
4
|
+
import { afterEach, beforeEach, chai, describe, it, vi } from 'vitest';
|
|
5
|
+
import { ClientError, Status } from 'nice-grpc';
|
|
6
|
+
import { a_reducer_whose_stream_ends } from './given/a_reducer_whose_stream_ends.js';
|
|
7
|
+
|
|
8
|
+
chai.should();
|
|
9
|
+
|
|
10
|
+
describe('when reconnecting a reducer observation stream and streams keep ending without working', () => {
|
|
11
|
+
let context: a_reducer_whose_stream_ends;
|
|
12
|
+
|
|
13
|
+
beforeEach(async () => {
|
|
14
|
+
context = new a_reducer_whose_stream_ends();
|
|
15
|
+
context.scripts.push({}, { error: new ClientError('/Reducers/Observe', Status.UNAVAILABLE, 'kernel stopping') });
|
|
16
|
+
await context.reducers.register();
|
|
17
|
+
await vi.advanceTimersByTimeAsync(60_000);
|
|
18
|
+
});
|
|
19
|
+
|
|
20
|
+
afterEach(() => context.dispose());
|
|
21
|
+
|
|
22
|
+
it('should double the wait each time a stream ends', () => context.waits.slice(0, 5).should.deep.equal([1_000, 2_000, 4_000, 8_000, 16_000]));
|
|
23
|
+
it('should not reconnect the client', () => context.reconnectRequests.should.be.empty);
|
|
24
|
+
});
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
// Copyright (c) Cratis. All rights reserved.
|
|
2
|
+
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
|
|
3
|
+
|
|
4
|
+
import { afterEach, beforeEach, chai, describe, it, vi } from 'vitest';
|
|
5
|
+
import { a_reducer_whose_stream_ends } from './given/a_reducer_whose_stream_ends.js';
|
|
6
|
+
|
|
7
|
+
chai.should();
|
|
8
|
+
|
|
9
|
+
describe('when reconnecting a reducer observation stream and the client reconnected after streams kept ending', () => {
|
|
10
|
+
let context: a_reducer_whose_stream_ends;
|
|
11
|
+
|
|
12
|
+
beforeEach(async () => {
|
|
13
|
+
context = new a_reducer_whose_stream_ends();
|
|
14
|
+
context.scripts.push({}, {}, {}, { untilAborted: true });
|
|
15
|
+
await context.reducers.register();
|
|
16
|
+
await vi.advanceTimersByTimeAsync(1_000);
|
|
17
|
+
await context.reconnect();
|
|
18
|
+
await vi.advanceTimersByTimeAsync(4_000);
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
afterEach(() => context.dispose());
|
|
22
|
+
|
|
23
|
+
it('should keep the wait it had grown to', () => context.waits.should.deep.equal([1_000, 0, 4_000]));
|
|
24
|
+
});
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
// Copyright (c) Cratis. All rights reserved.
|
|
2
|
+
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
|
|
3
|
+
|
|
4
|
+
import { afterEach, beforeEach, chai, describe, it, vi } from 'vitest';
|
|
5
|
+
import { a_reducer_whose_stream_ends } from './given/a_reducer_whose_stream_ends.js';
|
|
6
|
+
|
|
7
|
+
chai.should();
|
|
8
|
+
|
|
9
|
+
describe('when reconnecting a reducer observation stream and the client reconnected while it waited to reopen', () => {
|
|
10
|
+
let context: a_reducer_whose_stream_ends;
|
|
11
|
+
|
|
12
|
+
beforeEach(async () => {
|
|
13
|
+
context = new a_reducer_whose_stream_ends();
|
|
14
|
+
context.scripts.push({}, { untilAborted: true });
|
|
15
|
+
await context.reducers.register();
|
|
16
|
+
await vi.advanceTimersByTimeAsync(0);
|
|
17
|
+
await context.reconnect();
|
|
18
|
+
await vi.advanceTimersByTimeAsync(120_000);
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
afterEach(() => context.dispose());
|
|
22
|
+
|
|
23
|
+
it('should only open the stream the reconnect registered', () => context.openedAt.should.have.lengthOf(2));
|
|
24
|
+
it('should leave exactly one stream open', () => context.stream.open.should.equal(1));
|
|
25
|
+
});
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
// Copyright (c) Cratis. All rights reserved.
|
|
2
|
+
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
|
|
3
|
+
|
|
4
|
+
import { afterEach, beforeEach, chai, describe, it, vi } from 'vitest';
|
|
5
|
+
import { ClientError, Status } from 'nice-grpc';
|
|
6
|
+
import { a_reducer_whose_stream_ends } from './given/a_reducer_whose_stream_ends.js';
|
|
7
|
+
|
|
8
|
+
chai.should();
|
|
9
|
+
|
|
10
|
+
describe('when reconnecting a reducer observation stream and the kernel lost the session', () => {
|
|
11
|
+
let context: a_reducer_whose_stream_ends;
|
|
12
|
+
|
|
13
|
+
beforeEach(async () => {
|
|
14
|
+
context = new a_reducer_whose_stream_ends();
|
|
15
|
+
context.scripts.push({ error: new ClientError('/Reducers/Observe', Status.UNKNOWN, 'ClientIsNotConnected: connection is not connected') });
|
|
16
|
+
await context.reducers.register();
|
|
17
|
+
await vi.advanceTimersByTimeAsync(120_000);
|
|
18
|
+
});
|
|
19
|
+
|
|
20
|
+
afterEach(() => context.dispose());
|
|
21
|
+
|
|
22
|
+
it('should ask the whole client to reconnect', () => context.reconnectRequests.should.deep.equal(['reducer-session-lost']));
|
|
23
|
+
it('should not open the stream again with the same connection', () => context.openedAt.should.have.lengthOf(1));
|
|
24
|
+
});
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
// Copyright (c) Cratis. All rights reserved.
|
|
2
|
+
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
|
|
3
|
+
|
|
4
|
+
import { vi } from 'vitest';
|
|
5
|
+
import type { ReduceOperationMessage } from '@cratis/chronicle.contracts';
|
|
6
|
+
import { Reducers } from '../../../Reducers.js';
|
|
7
|
+
import { reducer } from '../../../reducer.js';
|
|
8
|
+
import { eventType } from '../../../../events/eventTypeDecorator.js';
|
|
9
|
+
import { ConnectionLifecycle } from '../../../../connection/ConnectionLifecycle.js';
|
|
10
|
+
import type { ChronicleConnection } from '../../../../connection/ChronicleConnection.js';
|
|
11
|
+
import type { IClientArtifactsProvider } from '../../../../artifacts/IClientArtifactsProvider.js';
|
|
12
|
+
import { a_scripted_stream, type StreamScript } from '../../../../observation/for_ObservationReconnectBackoff/given/a_scripted_stream.js';
|
|
13
|
+
|
|
14
|
+
@eventType('reconnecting-reducer-item')
|
|
15
|
+
class Item {}
|
|
16
|
+
class Model {}
|
|
17
|
+
|
|
18
|
+
@reducer('reconnecting-reducer', undefined, Model)
|
|
19
|
+
class Observer { item() { return new Model(); } }
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* A reducer whose observation streams follow a script, on fake timers and a jitter of zero, so every wait is
|
|
23
|
+
* exactly half of its ceiling.
|
|
24
|
+
*/
|
|
25
|
+
export class a_reducer_whose_stream_ends {
|
|
26
|
+
readonly lifecycle = new ConnectionLifecycle();
|
|
27
|
+
readonly reconnectRequests: string[] = [];
|
|
28
|
+
readonly stream = new a_scripted_stream<ReduceOperationMessage>();
|
|
29
|
+
readonly reducers: Reducers;
|
|
30
|
+
|
|
31
|
+
constructor() {
|
|
32
|
+
vi.useFakeTimers();
|
|
33
|
+
this.lifecycle.onReconnectRequested(reason => this.reconnectRequests.push(reason));
|
|
34
|
+
const artifacts = { eventTypes: [Item], reducers: [Observer], reactors: [], readModels: [Model], seeders: [], constraints: [],
|
|
35
|
+
projections: [], webhooks: [], eventTypeMigrations: [], globalForHandlers: [] } as IClientArtifactsProvider;
|
|
36
|
+
const connection = {
|
|
37
|
+
reducers: { observe: this.stream.observe },
|
|
38
|
+
readModels: { registerMany: vi.fn().mockResolvedValue({}) }
|
|
39
|
+
} as unknown as ChronicleConnection;
|
|
40
|
+
this.reducers = new Reducers(artifacts, connection, 'store', 'Default', this.lifecycle, 'sink',
|
|
41
|
+
undefined, undefined, undefined, false, { now: () => Date.now(), nextJitter: () => 0 });
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/** Gets the scripts the next streams follow. */
|
|
45
|
+
get scripts(): StreamScript<ReduceOperationMessage>[] {
|
|
46
|
+
return this.stream.scripts;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** Gets when each stream was opened. */
|
|
50
|
+
get openedAt(): number[] {
|
|
51
|
+
return this.stream.openedAt;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/** Gets the wait before each stream after the first was opened. */
|
|
55
|
+
get waits(): number[] {
|
|
56
|
+
return this.stream.waits;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/** Drops the session and registers again, as a client reconnect does. */
|
|
60
|
+
async reconnect(): Promise<void> {
|
|
61
|
+
await this.lifecycle.disconnected(() => undefined);
|
|
62
|
+
await this.reducers.register();
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
dispose(): void {
|
|
66
|
+
this.reducers.dispose();
|
|
67
|
+
vi.useRealTimers();
|
|
68
|
+
}
|
|
69
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
// Copyright (c) Cratis. All rights reserved.
|
|
2
|
+
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
|
|
3
|
+
|
|
4
|
+
import type { BatchOutcome } from '../observation/batches/BatchOutcome.js';
|
|
5
|
+
|
|
6
|
+
/** A partition a batch method of a scenario reactor reported as failed or deferred. */
|
|
7
|
+
export interface BatchPartitionOutcome extends BatchOutcome {
|
|
8
|
+
/** The name of the batch method that reported it. */
|
|
9
|
+
readonly methodName: string;
|
|
10
|
+
}
|
|
@@ -26,6 +26,8 @@ import { EventSequenceNumber } from '../eventSequences/EventSequenceNumber.js';
|
|
|
26
26
|
import type { IEventSequence } from '../eventSequences/IEventSequence.js';
|
|
27
27
|
import type { ITransactionalEventSequence } from '../eventSequences/ITransactionalEventSequence.js';
|
|
28
28
|
import type { CompleteStreamResult } from '../eventSequences/CompleteStreamResult.js';
|
|
29
|
+
import type { IEventImport } from '../imports/IEventImport.js';
|
|
30
|
+
import type { ImportResult } from '../imports/ImportResult.js';
|
|
29
31
|
import { prepareSingleAppend } from '../eventSequences/prepareSingleAppend.js';
|
|
30
32
|
import { toContractsGuid } from '../connection/Guid.js';
|
|
31
33
|
import { getRemovedConstraintNames } from '../events/constraints/removeConstraint.js';
|
|
@@ -398,6 +400,8 @@ export class InProcessEventSequence implements IEventSequence {
|
|
|
398
400
|
async redactForEventSource(): Promise<void> { throw this.unsupported('redactForEventSource', this.id.value, 'Redaction requires kernel storage.'); }
|
|
399
401
|
async copy(): Promise<Guid> { throw this.unsupported('copy', this.id.value, 'Copying events requires kernel jobs.'); }
|
|
400
402
|
async completeStream(): Promise<CompleteStreamResult> { throw this.unsupported('completeStream', this.id.value, 'Stream completion requires a kernel.'); }
|
|
403
|
+
async import(): Promise<ImportResult> { throw this.unsupported('import', this.id.value, 'Importing events requires kernel import sessions.'); }
|
|
404
|
+
async beginImport(): Promise<IEventImport> { throw this.unsupported('beginImport', this.id.value, 'Importing events requires kernel import sessions.'); }
|
|
401
405
|
|
|
402
406
|
private filterIds(types?: Constructor[]): string[] {
|
|
403
407
|
return (types ?? []).map(type => {
|
|
@@ -6,6 +6,8 @@ import type { EventContext } from '../events/EventContext.js';
|
|
|
6
6
|
/** Observable outcome of one ordered source-partition delivery (not a kernel checkpoint). */
|
|
7
7
|
export interface ReactorDeliveryResult {
|
|
8
8
|
readonly sourceId: string;
|
|
9
|
+
/** The partitions (event source identifiers) the delivery carried, in the order their first event appears. */
|
|
10
|
+
readonly partitions: readonly string[];
|
|
9
11
|
readonly handled: readonly EventContext[];
|
|
10
12
|
readonly skipped: readonly EventContext[];
|
|
11
13
|
readonly completed: boolean;
|