@powersync/service-core 1.20.4 → 1.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +53 -0
- package/dist/api/RouteAPI.d.ts +17 -3
- package/dist/api/api-index.d.ts +1 -1
- package/dist/api/api-index.js +1 -1
- package/dist/api/api-index.js.map +1 -1
- package/dist/api/api-metrics.js.map +1 -1
- package/dist/api/diagnostics.d.ts +1 -1
- package/dist/api/diagnostics.js +32 -14
- package/dist/api/diagnostics.js.map +1 -1
- package/dist/auth/CachedKeyCollector.js +1 -1
- package/dist/auth/CachedKeyCollector.js.map +1 -1
- package/dist/auth/CompoundKeyCollector.js.map +1 -1
- package/dist/auth/KeyStore.js.map +1 -1
- package/dist/auth/RemoteJWKSCollector.js.map +1 -1
- package/dist/auth/StaticKeyCollector.d.ts +1 -1
- package/dist/auth/StaticKeyCollector.js.map +1 -1
- package/dist/auth/StaticSupabaseKeyCollector.d.ts +1 -1
- package/dist/auth/StaticSupabaseKeyCollector.js.map +1 -1
- package/dist/entry/commands/teardown-action.js +2 -2
- package/dist/entry/commands/teardown-action.js.map +1 -1
- package/dist/entry/entry-index.d.ts +1 -1
- package/dist/entry/entry-index.js +1 -1
- package/dist/entry/entry-index.js.map +1 -1
- package/dist/events/EventsEngine.js +1 -1
- package/dist/events/EventsEngine.js.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/metrics/MetricsEngine.d.ts +1 -1
- package/dist/metrics/metrics-index.d.ts +3 -3
- package/dist/metrics/metrics-index.js +3 -3
- package/dist/metrics/metrics-index.js.map +1 -1
- package/dist/metrics/open-telemetry/util.js +1 -1
- package/dist/metrics/open-telemetry/util.js.map +1 -1
- package/dist/metrics/register-metrics.js +2 -2
- package/dist/metrics/register-metrics.js.map +1 -1
- package/dist/modules/AbstractModule.d.ts +2 -2
- package/dist/modules/AbstractModule.js.map +1 -1
- package/dist/modules/modules-index.d.ts +1 -1
- package/dist/modules/modules-index.js +1 -1
- package/dist/modules/modules-index.js.map +1 -1
- package/dist/replication/AbstractReplicationJob.d.ts +1 -1
- package/dist/replication/AbstractReplicationJob.js +1 -1
- package/dist/replication/AbstractReplicationJob.js.map +1 -1
- package/dist/replication/AbstractReplicator.d.ts +6 -6
- package/dist/replication/AbstractReplicator.js +21 -21
- package/dist/replication/AbstractReplicator.js.map +1 -1
- package/dist/replication/replication-index.d.ts +3 -3
- package/dist/replication/replication-index.js +3 -3
- package/dist/replication/replication-index.js.map +1 -1
- package/dist/replication/replication-metrics.js.map +1 -1
- package/dist/routes/configure-fastify.d.ts +59 -32
- package/dist/routes/endpoints/admin.d.ts +108 -54
- package/dist/routes/endpoints/admin.js +7 -3
- package/dist/routes/endpoints/admin.js.map +1 -1
- package/dist/routes/endpoints/checkpointing.js +1 -1
- package/dist/routes/endpoints/checkpointing.js.map +1 -1
- package/dist/routes/endpoints/socket-route.js +1 -1
- package/dist/routes/endpoints/socket-route.js.map +1 -1
- package/dist/routes/endpoints/sync-rules.js +10 -10
- package/dist/routes/endpoints/sync-rules.js.map +1 -1
- package/dist/routes/endpoints/sync-stream.d.ts +10 -10
- package/dist/routes/endpoints/sync-stream.js +2 -2
- package/dist/routes/endpoints/sync-stream.js.map +1 -1
- package/dist/routes/hooks.js +1 -1
- package/dist/routes/hooks.js.map +1 -1
- package/dist/routes/route-register.js.map +1 -1
- package/dist/runner/teardown.js +4 -4
- package/dist/runner/teardown.js.map +1 -1
- package/dist/storage/BucketStorage.d.ts +9 -9
- package/dist/storage/BucketStorage.js +9 -9
- package/dist/storage/BucketStorageBatch.d.ts +1 -1
- package/dist/storage/BucketStorageFactory.d.ts +27 -20
- package/dist/storage/BucketStorageFactory.js +19 -16
- package/dist/storage/BucketStorageFactory.js.map +1 -1
- package/dist/storage/ChecksumCache.js.map +1 -1
- package/dist/storage/PersistedSyncRulesContent.d.ts +3 -1
- package/dist/storage/PersistedSyncRulesContent.js +24 -5
- package/dist/storage/PersistedSyncRulesContent.js.map +1 -1
- package/dist/storage/ReplicationEventPayload.d.ts +1 -1
- package/dist/storage/SourceTable.d.ts +4 -4
- package/dist/storage/SourceTable.js +3 -3
- package/dist/storage/SourceTable.js.map +1 -1
- package/dist/storage/StorageVersionConfig.d.ts +1 -1
- package/dist/storage/StorageVersionConfig.js +1 -1
- package/dist/storage/SyncRulesBucketStorage.d.ts +38 -6
- package/dist/storage/SyncRulesBucketStorage.js +14 -0
- package/dist/storage/SyncRulesBucketStorage.js.map +1 -1
- package/dist/storage/WriteCheckpointAPI.d.ts +6 -6
- package/dist/storage/WriteCheckpointAPI.js +1 -1
- package/dist/storage/bson.d.ts +0 -1
- package/dist/storage/bson.js +0 -4
- package/dist/storage/bson.js.map +1 -1
- package/dist/storage/storage-index.d.ts +8 -8
- package/dist/storage/storage-index.js +8 -8
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/storage/storage-metrics.js.map +1 -1
- package/dist/streams/streams-index.d.ts +2 -2
- package/dist/streams/streams-index.js +2 -2
- package/dist/streams/streams-index.js.map +1 -1
- package/dist/sync/BucketChecksumState.d.ts +2 -5
- package/dist/sync/BucketChecksumState.js +119 -75
- package/dist/sync/BucketChecksumState.js.map +1 -1
- package/dist/sync/RequestTracker.js +1 -1
- package/dist/sync/RequestTracker.js.map +1 -1
- package/dist/sync/sync-index.d.ts +2 -2
- package/dist/sync/sync-index.js +2 -2
- package/dist/sync/sync-index.js.map +1 -1
- package/dist/sync/sync.js.map +1 -1
- package/dist/sync/util.js.map +1 -1
- package/dist/system/ServiceContext.d.ts +1 -1
- package/dist/system/ServiceContext.js +1 -1
- package/dist/system/ServiceContext.js.map +1 -1
- package/dist/tracing/PerformanceTracer.d.ts +44 -0
- package/dist/tracing/PerformanceTracer.js +102 -0
- package/dist/tracing/PerformanceTracer.js.map +1 -0
- package/dist/tracing/TraceWriter.d.ts +22 -0
- package/dist/tracing/TraceWriter.js +63 -0
- package/dist/tracing/TraceWriter.js.map +1 -0
- package/dist/util/checkpointing.js +1 -1
- package/dist/util/config/collectors/impl/base64-config-collector.d.ts +1 -1
- package/dist/util/config/collectors/impl/base64-config-collector.js.map +1 -1
- package/dist/util/config/collectors/impl/filesystem-config-collector.d.ts +1 -1
- package/dist/util/config/collectors/impl/filesystem-config-collector.js +1 -1
- package/dist/util/config/collectors/impl/filesystem-config-collector.js.map +1 -1
- package/dist/util/config/compound-config-collector.d.ts +1 -1
- package/dist/util/config/compound-config-collector.js +2 -2
- package/dist/util/config/compound-config-collector.js.map +1 -1
- package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js +1 -1
- package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js.map +1 -1
- package/dist/util/config/sync-rules/sync-rules-provider.js.map +1 -1
- package/dist/util/config.js +1 -1
- package/dist/util/config.js.map +1 -1
- package/dist/util/env.js +1 -1
- package/dist/util/errors.d.ts +3 -0
- package/dist/util/errors.js +15 -0
- package/dist/util/errors.js.map +1 -0
- package/dist/util/protocol-types.d.ts +3 -3
- package/dist/util/protocol-types.js +1 -1
- package/dist/util/util-index.d.ts +1 -1
- package/dist/util/util-index.js +1 -1
- package/dist/util/util-index.js.map +1 -1
- package/dist/util/utils.d.ts +1 -1
- package/package.json +11 -11
- package/src/api/RouteAPI.ts +20 -3
- package/src/api/api-index.ts +1 -1
- package/src/api/api-metrics.ts +1 -1
- package/src/api/diagnostics.ts +42 -20
- package/src/auth/CachedKeyCollector.ts +2 -3
- package/src/auth/CompoundKeyCollector.ts +2 -3
- package/src/auth/KeyStore.ts +1 -1
- package/src/auth/RemoteJWKSCollector.ts +0 -1
- package/src/auth/StaticKeyCollector.ts +1 -1
- package/src/auth/StaticSupabaseKeyCollector.ts +1 -1
- package/src/entry/commands/teardown-action.ts +2 -2
- package/src/entry/entry-index.ts +1 -1
- package/src/events/EventsEngine.ts +1 -1
- package/src/index.ts +2 -0
- package/src/metrics/MetricsEngine.ts +1 -1
- package/src/metrics/metrics-index.ts +3 -3
- package/src/metrics/open-telemetry/util.ts +1 -1
- package/src/metrics/register-metrics.ts +3 -3
- package/src/modules/AbstractModule.ts +2 -2
- package/src/modules/modules-index.ts +1 -1
- package/src/replication/AbstractReplicationJob.ts +2 -2
- package/src/replication/AbstractReplicator.ts +23 -23
- package/src/replication/replication-index.ts +3 -3
- package/src/replication/replication-metrics.ts +1 -1
- package/src/routes/endpoints/admin.ts +7 -3
- package/src/routes/endpoints/checkpointing.ts +1 -1
- package/src/routes/endpoints/socket-route.ts +1 -1
- package/src/routes/endpoints/sync-rules.ts +10 -12
- package/src/routes/endpoints/sync-stream.ts +2 -2
- package/src/routes/hooks.ts +2 -2
- package/src/routes/route-register.ts +2 -10
- package/src/runner/teardown.ts +4 -4
- package/src/storage/BucketStorage.ts +9 -9
- package/src/storage/BucketStorageBatch.ts +1 -1
- package/src/storage/BucketStorageFactory.ts +45 -34
- package/src/storage/ChecksumCache.ts +1 -1
- package/src/storage/PersistedSyncRulesContent.ts +30 -6
- package/src/storage/ReplicationEventPayload.ts +1 -1
- package/src/storage/SourceTable.ts +4 -4
- package/src/storage/StorageVersionConfig.ts +1 -1
- package/src/storage/SyncRulesBucketStorage.ts +46 -7
- package/src/storage/WriteCheckpointAPI.ts +6 -6
- package/src/storage/bson.ts +0 -5
- package/src/storage/storage-index.ts +8 -8
- package/src/storage/storage-metrics.ts +2 -2
- package/src/streams/streams-index.ts +2 -2
- package/src/sync/BucketChecksumState.ts +141 -93
- package/src/sync/RequestTracker.ts +1 -1
- package/src/sync/sync-index.ts +2 -2
- package/src/sync/sync.ts +2 -8
- package/src/sync/util.ts +1 -1
- package/src/system/ServiceContext.ts +1 -1
- package/src/tracing/PerformanceTracer.ts +126 -0
- package/src/tracing/TraceWriter.ts +67 -0
- package/src/util/checkpointing.ts +1 -1
- package/src/util/config/collectors/impl/base64-config-collector.ts +1 -1
- package/src/util/config/collectors/impl/filesystem-config-collector.ts +2 -2
- package/src/util/config/compound-config-collector.ts +3 -3
- package/src/util/config/sync-rules/impl/filesystem-sync-rules-collector.ts +1 -1
- package/src/util/config/sync-rules/sync-rules-provider.ts +1 -1
- package/src/util/config.ts +1 -1
- package/src/util/env.ts +1 -1
- package/src/util/errors.ts +21 -0
- package/src/util/protocol-types.ts +1 -1
- package/src/util/util-index.ts +1 -1
- package/src/util/utils.ts +1 -1
- package/test/src/auth.test.ts +115 -7
- package/test/src/diagnostics.test.ts +151 -0
- package/test/src/module-loader.test.ts +1 -1
- package/test/src/routes/mocks.ts +1 -1
- package/test/src/routes/stream.test.ts +1 -2
- package/test/src/sync/BucketChecksumState.test.ts +223 -67
- package/test/src/util/protocol_types.test.ts +1 -1
- package/test/tsconfig.json +0 -1
- package/tsconfig.tsbuildinfo +1 -1
- package/vitest.config.ts +1 -1
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import winston from 'winston';
|
|
2
|
+
import { MetricsEngine } from '../metrics/MetricsEngine.js';
|
|
2
3
|
import * as storage from '../storage/storage-index.js';
|
|
3
4
|
import { ErrorRateLimiter } from './ErrorRateLimiter.js';
|
|
4
|
-
import { MetricsEngine } from '../metrics/MetricsEngine.js';
|
|
5
5
|
export interface AbstractReplicationJobOptions {
|
|
6
6
|
id: string;
|
|
7
7
|
storage: storage.SyncRulesBucketStorage;
|
|
@@ -30,7 +30,7 @@ export class AbstractReplicationJob {
|
|
|
30
30
|
* Safely stop the replication process
|
|
31
31
|
*/
|
|
32
32
|
async stop() {
|
|
33
|
-
this.logger.info(`Stopping replication job
|
|
33
|
+
this.logger.info(`Stopping replication job`);
|
|
34
34
|
this.abortController.abort();
|
|
35
35
|
await this.isReplicatingPromise;
|
|
36
36
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"AbstractReplicationJob.js","sourceRoot":"","sources":["../../src/replication/AbstractReplicationJob.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AActE,MAAM,OAAgB,sBAAsB;IAKV;IAJtB,MAAM,CAAiB;IACvB,eAAe,GAAG,IAAI,eAAe,EAAE,CAAC;IACxC,oBAAoB,GAAyB,IAAI,CAAC;IAE5D,YAAgC,OAAsC;QAAtC,YAAO,GAAP,OAAO,CAA+B;QACpE,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,mBAAmB,IAAI,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;IACrE,CAAC;IAYD;;OAEG;IACI,KAAK,CAAC,KAAK;QAChB,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,SAAS,EAAE;aACzC,KAAK,CAAC,CAAC,EAAE,EAAE,EAAE;YACZ,SAAS,CAAC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,EAAE;gBACtC,QAAQ,EAAE;oBACR,UAAU,EAAE,IAAI,CAAC,EAAE;iBACpB;aACF,CAAC,CAAC;YACH,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,qBAAqB,EAAE,EAAE,CAAC,CAAC;QAC/C,CAAC,CAAC;aACD,OAAO,CAAC,KAAK,IAAI,EAAE;YAClB,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;YAC7B,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;QACpC,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,IAAI;QACf,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,
|
|
1
|
+
{"version":3,"file":"AbstractReplicationJob.js","sourceRoot":"","sources":["../../src/replication/AbstractReplicationJob.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AActE,MAAM,OAAgB,sBAAsB;IAKV;IAJtB,MAAM,CAAiB;IACvB,eAAe,GAAG,IAAI,eAAe,EAAE,CAAC;IACxC,oBAAoB,GAAyB,IAAI,CAAC;IAE5D,YAAgC,OAAsC;QAAtC,YAAO,GAAP,OAAO,CAA+B;QACpE,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,mBAAmB,IAAI,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;IACrE,CAAC;IAYD;;OAEG;IACI,KAAK,CAAC,KAAK;QAChB,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,SAAS,EAAE;aACzC,KAAK,CAAC,CAAC,EAAE,EAAE,EAAE;YACZ,SAAS,CAAC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,EAAE;gBACtC,QAAQ,EAAE;oBACR,UAAU,EAAE,IAAI,CAAC,EAAE;iBACpB;aACF,CAAC,CAAC;YACH,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,qBAAqB,EAAE,EAAE,CAAC,CAAC;QAC/C,CAAC,CAAC;aACD,OAAO,CAAC,KAAK,IAAI,EAAE;YAClB,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;YAC7B,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;QACpC,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,IAAI;QACf,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QAC7C,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;QAC7B,MAAM,IAAI,CAAC,oBAAoB,CAAC;IAClC,CAAC;IAED,IAAW,EAAE;QACX,OAAO,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;IACzB,CAAC;IAED,IAAW,OAAO;QAChB,OAAO,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC;IAC9B,CAAC;IAED,IAAc,IAAI;QAChB,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;IAC3B,CAAC;IAED,IAAc,WAAW;QACvB,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC;IAClC,CAAC;IAED,IAAW,SAAS;QAClB,OAAO,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,OAAO,CAAC;IAC7C,CAAC;CAMF"}
|
|
@@ -23,19 +23,19 @@ export interface AbstractReplicatorOptions {
|
|
|
23
23
|
/**
|
|
24
24
|
* A replicator manages the mechanics for replicating data from a data source to a storage bucket.
|
|
25
25
|
* This includes copying across the original data set and then keeping it in sync with the data source using Replication Jobs.
|
|
26
|
-
* It also handles any changes to the sync
|
|
26
|
+
* It also handles any changes to the sync config.
|
|
27
27
|
*/
|
|
28
28
|
export declare abstract class AbstractReplicator<T extends AbstractReplicationJob = AbstractReplicationJob> {
|
|
29
29
|
private options;
|
|
30
30
|
protected logger: winston.Logger;
|
|
31
31
|
private lockAlerted;
|
|
32
32
|
/**
|
|
33
|
-
* Map of replication jobs by
|
|
34
|
-
* transitioning to a new
|
|
33
|
+
* Map of replication jobs by replication stream id. Usually there is only one running job, but there could be two when
|
|
34
|
+
* transitioning to a new replication stream.
|
|
35
35
|
*/
|
|
36
36
|
private replicationJobs;
|
|
37
37
|
/**
|
|
38
|
-
* Map of
|
|
38
|
+
* Map of replciation stream ids to promises that are clearing the replication stream.
|
|
39
39
|
*
|
|
40
40
|
* We primarily do this to keep track of what we're currently clearing, but don't currently
|
|
41
41
|
* use the Promise value.
|
|
@@ -50,8 +50,8 @@ export declare abstract class AbstractReplicator<T extends AbstractReplicationJo
|
|
|
50
50
|
protected constructor(options: AbstractReplicatorOptions);
|
|
51
51
|
abstract createJob(options: CreateJobOptions): T;
|
|
52
52
|
/**
|
|
53
|
-
*
|
|
54
|
-
*
|
|
53
|
+
* Clean up any configuration or state for the specified replication stream on the datasource.
|
|
54
|
+
* Should be a no-op if the replication stream has already been cleared
|
|
55
55
|
*/
|
|
56
56
|
abstract cleanUp(syncRuleStorage: storage.SyncRulesBucketStorage): Promise<void>;
|
|
57
57
|
get id(): string;
|
|
@@ -7,19 +7,19 @@ const PING_INTERVAL = 1000000000n * 60n;
|
|
|
7
7
|
/**
|
|
8
8
|
* A replicator manages the mechanics for replicating data from a data source to a storage bucket.
|
|
9
9
|
* This includes copying across the original data set and then keeping it in sync with the data source using Replication Jobs.
|
|
10
|
-
* It also handles any changes to the sync
|
|
10
|
+
* It also handles any changes to the sync config.
|
|
11
11
|
*/
|
|
12
12
|
export class AbstractReplicator {
|
|
13
13
|
options;
|
|
14
14
|
logger;
|
|
15
15
|
lockAlerted = false;
|
|
16
16
|
/**
|
|
17
|
-
* Map of replication jobs by
|
|
18
|
-
* transitioning to a new
|
|
17
|
+
* Map of replication jobs by replication stream id. Usually there is only one running job, but there could be two when
|
|
18
|
+
* transitioning to a new replication stream.
|
|
19
19
|
*/
|
|
20
20
|
replicationJobs = new Map();
|
|
21
21
|
/**
|
|
22
|
-
* Map of
|
|
22
|
+
* Map of replciation stream ids to promises that are clearing the replication stream.
|
|
23
23
|
*
|
|
24
24
|
* We primarily do this to keep track of what we're currently clearing, but don't currently
|
|
25
25
|
* use the Promise value.
|
|
@@ -57,7 +57,7 @@ export class AbstractReplicator {
|
|
|
57
57
|
async start() {
|
|
58
58
|
this.abortController = new AbortController();
|
|
59
59
|
this.runLoop().catch((e) => {
|
|
60
|
-
this.logger.error('
|
|
60
|
+
this.logger.error('Fatal replication error', e);
|
|
61
61
|
container.reporter.captureException(e);
|
|
62
62
|
setTimeout(() => {
|
|
63
63
|
process.exit(1);
|
|
@@ -90,9 +90,9 @@ export class AbstractReplicator {
|
|
|
90
90
|
const syncRules = await this.syncRuleProvider.get();
|
|
91
91
|
let configuredLock = undefined;
|
|
92
92
|
if (syncRules != null) {
|
|
93
|
-
this.logger.info('Loaded sync
|
|
93
|
+
this.logger.info('Loaded sync config');
|
|
94
94
|
try {
|
|
95
|
-
// Configure new sync
|
|
95
|
+
// Configure new sync config, if they have changed.
|
|
96
96
|
// In that case, also immediately take out a lock, so that another process doesn't start replication on it.
|
|
97
97
|
const { lock } = await this.storage.configureSyncRules(storage.updateSyncRulesFromYaml(syncRules, { lock: true, validate: this.syncRuleProvider.exitOnError }));
|
|
98
98
|
if (lock) {
|
|
@@ -102,12 +102,12 @@ export class AbstractReplicator {
|
|
|
102
102
|
catch (e) {
|
|
103
103
|
// Log and re-raise to exit.
|
|
104
104
|
// Should only reach this due to validation errors if exit_on_error is true.
|
|
105
|
-
this.logger.error(`Failed to update sync
|
|
105
|
+
this.logger.error(`Failed to update sync config`, e);
|
|
106
106
|
throw e;
|
|
107
107
|
}
|
|
108
108
|
}
|
|
109
109
|
else {
|
|
110
|
-
this.logger.info('No sync rules configured - configure via API');
|
|
110
|
+
this.logger.info('No sync streams or rules configured - configure via API');
|
|
111
111
|
}
|
|
112
112
|
while (!this.stopped) {
|
|
113
113
|
await container.probes.touch();
|
|
@@ -158,7 +158,7 @@ export class AbstractReplicator {
|
|
|
158
158
|
existingJobs.delete(syncRules.id);
|
|
159
159
|
}
|
|
160
160
|
else {
|
|
161
|
-
// New sync
|
|
161
|
+
// New sync config was found (or resume after restart)
|
|
162
162
|
try {
|
|
163
163
|
let lock;
|
|
164
164
|
if (configuredLock?.sync_rules_id == syncRules.id) {
|
|
@@ -182,23 +182,23 @@ export class AbstractReplicator {
|
|
|
182
182
|
catch (e) {
|
|
183
183
|
if (e?.errorData?.code === ErrorCode.PSYNC_S1003) {
|
|
184
184
|
if (!this.lockAlerted) {
|
|
185
|
-
|
|
185
|
+
syncRules.logger.info(`[${e.errorData.code}] ${e.errorData.description}`);
|
|
186
186
|
this.lockAlerted = true;
|
|
187
187
|
}
|
|
188
188
|
}
|
|
189
189
|
else {
|
|
190
|
-
// Could be a sync
|
|
190
|
+
// Could be a sync config parse error,
|
|
191
191
|
// for example from stricter validation that was added.
|
|
192
192
|
// This will be retried every couple of seconds.
|
|
193
|
-
// When new (valid) sync
|
|
194
|
-
|
|
193
|
+
// When new (valid) sync config is deployed and processed, this one be disabled.
|
|
194
|
+
syncRules.logger.error('Failed to start replication for new sync config', e);
|
|
195
195
|
}
|
|
196
196
|
}
|
|
197
197
|
}
|
|
198
198
|
}
|
|
199
199
|
this.replicationJobs = newJobs;
|
|
200
200
|
this.activeReplicationJob = activeJob;
|
|
201
|
-
// Stop any orphaned jobs that no longer have
|
|
201
|
+
// Stop any orphaned jobs that no longer have a replication stream.
|
|
202
202
|
// Termination happens below
|
|
203
203
|
for (let job of existingJobs.values()) {
|
|
204
204
|
// Old - stop and clean up
|
|
@@ -207,10 +207,10 @@ export class AbstractReplicator {
|
|
|
207
207
|
}
|
|
208
208
|
catch (e) {
|
|
209
209
|
// This will be retried
|
|
210
|
-
|
|
210
|
+
job.storage.logger.warn('Failed to stop old replication job', e);
|
|
211
211
|
}
|
|
212
212
|
}
|
|
213
|
-
//
|
|
213
|
+
// Replication stream stopped previously, including by a different process.
|
|
214
214
|
const stopped = await this.storage.getStoppedSyncRules();
|
|
215
215
|
for (let syncRules of stopped) {
|
|
216
216
|
if (this.clearingJobs.has(syncRules.id)) {
|
|
@@ -219,11 +219,11 @@ export class AbstractReplicator {
|
|
|
219
219
|
}
|
|
220
220
|
// We clear storage asynchronously.
|
|
221
221
|
// It is important to be able to continue running the refresh loop, otherwise we cannot
|
|
222
|
-
// retry locked
|
|
222
|
+
// retry locked replication stream, for example.
|
|
223
223
|
const syncRuleStorage = this.storage.getInstance(syncRules, { skipLifecycleHooks: true });
|
|
224
224
|
const promise = this.terminateSyncRules(syncRuleStorage)
|
|
225
225
|
.catch((e) => {
|
|
226
|
-
|
|
226
|
+
syncRuleStorage.logger.warn(`Failed clean up replication config`, e);
|
|
227
227
|
})
|
|
228
228
|
.finally(() => {
|
|
229
229
|
this.clearingJobs.delete(syncRules.id);
|
|
@@ -235,12 +235,12 @@ export class AbstractReplicator {
|
|
|
235
235
|
return `${this.id}-${syncRuleId}`;
|
|
236
236
|
}
|
|
237
237
|
async terminateSyncRules(syncRuleStorage) {
|
|
238
|
-
|
|
238
|
+
syncRuleStorage.logger.info(`Terminating replication stream...`);
|
|
239
239
|
// This deletes postgres replication slots - should complete quickly.
|
|
240
240
|
// It is safe to do before or after clearing the data in the storage.
|
|
241
241
|
await this.cleanUp(syncRuleStorage);
|
|
242
242
|
await syncRuleStorage.terminate({ signal: this.abortController?.signal, clearStorage: true });
|
|
243
|
-
|
|
243
|
+
syncRuleStorage.logger.info(`Successfully terminated replication stream`);
|
|
244
244
|
}
|
|
245
245
|
/**
|
|
246
246
|
* Measure replication lag in milliseconds.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"AbstractReplicator.js","sourceRoot":"","sources":["../../src/replication/AbstractReplicator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AACjF,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAC7D,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAGtC,OAAO,KAAK,OAAO,MAAM,6BAA6B,CAAC;AAOvD,WAAW;AACX,MAAM,aAAa,GAAG,WAAc,GAAG,GAAG,CAAC;AAkB3C;;;;GAIG;AACH,MAAM,OAAgB,kBAAkB;IA2BR;IA1BpB,MAAM,CAAiB;IACzB,WAAW,GAAY,KAAK,CAAC;IACrC;;;OAGG;IACK,eAAe,GAAG,IAAI,GAAG,EAAa,CAAC;IAE/C;;;;;OAKG;IACK,YAAY,GAAG,IAAI,GAAG,EAAyB,CAAC;IAExD;;OAEG;IACK,oBAAoB,GAAkB,SAAS,CAAC;IAExD,wDAAwD;IAChD,QAAQ,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;IAE3B,eAAe,CAA8B;IAErD,YAA8B,OAAkC;QAAlC,YAAO,GAAP,OAAO,CAA2B;QAC9D,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,cAAc,OAAO,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;IACnE,CAAC;IAUD,IAAW,EAAE;QACX,OAAO,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;IACzB,CAAC;IAED,IAAc,OAAO;QACnB,OAAO,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,mBAAmB,CAAC;IACxD,CAAC;IAED,IAAc,gBAAgB;QAC5B,OAAO,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC;IACvC,CAAC;IAED,IAAc,WAAW;QACvB,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC;IAClC,CAAC;IAED,IAAc,OAAO;QACnB,OAAO,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC;IACpC,CAAC;IAED,IAAc,OAAO;QACnB,OAAO,IAAI,CAAC,eAAe,EAAE,MAAM,CAAC,OAAO,CAAC;IAC9C,CAAC;IAEM,KAAK,CAAC,KAAK;QAChB,IAAI,CAAC,eAAe,GAAG,IAAI,eAAe,EAAE,CAAC;QAC7C,IAAI,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;YACzB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,
|
|
1
|
+
{"version":3,"file":"AbstractReplicator.js","sourceRoot":"","sources":["../../src/replication/AbstractReplicator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AACjF,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAC7D,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAGtC,OAAO,KAAK,OAAO,MAAM,6BAA6B,CAAC;AAOvD,WAAW;AACX,MAAM,aAAa,GAAG,WAAc,GAAG,GAAG,CAAC;AAkB3C;;;;GAIG;AACH,MAAM,OAAgB,kBAAkB;IA2BR;IA1BpB,MAAM,CAAiB;IACzB,WAAW,GAAY,KAAK,CAAC;IACrC;;;OAGG;IACK,eAAe,GAAG,IAAI,GAAG,EAAa,CAAC;IAE/C;;;;;OAKG;IACK,YAAY,GAAG,IAAI,GAAG,EAAyB,CAAC;IAExD;;OAEG;IACK,oBAAoB,GAAkB,SAAS,CAAC;IAExD,wDAAwD;IAChD,QAAQ,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;IAE3B,eAAe,CAA8B;IAErD,YAA8B,OAAkC;QAAlC,YAAO,GAAP,OAAO,CAA2B;QAC9D,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,cAAc,OAAO,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;IACnE,CAAC;IAUD,IAAW,EAAE;QACX,OAAO,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;IACzB,CAAC;IAED,IAAc,OAAO;QACnB,OAAO,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,mBAAmB,CAAC;IACxD,CAAC;IAED,IAAc,gBAAgB;QAC5B,OAAO,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC;IACvC,CAAC;IAED,IAAc,WAAW;QACvB,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC;IAClC,CAAC;IAED,IAAc,OAAO;QACnB,OAAO,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC;IACpC,CAAC;IAED,IAAc,OAAO;QACnB,OAAO,IAAI,CAAC,eAAe,EAAE,MAAM,CAAC,OAAO,CAAC;IAC9C,CAAC;IAEM,KAAK,CAAC,KAAK;QAChB,IAAI,CAAC,eAAe,GAAG,IAAI,eAAe,EAAE,CAAC;QAC7C,IAAI,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;YACzB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,yBAAyB,EAAE,CAAC,CAAC,CAAC;YAChD,SAAS,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC;YACvC,UAAU,CAAC,GAAG,EAAE;gBACd,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC,EAAE,IAAI,CAAC,CAAC;QACX,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,iBAAiB,CAAC,uBAAuB,CAAC,CAAC,gBAAgB,CAAC,KAAK,IAAI,EAAE;YACrG,IAAI,CAAC;gBACH,MAAM,GAAG,GAAG,IAAI,CAAC,uBAAuB,EAAE,CAAC;gBAC3C,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;oBAChB,OAAO,SAAS,CAAC;gBACnB,CAAC;gBACD,gBAAgB;gBAChB,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,IAAI,CAAC,CAAC;YAChC,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,+BAA+B,EAAE,CAAC,CAAC,CAAC;gBACtD,OAAO,SAAS,CAAC;YACnB,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAEM,KAAK,CAAC,IAAI;QACf,IAAI,CAAC,eAAe,EAAE,KAAK,EAAE,CAAC;QAC9B,IAAI,QAAQ,GAAoB,EAAE,CAAC;QACnC,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,EAAE,CAAC;YAChD,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;QAC5B,CAAC;QACD,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC9B,CAAC;IAEO,KAAK,CAAC,OAAO;QACnB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,EAAE,CAAC;QAEpD,IAAI,cAAc,GAAwC,SAAS,CAAC;QACpE,IAAI,SAAS,IAAI,IAAI,EAAE,CAAC;YACtB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;YACvC,IAAI,CAAC;gBACH,mDAAmD;gBACnD,2GAA2G;gBAE3G,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB,CACpD,OAAO,CAAC,uBAAuB,CAAC,SAAS,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,gBAAgB,CAAC,WAAW,EAAE,CAAC,CACxG,CAAC;gBACF,IAAI,IAAI,EAAE,CAAC;oBACT,cAAc,GAAG,IAAI,CAAC;gBACxB,CAAC;YACH,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,4BAA4B;gBAC5B,4EAA4E;gBAC5E,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,8BAA8B,EAAE,CAAC,CAAC,CAAC;gBACrD,MAAM,CAAC,CAAC;YACV,CAAC;QACH,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,yDAAyD,CAAC,CAAC;QAC9E,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YACrB,MAAM,SAAS,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;YAC/B,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,OAAO,CAAC,EAAE,eAAe,EAAE,cAAc,EAAE,CAAC,CAAC;gBACxD,+CAA+C;gBAC/C,cAAc,GAAG,SAAS,CAAC;gBAE3B,gEAAgE;gBAChE,6EAA6E;gBAC7E,IAAI,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,EAAE,CAAC;oBAC/B,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;oBAC5B,IAAI,GAAG,GAAG,IAAI,CAAC,QAAQ,IAAI,aAAa,EAAE,CAAC;wBACzC,KAAK,MAAM,SAAS,IAAI,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,EAAE,CAAC;4BACtD,MAAM,SAAS,CAAC,SAAS,EAAE,CAAC;wBAC9B,CAAC;wBAED,IAAI,CAAC,QAAQ,GAAG,GAAG,CAAC;oBACtB,CAAC;gBACH,CAAC;YACH,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,oCAAoC,EAAE,CAAC,CAAC,CAAC;YAC7D,CAAC;YACD,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;QAC5D,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,OAAO,CAAC,OAAuD;QAC3E,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,OAAO;QACT,CAAC;QAED,IAAI,cAAc,GAAG,OAAO,EAAE,eAAe,CAAC;QAE9C,MAAM,YAAY,GAAG,IAAI,GAAG,CAAY,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC,CAAC;QACxE,MAAM,oBAAoB,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,uBAAuB,EAAE,CAAC;QAC1E,MAAM,OAAO,GAAG,IAAI,GAAG,EAAa,CAAC;QACrC,IAAI,SAAS,GAAkB,SAAS,CAAC;QACzC,KAAK,IAAI,SAAS,IAAI,oBAAoB,EAAE,CAAC;YAC3C,MAAM,WAAW,GAAG,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;YACnD,IAAI,SAAS,CAAC,MAAM,IAAI,SAAS,IAAI,IAAI,EAAE,CAAC;gBAC1C,SAAS,GAAG,WAAW,CAAC;YAC1B,CAAC;YACD,IAAI,WAAW,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE,CAAC;gBAC1C,YAAY;gBACZ,YAAY,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;gBAClC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC;YACzC,CAAC;iBAAM,IAAI,WAAW,IAAI,WAAW,CAAC,SAAS,EAAE,CAAC;gBAChD,8BAA8B;gBAC9B,gEAAgE;gBAChE,YAAY,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;YACpC,CAAC;iBAAM,CAAC;gBACN,sDAAsD;gBACtD,IAAI,CAAC;oBACH,IAAI,IAA6B,CAAC;oBAClC,IAAI,cAAc,EAAE,aAAa,IAAI,SAAS,CAAC,EAAE,EAAE,CAAC;wBAClD,IAAI,GAAG,cAAc,CAAC;oBACxB,CAAC;yBAAM,CAAC;wBACN,IAAI,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,CAAC;oBAChC,CAAC;oBACD,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;oBACpD,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC;wBAC5B,IAAI,EAAE,IAAI;wBACV,OAAO,EAAE,OAAO;qBACjB,CAAC,CAAC;oBAEH,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;oBAClC,MAAM,CAAC,KAAK,EAAE,CAAC;oBACf,IAAI,SAAS,CAAC,MAAM,EAAE,CAAC;wBACrB,SAAS,GAAG,MAAM,CAAC;oBACrB,CAAC;oBACD,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;gBAC3B,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACX,IAAI,CAAC,EAAE,SAAS,EAAE,IAAI,KAAK,SAAS,CAAC,WAAW,EAAE,CAAC;wBACjD,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;4BACtB,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC,CAAC;4BAC1E,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;wBAC1B,CAAC;oBACH,CAAC;yBAAM,CAAC;wBACN,sCAAsC;wBACtC,uDAAuD;wBACvD,gDAAgD;wBAChD,gFAAgF;wBAChF,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,iDAAiD,EAAE,CAAC,CAAC,CAAC;oBAC/E,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,CAAC,eAAe,GAAG,OAAO,CAAC;QAC/B,IAAI,CAAC,oBAAoB,GAAG,SAAS,CAAC;QAEtC,mEAAmE;QACnE,4BAA4B;QAC5B,KAAK,IAAI,GAAG,IAAI,YAAY,CAAC,MAAM,EAAE,EAAE,CAAC;YACtC,0BAA0B;YAC1B,IAAI,CAAC;gBACH,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;YACnB,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,uBAAuB;gBACvB,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,oCAAoC,EAAE,CAAC,CAAC,CAAC;YACnE,CAAC;QACH,CAAC;QAED,2EAA2E;QAC3E,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,mBAAmB,EAAE,CAAC;QACzD,KAAK,IAAI,SAAS,IAAI,OAAO,EAAE,CAAC;YAC9B,IAAI,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,EAAE,CAAC;gBACxC,sBAAsB;gBACtB,SAAS;YACX,CAAC;YAED,mCAAmC;YACnC,uFAAuF;YACvF,gDAAgD;YAChD,MAAM,eAAe,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,SAAS,EAAE,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC,CAAC;YAC1F,MAAM,OAAO,GAAG,IAAI,CAAC,kBAAkB,CAAC,eAAe,CAAC;iBACrD,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;gBACX,eAAe,CAAC,MAAM,CAAC,IAAI,CAAC,oCAAoC,EAAE,CAAC,CAAC,CAAC;YACvE,CAAC,CAAC;iBACD,OAAO,CAAC,GAAG,EAAE;gBACZ,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;YACzC,CAAC,CAAC,CAAC;YACL,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;QAC/C,CAAC;IACH,CAAC;IAES,WAAW,CAAC,UAAkB;QACtC,OAAO,GAAG,IAAI,CAAC,EAAE,IAAI,UAAU,EAAE,CAAC;IACpC,CAAC;IAES,KAAK,CAAC,kBAAkB,CAAC,eAA+C;QAChF,eAAe,CAAC,MAAM,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAC;QACjE,qEAAqE;QACrE,qEAAqE;QACrE,MAAM,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC;QACpC,MAAM,eAAe,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,eAAe,EAAE,MAAM,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9F,eAAe,CAAC,MAAM,CAAC,IAAI,CAAC,4CAA4C,CAAC,CAAC;IAC5E,CAAC;IAID;;;;;;;;;;;;;;;;;OAiBG;IACH,uBAAuB;QACrB,OAAO,IAAI,CAAC,oBAAoB,EAAE,uBAAuB,EAAE,CAAC;IAC9D,CAAC;CACF"}
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
export * from './AbstractReplicationJob.js';
|
|
2
2
|
export * from './AbstractReplicator.js';
|
|
3
3
|
export * from './ErrorRateLimiter.js';
|
|
4
|
-
export * from './
|
|
4
|
+
export * from './RelationCache.js';
|
|
5
|
+
export * from './replication-metrics.js';
|
|
5
6
|
export * from './ReplicationEngine.js';
|
|
7
|
+
export * from './ReplicationLagTracker.js';
|
|
6
8
|
export * from './ReplicationModule.js';
|
|
7
|
-
export * from './replication-metrics.js';
|
|
8
|
-
export * from './RelationCache.js';
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
export * from './AbstractReplicationJob.js';
|
|
2
2
|
export * from './AbstractReplicator.js';
|
|
3
3
|
export * from './ErrorRateLimiter.js';
|
|
4
|
-
export * from './
|
|
4
|
+
export * from './RelationCache.js';
|
|
5
|
+
export * from './replication-metrics.js';
|
|
5
6
|
export * from './ReplicationEngine.js';
|
|
7
|
+
export * from './ReplicationLagTracker.js';
|
|
6
8
|
export * from './ReplicationModule.js';
|
|
7
|
-
export * from './replication-metrics.js';
|
|
8
|
-
export * from './RelationCache.js';
|
|
9
9
|
//# sourceMappingURL=replication-index.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"replication-index.js","sourceRoot":"","sources":["../../src/replication/replication-index.ts"],"names":[],"mappings":"AAAA,cAAc,6BAA6B,CAAC;AAC5C,cAAc,yBAAyB,CAAC;AACxC,cAAc,uBAAuB,CAAC;AACtC,cAAc,
|
|
1
|
+
{"version":3,"file":"replication-index.js","sourceRoot":"","sources":["../../src/replication/replication-index.ts"],"names":[],"mappings":"AAAA,cAAc,6BAA6B,CAAC;AAC5C,cAAc,yBAAyB,CAAC;AACxC,cAAc,uBAAuB,CAAC;AACtC,cAAc,oBAAoB,CAAC;AACnC,cAAc,0BAA0B,CAAC;AACzC,cAAc,wBAAwB,CAAC;AACvC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,wBAAwB,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"replication-metrics.js","sourceRoot":"","sources":["../../src/replication/replication-metrics.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"replication-metrics.js","sourceRoot":"","sources":["../../src/replication/replication-metrics.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAG7D;;;GAGG;AACH,MAAM,UAAU,4BAA4B,CAAC,MAAqB;IAChE,MAAM,CAAC,aAAa,CAAC;QACnB,IAAI,EAAE,iBAAiB,CAAC,qBAAqB;QAC7C,WAAW,EAAE,sCAAsC;QACnD,IAAI,EAAE,OAAO;KACd,CAAC,CAAC;IAEH,MAAM,CAAC,aAAa,CAAC;QACnB,IAAI,EAAE,iBAAiB,CAAC,eAAe;QACvC,WAAW,EAAE,iCAAiC;KAC/C,CAAC,CAAC;IAEH,MAAM,CAAC,aAAa,CAAC;QACnB,IAAI,EAAE,iBAAiB,CAAC,uBAAuB;QAC/C,WAAW,EAAE,yCAAyC;KACvD,CAAC,CAAC;IAEH,MAAM,CAAC,aAAa,CAAC;QACnB,IAAI,EAAE,iBAAiB,CAAC,iBAAiB;QACzC,WAAW,EAAE,oCAAoC;KAClD,CAAC,CAAC;IAEH,MAAM,CAAC,qBAAqB,CAAC;QAC3B,IAAI,EAAE,iBAAiB,CAAC,uBAAuB;QAC/C,WAAW,EAAE,oEAAoE;KAClF,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,gCAAgC,CAAC,MAAqB;IACpE,MAAM,qBAAqB,GAAG,MAAM,CAAC,UAAU,CAAC,iBAAiB,CAAC,qBAAqB,CAAC,CAAC;IACzF,MAAM,qBAAqB,GAAG,MAAM,CAAC,UAAU,CAAC,iBAAiB,CAAC,eAAe,CAAC,CAAC;IACnF,MAAM,6BAA6B,GAAG,MAAM,CAAC,UAAU,CAAC,iBAAiB,CAAC,uBAAuB,CAAC,CAAC;IACnG,MAAM,uBAAuB,GAAG,MAAM,CAAC,UAAU,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,CAAC;IAEvF,qBAAqB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IAC7B,qBAAqB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IAC7B,6BAA6B,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IACrC,uBAAuB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IAC/B,4GAA4G;IAC5G,eAAe;AACjB,CAAC"}
|
|
@@ -92,8 +92,8 @@ export declare const DEFAULT_ROUTE_OPTIONS: {
|
|
|
92
92
|
}, {
|
|
93
93
|
connections: {
|
|
94
94
|
errors: {
|
|
95
|
-
message: string;
|
|
96
95
|
level: "warning" | "fatal";
|
|
96
|
+
message: string;
|
|
97
97
|
location?: {
|
|
98
98
|
start_offset: number;
|
|
99
99
|
end_offset: number;
|
|
@@ -140,6 +140,9 @@ export declare const DEFAULT_ROUTE_OPTIONS: {
|
|
|
140
140
|
last_keepalive_ts?: string | undefined;
|
|
141
141
|
last_checkpoint_ts?: string | undefined;
|
|
142
142
|
replication_lag_bytes?: number | undefined;
|
|
143
|
+
wal_status?: string | undefined;
|
|
144
|
+
safe_wal_size?: number | undefined;
|
|
145
|
+
max_slot_wal_keep_size?: number | undefined;
|
|
143
146
|
}[];
|
|
144
147
|
content?: string | undefined;
|
|
145
148
|
} | undefined;
|
|
@@ -179,6 +182,9 @@ export declare const DEFAULT_ROUTE_OPTIONS: {
|
|
|
179
182
|
last_keepalive_ts?: string | undefined;
|
|
180
183
|
last_checkpoint_ts?: string | undefined;
|
|
181
184
|
replication_lag_bytes?: number | undefined;
|
|
185
|
+
wal_status?: string | undefined;
|
|
186
|
+
safe_wal_size?: number | undefined;
|
|
187
|
+
max_slot_wal_keep_size?: number | undefined;
|
|
182
188
|
}[];
|
|
183
189
|
content?: string | undefined;
|
|
184
190
|
} | undefined;
|
|
@@ -189,8 +195,8 @@ export declare const DEFAULT_ROUTE_OPTIONS: {
|
|
|
189
195
|
}, import("./router.js").Context, import("./router.js").BasicRouterRequest>, {
|
|
190
196
|
connections: {
|
|
191
197
|
errors: {
|
|
192
|
-
message: string;
|
|
193
198
|
level: "warning" | "fatal";
|
|
199
|
+
message: string;
|
|
194
200
|
location?: {
|
|
195
201
|
start_offset: number;
|
|
196
202
|
end_offset: number;
|
|
@@ -237,6 +243,9 @@ export declare const DEFAULT_ROUTE_OPTIONS: {
|
|
|
237
243
|
last_keepalive_ts?: string | undefined;
|
|
238
244
|
last_checkpoint_ts?: string | undefined;
|
|
239
245
|
replication_lag_bytes?: number | undefined;
|
|
246
|
+
wal_status?: string | undefined;
|
|
247
|
+
safe_wal_size?: number | undefined;
|
|
248
|
+
max_slot_wal_keep_size?: number | undefined;
|
|
240
249
|
}[];
|
|
241
250
|
content?: string | undefined;
|
|
242
251
|
} | undefined;
|
|
@@ -276,6 +285,9 @@ export declare const DEFAULT_ROUTE_OPTIONS: {
|
|
|
276
285
|
last_keepalive_ts?: string | undefined;
|
|
277
286
|
last_checkpoint_ts?: string | undefined;
|
|
278
287
|
replication_lag_bytes?: number | undefined;
|
|
288
|
+
wal_status?: string | undefined;
|
|
289
|
+
safe_wal_size?: number | undefined;
|
|
290
|
+
max_slot_wal_keep_size?: number | undefined;
|
|
279
291
|
}[];
|
|
280
292
|
content?: string | undefined;
|
|
281
293
|
} | undefined;
|
|
@@ -297,8 +309,8 @@ export declare const DEFAULT_ROUTE_OPTIONS: {
|
|
|
297
309
|
}, import("./router.js").Context, import("./router.js").BasicRouterRequest>) => Promise<{
|
|
298
310
|
connections: {
|
|
299
311
|
errors: {
|
|
300
|
-
message: string;
|
|
301
312
|
level: "warning" | "fatal";
|
|
313
|
+
message: string;
|
|
302
314
|
location?: {
|
|
303
315
|
start_offset: number;
|
|
304
316
|
end_offset: number;
|
|
@@ -345,6 +357,9 @@ export declare const DEFAULT_ROUTE_OPTIONS: {
|
|
|
345
357
|
last_keepalive_ts?: string | undefined;
|
|
346
358
|
last_checkpoint_ts?: string | undefined;
|
|
347
359
|
replication_lag_bytes?: number | undefined;
|
|
360
|
+
wal_status?: string | undefined;
|
|
361
|
+
safe_wal_size?: number | undefined;
|
|
362
|
+
max_slot_wal_keep_size?: number | undefined;
|
|
348
363
|
}[];
|
|
349
364
|
content?: string | undefined;
|
|
350
365
|
} | undefined;
|
|
@@ -384,6 +399,9 @@ export declare const DEFAULT_ROUTE_OPTIONS: {
|
|
|
384
399
|
last_keepalive_ts?: string | undefined;
|
|
385
400
|
last_checkpoint_ts?: string | undefined;
|
|
386
401
|
replication_lag_bytes?: number | undefined;
|
|
402
|
+
wal_status?: string | undefined;
|
|
403
|
+
safe_wal_size?: number | undefined;
|
|
404
|
+
max_slot_wal_keep_size?: number | undefined;
|
|
387
405
|
}[];
|
|
388
406
|
content?: string | undefined;
|
|
389
407
|
} | undefined;
|
|
@@ -396,10 +414,10 @@ export declare const DEFAULT_ROUTE_OPTIONS: {
|
|
|
396
414
|
tables: {
|
|
397
415
|
name: string;
|
|
398
416
|
columns: {
|
|
399
|
-
type: string;
|
|
400
417
|
name: string;
|
|
401
418
|
sqlite_type: number | import("@powersync/service-types").SqliteSchemaTypeText;
|
|
402
419
|
internal_type: string;
|
|
420
|
+
type: string;
|
|
403
421
|
pg_type: string;
|
|
404
422
|
description?: string | undefined;
|
|
405
423
|
}[];
|
|
@@ -417,10 +435,10 @@ export declare const DEFAULT_ROUTE_OPTIONS: {
|
|
|
417
435
|
tables: {
|
|
418
436
|
name: string;
|
|
419
437
|
columns: {
|
|
420
|
-
type: string;
|
|
421
438
|
name: string;
|
|
422
439
|
sqlite_type: number | import("@powersync/service-types").SqliteSchemaTypeText;
|
|
423
440
|
internal_type: string;
|
|
441
|
+
type: string;
|
|
424
442
|
pg_type: string;
|
|
425
443
|
description?: string | undefined;
|
|
426
444
|
}[];
|
|
@@ -449,10 +467,10 @@ export declare const DEFAULT_ROUTE_OPTIONS: {
|
|
|
449
467
|
tables: {
|
|
450
468
|
name: string;
|
|
451
469
|
columns: {
|
|
452
|
-
type: string;
|
|
453
470
|
name: string;
|
|
454
471
|
sqlite_type: number | import("@powersync/service-types").SqliteSchemaTypeText;
|
|
455
472
|
internal_type: string;
|
|
473
|
+
type: string;
|
|
456
474
|
pg_type: string;
|
|
457
475
|
description?: string | undefined;
|
|
458
476
|
}[];
|
|
@@ -497,8 +515,8 @@ export declare const DEFAULT_ROUTE_OPTIONS: {
|
|
|
497
515
|
sync_rules: string;
|
|
498
516
|
}, {
|
|
499
517
|
errors: {
|
|
500
|
-
message: string;
|
|
501
518
|
level: "warning" | "fatal";
|
|
519
|
+
message: string;
|
|
502
520
|
location?: {
|
|
503
521
|
start_offset: number;
|
|
504
522
|
end_offset: number;
|
|
@@ -511,26 +529,29 @@ export declare const DEFAULT_ROUTE_OPTIONS: {
|
|
|
511
529
|
slot_name: string;
|
|
512
530
|
initial_replication_done: boolean;
|
|
513
531
|
tables: {
|
|
532
|
+
name: string;
|
|
533
|
+
schema: string;
|
|
534
|
+
replication_id: string[];
|
|
535
|
+
data_queries: boolean;
|
|
536
|
+
parameter_queries: boolean;
|
|
514
537
|
errors: {
|
|
515
|
-
message: string;
|
|
516
538
|
level: "warning" | "fatal";
|
|
539
|
+
message: string;
|
|
517
540
|
location?: {
|
|
518
541
|
start_offset: number;
|
|
519
542
|
end_offset: number;
|
|
520
543
|
} | undefined;
|
|
521
544
|
ts?: string | undefined;
|
|
522
545
|
}[];
|
|
523
|
-
name: string;
|
|
524
|
-
schema: string;
|
|
525
|
-
replication_id: string[];
|
|
526
|
-
data_queries: boolean;
|
|
527
|
-
parameter_queries: boolean;
|
|
528
546
|
pattern?: string | undefined;
|
|
529
547
|
}[];
|
|
530
548
|
last_lsn?: string | undefined;
|
|
531
549
|
last_keepalive_ts?: string | undefined;
|
|
532
550
|
last_checkpoint_ts?: string | undefined;
|
|
533
551
|
replication_lag_bytes?: number | undefined;
|
|
552
|
+
wal_status?: string | undefined;
|
|
553
|
+
safe_wal_size?: number | undefined;
|
|
554
|
+
max_slot_wal_keep_size?: number | undefined;
|
|
534
555
|
}[];
|
|
535
556
|
content?: string | undefined;
|
|
536
557
|
}, import("./router.js").Context, import("./router.js").RequestEndpointHandlerPayload<{
|
|
@@ -539,8 +560,8 @@ export declare const DEFAULT_ROUTE_OPTIONS: {
|
|
|
539
560
|
sync_rules: string;
|
|
540
561
|
}, import("./router.js").Context, import("./router.js").BasicRouterRequest>, {
|
|
541
562
|
errors: {
|
|
542
|
-
message: string;
|
|
543
563
|
level: "warning" | "fatal";
|
|
564
|
+
message: string;
|
|
544
565
|
location?: {
|
|
545
566
|
start_offset: number;
|
|
546
567
|
end_offset: number;
|
|
@@ -553,26 +574,29 @@ export declare const DEFAULT_ROUTE_OPTIONS: {
|
|
|
553
574
|
slot_name: string;
|
|
554
575
|
initial_replication_done: boolean;
|
|
555
576
|
tables: {
|
|
577
|
+
name: string;
|
|
578
|
+
schema: string;
|
|
579
|
+
replication_id: string[];
|
|
580
|
+
data_queries: boolean;
|
|
581
|
+
parameter_queries: boolean;
|
|
556
582
|
errors: {
|
|
557
|
-
message: string;
|
|
558
583
|
level: "warning" | "fatal";
|
|
584
|
+
message: string;
|
|
559
585
|
location?: {
|
|
560
586
|
start_offset: number;
|
|
561
587
|
end_offset: number;
|
|
562
588
|
} | undefined;
|
|
563
589
|
ts?: string | undefined;
|
|
564
590
|
}[];
|
|
565
|
-
name: string;
|
|
566
|
-
schema: string;
|
|
567
|
-
replication_id: string[];
|
|
568
|
-
data_queries: boolean;
|
|
569
|
-
parameter_queries: boolean;
|
|
570
591
|
pattern?: string | undefined;
|
|
571
592
|
}[];
|
|
572
593
|
last_lsn?: string | undefined;
|
|
573
594
|
last_keepalive_ts?: string | undefined;
|
|
574
595
|
last_checkpoint_ts?: string | undefined;
|
|
575
596
|
replication_lag_bytes?: number | undefined;
|
|
597
|
+
wal_status?: string | undefined;
|
|
598
|
+
safe_wal_size?: number | undefined;
|
|
599
|
+
max_slot_wal_keep_size?: number | undefined;
|
|
576
600
|
}[];
|
|
577
601
|
content?: string | undefined;
|
|
578
602
|
}>> & {
|
|
@@ -592,8 +616,8 @@ export declare const DEFAULT_ROUTE_OPTIONS: {
|
|
|
592
616
|
sync_rules: string;
|
|
593
617
|
}, import("./router.js").Context, import("./router.js").BasicRouterRequest>) => Promise<{
|
|
594
618
|
errors: {
|
|
595
|
-
message: string;
|
|
596
619
|
level: "warning" | "fatal";
|
|
620
|
+
message: string;
|
|
597
621
|
location?: {
|
|
598
622
|
start_offset: number;
|
|
599
623
|
end_offset: number;
|
|
@@ -606,26 +630,29 @@ export declare const DEFAULT_ROUTE_OPTIONS: {
|
|
|
606
630
|
slot_name: string;
|
|
607
631
|
initial_replication_done: boolean;
|
|
608
632
|
tables: {
|
|
633
|
+
name: string;
|
|
634
|
+
schema: string;
|
|
635
|
+
replication_id: string[];
|
|
636
|
+
data_queries: boolean;
|
|
637
|
+
parameter_queries: boolean;
|
|
609
638
|
errors: {
|
|
610
|
-
message: string;
|
|
611
639
|
level: "warning" | "fatal";
|
|
640
|
+
message: string;
|
|
612
641
|
location?: {
|
|
613
642
|
start_offset: number;
|
|
614
643
|
end_offset: number;
|
|
615
644
|
} | undefined;
|
|
616
645
|
ts?: string | undefined;
|
|
617
646
|
}[];
|
|
618
|
-
name: string;
|
|
619
|
-
schema: string;
|
|
620
|
-
replication_id: string[];
|
|
621
|
-
data_queries: boolean;
|
|
622
|
-
parameter_queries: boolean;
|
|
623
647
|
pattern?: string | undefined;
|
|
624
648
|
}[];
|
|
625
649
|
last_lsn?: string | undefined;
|
|
626
650
|
last_keepalive_ts?: string | undefined;
|
|
627
651
|
last_checkpoint_ts?: string | undefined;
|
|
628
652
|
replication_lag_bytes?: number | undefined;
|
|
653
|
+
wal_status?: string | undefined;
|
|
654
|
+
safe_wal_size?: number | undefined;
|
|
655
|
+
max_slot_wal_keep_size?: number | undefined;
|
|
629
656
|
}[];
|
|
630
657
|
content?: string | undefined;
|
|
631
658
|
}>;
|
|
@@ -792,8 +819,8 @@ export declare const DEFAULT_ROUTE_OPTIONS: {
|
|
|
792
819
|
client_id?: string | undefined;
|
|
793
820
|
streams?: {
|
|
794
821
|
subscriptions: {
|
|
795
|
-
parameters: Record<string, any> | null;
|
|
796
822
|
stream: string;
|
|
823
|
+
parameters: Record<string, any> | null;
|
|
797
824
|
override_priority: number | null;
|
|
798
825
|
}[];
|
|
799
826
|
include_defaults?: boolean | undefined;
|
|
@@ -811,8 +838,8 @@ export declare const DEFAULT_ROUTE_OPTIONS: {
|
|
|
811
838
|
client_id?: string | undefined;
|
|
812
839
|
streams?: {
|
|
813
840
|
subscriptions: {
|
|
814
|
-
parameters: Record<string, any> | null;
|
|
815
841
|
stream: string;
|
|
842
|
+
parameters: Record<string, any> | null;
|
|
816
843
|
override_priority: number | null;
|
|
817
844
|
}[];
|
|
818
845
|
include_defaults?: boolean | undefined;
|
|
@@ -830,8 +857,8 @@ export declare const DEFAULT_ROUTE_OPTIONS: {
|
|
|
830
857
|
client_id?: string | undefined;
|
|
831
858
|
streams?: {
|
|
832
859
|
subscriptions: {
|
|
833
|
-
parameters: Record<string, any> | null;
|
|
834
860
|
stream: string;
|
|
861
|
+
parameters: Record<string, any> | null;
|
|
835
862
|
override_priority: number | null;
|
|
836
863
|
}[];
|
|
837
864
|
include_defaults?: boolean | undefined;
|
|
@@ -853,8 +880,8 @@ export declare const DEFAULT_ROUTE_OPTIONS: {
|
|
|
853
880
|
client_id?: string | undefined;
|
|
854
881
|
streams?: {
|
|
855
882
|
subscriptions: {
|
|
856
|
-
parameters: Record<string, any> | null;
|
|
857
883
|
stream: string;
|
|
884
|
+
parameters: Record<string, any> | null;
|
|
858
885
|
override_priority: number | null;
|
|
859
886
|
}[];
|
|
860
887
|
include_defaults?: boolean | undefined;
|
|
@@ -873,8 +900,8 @@ export declare const DEFAULT_ROUTE_OPTIONS: {
|
|
|
873
900
|
client_id?: string | undefined;
|
|
874
901
|
streams?: {
|
|
875
902
|
subscriptions: {
|
|
876
|
-
parameters: Record<string, any> | null;
|
|
877
903
|
stream: string;
|
|
904
|
+
parameters: Record<string, any> | null;
|
|
878
905
|
override_priority: number | null;
|
|
879
906
|
}[];
|
|
880
907
|
include_defaults?: boolean | undefined;
|