@powersync/service-core 0.0.0-dev-20260602073133 → 0.0.0-dev-20260909133214
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +152 -4
- package/dist/api/RouteAPI.d.ts +42 -14
- package/dist/api/diagnostics.d.ts +12 -1
- package/dist/api/diagnostics.js +36 -22
- package/dist/api/diagnostics.js.map +1 -1
- package/dist/entry/commands/compact-action.js +30 -22
- package/dist/entry/commands/compact-action.js.map +1 -1
- package/dist/metrics/metrics-interfaces.d.ts +5 -1
- package/dist/metrics/metrics-interfaces.js.map +1 -1
- package/dist/metrics/open-telemetry/MetricAttributeFilteringExporter.d.ts +36 -0
- package/dist/metrics/open-telemetry/MetricAttributeFilteringExporter.js +70 -0
- package/dist/metrics/open-telemetry/MetricAttributeFilteringExporter.js.map +1 -0
- package/dist/metrics/open-telemetry/OpenTelemetryMetricsFactory.js +6 -1
- package/dist/metrics/open-telemetry/OpenTelemetryMetricsFactory.js.map +1 -1
- package/dist/metrics/open-telemetry/util.js +7 -3
- package/dist/metrics/open-telemetry/util.js.map +1 -1
- package/dist/modules/AbstractModule.d.ts +6 -2
- package/dist/modules/AbstractModule.js.map +1 -1
- package/dist/replication/AbstractReplicator.d.ts +31 -7
- package/dist/replication/AbstractReplicator.js +168 -74
- package/dist/replication/AbstractReplicator.js.map +1 -1
- package/dist/replication/ReplicationModule.d.ts +3 -3
- package/dist/replication/ReplicationModule.js.map +1 -1
- package/dist/routes/RouterEngine.d.ts +1 -0
- package/dist/routes/RouterEngine.js +6 -1
- package/dist/routes/RouterEngine.js.map +1 -1
- package/dist/routes/configure-fastify.d.ts +104 -56
- package/dist/routes/configure-fastify.js +15 -2
- package/dist/routes/configure-fastify.js.map +1 -1
- package/dist/routes/endpoints/admin.d.ts +42 -24
- package/dist/routes/endpoints/admin.js +49 -36
- package/dist/routes/endpoints/admin.js.map +1 -1
- package/dist/routes/endpoints/checkpointing.d.ts +62 -0
- package/dist/routes/endpoints/checkpointing.js +65 -7
- package/dist/routes/endpoints/checkpointing.js.map +1 -1
- package/dist/routes/endpoints/socket-route.js +3 -2
- package/dist/routes/endpoints/socket-route.js.map +1 -1
- package/dist/routes/endpoints/sync-rules.js +25 -14
- package/dist/routes/endpoints/sync-rules.js.map +1 -1
- package/dist/routes/endpoints/sync-stream.js +1 -1
- package/dist/routes/endpoints/sync-stream.js.map +1 -1
- package/dist/runner/teardown.js +8 -8
- package/dist/runner/teardown.js.map +1 -1
- package/dist/storage/BucketStorageBatch.d.ts +64 -17
- package/dist/storage/BucketStorageBatch.js.map +1 -1
- package/dist/storage/BucketStorageFactory.d.ts +50 -31
- package/dist/storage/BucketStorageFactory.js +10 -20
- package/dist/storage/BucketStorageFactory.js.map +1 -1
- package/dist/storage/CheckpointChecksumInvalidatedError.d.ts +12 -0
- package/dist/storage/CheckpointChecksumInvalidatedError.js +17 -0
- package/dist/storage/CheckpointChecksumInvalidatedError.js.map +1 -0
- package/dist/storage/ChecksumCache.d.ts +11 -6
- package/dist/storage/ChecksumCache.js +5 -4
- package/dist/storage/ChecksumCache.js.map +1 -1
- package/dist/storage/ParsedSyncConfigSet.d.ts +23 -0
- package/dist/storage/ParsedSyncConfigSet.js +2 -0
- package/dist/storage/ParsedSyncConfigSet.js.map +1 -0
- package/dist/storage/PersistedReplicationStream.d.ts +32 -0
- package/dist/storage/PersistedReplicationStream.js +26 -0
- package/dist/storage/PersistedReplicationStream.js.map +1 -0
- package/dist/storage/PersistedSyncConfigContent.d.ts +65 -0
- package/dist/storage/PersistedSyncConfigContent.js +138 -0
- package/dist/storage/PersistedSyncConfigContent.js.map +1 -0
- package/dist/storage/PersistedSyncConfigStatus.d.ts +13 -0
- package/dist/storage/PersistedSyncConfigStatus.js +2 -0
- package/dist/storage/PersistedSyncConfigStatus.js.map +1 -0
- package/dist/storage/ReplicationEventPayload.d.ts +6 -1
- package/dist/storage/SourceEntity.d.ts +13 -2
- package/dist/storage/SourceTable.d.ts +53 -4
- package/dist/storage/SourceTable.js +62 -7
- package/dist/storage/SourceTable.js.map +1 -1
- package/dist/storage/SourceTableReconciler.d.ts +83 -0
- package/dist/storage/SourceTableReconciler.js +95 -0
- package/dist/storage/SourceTableReconciler.js.map +1 -0
- package/dist/storage/StorageVersionConfig.d.ts +4 -0
- package/dist/storage/StorageVersionConfig.js +10 -1
- package/dist/storage/StorageVersionConfig.js.map +1 -1
- package/dist/storage/SyncRulesBucketStorage.d.ts +162 -26
- package/dist/storage/SyncRulesBucketStorage.js +3 -0
- package/dist/storage/SyncRulesBucketStorage.js.map +1 -1
- package/dist/storage/WriteCheckpointAPI.d.ts +85 -6
- package/dist/storage/WriteCheckpointAPI.js +33 -0
- package/dist/storage/WriteCheckpointAPI.js.map +1 -1
- package/dist/storage/implementation/BucketDefinitionMapping.d.ts +133 -0
- package/dist/storage/implementation/BucketDefinitionMapping.js +351 -0
- package/dist/storage/implementation/BucketDefinitionMapping.js.map +1 -0
- package/dist/storage/implementation/IncrementalReprocessingCompatibility.d.ts +6 -0
- package/dist/storage/implementation/IncrementalReprocessingCompatibility.js +31 -0
- package/dist/storage/implementation/IncrementalReprocessingCompatibility.js.map +1 -0
- package/dist/storage/implementation/IncrementalReprocessingSyncConfigLog.d.ts +17 -0
- package/dist/storage/implementation/IncrementalReprocessingSyncConfigLog.js +70 -0
- package/dist/storage/implementation/IncrementalReprocessingSyncConfigLog.js.map +1 -0
- package/dist/storage/implementation/ReplicationStreamStorageIds.d.ts +15 -0
- package/dist/storage/implementation/ReplicationStreamStorageIds.js +20 -0
- package/dist/storage/implementation/ReplicationStreamStorageIds.js.map +1 -0
- package/dist/storage/storage-index.d.ts +10 -1
- package/dist/storage/storage-index.js +10 -1
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/storage/storage-metrics.js +120 -4
- package/dist/storage/storage-metrics.js.map +1 -1
- package/dist/sync/BucketChecksumState.d.ts +12 -2
- package/dist/sync/BucketChecksumState.js +69 -21
- package/dist/sync/BucketChecksumState.js.map +1 -1
- package/dist/sync/RequestTracker.d.ts +7 -0
- package/dist/sync/RequestTracker.js +12 -0
- package/dist/sync/RequestTracker.js.map +1 -1
- package/dist/sync/sync.js +184 -43
- package/dist/sync/sync.js.map +1 -1
- package/dist/sync/util.d.ts +5 -0
- package/dist/sync/util.js +19 -1
- package/dist/sync/util.js.map +1 -1
- package/dist/system/ServiceContext.d.ts +2 -0
- package/dist/system/ServiceContext.js +2 -0
- package/dist/system/ServiceContext.js.map +1 -1
- package/dist/tracing/PerformanceTracer.d.ts +23 -1
- package/dist/tracing/PerformanceTracer.js +11 -0
- package/dist/tracing/PerformanceTracer.js.map +1 -1
- package/dist/util/checkpointing.d.ts +4 -8
- package/dist/util/checkpointing.js +1 -16
- package/dist/util/checkpointing.js.map +1 -1
- package/dist/util/config/compound-config-collector.js +23 -2
- package/dist/util/config/compound-config-collector.js.map +1 -1
- package/dist/util/config/defaults.d.ts +2 -0
- package/dist/util/config/defaults.js +2 -0
- package/dist/util/config/defaults.js.map +1 -1
- package/dist/util/config/sync-rules/impl/base64-sync-rules-collector.js +1 -0
- package/dist/util/config/sync-rules/impl/base64-sync-rules-collector.js.map +1 -1
- package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js +1 -0
- package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js.map +1 -1
- package/dist/util/config/sync-rules/sync-rules-provider.d.ts +2 -0
- package/dist/util/config/sync-rules/sync-rules-provider.js +3 -0
- package/dist/util/config/sync-rules/sync-rules-provider.js.map +1 -1
- package/dist/util/config/types.d.ts +4 -1
- package/dist/util/lsn.d.ts +4 -0
- package/dist/util/lsn.js +10 -0
- package/dist/util/lsn.js.map +1 -1
- package/dist/util/protocol-types.d.ts +4 -4
- package/dist/util/protocol-types.js +3 -2
- package/dist/util/protocol-types.js.map +1 -1
- package/dist/util/util-index.d.ts +1 -0
- package/dist/util/util-index.js +1 -0
- package/dist/util/util-index.js.map +1 -1
- package/dist/util/utils.d.ts +3 -2
- package/dist/util/utils.js +17 -2
- package/dist/util/utils.js.map +1 -1
- package/dist/util/write-checkpoint-batcher.d.ts +19 -0
- package/dist/util/write-checkpoint-batcher.js +100 -0
- package/dist/util/write-checkpoint-batcher.js.map +1 -0
- package/package.json +7 -7
- package/src/api/RouteAPI.ts +43 -14
- package/src/api/diagnostics.ts +45 -26
- package/src/entry/commands/compact-action.ts +37 -23
- package/src/metrics/metrics-interfaces.ts +6 -1
- package/src/metrics/open-telemetry/MetricAttributeFilteringExporter.ts +85 -0
- package/src/metrics/open-telemetry/OpenTelemetryMetricsFactory.ts +7 -2
- package/src/metrics/open-telemetry/util.ts +7 -3
- package/src/modules/AbstractModule.ts +6 -2
- package/src/replication/AbstractReplicator.ts +212 -78
- package/src/replication/ReplicationModule.ts +2 -3
- package/src/routes/RouterEngine.ts +7 -1
- package/src/routes/configure-fastify.ts +25 -2
- package/src/routes/endpoints/admin.ts +75 -51
- package/src/routes/endpoints/checkpointing.ts +79 -8
- package/src/routes/endpoints/socket-route.ts +3 -2
- package/src/routes/endpoints/sync-rules.ts +26 -14
- package/src/routes/endpoints/sync-stream.ts +1 -1
- package/src/runner/teardown.ts +8 -8
- package/src/storage/BucketStorageBatch.ts +65 -18
- package/src/storage/BucketStorageFactory.ts +73 -46
- package/src/storage/CheckpointChecksumInvalidatedError.ts +17 -0
- package/src/storage/ChecksumCache.ts +27 -14
- package/src/storage/ParsedSyncConfigSet.ts +25 -0
- package/src/storage/PersistedReplicationStream.ts +55 -0
- package/src/storage/PersistedSyncConfigContent.ts +207 -0
- package/src/storage/PersistedSyncConfigStatus.ts +14 -0
- package/src/storage/ReplicationEventPayload.ts +6 -1
- package/src/storage/SourceEntity.ts +12 -2
- package/src/storage/SourceTable.ts +96 -8
- package/src/storage/SourceTableReconciler.ts +188 -0
- package/src/storage/StorageVersionConfig.ts +11 -1
- package/src/storage/SyncRulesBucketStorage.ts +189 -27
- package/src/storage/WriteCheckpointAPI.ts +140 -6
- package/src/storage/implementation/BucketDefinitionMapping.ts +539 -0
- package/src/storage/implementation/IncrementalReprocessingCompatibility.ts +41 -0
- package/src/storage/implementation/IncrementalReprocessingSyncConfigLog.ts +126 -0
- package/src/storage/implementation/ReplicationStreamStorageIds.ts +21 -0
- package/src/storage/storage-index.ts +11 -1
- package/src/storage/storage-metrics.ts +154 -8
- package/src/sync/BucketChecksumState.ts +105 -21
- package/src/sync/RequestTracker.ts +20 -0
- package/src/sync/sync.ts +214 -50
- package/src/sync/util.ts +33 -8
- package/src/system/ServiceContext.ts +7 -0
- package/src/tracing/PerformanceTracer.ts +41 -1
- package/src/util/checkpointing.ts +4 -22
- package/src/util/config/compound-config-collector.ts +31 -1
- package/src/util/config/defaults.ts +2 -0
- package/src/util/config/sync-rules/impl/base64-sync-rules-collector.ts +1 -0
- package/src/util/config/sync-rules/impl/filesystem-sync-rules-collector.ts +1 -0
- package/src/util/config/sync-rules/sync-rules-provider.ts +5 -0
- package/src/util/config/types.ts +4 -1
- package/src/util/lsn.ts +9 -0
- package/src/util/protocol-types.ts +3 -2
- package/src/util/util-index.ts +1 -0
- package/src/util/utils.ts +16 -2
- package/src/util/write-checkpoint-batcher.ts +142 -0
- package/test/src/AbstractReplicator.test.ts +177 -0
- package/test/src/MetricAttributeFilteringExporter.test.ts +96 -0
- package/test/src/PersistedSyncConfigContent.test.ts +152 -0
- package/test/src/checksum_cache.test.ts +22 -0
- package/test/src/config.test.ts +142 -0
- package/test/src/diagnostics.test.ts +87 -34
- package/test/src/module-loader.test.ts +33 -1
- package/test/src/routes/admin.test.ts +188 -18
- package/test/src/routes/checkpointing.test.ts +54 -0
- package/test/src/routes/mocks.ts +6 -2
- package/test/src/routes/stream.test.ts +123 -2
- package/test/src/source-table-reconciler.test.ts +244 -0
- package/test/src/storage/SourceTable.test.ts +92 -0
- package/test/src/storage-metrics.test.ts +148 -0
- package/test/src/sync/BucketChecksumState.test.ts +98 -12
- package/test/src/sync/util.test.ts +28 -1
- package/test/src/util/checkpointing.test.ts +249 -0
- package/test/src/util/utils.test.ts +23 -0
- package/tsconfig.json +1 -4
- package/tsconfig.tsbuildinfo +1 -1
- package/dist/storage/PersistedSyncRulesContent.d.ts +0 -59
- package/dist/storage/PersistedSyncRulesContent.js +0 -100
- package/dist/storage/PersistedSyncRulesContent.js.map +0 -1
- package/src/storage/PersistedSyncRulesContent.ts +0 -174
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { logger, LookupOptions } from '@powersync/lib-services-framework';
|
|
2
|
+
import { validateStorageVersion } from '@powersync/service-sync-rules';
|
|
2
3
|
import { configFile } from '@powersync/service-types';
|
|
3
4
|
import * as auth from '../../auth/auth-index.js';
|
|
4
5
|
import { ConfigCollector } from './collectors/config-collector.js';
|
|
@@ -6,6 +7,8 @@ import { Base64ConfigCollector } from './collectors/impl/base64-config-collector
|
|
|
6
7
|
import { FallbackConfigCollector } from './collectors/impl/fallback-config-collector.js';
|
|
7
8
|
import { FileSystemConfigCollector } from './collectors/impl/filesystem-config-collector.js';
|
|
8
9
|
import {
|
|
10
|
+
DEFAULT_CHECKPOINT_REQUEST_RETENTION_MINUTES,
|
|
11
|
+
DEFAULT_CHECKSUM_CACHE_TTL_MINUTES,
|
|
9
12
|
DEFAULT_MAX_BUCKETS_PER_CONNECTION,
|
|
10
13
|
DEFAULT_MAX_CONCURRENT_CONNECTIONS,
|
|
11
14
|
DEFAULT_MAX_DATA_FETCH_CONCURRENCY,
|
|
@@ -60,6 +63,11 @@ export class CompoundConfigCollector {
|
|
|
60
63
|
async collectConfig(runnerConfig: RunnerConfig = {}): Promise<ResolvedPowerSyncConfig> {
|
|
61
64
|
const baseConfig = await this.collectBaseConfig(runnerConfig);
|
|
62
65
|
|
|
66
|
+
const defaultStorageVersion = baseConfig.storage?.default_storage_version;
|
|
67
|
+
if (defaultStorageVersion != null && validateStorageVersion(defaultStorageVersion) == null) {
|
|
68
|
+
throw new Error(`Storage version ${defaultStorageVersion} is not supported`);
|
|
69
|
+
}
|
|
70
|
+
|
|
63
71
|
const dataSources = baseConfig.replication?.connections ?? [];
|
|
64
72
|
if (dataSources.length > 1) {
|
|
65
73
|
throw new Error('Only a single replication data source is supported currently');
|
|
@@ -193,7 +201,13 @@ export class CompoundConfigCollector {
|
|
|
193
201
|
max_concurrent_connections:
|
|
194
202
|
baseConfig.api?.parameters?.max_concurrent_connections ?? DEFAULT_MAX_CONCURRENT_CONNECTIONS,
|
|
195
203
|
max_data_fetch_concurrency:
|
|
196
|
-
baseConfig.api?.parameters?.max_data_fetch_concurrency ?? DEFAULT_MAX_DATA_FETCH_CONCURRENCY
|
|
204
|
+
baseConfig.api?.parameters?.max_data_fetch_concurrency ?? DEFAULT_MAX_DATA_FETCH_CONCURRENCY,
|
|
205
|
+
checkpoint_request_retention_minutes: normalizeCheckpointRequestRetentionMinutes(
|
|
206
|
+
baseConfig.api?.parameters?.checkpoint_request_retention_minutes
|
|
207
|
+
),
|
|
208
|
+
bucket_count_cache_ttl_minutes: normalizeChecksumCacheTtlMinutes(
|
|
209
|
+
baseConfig.api?.parameters?.bucket_count_cache_ttl_minutes
|
|
210
|
+
)
|
|
197
211
|
},
|
|
198
212
|
// TODO maybe move this out of the connection or something
|
|
199
213
|
slot_name_prefix: baseConfig.replication?.connections?.[0]?.slot_name_prefix ?? 'powersync_',
|
|
@@ -249,3 +263,19 @@ export class CompoundConfigCollector {
|
|
|
249
263
|
};
|
|
250
264
|
}
|
|
251
265
|
}
|
|
266
|
+
|
|
267
|
+
function normalizeChecksumCacheTtlMinutes(ttlMinutes: number | undefined): number {
|
|
268
|
+
const normalized = ttlMinutes ?? DEFAULT_CHECKSUM_CACHE_TTL_MINUTES;
|
|
269
|
+
if (!Number.isFinite(normalized) || !Number.isInteger(normalized) || normalized < 1) {
|
|
270
|
+
throw new Error('api.parameters.bucket_count_cache_ttl_minutes must be a positive integer');
|
|
271
|
+
}
|
|
272
|
+
return normalized;
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
function normalizeCheckpointRequestRetentionMinutes(retentionMinutes: number | undefined): number {
|
|
276
|
+
const normalized = retentionMinutes ?? DEFAULT_CHECKPOINT_REQUEST_RETENTION_MINUTES;
|
|
277
|
+
if (!Number.isFinite(normalized) || !Number.isInteger(normalized) || normalized < 1) {
|
|
278
|
+
throw new Error('api.parameters.checkpoint_request_retention_minutes must be a positive integer');
|
|
279
|
+
}
|
|
280
|
+
return normalized;
|
|
281
|
+
}
|
|
@@ -3,3 +3,5 @@ export const DEFAULT_MAX_CONCURRENT_CONNECTIONS = 200;
|
|
|
3
3
|
export const DEFAULT_MAX_DATA_FETCH_CONCURRENCY = 10;
|
|
4
4
|
export const DEFAULT_MAX_BUCKETS_PER_CONNECTION = 1000;
|
|
5
5
|
export const DEFAULT_MAX_PARAMETER_QUERY_RESULTS = 1000;
|
|
6
|
+
export const DEFAULT_CHECKPOINT_REQUEST_RETENTION_MINUTES = 60;
|
|
7
|
+
export const DEFAULT_CHECKSUM_CACHE_TTL_MINUTES = 1 * 60;
|
|
@@ -15,6 +15,7 @@ export class Base64SyncRulesCollector extends SyncRulesCollector {
|
|
|
15
15
|
|
|
16
16
|
return {
|
|
17
17
|
present: true,
|
|
18
|
+
version_label: baseConfig.sync_config?.version_label,
|
|
18
19
|
exit_on_error: baseConfig.sync_config?.exit_on_error ?? true,
|
|
19
20
|
content: Buffer.from(sync_config_base64, 'base64').toString()
|
|
20
21
|
};
|
|
@@ -20,6 +20,7 @@ export class FileSystemSyncRulesCollector extends SyncRulesCollector {
|
|
|
20
20
|
// Only persist the path here, and load on demand using `loadSyncRules()`.
|
|
21
21
|
return {
|
|
22
22
|
present: true,
|
|
23
|
+
version_label: baseConfig.sync_config?.version_label,
|
|
23
24
|
exit_on_error: baseConfig.sync_config?.exit_on_error ?? true,
|
|
24
25
|
path: config_path ? path.resolve(path.dirname(config_path), sync_path) : sync_path
|
|
25
26
|
};
|
|
@@ -5,6 +5,7 @@ export interface SyncRulesProvider {
|
|
|
5
5
|
get(): Promise<string | undefined>;
|
|
6
6
|
|
|
7
7
|
readonly exitOnError: boolean;
|
|
8
|
+
readonly versionLabel: string | undefined;
|
|
8
9
|
}
|
|
9
10
|
|
|
10
11
|
export class ConfigurationFileSyncRulesProvider implements SyncRulesProvider {
|
|
@@ -21,4 +22,8 @@ export class ConfigurationFileSyncRulesProvider implements SyncRulesProvider {
|
|
|
21
22
|
get exitOnError() {
|
|
22
23
|
return this.config.exit_on_error;
|
|
23
24
|
}
|
|
25
|
+
|
|
26
|
+
get versionLabel() {
|
|
27
|
+
return this.config.version_label;
|
|
28
|
+
}
|
|
24
29
|
}
|
package/src/util/config/types.ts
CHANGED
|
@@ -22,6 +22,7 @@ export type Runner = (config: RunnerConfig) => Promise<void>;
|
|
|
22
22
|
|
|
23
23
|
export type SyncRulesConfig = {
|
|
24
24
|
present: boolean;
|
|
25
|
+
version_label?: string;
|
|
25
26
|
content?: string;
|
|
26
27
|
path?: string;
|
|
27
28
|
exit_on_error: boolean;
|
|
@@ -53,9 +54,11 @@ export type ResolvedPowerSyncConfig = {
|
|
|
53
54
|
max_data_fetch_concurrency: number;
|
|
54
55
|
max_buckets_per_connection: number;
|
|
55
56
|
max_parameter_query_results: number;
|
|
57
|
+
checkpoint_request_retention_minutes: number;
|
|
58
|
+
bucket_count_cache_ttl_minutes: number;
|
|
56
59
|
};
|
|
57
60
|
|
|
58
|
-
/** Prefix for postgres replication slot names
|
|
61
|
+
/** Prefix for postgres replication slot names, as well as replication stream names. */
|
|
59
62
|
slot_name_prefix: string;
|
|
60
63
|
|
|
61
64
|
healthcheck: {
|
package/src/util/lsn.ts
CHANGED
|
@@ -6,3 +6,12 @@ export function maxLsn(a: string | null | undefined, b: string | null | undefine
|
|
|
6
6
|
if (b == null) return a;
|
|
7
7
|
return a > b ? a : b;
|
|
8
8
|
}
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Return the smaller of two LSNs, ignoring nulls.
|
|
12
|
+
*/
|
|
13
|
+
export function minLsn(a: string | null | undefined, b: string | null | undefined): string | null {
|
|
14
|
+
if (a == null) return b ?? null;
|
|
15
|
+
if (b == null) return a;
|
|
16
|
+
return a < b ? a : b;
|
|
17
|
+
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { JsonContainer } from '@powersync/service-jsonbig';
|
|
2
2
|
import { BucketPriority, SqliteJsonRow } from '@powersync/service-sync-rules';
|
|
3
|
+
import { orNull } from '@powersync/service-types';
|
|
3
4
|
import * as t from 'ts-codec';
|
|
4
5
|
|
|
5
6
|
export const BucketRequest = t.object({
|
|
@@ -24,14 +25,14 @@ export const RequestedStreamSubscription = t.object({
|
|
|
24
25
|
/**
|
|
25
26
|
* An optional dictionary of parameters to pass to this specific stream.
|
|
26
27
|
*/
|
|
27
|
-
parameters:
|
|
28
|
+
parameters: orNull(t.record(t.any)),
|
|
28
29
|
/**
|
|
29
30
|
* Set when the client wishes to re-assign a different priority to this stream.
|
|
30
31
|
*
|
|
31
32
|
* Streams and sync rules can also assign a default priority, but clients are allowed to override those. This can be
|
|
32
33
|
* useful when the priority for partial syncs depends on e.g. the current page opened in a client.
|
|
33
34
|
*/
|
|
34
|
-
override_priority:
|
|
35
|
+
override_priority: orNull(t.number)
|
|
35
36
|
});
|
|
36
37
|
|
|
37
38
|
export type RequestedStreamSubscription = t.Decoded<typeof RequestedStreamSubscription>;
|
package/src/util/util-index.ts
CHANGED
|
@@ -8,6 +8,7 @@ export * from './protocol-types.js';
|
|
|
8
8
|
export * from './secs.js';
|
|
9
9
|
export * from './utils.js';
|
|
10
10
|
export * from './version.js';
|
|
11
|
+
export * from './write-checkpoint-batcher.js';
|
|
11
12
|
|
|
12
13
|
export * from './config.js';
|
|
13
14
|
export * from './config/compound-config-collector.js';
|
package/src/util/utils.ts
CHANGED
|
@@ -33,7 +33,7 @@ export interface PartialChecksum {
|
|
|
33
33
|
*/
|
|
34
34
|
export type InternalOpId = bigint;
|
|
35
35
|
|
|
36
|
-
export const ID_NAMESPACE = 'a396dd91-09fc-4017-a28d-3df722f651e9';
|
|
36
|
+
export const ID_NAMESPACE = uuid.parse('a396dd91-09fc-4017-a28d-3df722f651e9');
|
|
37
37
|
|
|
38
38
|
export function escapeIdentifier(identifier: string) {
|
|
39
39
|
return `"${identifier.replace(/"/g, '""').replace(/\./g, '"."')}"`;
|
|
@@ -223,7 +223,7 @@ export function isCompleteRow<V>(
|
|
|
223
223
|
* All other operations are replaced with a single CLEAR operation,
|
|
224
224
|
* summing their checksums, and using a 0 as an op_id.
|
|
225
225
|
*
|
|
226
|
-
* This is the function $r(B)$, as described in /docs/bucket-properties.md.
|
|
226
|
+
* This is the function $r(B)$, as described in /docs/storage/bucket-properties.md.
|
|
227
227
|
*
|
|
228
228
|
* Used for tests.
|
|
229
229
|
*/
|
|
@@ -309,3 +309,17 @@ export function estimateRowSize(record: sync_rules.ToastableSqliteRow | undefine
|
|
|
309
309
|
}
|
|
310
310
|
return size;
|
|
311
311
|
}
|
|
312
|
+
|
|
313
|
+
export function formatBytes(bytes: number | bigint): string {
|
|
314
|
+
if (typeof bytes == 'bigint') {
|
|
315
|
+
bytes = Number(bytes); // We round either way
|
|
316
|
+
}
|
|
317
|
+
if (bytes >= 1024 * 1024 * 1024) {
|
|
318
|
+
return `${(bytes / (1024 * 1024 * 1024)).toFixed(1)}GB`;
|
|
319
|
+
} else if (bytes >= 1024 * 1024) {
|
|
320
|
+
return `${(bytes / (1024 * 1024)).toFixed(1)}MB`;
|
|
321
|
+
} else if (bytes >= 1024) {
|
|
322
|
+
return `${(bytes / 1024).toFixed(1)}KB`;
|
|
323
|
+
}
|
|
324
|
+
return `${bytes}B`;
|
|
325
|
+
}
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
import { ErrorCode, ServiceAssertionError, ServiceError } from '@powersync/lib-services-framework';
|
|
2
|
+
import { RouteAPI } from '../api/RouteAPI.js';
|
|
3
|
+
import {
|
|
4
|
+
BucketStorageFactory,
|
|
5
|
+
CreateManagedWriteCheckpointsResult,
|
|
6
|
+
ManagedWriteCheckpointOptions,
|
|
7
|
+
SyncRulesBucketStorage
|
|
8
|
+
} from '../storage/storage-index.js';
|
|
9
|
+
|
|
10
|
+
// Keep up to three source-head/storage batches executing concurrently under load.
|
|
11
|
+
// There is intentionally no explicit batch-size cap here: the HTTP request queue
|
|
12
|
+
// already bounds how many requests can be waiting in this process.
|
|
13
|
+
// Smaller values here have better efficiency, while larger numbers here may give slight improvements in latency.
|
|
14
|
+
// We want to limit the number of database connections in use for write checkpoints, so we keep the numbers low here.
|
|
15
|
+
const MAX_IN_FLIGHT_WRITE_CHECKPOINT_BATCHES = 3;
|
|
16
|
+
|
|
17
|
+
export interface CreateWriteCheckpointResult {
|
|
18
|
+
writeCheckpoint: string;
|
|
19
|
+
replicationHead: string;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
interface QueuedWriteCheckpoint {
|
|
23
|
+
userId: string;
|
|
24
|
+
checkpointRequestId: bigint | undefined;
|
|
25
|
+
resolvers: PromiseWithResolvers<CreateWriteCheckpointResult>;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export class WriteCheckpointBatcher {
|
|
29
|
+
private pending: QueuedWriteCheckpoint[] = [];
|
|
30
|
+
private inFlight = 0;
|
|
31
|
+
private scheduledPump: NodeJS.Timeout | undefined;
|
|
32
|
+
|
|
33
|
+
constructor(
|
|
34
|
+
private readonly getAPI: () => RouteAPI,
|
|
35
|
+
private readonly getStorage: () => BucketStorageFactory
|
|
36
|
+
) {}
|
|
37
|
+
|
|
38
|
+
enqueue(userId: string, checkpointRequestId?: bigint): Promise<CreateWriteCheckpointResult> {
|
|
39
|
+
const resolvers = Promise.withResolvers<CreateWriteCheckpointResult>();
|
|
40
|
+
this.pending.push({ userId, checkpointRequestId, resolvers });
|
|
41
|
+
this.schedulePump();
|
|
42
|
+
return resolvers.promise;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
private schedulePump() {
|
|
46
|
+
if (this.scheduledPump != null) {
|
|
47
|
+
return;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
// Delay dispatch by one timer turn so requests arriving together can share
|
|
51
|
+
// one source head and one storage write batch.
|
|
52
|
+
// While normally we wouldn't have http requests arriving at the exact same time, this is relevant when
|
|
53
|
+
// requests are queued by fastify: All current requests are typically batched together, and finish at the
|
|
54
|
+
// same time. Then we get many requests from the queue arriving at the same time. Without this delayed dispatch,
|
|
55
|
+
// the first request would form its own batch, and the rest will then wait for a new batch to become available.
|
|
56
|
+
// The delayed dispatch allow all those requests to be batched together.
|
|
57
|
+
this.scheduledPump = setTimeout(() => {
|
|
58
|
+
this.scheduledPump = undefined;
|
|
59
|
+
this.pump();
|
|
60
|
+
}, 0);
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
private pump() {
|
|
64
|
+
while (this.inFlight < MAX_IN_FLIGHT_WRITE_CHECKPOINT_BATCHES && this.pending.length > 0) {
|
|
65
|
+
const batch = this.pending;
|
|
66
|
+
this.pending = [];
|
|
67
|
+
this.inFlight++;
|
|
68
|
+
|
|
69
|
+
void this.executeBatch(batch).finally(() => {
|
|
70
|
+
this.inFlight--;
|
|
71
|
+
this.schedulePump();
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
private async executeBatch(batch: QueuedWriteCheckpoint[]) {
|
|
77
|
+
try {
|
|
78
|
+
const syncBucketStorage = (await this.getStorage().getActiveSyncConfig())?.storage;
|
|
79
|
+
if (!syncBucketStorage) {
|
|
80
|
+
throw new ServiceError(ErrorCode.PSYNC_S2302, `Cannot create Write Checkpoint since no sync config is active.`);
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
// The source adapter reads the head, hands it to this callback to persist the
|
|
84
|
+
// write-checkpoint mapping, then forces a source marker only when storage
|
|
85
|
+
// reports an advance. Keeping the marker inside the callback means it is
|
|
86
|
+
// causally ordered after the head within a single source session.
|
|
87
|
+
const { writeCheckpoints, currentCheckpoint } = await this.getAPI().createReplicationHead(
|
|
88
|
+
async (currentCheckpoint) => {
|
|
89
|
+
const { writeCheckpoints, shouldAdvance } = await this.createBatchWriteCheckpoints(
|
|
90
|
+
syncBucketStorage,
|
|
91
|
+
batch,
|
|
92
|
+
currentCheckpoint
|
|
93
|
+
);
|
|
94
|
+
return { response: { writeCheckpoints, currentCheckpoint }, shouldAdvance };
|
|
95
|
+
}
|
|
96
|
+
);
|
|
97
|
+
|
|
98
|
+
const results = batch.map((request) => {
|
|
99
|
+
const writeCheckpoint = writeCheckpoints.get(request.userId);
|
|
100
|
+
if (writeCheckpoint == null) {
|
|
101
|
+
throw new ServiceAssertionError(
|
|
102
|
+
`Write checkpoint storage did not return a checkpoint for user ${request.userId}`
|
|
103
|
+
);
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
return { request, writeCheckpoint };
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
for (const { request, writeCheckpoint } of results) {
|
|
110
|
+
request.resolvers.resolve({
|
|
111
|
+
writeCheckpoint: String(writeCheckpoint),
|
|
112
|
+
replicationHead: currentCheckpoint
|
|
113
|
+
});
|
|
114
|
+
}
|
|
115
|
+
} catch (error) {
|
|
116
|
+
// Do not retry here. The route should pass failures through so clients can
|
|
117
|
+
// apply their normal write-checkpoint retry policy.
|
|
118
|
+
for (const request of batch) {
|
|
119
|
+
request.resolvers.reject(error);
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
private async createBatchWriteCheckpoints(
|
|
125
|
+
syncBucketStorage: SyncRulesBucketStorage,
|
|
126
|
+
batch: QueuedWriteCheckpoint[],
|
|
127
|
+
currentCheckpoint: string
|
|
128
|
+
): Promise<CreateManagedWriteCheckpointsResult> {
|
|
129
|
+
return syncBucketStorage.createManagedWriteCheckpoints(
|
|
130
|
+
batch.map((request) => {
|
|
131
|
+
const checkpoint: ManagedWriteCheckpointOptions = {
|
|
132
|
+
user_id: request.userId,
|
|
133
|
+
heads: { '1': currentCheckpoint }
|
|
134
|
+
};
|
|
135
|
+
if (request.checkpointRequestId != null) {
|
|
136
|
+
checkpoint.checkpoint_request_id = request.checkpointRequestId;
|
|
137
|
+
}
|
|
138
|
+
return checkpoint;
|
|
139
|
+
})
|
|
140
|
+
);
|
|
141
|
+
}
|
|
142
|
+
}
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
import { AbstractReplicationJob } from '@/replication/AbstractReplicationJob.js';
|
|
2
|
+
import { AbstractReplicator, AbstractReplicatorOptions, CreateJobOptions } from '@/replication/AbstractReplicator.js';
|
|
3
|
+
import { PersistedReplicationStream } from '@/storage/PersistedReplicationStream.js';
|
|
4
|
+
import { SyncRulesBucketStorage } from '@/storage/SyncRulesBucketStorage.js';
|
|
5
|
+
import { describe, expect, it, vi } from 'vitest';
|
|
6
|
+
|
|
7
|
+
class TestReplicator extends AbstractReplicator {
|
|
8
|
+
constructor(
|
|
9
|
+
private readonly cleanup: (storage: SyncRulesBucketStorage) => Promise<void>,
|
|
10
|
+
options: AbstractReplicatorOptions = {
|
|
11
|
+
id: 'test',
|
|
12
|
+
storageEngine: {} as AbstractReplicatorOptions['storageEngine'],
|
|
13
|
+
metricsEngine: {} as AbstractReplicatorOptions['metricsEngine'],
|
|
14
|
+
syncRuleProvider: {} as AbstractReplicatorOptions['syncRuleProvider'],
|
|
15
|
+
rateLimiter: {} as AbstractReplicatorOptions['rateLimiter']
|
|
16
|
+
}
|
|
17
|
+
) {
|
|
18
|
+
super(options);
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
createJob(_options: CreateJobOptions): AbstractReplicationJob {
|
|
22
|
+
throw new Error('Not implemented');
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
cleanUp(storage: SyncRulesBucketStorage): Promise<void> {
|
|
26
|
+
return this.cleanup(storage);
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
async testConnection() {
|
|
30
|
+
return { connectionDescription: 'test' };
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
terminateStoppedStream(
|
|
34
|
+
replicationStream: PersistedReplicationStream,
|
|
35
|
+
syncRuleStorage: SyncRulesBucketStorage
|
|
36
|
+
): Promise<void> {
|
|
37
|
+
return this.terminateStoppedReplicationStream(replicationStream, syncRuleStorage);
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
addClearingJob(replicationStreamId: number, promise: Promise<void>): void {
|
|
41
|
+
this.clearingJobs.set(replicationStreamId, promise);
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
get heartbeatIntervalNanosForTest(): bigint | null {
|
|
45
|
+
return (this as any).heartbeatIntervalNanos;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
shouldHandleStreamForTest(
|
|
49
|
+
replicationStream: PersistedReplicationStream,
|
|
50
|
+
loadedSyncRules: string | undefined,
|
|
51
|
+
loadedVersionLabel: string | undefined
|
|
52
|
+
): boolean {
|
|
53
|
+
return (this as any).shouldHandleReplicationStream(replicationStream, loadedSyncRules, loadedVersionLabel);
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
describe('AbstractReplicator heartbeat interval', () => {
|
|
58
|
+
const options: AbstractReplicatorOptions = {
|
|
59
|
+
id: 'test',
|
|
60
|
+
storageEngine: {} as AbstractReplicatorOptions['storageEngine'],
|
|
61
|
+
metricsEngine: {} as AbstractReplicatorOptions['metricsEngine'],
|
|
62
|
+
syncRuleProvider: {} as AbstractReplicatorOptions['syncRuleProvider'],
|
|
63
|
+
rateLimiter: {} as AbstractReplicatorOptions['rateLimiter']
|
|
64
|
+
};
|
|
65
|
+
|
|
66
|
+
it.each([undefined, null])('uses the default for %s', (heartbeatIntervalSeconds) => {
|
|
67
|
+
const replicator = new TestReplicator(async () => {}, { ...options, heartbeatIntervalSeconds });
|
|
68
|
+
|
|
69
|
+
expect(replicator.heartbeatIntervalNanosForTest).toBe(60_000_000_000n);
|
|
70
|
+
});
|
|
71
|
+
|
|
72
|
+
it('disables the heartbeat interval with 0', () => {
|
|
73
|
+
const replicator = new TestReplicator(async () => {}, { ...options, heartbeatIntervalSeconds: 0 });
|
|
74
|
+
|
|
75
|
+
expect(replicator.heartbeatIntervalNanosForTest).toBeNull();
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
it('converts a positive heartbeat interval to nanoseconds', () => {
|
|
79
|
+
const replicator = new TestReplicator(async () => {}, { ...options, heartbeatIntervalSeconds: 5 });
|
|
80
|
+
|
|
81
|
+
expect(replicator.heartbeatIntervalNanosForTest).toBe(5_000_000_000n);
|
|
82
|
+
});
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
describe('AbstractReplicator stopped stream cleanup', () => {
|
|
86
|
+
it('holds the replication stream lock across source and storage cleanup', async () => {
|
|
87
|
+
const calls: string[] = [];
|
|
88
|
+
const release = vi.fn(async () => {
|
|
89
|
+
calls.push('release');
|
|
90
|
+
});
|
|
91
|
+
const replicationStream = {
|
|
92
|
+
async lock() {
|
|
93
|
+
calls.push('lock');
|
|
94
|
+
return { sync_rules_id: 1, release };
|
|
95
|
+
}
|
|
96
|
+
} as unknown as PersistedReplicationStream;
|
|
97
|
+
const syncRuleStorage = {
|
|
98
|
+
logger: { info: vi.fn() },
|
|
99
|
+
async terminate() {
|
|
100
|
+
calls.push('terminate');
|
|
101
|
+
}
|
|
102
|
+
} as unknown as SyncRulesBucketStorage;
|
|
103
|
+
const replicator = new TestReplicator(async () => {
|
|
104
|
+
calls.push('cleanup');
|
|
105
|
+
});
|
|
106
|
+
|
|
107
|
+
await replicator.terminateStoppedStream(replicationStream, syncRuleStorage);
|
|
108
|
+
|
|
109
|
+
expect(calls).toEqual(['lock', 'cleanup', 'terminate', 'release']);
|
|
110
|
+
expect(release).toHaveBeenCalledOnce();
|
|
111
|
+
});
|
|
112
|
+
|
|
113
|
+
it('releases the replication stream lock when cleanup fails', async () => {
|
|
114
|
+
const cleanupError = new Error('cleanup failed');
|
|
115
|
+
const release = vi.fn(async () => {});
|
|
116
|
+
const replicationStream = {
|
|
117
|
+
async lock() {
|
|
118
|
+
return { sync_rules_id: 1, release };
|
|
119
|
+
}
|
|
120
|
+
} as unknown as PersistedReplicationStream;
|
|
121
|
+
const terminate = vi.fn(async () => {});
|
|
122
|
+
const syncRuleStorage = {
|
|
123
|
+
logger: { info: vi.fn() },
|
|
124
|
+
terminate
|
|
125
|
+
} as unknown as SyncRulesBucketStorage;
|
|
126
|
+
const replicator = new TestReplicator(async () => {
|
|
127
|
+
throw cleanupError;
|
|
128
|
+
});
|
|
129
|
+
|
|
130
|
+
await expect(replicator.terminateStoppedStream(replicationStream, syncRuleStorage)).rejects.toBe(cleanupError);
|
|
131
|
+
|
|
132
|
+
expect(terminate).not.toHaveBeenCalled();
|
|
133
|
+
expect(release).toHaveBeenCalledOnce();
|
|
134
|
+
});
|
|
135
|
+
|
|
136
|
+
it('waits for stopped stream cleanup when stopping', async () => {
|
|
137
|
+
let finishCleanup: () => void;
|
|
138
|
+
const cleanup = new Promise<void>((resolve) => {
|
|
139
|
+
finishCleanup = resolve;
|
|
140
|
+
});
|
|
141
|
+
const replicator = new TestReplicator(async () => {});
|
|
142
|
+
replicator.addClearingJob(1, cleanup);
|
|
143
|
+
|
|
144
|
+
let stopped = false;
|
|
145
|
+
const stop = replicator.stop().then(() => {
|
|
146
|
+
stopped = true;
|
|
147
|
+
});
|
|
148
|
+
await Promise.resolve();
|
|
149
|
+
expect(stopped).toBe(false);
|
|
150
|
+
|
|
151
|
+
finishCleanup!();
|
|
152
|
+
await stop;
|
|
153
|
+
expect(stopped).toBe(true);
|
|
154
|
+
});
|
|
155
|
+
});
|
|
156
|
+
|
|
157
|
+
describe('AbstractReplicator rolling deployment config matching', () => {
|
|
158
|
+
const processingStream = (version_label: string | undefined): PersistedReplicationStream =>
|
|
159
|
+
({
|
|
160
|
+
syncConfigContent: [
|
|
161
|
+
{
|
|
162
|
+
syncConfigState: 'PROCESSING',
|
|
163
|
+
sync_rules_content: 'bucket_definitions: {}',
|
|
164
|
+
version_label
|
|
165
|
+
}
|
|
166
|
+
]
|
|
167
|
+
}) as unknown as PersistedReplicationStream;
|
|
168
|
+
|
|
169
|
+
it('requires both YAML and version label to match a processing config', () => {
|
|
170
|
+
const replicator = new TestReplicator(async () => {});
|
|
171
|
+
const stream = processingStream('v2');
|
|
172
|
+
|
|
173
|
+
expect(replicator.shouldHandleStreamForTest(stream, 'bucket_definitions: {}', 'v2')).toBe(true);
|
|
174
|
+
expect(replicator.shouldHandleStreamForTest(stream, 'bucket_definitions: {}', 'v1')).toBe(false);
|
|
175
|
+
expect(replicator.shouldHandleStreamForTest(stream, 'bucket_definitions: {}', undefined)).toBe(false);
|
|
176
|
+
});
|
|
177
|
+
});
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
import { MetricAttributeFilteringExporter } from '@/metrics/open-telemetry/MetricAttributeFilteringExporter.js';
|
|
2
|
+
import { Attributes } from '@opentelemetry/api';
|
|
3
|
+
import { ExportResult } from '@opentelemetry/core';
|
|
4
|
+
import { DataPointType, PushMetricExporter, ResourceMetrics } from '@opentelemetry/sdk-metrics';
|
|
5
|
+
import { describe, expect, it, vi } from 'vitest';
|
|
6
|
+
|
|
7
|
+
type ExportMetrics = (metrics: ResourceMetrics, resultCallback: (result: ExportResult) => void) => void;
|
|
8
|
+
|
|
9
|
+
describe('MetricAttributeFilteringExporter', () => {
|
|
10
|
+
it('filters configured attributes without changing the collected metrics', () => {
|
|
11
|
+
const exportMetrics = vi.fn<ExportMetrics>();
|
|
12
|
+
const delegate = {
|
|
13
|
+
export: exportMetrics,
|
|
14
|
+
forceFlush: vi.fn(async () => {}),
|
|
15
|
+
shutdown: vi.fn(async () => {})
|
|
16
|
+
} satisfies PushMetricExporter;
|
|
17
|
+
const exporter = new MetricAttributeFilteringExporter(delegate, new Set(['version_label']));
|
|
18
|
+
const attributes: Attributes = {
|
|
19
|
+
sync_config_id: 'config-a',
|
|
20
|
+
sync_config_state: 'ACTIVE',
|
|
21
|
+
version_label: 'customer-version'
|
|
22
|
+
};
|
|
23
|
+
const metrics = {
|
|
24
|
+
resource: {},
|
|
25
|
+
scopeMetrics: [
|
|
26
|
+
{
|
|
27
|
+
scope: {},
|
|
28
|
+
metrics: [
|
|
29
|
+
{
|
|
30
|
+
dataPointType: DataPointType.GAUGE,
|
|
31
|
+
dataPoints: [{ attributes, value: 100 }]
|
|
32
|
+
}
|
|
33
|
+
]
|
|
34
|
+
}
|
|
35
|
+
]
|
|
36
|
+
} as unknown as ResourceMetrics;
|
|
37
|
+
const callback = vi.fn();
|
|
38
|
+
|
|
39
|
+
exporter.export(metrics, callback);
|
|
40
|
+
|
|
41
|
+
expect(exportMetrics).toHaveBeenCalledWith(
|
|
42
|
+
expect.objectContaining({
|
|
43
|
+
scopeMetrics: [
|
|
44
|
+
expect.objectContaining({
|
|
45
|
+
metrics: [
|
|
46
|
+
expect.objectContaining({
|
|
47
|
+
dataPoints: [
|
|
48
|
+
expect.objectContaining({
|
|
49
|
+
attributes: { sync_config_id: 'config-a', sync_config_state: 'ACTIVE' }
|
|
50
|
+
})
|
|
51
|
+
]
|
|
52
|
+
})
|
|
53
|
+
]
|
|
54
|
+
})
|
|
55
|
+
]
|
|
56
|
+
}),
|
|
57
|
+
callback
|
|
58
|
+
);
|
|
59
|
+
expect(metrics.scopeMetrics[0].metrics[0].dataPoints[0].attributes).toBe(attributes);
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
it('forwards rather than merges data points whose attributes become identical', () => {
|
|
63
|
+
const exportMetrics = vi.fn<ExportMetrics>();
|
|
64
|
+
const delegate = {
|
|
65
|
+
export: exportMetrics,
|
|
66
|
+
forceFlush: vi.fn(async () => {}),
|
|
67
|
+
shutdown: vi.fn(async () => {})
|
|
68
|
+
} satisfies PushMetricExporter;
|
|
69
|
+
const exporter = new MetricAttributeFilteringExporter(delegate, new Set(['filtered']));
|
|
70
|
+
const metrics = {
|
|
71
|
+
resource: {},
|
|
72
|
+
scopeMetrics: [
|
|
73
|
+
{
|
|
74
|
+
scope: {},
|
|
75
|
+
metrics: [
|
|
76
|
+
{
|
|
77
|
+
dataPointType: DataPointType.GAUGE,
|
|
78
|
+
dataPoints: [
|
|
79
|
+
{ attributes: { retained: 'same', filtered: 'a' }, value: 10 },
|
|
80
|
+
{ attributes: { retained: 'same', filtered: 'b' }, value: 20 }
|
|
81
|
+
]
|
|
82
|
+
}
|
|
83
|
+
]
|
|
84
|
+
}
|
|
85
|
+
]
|
|
86
|
+
} as unknown as ResourceMetrics;
|
|
87
|
+
|
|
88
|
+
exporter.export(metrics, vi.fn());
|
|
89
|
+
|
|
90
|
+
const exported = exportMetrics.mock.calls[0][0];
|
|
91
|
+
expect(exported.scopeMetrics[0].metrics[0].dataPoints).toMatchObject([
|
|
92
|
+
{ attributes: { retained: 'same' }, value: 10 },
|
|
93
|
+
{ attributes: { retained: 'same' }, value: 20 }
|
|
94
|
+
]);
|
|
95
|
+
});
|
|
96
|
+
});
|