@powersync/service-core 0.0.0-dev-20260602073133 → 0.0.0-dev-20260909133214
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +152 -4
- package/dist/api/RouteAPI.d.ts +42 -14
- package/dist/api/diagnostics.d.ts +12 -1
- package/dist/api/diagnostics.js +36 -22
- package/dist/api/diagnostics.js.map +1 -1
- package/dist/entry/commands/compact-action.js +30 -22
- package/dist/entry/commands/compact-action.js.map +1 -1
- package/dist/metrics/metrics-interfaces.d.ts +5 -1
- package/dist/metrics/metrics-interfaces.js.map +1 -1
- package/dist/metrics/open-telemetry/MetricAttributeFilteringExporter.d.ts +36 -0
- package/dist/metrics/open-telemetry/MetricAttributeFilteringExporter.js +70 -0
- package/dist/metrics/open-telemetry/MetricAttributeFilteringExporter.js.map +1 -0
- package/dist/metrics/open-telemetry/OpenTelemetryMetricsFactory.js +6 -1
- package/dist/metrics/open-telemetry/OpenTelemetryMetricsFactory.js.map +1 -1
- package/dist/metrics/open-telemetry/util.js +7 -3
- package/dist/metrics/open-telemetry/util.js.map +1 -1
- package/dist/modules/AbstractModule.d.ts +6 -2
- package/dist/modules/AbstractModule.js.map +1 -1
- package/dist/replication/AbstractReplicator.d.ts +31 -7
- package/dist/replication/AbstractReplicator.js +168 -74
- package/dist/replication/AbstractReplicator.js.map +1 -1
- package/dist/replication/ReplicationModule.d.ts +3 -3
- package/dist/replication/ReplicationModule.js.map +1 -1
- package/dist/routes/RouterEngine.d.ts +1 -0
- package/dist/routes/RouterEngine.js +6 -1
- package/dist/routes/RouterEngine.js.map +1 -1
- package/dist/routes/configure-fastify.d.ts +104 -56
- package/dist/routes/configure-fastify.js +15 -2
- package/dist/routes/configure-fastify.js.map +1 -1
- package/dist/routes/endpoints/admin.d.ts +42 -24
- package/dist/routes/endpoints/admin.js +49 -36
- package/dist/routes/endpoints/admin.js.map +1 -1
- package/dist/routes/endpoints/checkpointing.d.ts +62 -0
- package/dist/routes/endpoints/checkpointing.js +65 -7
- package/dist/routes/endpoints/checkpointing.js.map +1 -1
- package/dist/routes/endpoints/socket-route.js +3 -2
- package/dist/routes/endpoints/socket-route.js.map +1 -1
- package/dist/routes/endpoints/sync-rules.js +25 -14
- package/dist/routes/endpoints/sync-rules.js.map +1 -1
- package/dist/routes/endpoints/sync-stream.js +1 -1
- package/dist/routes/endpoints/sync-stream.js.map +1 -1
- package/dist/runner/teardown.js +8 -8
- package/dist/runner/teardown.js.map +1 -1
- package/dist/storage/BucketStorageBatch.d.ts +64 -17
- package/dist/storage/BucketStorageBatch.js.map +1 -1
- package/dist/storage/BucketStorageFactory.d.ts +50 -31
- package/dist/storage/BucketStorageFactory.js +10 -20
- package/dist/storage/BucketStorageFactory.js.map +1 -1
- package/dist/storage/CheckpointChecksumInvalidatedError.d.ts +12 -0
- package/dist/storage/CheckpointChecksumInvalidatedError.js +17 -0
- package/dist/storage/CheckpointChecksumInvalidatedError.js.map +1 -0
- package/dist/storage/ChecksumCache.d.ts +11 -6
- package/dist/storage/ChecksumCache.js +5 -4
- package/dist/storage/ChecksumCache.js.map +1 -1
- package/dist/storage/ParsedSyncConfigSet.d.ts +23 -0
- package/dist/storage/ParsedSyncConfigSet.js +2 -0
- package/dist/storage/ParsedSyncConfigSet.js.map +1 -0
- package/dist/storage/PersistedReplicationStream.d.ts +32 -0
- package/dist/storage/PersistedReplicationStream.js +26 -0
- package/dist/storage/PersistedReplicationStream.js.map +1 -0
- package/dist/storage/PersistedSyncConfigContent.d.ts +65 -0
- package/dist/storage/PersistedSyncConfigContent.js +138 -0
- package/dist/storage/PersistedSyncConfigContent.js.map +1 -0
- package/dist/storage/PersistedSyncConfigStatus.d.ts +13 -0
- package/dist/storage/PersistedSyncConfigStatus.js +2 -0
- package/dist/storage/PersistedSyncConfigStatus.js.map +1 -0
- package/dist/storage/ReplicationEventPayload.d.ts +6 -1
- package/dist/storage/SourceEntity.d.ts +13 -2
- package/dist/storage/SourceTable.d.ts +53 -4
- package/dist/storage/SourceTable.js +62 -7
- package/dist/storage/SourceTable.js.map +1 -1
- package/dist/storage/SourceTableReconciler.d.ts +83 -0
- package/dist/storage/SourceTableReconciler.js +95 -0
- package/dist/storage/SourceTableReconciler.js.map +1 -0
- package/dist/storage/StorageVersionConfig.d.ts +4 -0
- package/dist/storage/StorageVersionConfig.js +10 -1
- package/dist/storage/StorageVersionConfig.js.map +1 -1
- package/dist/storage/SyncRulesBucketStorage.d.ts +162 -26
- package/dist/storage/SyncRulesBucketStorage.js +3 -0
- package/dist/storage/SyncRulesBucketStorage.js.map +1 -1
- package/dist/storage/WriteCheckpointAPI.d.ts +85 -6
- package/dist/storage/WriteCheckpointAPI.js +33 -0
- package/dist/storage/WriteCheckpointAPI.js.map +1 -1
- package/dist/storage/implementation/BucketDefinitionMapping.d.ts +133 -0
- package/dist/storage/implementation/BucketDefinitionMapping.js +351 -0
- package/dist/storage/implementation/BucketDefinitionMapping.js.map +1 -0
- package/dist/storage/implementation/IncrementalReprocessingCompatibility.d.ts +6 -0
- package/dist/storage/implementation/IncrementalReprocessingCompatibility.js +31 -0
- package/dist/storage/implementation/IncrementalReprocessingCompatibility.js.map +1 -0
- package/dist/storage/implementation/IncrementalReprocessingSyncConfigLog.d.ts +17 -0
- package/dist/storage/implementation/IncrementalReprocessingSyncConfigLog.js +70 -0
- package/dist/storage/implementation/IncrementalReprocessingSyncConfigLog.js.map +1 -0
- package/dist/storage/implementation/ReplicationStreamStorageIds.d.ts +15 -0
- package/dist/storage/implementation/ReplicationStreamStorageIds.js +20 -0
- package/dist/storage/implementation/ReplicationStreamStorageIds.js.map +1 -0
- package/dist/storage/storage-index.d.ts +10 -1
- package/dist/storage/storage-index.js +10 -1
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/storage/storage-metrics.js +120 -4
- package/dist/storage/storage-metrics.js.map +1 -1
- package/dist/sync/BucketChecksumState.d.ts +12 -2
- package/dist/sync/BucketChecksumState.js +69 -21
- package/dist/sync/BucketChecksumState.js.map +1 -1
- package/dist/sync/RequestTracker.d.ts +7 -0
- package/dist/sync/RequestTracker.js +12 -0
- package/dist/sync/RequestTracker.js.map +1 -1
- package/dist/sync/sync.js +184 -43
- package/dist/sync/sync.js.map +1 -1
- package/dist/sync/util.d.ts +5 -0
- package/dist/sync/util.js +19 -1
- package/dist/sync/util.js.map +1 -1
- package/dist/system/ServiceContext.d.ts +2 -0
- package/dist/system/ServiceContext.js +2 -0
- package/dist/system/ServiceContext.js.map +1 -1
- package/dist/tracing/PerformanceTracer.d.ts +23 -1
- package/dist/tracing/PerformanceTracer.js +11 -0
- package/dist/tracing/PerformanceTracer.js.map +1 -1
- package/dist/util/checkpointing.d.ts +4 -8
- package/dist/util/checkpointing.js +1 -16
- package/dist/util/checkpointing.js.map +1 -1
- package/dist/util/config/compound-config-collector.js +23 -2
- package/dist/util/config/compound-config-collector.js.map +1 -1
- package/dist/util/config/defaults.d.ts +2 -0
- package/dist/util/config/defaults.js +2 -0
- package/dist/util/config/defaults.js.map +1 -1
- package/dist/util/config/sync-rules/impl/base64-sync-rules-collector.js +1 -0
- package/dist/util/config/sync-rules/impl/base64-sync-rules-collector.js.map +1 -1
- package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js +1 -0
- package/dist/util/config/sync-rules/impl/filesystem-sync-rules-collector.js.map +1 -1
- package/dist/util/config/sync-rules/sync-rules-provider.d.ts +2 -0
- package/dist/util/config/sync-rules/sync-rules-provider.js +3 -0
- package/dist/util/config/sync-rules/sync-rules-provider.js.map +1 -1
- package/dist/util/config/types.d.ts +4 -1
- package/dist/util/lsn.d.ts +4 -0
- package/dist/util/lsn.js +10 -0
- package/dist/util/lsn.js.map +1 -1
- package/dist/util/protocol-types.d.ts +4 -4
- package/dist/util/protocol-types.js +3 -2
- package/dist/util/protocol-types.js.map +1 -1
- package/dist/util/util-index.d.ts +1 -0
- package/dist/util/util-index.js +1 -0
- package/dist/util/util-index.js.map +1 -1
- package/dist/util/utils.d.ts +3 -2
- package/dist/util/utils.js +17 -2
- package/dist/util/utils.js.map +1 -1
- package/dist/util/write-checkpoint-batcher.d.ts +19 -0
- package/dist/util/write-checkpoint-batcher.js +100 -0
- package/dist/util/write-checkpoint-batcher.js.map +1 -0
- package/package.json +7 -7
- package/src/api/RouteAPI.ts +43 -14
- package/src/api/diagnostics.ts +45 -26
- package/src/entry/commands/compact-action.ts +37 -23
- package/src/metrics/metrics-interfaces.ts +6 -1
- package/src/metrics/open-telemetry/MetricAttributeFilteringExporter.ts +85 -0
- package/src/metrics/open-telemetry/OpenTelemetryMetricsFactory.ts +7 -2
- package/src/metrics/open-telemetry/util.ts +7 -3
- package/src/modules/AbstractModule.ts +6 -2
- package/src/replication/AbstractReplicator.ts +212 -78
- package/src/replication/ReplicationModule.ts +2 -3
- package/src/routes/RouterEngine.ts +7 -1
- package/src/routes/configure-fastify.ts +25 -2
- package/src/routes/endpoints/admin.ts +75 -51
- package/src/routes/endpoints/checkpointing.ts +79 -8
- package/src/routes/endpoints/socket-route.ts +3 -2
- package/src/routes/endpoints/sync-rules.ts +26 -14
- package/src/routes/endpoints/sync-stream.ts +1 -1
- package/src/runner/teardown.ts +8 -8
- package/src/storage/BucketStorageBatch.ts +65 -18
- package/src/storage/BucketStorageFactory.ts +73 -46
- package/src/storage/CheckpointChecksumInvalidatedError.ts +17 -0
- package/src/storage/ChecksumCache.ts +27 -14
- package/src/storage/ParsedSyncConfigSet.ts +25 -0
- package/src/storage/PersistedReplicationStream.ts +55 -0
- package/src/storage/PersistedSyncConfigContent.ts +207 -0
- package/src/storage/PersistedSyncConfigStatus.ts +14 -0
- package/src/storage/ReplicationEventPayload.ts +6 -1
- package/src/storage/SourceEntity.ts +12 -2
- package/src/storage/SourceTable.ts +96 -8
- package/src/storage/SourceTableReconciler.ts +188 -0
- package/src/storage/StorageVersionConfig.ts +11 -1
- package/src/storage/SyncRulesBucketStorage.ts +189 -27
- package/src/storage/WriteCheckpointAPI.ts +140 -6
- package/src/storage/implementation/BucketDefinitionMapping.ts +539 -0
- package/src/storage/implementation/IncrementalReprocessingCompatibility.ts +41 -0
- package/src/storage/implementation/IncrementalReprocessingSyncConfigLog.ts +126 -0
- package/src/storage/implementation/ReplicationStreamStorageIds.ts +21 -0
- package/src/storage/storage-index.ts +11 -1
- package/src/storage/storage-metrics.ts +154 -8
- package/src/sync/BucketChecksumState.ts +105 -21
- package/src/sync/RequestTracker.ts +20 -0
- package/src/sync/sync.ts +214 -50
- package/src/sync/util.ts +33 -8
- package/src/system/ServiceContext.ts +7 -0
- package/src/tracing/PerformanceTracer.ts +41 -1
- package/src/util/checkpointing.ts +4 -22
- package/src/util/config/compound-config-collector.ts +31 -1
- package/src/util/config/defaults.ts +2 -0
- package/src/util/config/sync-rules/impl/base64-sync-rules-collector.ts +1 -0
- package/src/util/config/sync-rules/impl/filesystem-sync-rules-collector.ts +1 -0
- package/src/util/config/sync-rules/sync-rules-provider.ts +5 -0
- package/src/util/config/types.ts +4 -1
- package/src/util/lsn.ts +9 -0
- package/src/util/protocol-types.ts +3 -2
- package/src/util/util-index.ts +1 -0
- package/src/util/utils.ts +16 -2
- package/src/util/write-checkpoint-batcher.ts +142 -0
- package/test/src/AbstractReplicator.test.ts +177 -0
- package/test/src/MetricAttributeFilteringExporter.test.ts +96 -0
- package/test/src/PersistedSyncConfigContent.test.ts +152 -0
- package/test/src/checksum_cache.test.ts +22 -0
- package/test/src/config.test.ts +142 -0
- package/test/src/diagnostics.test.ts +87 -34
- package/test/src/module-loader.test.ts +33 -1
- package/test/src/routes/admin.test.ts +188 -18
- package/test/src/routes/checkpointing.test.ts +54 -0
- package/test/src/routes/mocks.ts +6 -2
- package/test/src/routes/stream.test.ts +123 -2
- package/test/src/source-table-reconciler.test.ts +244 -0
- package/test/src/storage/SourceTable.test.ts +92 -0
- package/test/src/storage-metrics.test.ts +148 -0
- package/test/src/sync/BucketChecksumState.test.ts +98 -12
- package/test/src/sync/util.test.ts +28 -1
- package/test/src/util/checkpointing.test.ts +249 -0
- package/test/src/util/utils.test.ts +23 -0
- package/tsconfig.json +1 -4
- package/tsconfig.tsbuildinfo +1 -1
- package/dist/storage/PersistedSyncRulesContent.d.ts +0 -59
- package/dist/storage/PersistedSyncRulesContent.js +0 -100
- package/dist/storage/PersistedSyncRulesContent.js.map +0 -1
- package/src/storage/PersistedSyncRulesContent.ts +0 -174
package/CHANGELOG.md
CHANGED
|
@@ -1,15 +1,163 @@
|
|
|
1
1
|
# @powersync/service-core
|
|
2
2
|
|
|
3
|
-
## 0.0.0-dev-
|
|
3
|
+
## 0.0.0-dev-20260909133214
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- Updated dependencies [70b00a4]
|
|
8
|
+
- @powersync/service-sync-rules@0.0.0-dev-20260909133214
|
|
9
|
+
- @powersync/lib-services-framework@0.0.0-dev-20260909133214
|
|
10
|
+
- @powersync/service-rsocket-router@0.0.0-dev-20260909133214
|
|
11
|
+
|
|
12
|
+
## 1.26.0
|
|
13
|
+
|
|
14
|
+
### Minor Changes
|
|
15
|
+
|
|
16
|
+
- 189bd9e: Add storage_version: 4 as a stable storage version.
|
|
17
|
+
- bb08068: Compile event payload queries into serialized sync plans and evaluate them through the shared row-projection implementation. Existing event deployments use one basic, direct-projection query per physical table; filters and other advanced SQL forms are not currently documented or recommended.
|
|
18
|
+
|
|
19
|
+
Edition 3 now validates and applies event `WHERE` clauses. Existing persisted plans and editions 1 and 2 retain the legacy behavior of ignoring filters and emit a warning; deploy a new sync config to enable compiled filtering. Remove event `WHERE` clauses before rolling the service back to an older version.
|
|
20
|
+
|
|
21
|
+
Event evaluation now follows sync stream semantics by returning all payloads from matching queries, including overlapping exact and wildcard sources, or no payloads when filters do not match. Payload query order does not affect persisted equality. Compiled `SELECT *` payloads also include underscore-prefixed columns, matching sync streams.
|
|
22
|
+
|
|
23
|
+
- 332d649: [MongoDB Storage] Support incremental parameter compacting jobs.
|
|
24
|
+
- 33a3c23: Add optional version_label for sync configs.
|
|
25
|
+
- fccdff2: Add object storage metrics, and storage metrics split by sync config.
|
|
26
|
+
- 989d9e1: Scope MongoDB v3 custom write checkpoint records and reads to the stream-assigned event definition that created them. Custom checkpoint mode names the event whose id is resolved through the active sync config's persisted mapping, while event handlers copy the assigned id from replication event payloads when writing checkpoints. This keeps active and incrementally processing checkpoint events separate when their definitions change.
|
|
27
|
+
- 646c0de: Track replication events as MongoDB storage v3 source-table memberships so unchanged event definitions retain their assigned ids while new or changed definitions are resnapshotted during incremental reprocessing.
|
|
28
|
+
- 2407f71: Apply timeouts and abortSignal handling to all S3 operations. The broad timeouts can be configured using the new `storage.object_storage.defaults_mode` option, or the `AWS_DEFAULTS_MODE` environment variable. Upgrade S3 SDK to fix further timeout issues.
|
|
29
|
+
- a997c88: Restructure MongoDB V3 bucket compacting.
|
|
30
|
+
|
|
31
|
+
### Patch Changes
|
|
32
|
+
|
|
33
|
+
- e11c54b: Add S3 timing info for sync checkpoints.
|
|
34
|
+
- 0077a8d: Fix connection stall after data fetch errors.
|
|
35
|
+
- cc121b3: [MongoDB Storage V3] Force create new replication stream when the old one is invalidated, and fix other consistency issues with V3 replication stream updates.
|
|
36
|
+
- 4e4063b: Optimize runtime type checks, closes https://github.com/powersync-ja/powersync-service/issues/771.
|
|
37
|
+
- 10131ca: Log checkpoint_interrupted when a stream errors while fetching data.
|
|
38
|
+
- 4037e8f: Add storage.default_storage_version config option.
|
|
39
|
+
- dd7a22f: Add the sync config version label to attributed storage metrics while excluding it from shared OTLP telemetry.
|
|
40
|
+
- Updated dependencies [4b6a23e]
|
|
41
|
+
- Updated dependencies [189bd9e]
|
|
42
|
+
- Updated dependencies [bb08068]
|
|
43
|
+
- Updated dependencies [33a3c23]
|
|
44
|
+
- Updated dependencies [fccdff2]
|
|
45
|
+
- Updated dependencies [646c0de]
|
|
46
|
+
- Updated dependencies [4e4063b]
|
|
47
|
+
- Updated dependencies [4037e8f]
|
|
48
|
+
- Updated dependencies [3972bac]
|
|
49
|
+
- @powersync/service-sync-rules@0.41.0
|
|
50
|
+
- @powersync/service-types@0.18.0
|
|
51
|
+
- @powersync/lib-services-framework@0.10.1
|
|
52
|
+
- @powersync/service-rsocket-router@0.2.26
|
|
53
|
+
|
|
54
|
+
## 1.25.0
|
|
55
|
+
|
|
56
|
+
### Minor Changes
|
|
57
|
+
|
|
58
|
+
- 798d739: Add source-owned `SourceTable` reconciliation.
|
|
59
|
+
|
|
60
|
+
`resolveTables()` now queries all overlapping persisted candidates and passes them to a
|
|
61
|
+
source-provided reconciler that returns compatible and incompatible tables, can return
|
|
62
|
+
modified compatible copies, and supplies values used for potential new records. Storage persists allowlisted
|
|
63
|
+
source metadata differences but never interprets them. MongoDB v1/v3 and PostgreSQL storage were
|
|
64
|
+
refactored to this candidate-first model (PostgreSQL gains a nullable `source_metadata` JSONB column
|
|
65
|
+
via migration).
|
|
66
|
+
|
|
67
|
+
### Patch Changes
|
|
68
|
+
|
|
69
|
+
- 27b56cb: Optimization for generating subkey UUIDs.
|
|
70
|
+
|
|
71
|
+
## 1.24.0
|
|
72
|
+
|
|
73
|
+
### Minor Changes
|
|
74
|
+
|
|
75
|
+
- 087b61e: Configurable heartbeat_interval_seconds for MongoDB, Postgres, SQL Server.
|
|
76
|
+
- 2189250: Add `/sync/checkpoint-request` for client-supplied checkpoint request ids, previously called write checkpoint ids. The route returns the stored `checkpoint_request_id`, storage now treats managed request ids as monotonic per user/client, custom checkpoint request ids continue to use the existing `checkpoint` field for backwards compatibility, and `checkpoint_requested_at` metadata lets compact jobs remove expired request-derived checkpoint records.
|
|
77
|
+
|
|
78
|
+
This release includes storage migrations for the checkpoint request metadata. Self-hosters should run migrations as part of the upgrade.
|
|
79
|
+
|
|
80
|
+
- 922f974: Add experimental support for storing bucket data chunks on S3.
|
|
81
|
+
|
|
82
|
+
### Patch Changes
|
|
83
|
+
|
|
84
|
+
- c4860c9: Improve Postgres sync throughput by reading bucket data with ordered, per-bucket index range scans that stop once the requested batch is full.
|
|
85
|
+
- 483415d: Clarify whether too-many-buckets log breakdowns are grouped by sync stream or legacy bucket definition, and include up
|
|
86
|
+
to 100 entries.
|
|
87
|
+
- aab068b: [MongoDB Storage V3] Add has_clear_op field and simplify checksum queries.
|
|
88
|
+
- 37591e9: Lock replication slots before clearing data, preventing concurrent clearing.
|
|
89
|
+
- Updated dependencies [2189250]
|
|
90
|
+
- Updated dependencies [8daa300]
|
|
91
|
+
- Updated dependencies [df5663b]
|
|
92
|
+
- Updated dependencies [be42e25]
|
|
93
|
+
- Updated dependencies [cb4c627]
|
|
94
|
+
- @powersync/lib-services-framework@0.10.0
|
|
95
|
+
- @powersync/service-types@0.17.0
|
|
96
|
+
- @powersync/service-sync-rules@0.40.0
|
|
97
|
+
- @powersync/service-rsocket-router@0.2.25
|
|
98
|
+
|
|
99
|
+
## 1.23.3
|
|
100
|
+
|
|
101
|
+
### Patch Changes
|
|
102
|
+
|
|
103
|
+
- ea31f64: Potential fix and improved stack trace for 'evicted' errors.
|
|
104
|
+
- Updated dependencies [ea71bf3]
|
|
105
|
+
- Updated dependencies [edc6ed4]
|
|
106
|
+
- @powersync/service-sync-rules@0.39.0
|
|
107
|
+
- @powersync/lib-services-framework@0.9.8
|
|
108
|
+
- @powersync/service-rsocket-router@0.2.24
|
|
109
|
+
|
|
110
|
+
## 1.23.2
|
|
111
|
+
|
|
112
|
+
### Patch Changes
|
|
113
|
+
|
|
114
|
+
- 71d4a0a: Add logs for checkpoint_complete to assist with debugging. Remove "sync lock" logs.
|
|
115
|
+
- e4f683d: [MongoDB Storage] Add experimental option to allow reading data from secondaries.
|
|
116
|
+
- 71d4a0a: Fix edge case where the service can return incomplete data. This could happen when using bucket priorities, and a checkpoint is invalidated by the compact process while syncing.
|
|
117
|
+
- c2edf86: Add `api.parameters.bucket_count_cache_ttl_minutes` configuration option.
|
|
118
|
+
- df9ab1e: Batch concurrent write checkpoint requests and increase request limits to improve throughput.
|
|
119
|
+
- Updated dependencies [a6ae678]
|
|
120
|
+
- Updated dependencies [c2edf86]
|
|
121
|
+
- @powersync/service-sync-rules@0.38.1
|
|
122
|
+
- @powersync/service-types@0.16.1
|
|
123
|
+
- @powersync/lib-services-framework@0.9.7
|
|
124
|
+
- @powersync/service-rsocket-router@0.2.23
|
|
125
|
+
|
|
126
|
+
## 1.23.1
|
|
127
|
+
|
|
128
|
+
### Patch Changes
|
|
129
|
+
|
|
130
|
+
- 7e65360: Restructure replication locks for updated sync config.
|
|
131
|
+
|
|
132
|
+
## 1.23.0
|
|
133
|
+
|
|
134
|
+
### Minor Changes
|
|
135
|
+
|
|
136
|
+
- a91a08f: [Experimental] Enable incremental reprocessing for MongoDB source + MongoDB storage. This includes significant changes to the v3 storage format.
|
|
137
|
+
|
|
138
|
+
### Patch Changes
|
|
139
|
+
|
|
140
|
+
- Updated dependencies [a91a08f]
|
|
141
|
+
- Updated dependencies [184c39f]
|
|
142
|
+
- Updated dependencies [c3f75df]
|
|
143
|
+
- Updated dependencies [4bd35ea]
|
|
144
|
+
- @powersync/service-types@0.16.0
|
|
145
|
+
- @powersync/service-sync-rules@0.38.0
|
|
146
|
+
- @powersync/lib-services-framework@0.9.6
|
|
147
|
+
- @powersync/service-rsocket-router@0.2.22
|
|
148
|
+
|
|
149
|
+
## 1.22.0
|
|
4
150
|
|
|
5
151
|
### Minor Changes
|
|
6
152
|
|
|
7
153
|
- e2bf1ad: [Internal] rework resolveTables to handle multiple SourceTables.
|
|
8
154
|
- 92cc83b: Add the experimental `unstable_sqlite_expression_engine` sync config option to evaluate Sync Streams with SQLite.
|
|
9
155
|
- 15e2466: [MongoDB] Support snapshotting concurrently with streaming in storage v3+.
|
|
156
|
+
- ebeaa3b: [Postgres] Skip storing a redundant copy of replicated rows in `current_data` for tables with `REPLICA IDENTITY FULL`.
|
|
10
157
|
|
|
11
158
|
### Patch Changes
|
|
12
159
|
|
|
160
|
+
- 17fd96b: Fix duration logs for "Processing batch"
|
|
13
161
|
- 6e2a57e: Refactor HydratedSyncConfig to support multiple SyncConfigs.
|
|
14
162
|
- ec6df9f: Add a custom Fastify error handler so uncaught errors honour the negotiated `Content-Encoding` (gzip/zstd) instead of returning a header that doesn't match the body.
|
|
15
163
|
- 99d33d5: Normalize socket addresses to bare hostnames before IP-range validation, so direct-IP literals with any port form are recognized as IPs.
|
|
@@ -25,9 +173,9 @@
|
|
|
25
173
|
- Updated dependencies [92cc83b]
|
|
26
174
|
- Updated dependencies [0aab0f9]
|
|
27
175
|
- Updated dependencies [a94b6c3]
|
|
28
|
-
- @powersync/service-sync-rules@0.
|
|
29
|
-
- @powersync/lib-services-framework@0.
|
|
30
|
-
- @powersync/service-rsocket-router@0.
|
|
176
|
+
- @powersync/service-sync-rules@0.37.0
|
|
177
|
+
- @powersync/lib-services-framework@0.9.5
|
|
178
|
+
- @powersync/service-rsocket-router@0.2.21
|
|
31
179
|
|
|
32
180
|
## 1.21.0
|
|
33
181
|
|
package/dist/api/RouteAPI.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { SyncConfig, TablePattern } from '@powersync/service-sync-rules';
|
|
2
2
|
import * as types from '@powersync/service-types';
|
|
3
|
-
import {
|
|
3
|
+
import { ParseSyncConfigOptions, SyncRulesBucketStorage } from '../storage/storage-index.js';
|
|
4
4
|
export interface PatternResult {
|
|
5
5
|
schema: string;
|
|
6
6
|
pattern: string;
|
|
@@ -20,7 +20,8 @@ export interface SlotWalBudgetOptions {
|
|
|
20
20
|
slotName: string;
|
|
21
21
|
}
|
|
22
22
|
/**
|
|
23
|
-
*
|
|
23
|
+
* Source-specific capabilities used by admin routes, sync config validation,
|
|
24
|
+
* checkpointing, diagnostics, and the sync API.
|
|
24
25
|
*/
|
|
25
26
|
export interface RouteAPI {
|
|
26
27
|
/**
|
|
@@ -33,16 +34,17 @@ export interface RouteAPI {
|
|
|
33
34
|
*/
|
|
34
35
|
getConnectionStatus(): Promise<types.ConnectionStatusV2>;
|
|
35
36
|
/**
|
|
36
|
-
*
|
|
37
|
+
* Expand sync config table patterns into table metadata from the source connection.
|
|
38
|
+
*
|
|
39
|
+
* This is used by validation and diagnostics paths to explain what will be
|
|
40
|
+
* replicated for the current sync config.
|
|
37
41
|
*
|
|
38
42
|
* @param tablePatterns A set of table patterns which typically come from
|
|
39
43
|
* the tables listed in sync config definitions.
|
|
40
44
|
*
|
|
41
45
|
* @param sqlSyncRules
|
|
42
|
-
* @returns
|
|
43
|
-
*
|
|
44
|
-
* tables to ensure syncing should function according to the input
|
|
45
|
-
* pattern. Debug errors and warnings are reported per table.
|
|
46
|
+
* @returns Matching tables or collections, their columns or fields where available,
|
|
47
|
+
* and per-table warnings or errors for the requested table patterns.
|
|
46
48
|
*/
|
|
47
49
|
getDebugTablesInfo(tablePatterns: TablePattern[], sqlSyncRules: SyncConfig): Promise<PatternResult[]>;
|
|
48
50
|
/**
|
|
@@ -59,13 +61,26 @@ export interface RouteAPI {
|
|
|
59
61
|
/**
|
|
60
62
|
* Get the current LSN or equivalent replication HEAD position identifier.
|
|
61
63
|
*
|
|
62
|
-
* The position is provided to the callback
|
|
63
|
-
*
|
|
64
|
+
* The position is provided to the callback so the caller can persist its
|
|
65
|
+
* write-checkpoint mapping before the source adapter forces any required
|
|
66
|
+
* source-side marker or keepalive. The callback returns whether storage
|
|
67
|
+
* actually needs a source marker: when `shouldAdvance` is true the adapter
|
|
68
|
+
* must ensure that the replication stream will observe this position or a
|
|
69
|
+
* greater one, even when the source is otherwise idle. When it is false, the
|
|
70
|
+
* adapter may skip the source marker.
|
|
71
|
+
*
|
|
72
|
+
* Reading the head and forcing the marker happen within a single source
|
|
73
|
+
* session/connection where applicable, so the marker is causally ordered
|
|
74
|
+
* after the head that was handed to the callback.
|
|
64
75
|
*/
|
|
65
76
|
createReplicationHead<T>(callback: ReplicationHeadCallback<T>): Promise<T>;
|
|
66
77
|
/**
|
|
67
|
-
* @returns The
|
|
68
|
-
*
|
|
78
|
+
* @returns The connected source schema in service-friendly table and column
|
|
79
|
+
* formats. Strict-schema sources should include actual tables,
|
|
80
|
+
* collections, columns, and types. Schemaless sources should return
|
|
81
|
+
* the best metadata available and document validation limitations in
|
|
82
|
+
* their adapter behavior. This is typically used to validate sync
|
|
83
|
+
* config and generate client schemas.
|
|
69
84
|
*/
|
|
70
85
|
getConnectionSchema(): Promise<types.DatabaseSchema[]>;
|
|
71
86
|
/**
|
|
@@ -78,8 +93,21 @@ export interface RouteAPI {
|
|
|
78
93
|
*/
|
|
79
94
|
shutdown(): Promise<void>;
|
|
80
95
|
/**
|
|
81
|
-
* Get
|
|
96
|
+
* Get source-specific sync config parsing defaults, such as the default
|
|
97
|
+
* schema or database to use when sync config references an unqualified table
|
|
98
|
+
* name.
|
|
99
|
+
*
|
|
100
|
+
* This is used wherever sync config is parsed for this source, including
|
|
101
|
+
* validation, diagnostics, and sync routing.
|
|
102
|
+
*/
|
|
103
|
+
getParseSyncRulesOptions(): ParseSyncConfigOptions;
|
|
104
|
+
}
|
|
105
|
+
export interface ReplicationHeadResult<T> {
|
|
106
|
+
response: T;
|
|
107
|
+
/**
|
|
108
|
+
* True when storage needs the source to force a later observable position.
|
|
109
|
+
* False when storage can prove no marker is needed.
|
|
82
110
|
*/
|
|
83
|
-
|
|
111
|
+
shouldAdvance: boolean;
|
|
84
112
|
}
|
|
85
|
-
export type ReplicationHeadCallback<T> = (head: string) => Promise<T
|
|
113
|
+
export type ReplicationHeadCallback<T> = (head: string) => Promise<ReplicationHeadResult<T>>;
|
|
@@ -12,10 +12,21 @@ export interface DiagnosticsOptions {
|
|
|
12
12
|
* If false, uses placeholder values for e.g. initial_replication_done.
|
|
13
13
|
*/
|
|
14
14
|
live_status: boolean;
|
|
15
|
+
/**
|
|
16
|
+
* Whether this is the active sync config or not.
|
|
17
|
+
*/
|
|
18
|
+
active: boolean;
|
|
15
19
|
/**
|
|
16
20
|
* Check against the source postgres connection.
|
|
17
21
|
*/
|
|
18
22
|
check_connection: boolean;
|
|
19
23
|
}
|
|
20
24
|
export declare const DEFAULT_DATASOURCE_ID = "default";
|
|
21
|
-
export declare function getSyncRulesStatus(
|
|
25
|
+
export declare function getSyncRulesStatus(apiHandler: RouteAPI, syncConfig: storage.PersistedSyncConfigContent | null, options: DiagnosticsOptions,
|
|
26
|
+
/**
|
|
27
|
+
* Storage instance for the replication stream of this config.
|
|
28
|
+
*
|
|
29
|
+
* Required to populate live status (snapshot/checkpoint info). The content object
|
|
30
|
+
* itself is no longer a replication stream, so the caller must resolve this.
|
|
31
|
+
*/
|
|
32
|
+
systemStorage?: storage.SyncRulesBucketStorage): Promise<SyncRulesStatus | undefined>;
|
package/dist/api/diagnostics.js
CHANGED
|
@@ -1,9 +1,16 @@
|
|
|
1
|
-
import { logger } from '@powersync/lib-services-framework';
|
|
1
|
+
import { logger, ServiceAssertionError } from '@powersync/lib-services-framework';
|
|
2
2
|
import { DEFAULT_TAG } from '@powersync/service-sync-rules';
|
|
3
3
|
import { syncConfigYamlErrorToReplicationError } from '../util/errors.js';
|
|
4
4
|
export const DEFAULT_DATASOURCE_ID = 'default';
|
|
5
|
-
export async function getSyncRulesStatus(
|
|
6
|
-
|
|
5
|
+
export async function getSyncRulesStatus(apiHandler, syncConfig, options,
|
|
6
|
+
/**
|
|
7
|
+
* Storage instance for the replication stream of this config.
|
|
8
|
+
*
|
|
9
|
+
* Required to populate live status (snapshot/checkpoint info). The content object
|
|
10
|
+
* itself is no longer a replication stream, so the caller must resolve this.
|
|
11
|
+
*/
|
|
12
|
+
systemStorage) {
|
|
13
|
+
if (syncConfig == null) {
|
|
7
14
|
return undefined;
|
|
8
15
|
}
|
|
9
16
|
const include_content = options.include_content ?? false;
|
|
@@ -13,12 +20,18 @@ export async function getSyncRulesStatus(bucketStorage, apiHandler, sync_rules,
|
|
|
13
20
|
let parsed;
|
|
14
21
|
let persisted;
|
|
15
22
|
try {
|
|
16
|
-
persisted =
|
|
17
|
-
parsed
|
|
23
|
+
persisted = syncConfig.parsed(apiHandler.getParseSyncRulesOptions());
|
|
24
|
+
// A content object represents a single sync config, so its parsed result has exactly one entry.
|
|
25
|
+
const [singleConfig] = persisted.syncConfigs;
|
|
26
|
+
if (singleConfig == null) {
|
|
27
|
+
throw new ServiceAssertionError('Expected one sync config');
|
|
28
|
+
}
|
|
29
|
+
parsed = singleConfig;
|
|
18
30
|
}
|
|
19
31
|
catch (e) {
|
|
20
32
|
return {
|
|
21
|
-
|
|
33
|
+
version_label: syncConfig.version_label,
|
|
34
|
+
content: include_content ? syncConfig.sync_rules_content : undefined,
|
|
22
35
|
connections: [],
|
|
23
36
|
errors: [{ level: 'fatal', message: e.message, ts: now }]
|
|
24
37
|
};
|
|
@@ -28,8 +41,7 @@ export async function getSyncRulesStatus(bucketStorage, apiHandler, sync_rules,
|
|
|
28
41
|
// This method can run under some situations if no connection is configured yet.
|
|
29
42
|
// It will return a default tag in such a case. This default tag is not module specific.
|
|
30
43
|
const tag = sourceConfig.tag ?? DEFAULT_TAG;
|
|
31
|
-
const
|
|
32
|
-
const status = await systemStorage?.getStatus();
|
|
44
|
+
const status = live_status ? await systemStorage?.getStatus() : undefined;
|
|
33
45
|
let replication_lag_bytes = undefined;
|
|
34
46
|
let slot_wal_budget = undefined;
|
|
35
47
|
let tables_flat = [];
|
|
@@ -57,10 +69,10 @@ export async function getSyncRulesStatus(bucketStorage, apiHandler, sync_rules,
|
|
|
57
69
|
// Ignore
|
|
58
70
|
logger.warn(`Unable to get replication lag`, e);
|
|
59
71
|
}
|
|
60
|
-
if (apiHandler.getSlotWalBudget &&
|
|
72
|
+
if (apiHandler.getSlotWalBudget && syncConfig.replicationStreamName) {
|
|
61
73
|
try {
|
|
62
74
|
slot_wal_budget = await apiHandler.getSlotWalBudget({
|
|
63
|
-
slotName:
|
|
75
|
+
slotName: syncConfig.replicationStreamName
|
|
64
76
|
});
|
|
65
77
|
}
|
|
66
78
|
catch (e) {
|
|
@@ -103,11 +115,12 @@ export async function getSyncRulesStatus(bucketStorage, apiHandler, sync_rules,
|
|
|
103
115
|
});
|
|
104
116
|
}
|
|
105
117
|
const errors = tables_flat.flatMap((info) => info.errors);
|
|
106
|
-
|
|
118
|
+
const statusSource = await syncConfig.getSyncConfigStatus();
|
|
119
|
+
if (statusSource?.last_fatal_error) {
|
|
107
120
|
errors.push({
|
|
108
121
|
level: 'fatal',
|
|
109
|
-
message:
|
|
110
|
-
ts:
|
|
122
|
+
message: statusSource.last_fatal_error,
|
|
123
|
+
ts: statusSource.last_fatal_error_ts?.toISOString()
|
|
111
124
|
});
|
|
112
125
|
}
|
|
113
126
|
errors.push(...syncRuleErrors.map((error) => syncConfigYamlErrorToReplicationError(error, now)));
|
|
@@ -127,10 +140,10 @@ export async function getSyncRulesStatus(bucketStorage, apiHandler, sync_rules,
|
|
|
127
140
|
});
|
|
128
141
|
}
|
|
129
142
|
}
|
|
130
|
-
if (live_status &&
|
|
143
|
+
if (live_status && options.active) {
|
|
131
144
|
// Check replication lag for active replication stream.
|
|
132
145
|
// Right now we exclude mysql, since it we don't have consistent keepalives for it.
|
|
133
|
-
if (
|
|
146
|
+
if (statusSource?.last_checkpoint_ts == null && statusSource?.last_keepalive_ts == null) {
|
|
134
147
|
errors.push({
|
|
135
148
|
level: 'warning',
|
|
136
149
|
message: 'No checkpoint found, cannot calculate replication lag',
|
|
@@ -138,7 +151,7 @@ export async function getSyncRulesStatus(bucketStorage, apiHandler, sync_rules,
|
|
|
138
151
|
});
|
|
139
152
|
}
|
|
140
153
|
else {
|
|
141
|
-
const lastTime = Math.max(
|
|
154
|
+
const lastTime = Math.max(statusSource.last_checkpoint_ts?.getTime() ?? 0, statusSource.last_keepalive_ts?.getTime() ?? 0);
|
|
142
155
|
const lagSeconds = Math.round((Date.now() - lastTime) / 1000);
|
|
143
156
|
// On idle instances, keepalive messages are only persisted every 60 seconds.
|
|
144
157
|
// So we use 5 minutes as a threshold for warnings, and 15 minutes for critical.
|
|
@@ -161,17 +174,18 @@ export async function getSyncRulesStatus(bucketStorage, apiHandler, sync_rules,
|
|
|
161
174
|
}
|
|
162
175
|
}
|
|
163
176
|
return {
|
|
164
|
-
|
|
177
|
+
version_label: syncConfig.version_label,
|
|
178
|
+
content: include_content ? syncConfig.sync_rules_content : undefined,
|
|
165
179
|
connections: [
|
|
166
180
|
{
|
|
167
181
|
id: sourceConfig.id ?? DEFAULT_DATASOURCE_ID,
|
|
168
182
|
tag: tag,
|
|
169
|
-
slot_name:
|
|
170
|
-
initial_replication_done: status?.
|
|
183
|
+
slot_name: syncConfig.replicationStreamName,
|
|
184
|
+
initial_replication_done: status?.snapshotDone ?? false,
|
|
171
185
|
// TODO: Rename?
|
|
172
|
-
last_lsn: status?.
|
|
173
|
-
last_checkpoint_ts:
|
|
174
|
-
last_keepalive_ts:
|
|
186
|
+
last_lsn: status?.resumeLsn ?? undefined,
|
|
187
|
+
last_checkpoint_ts: statusSource?.last_checkpoint_ts?.toISOString(),
|
|
188
|
+
last_keepalive_ts: statusSource?.last_keepalive_ts?.toISOString(),
|
|
175
189
|
replication_lag_bytes: replication_lag_bytes,
|
|
176
190
|
wal_status: slot_wal_budget?.wal_status,
|
|
177
191
|
safe_wal_size: slot_wal_budget?.safe_wal_size,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"diagnostics.js","sourceRoot":"","sources":["../../src/api/diagnostics.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;
|
|
1
|
+
{"version":3,"file":"diagnostics.js","sourceRoot":"","sources":["../../src/api/diagnostics.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,qBAAqB,EAAE,MAAM,mCAAmC,CAAC;AAClF,OAAO,EAAE,WAAW,EAAwC,MAAM,+BAA+B,CAAC;AAIlG,OAAO,EAAE,qCAAqC,EAAE,MAAM,mBAAmB,CAAC;AA2B1E,MAAM,CAAC,MAAM,qBAAqB,GAAG,SAAS,CAAC;AAE/C,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,UAAoB,EACpB,UAAqD,EACrD,OAA2B;AAC3B;;;;;GAKG;AACH,aAA8C;IAE9C,IAAI,UAAU,IAAI,IAAI,EAAE,CAAC;QACvB,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,eAAe,GAAG,OAAO,CAAC,eAAe,IAAI,KAAK,CAAC;IACzD,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,KAAK,CAAC;IACjD,MAAM,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,IAAI,KAAK,CAAC;IAC3D,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAErC,IAAI,MAA4B,CAAC;IACjC,IAAI,SAAsC,CAAC;IAC3C,IAAI,CAAC;QACH,SAAS,GAAG,UAAU,CAAC,MAAM,CAAC,UAAU,CAAC,wBAAwB,EAAE,CAAC,CAAC;QACrE,gGAAgG;QAChG,MAAM,CAAC,YAAY,CAAC,GAAG,SAAS,CAAC,WAAW,CAAC;QAC7C,IAAI,YAAY,IAAI,IAAI,EAAE,CAAC;YACzB,MAAM,IAAI,qBAAqB,CAAC,0BAA0B,CAAC,CAAC;QAC9D,CAAC;QACD,MAAM,GAAG,YAAY,CAAC;IACxB,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO;YACL,aAAa,EAAE,UAAU,CAAC,aAAa;YACvC,OAAO,EAAE,eAAe,CAAC,CAAC,CAAC,UAAU,CAAC,kBAAkB,CAAC,CAAC,CAAC,SAAS;YACpE,WAAW,EAAE,EAAE;YACf,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC;SAC1D,CAAC;IACJ,CAAC;IAED,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,CAAC;IACzD,MAAM,YAAY,GAAG,MAAM,UAAU,CAAC,eAAe,EAAE,CAAC;IACxD,gFAAgF;IAChF,wFAAwF;IACxF,MAAM,GAAG,GAAG,YAAY,CAAC,GAAG,IAAI,WAAW,CAAC;IAC5C,MAAM,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC,MAAM,aAAa,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;IAC1E,IAAI,qBAAqB,GAAuB,SAAS,CAAC;IAC1D,IAAI,eAAe,GAAkC,SAAS,CAAC;IAE/D,IAAI,WAAW,GAAgB,EAAE,CAAC;IAElC,IAAI,gBAAgB,EAAE,CAAC;QACrB,MAAM,qBAAqB,GAAG,KAAK,CAAC,eAAe,EAAE,CAAC;QACtD,MAAM,eAAe,GAAG,MAAM,UAAU,CAAC,kBAAkB,CAAC,qBAAqB,EAAE,KAAK,CAAC,CAAC;QAC1F,WAAW,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;YAC7C,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACtB,CAAC;iBAAM,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBACvB,OAAO,IAAI,CAAC,MAAM,CAAC;YACrB,CAAC;iBAAM,CAAC;gBACN,OAAO,EAAE,CAAC;YACZ,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,IAAI,aAAa,EAAE,CAAC;YAClB,IAAI,CAAC;gBACH,qBAAqB,GAAG,MAAM,UAAU,CAAC,sBAAsB,CAAC;oBAC9D,aAAa,EAAE,aAAa;iBAC7B,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,SAAS;gBACT,MAAM,CAAC,IAAI,CAAC,+BAA+B,EAAE,CAAC,CAAC,CAAC;YAClD,CAAC;YAED,IAAI,UAAU,CAAC,gBAAgB,IAAI,UAAU,CAAC,qBAAqB,EAAE,CAAC;gBACpE,IAAI,CAAC;oBACH,eAAe,GAAG,MAAM,UAAU,CAAC,gBAAgB,CAAC;wBAClD,QAAQ,EAAE,UAAU,CAAC,qBAAqB;qBAC3C,CAAC,CAAC;gBACL,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACX,MAAM,CAAC,IAAI,CAAC,0BAA0B,EAAE,CAAC,CAAC,CAAC;gBAC7C,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;SAAM,CAAC;QACN,MAAM,qBAAqB,GAAG,KAAK,CAAC,eAAe,EAAE,CAAC;QAEtD,WAAW,GAAG,qBAAqB,CAAC,GAAG,CAAC,CAAC,OAAO,EAAa,EAAE;YAC7D,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;gBACvB,OAAO;oBACL,MAAM,EAAE,OAAO,CAAC,MAAM;oBACtB,IAAI,EAAE,OAAO,CAAC,WAAW;oBACzB,OAAO,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS;oBAE9D,YAAY,EAAE,KAAK;oBACnB,iBAAiB,EAAE,KAAK;oBACxB,cAAc,EAAE,EAAE;oBAClB,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,mBAAmB,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC;iBACpE,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,MAAM,MAAM,GAAmB;oBAC7B,aAAa,EAAE,GAAG;oBAClB,MAAM,EAAE,OAAO,CAAC,MAAM;oBACtB,IAAI,EAAE,OAAO,CAAC,YAAY;iBAC3B,CAAC;gBACF,MAAM,QAAQ,GAAG,KAAK,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;gBAC9C,MAAM,cAAc,GAAG,KAAK,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAC;gBAC1D,OAAO;oBACL,MAAM,EAAE,OAAO,CAAC,MAAM;oBACtB,IAAI,EAAE,OAAO,CAAC,IAAI;oBAClB,YAAY,EAAE,QAAQ;oBACtB,iBAAiB,EAAE,cAAc;oBACjC,cAAc,EAAE,EAAE;oBAClB,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,mBAAmB,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC;iBACpE,CAAC;YACJ,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED,MAAM,MAAM,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC1D,MAAM,YAAY,GAAG,MAAM,UAAU,CAAC,mBAAmB,EAAE,CAAC;IAE5D,IAAI,YAAY,EAAE,gBAAgB,EAAE,CAAC;QACnC,MAAM,CAAC,IAAI,CAAC;YACV,KAAK,EAAE,OAAO;YACd,OAAO,EAAE,YAAY,CAAC,gBAAgB;YACtC,EAAE,EAAE,YAAY,CAAC,mBAAmB,EAAE,WAAW,EAAE;SACpD,CAAC,CAAC;IACL,CAAC;IACD,MAAM,CAAC,IAAI,CAAC,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,qCAAqC,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;IAEjG,IACE,eAAe;QACf,eAAe,CAAC,UAAU,KAAK,MAAM;QACrC,eAAe,CAAC,aAAa,IAAI,IAAI;QACrC,eAAe,CAAC,sBAAsB,IAAI,IAAI;QAC9C,eAAe,CAAC,sBAAsB,GAAG,CAAC,EAC1C,CAAC;QACD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CACxB,CAAC,EACD,IAAI,CAAC,KAAK,CAAC,CAAC,eAAe,CAAC,aAAa,GAAG,eAAe,CAAC,sBAAsB,CAAC,GAAG,GAAG,CAAC,CAC3F,CAAC;QACF,IAAI,SAAS,IAAI,EAAE,EAAE,CAAC;YACpB,MAAM,CAAC,IAAI,CAAC;gBACV,KAAK,EAAE,SAAS;gBAChB,OAAO,EACL,sBAAsB,SAAS,eAAe;oBAC9C,6DAA6D;oBAC7D,qEAAqE;gBACvE,EAAE,EAAE,GAAG;aACR,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,IAAI,WAAW,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QAClC,uDAAuD;QACvD,mFAAmF;QACnF,IAAI,YAAY,EAAE,kBAAkB,IAAI,IAAI,IAAI,YAAY,EAAE,iBAAiB,IAAI,IAAI,EAAE,CAAC;YACxF,MAAM,CAAC,IAAI,CAAC;gBACV,KAAK,EAAE,SAAS;gBAChB,OAAO,EAAE,uDAAuD;gBAChE,EAAE,EAAE,GAAG;aACR,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CACvB,YAAY,CAAC,kBAAkB,EAAE,OAAO,EAAE,IAAI,CAAC,EAC/C,YAAY,CAAC,iBAAiB,EAAE,OAAO,EAAE,IAAI,CAAC,CAC/C,CAAC;YACF,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,CAAC,GAAG,IAAI,CAAC,CAAC;YAC9D,6EAA6E;YAC7E,gFAAgF;YAChF,mGAAmG;YACnG,6FAA6F;YAC7F,IAAI,UAAU,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC;gBACzB,MAAM,CAAC,IAAI,CAAC;oBACV,KAAK,EAAE,OAAO;oBACd,OAAO,EAAE,qCAAqC,UAAU,GAAG;oBAC3D,EAAE,EAAE,GAAG;iBACR,CAAC,CAAC;YACL,CAAC;iBAAM,IAAI,UAAU,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC;gBAC/B,MAAM,CAAC,IAAI,CAAC;oBACV,KAAK,EAAE,SAAS;oBAChB,OAAO,EAAE,qCAAqC,UAAU,GAAG;oBAC3D,EAAE,EAAE,GAAG;iBACR,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO;QACL,aAAa,EAAE,UAAU,CAAC,aAAa;QACvC,OAAO,EAAE,eAAe,CAAC,CAAC,CAAC,UAAU,CAAC,kBAAkB,CAAC,CAAC,CAAC,SAAS;QACpE,WAAW,EAAE;YACX;gBACE,EAAE,EAAE,YAAY,CAAC,EAAE,IAAI,qBAAqB;gBAC5C,GAAG,EAAE,GAAG;gBACR,SAAS,EAAE,UAAU,CAAC,qBAAqB;gBAC3C,wBAAwB,EAAE,MAAM,EAAE,YAAY,IAAI,KAAK;gBACvD,gBAAgB;gBAChB,QAAQ,EAAE,MAAM,EAAE,SAAS,IAAI,SAAS;gBACxC,kBAAkB,EAAE,YAAY,EAAE,kBAAkB,EAAE,WAAW,EAAE;gBACnE,iBAAiB,EAAE,YAAY,EAAE,iBAAiB,EAAE,WAAW,EAAE;gBACjE,qBAAqB,EAAE,qBAAqB;gBAC5C,UAAU,EAAE,eAAe,EAAE,UAAU;gBACvC,aAAa,EAAE,eAAe,EAAE,aAAa;gBAC7C,sBAAsB,EAAE,eAAe,EAAE,sBAAsB;gBAC/D,MAAM,EAAE,WAAW;aACpB;SACF;QACD,MAAM,EAAE,WAAW,CAAC,MAAM,CAAC;KAC5B,CAAC;AACJ,CAAC;AAED,SAAS,WAAW,CAAC,MAA0B;IAC7C,IAAI,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC7B,IAAI,MAAM,GAAuB,EAAE,CAAC;IACpC,KAAK,IAAI,KAAK,IAAI,MAAM,EAAE,CAAC;QACzB,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QAClC,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAClB,SAAS;QACX,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACd,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACrB,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -15,12 +15,14 @@ const HEAP_LIMIT = v8.getHeapStatistics().heap_size_limit;
|
|
|
15
15
|
* Limit to 1024MB overall.
|
|
16
16
|
*/
|
|
17
17
|
const COMPACT_MEMORY_LIMIT_MB = Math.min(HEAP_LIMIT / 1024 / 1024 - 128, 1024);
|
|
18
|
+
const MINUTE_MS = 60 * 1000;
|
|
18
19
|
export function registerCompactAction(program) {
|
|
19
20
|
const compactCommand = program
|
|
20
21
|
.command(COMMAND_NAME)
|
|
21
22
|
.option(`-b, --buckets [buckets]`, 'Full bucket names, comma-separated (e.g., "global[],mybucket[\\"user1\\"]")')
|
|
22
23
|
.option('-p, --parameter-indexes', 'Compacting parameter indexes. Defaults to set unless --buckets is provided.')
|
|
23
|
-
.option('--no-parameter-indexes', 'Disabling compacting parameter indexes.')
|
|
24
|
+
.option('--no-parameter-indexes', 'Disabling compacting parameter indexes.')
|
|
25
|
+
.option('--incremental-only', '[EXPERIMENTAL] Perform incremental compacting only. Parameter compaction runs on supported storage versions.');
|
|
24
26
|
wrapConfigCommand(compactCommand);
|
|
25
27
|
return compactCommand.description('Compact storage').action(async (options) => {
|
|
26
28
|
const buckets = options.buckets
|
|
@@ -35,6 +37,7 @@ export function registerCompactAction(program) {
|
|
|
35
37
|
process.exit(1);
|
|
36
38
|
}
|
|
37
39
|
}
|
|
40
|
+
const incremental = options.incrementalOnly ?? false;
|
|
38
41
|
let compactParameters = options.parameterIndexes;
|
|
39
42
|
if (buckets == null) {
|
|
40
43
|
logger.info('Compacting storage for all buckets...');
|
|
@@ -66,28 +69,33 @@ export function registerCompactAction(program) {
|
|
|
66
69
|
// Start the storage engine in order to create the appropriate BucketStorage
|
|
67
70
|
await serviceContext.lifeCycleEngine.start();
|
|
68
71
|
const bucketStorage = serviceContext.storageEngine.activeBucketStorage;
|
|
69
|
-
const
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
72
|
+
const deleteCheckpointRequestsBefore = new Date(Date.now() - config.api_parameters.checkpoint_request_retention_minutes * MINUTE_MS);
|
|
73
|
+
const streams = await bucketStorage.getReplicatingReplicationStreams();
|
|
74
|
+
for (let stream of streams) {
|
|
75
|
+
const storage = bucketStorage.getInstance(stream);
|
|
76
|
+
const compactParameterData = (compactParameters ?? buckets == null) && (!incremental || storage.supportsIncrementalParameterCompaction());
|
|
77
|
+
logger.info(`[${stream.replicationStreamName}] Performing compaction...`);
|
|
78
|
+
if (buckets != null) {
|
|
79
|
+
await storage.compact({
|
|
80
|
+
memoryLimitMB: COMPACT_MEMORY_LIMIT_MB,
|
|
81
|
+
compactBuckets: buckets,
|
|
82
|
+
compactParameterData,
|
|
83
|
+
incrementalOnly: incremental,
|
|
84
|
+
deleteCheckpointRequestsBefore,
|
|
85
|
+
signal: abortController.signal
|
|
86
|
+
});
|
|
87
|
+
}
|
|
88
|
+
else {
|
|
89
|
+
await storage.compact({
|
|
90
|
+
memoryLimitMB: COMPACT_MEMORY_LIMIT_MB,
|
|
91
|
+
compactParameterData,
|
|
92
|
+
incrementalOnly: incremental,
|
|
93
|
+
deleteCheckpointRequestsBefore,
|
|
94
|
+
signal: abortController.signal
|
|
95
|
+
});
|
|
96
|
+
}
|
|
97
|
+
logger.info(`[${stream.replicationStreamName}] Successfully compacted storage.`);
|
|
73
98
|
}
|
|
74
|
-
if (buckets != null) {
|
|
75
|
-
logger.info('Performing compaction...');
|
|
76
|
-
await active.compact({
|
|
77
|
-
memoryLimitMB: COMPACT_MEMORY_LIMIT_MB,
|
|
78
|
-
compactBuckets: buckets,
|
|
79
|
-
compactParameterData: compactParameters ?? false,
|
|
80
|
-
signal: abortController.signal
|
|
81
|
-
});
|
|
82
|
-
}
|
|
83
|
-
else {
|
|
84
|
-
await active.compact({
|
|
85
|
-
memoryLimitMB: COMPACT_MEMORY_LIMIT_MB,
|
|
86
|
-
compactParameterData: compactParameters ?? true,
|
|
87
|
-
signal: abortController.signal
|
|
88
|
-
});
|
|
89
|
-
}
|
|
90
|
-
logger.info('Successfully compacted storage.');
|
|
91
99
|
}
|
|
92
100
|
catch (e) {
|
|
93
101
|
logger.error(`Failed to compact:`, e);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"compact-action.js","sourceRoot":"","sources":["../../../src/entry/commands/compact-action.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AACtE,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,MAAM,MAAM,8BAA8B,CAAC;AACvD,OAAO,KAAK,KAAK,MAAM,0BAA0B,CAAC;AAElD,OAAO,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AACzC,OAAO,EAAE,oBAAoB,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAE9E,MAAM,YAAY,GAAG,SAAS,CAAC;AAE/B;;GAEG;AACH,MAAM,UAAU,GAAG,EAAE,CAAC,iBAAiB,EAAE,CAAC,eAAe,CAAC;AAE1D;;;;GAIG;AACH,MAAM,uBAAuB,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,GAAG,IAAI,GAAG,IAAI,GAAG,GAAG,EAAE,IAAI,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"compact-action.js","sourceRoot":"","sources":["../../../src/entry/commands/compact-action.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AACtE,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,MAAM,MAAM,8BAA8B,CAAC;AACvD,OAAO,KAAK,KAAK,MAAM,0BAA0B,CAAC;AAElD,OAAO,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AACzC,OAAO,EAAE,oBAAoB,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAE9E,MAAM,YAAY,GAAG,SAAS,CAAC;AAE/B;;GAEG;AACH,MAAM,UAAU,GAAG,EAAE,CAAC,iBAAiB,EAAE,CAAC,eAAe,CAAC;AAE1D;;;;GAIG;AACH,MAAM,uBAAuB,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,GAAG,IAAI,GAAG,IAAI,GAAG,GAAG,EAAE,IAAI,CAAC,CAAC;AAC/E,MAAM,SAAS,GAAG,EAAE,GAAG,IAAI,CAAC;AAE5B,MAAM,UAAU,qBAAqB,CAAC,OAAgB;IACpD,MAAM,cAAc,GAAG,OAAO;SAC3B,OAAO,CAAC,YAAY,CAAC;SACrB,MAAM,CAAC,yBAAyB,EAAE,6EAA6E,CAAC;SAChH,MAAM,CAAC,yBAAyB,EAAE,6EAA6E,CAAC;SAChH,MAAM,CAAC,wBAAwB,EAAE,yCAAyC,CAAC;SAC3E,MAAM,CACL,oBAAoB,EACpB,8GAA8G,CAC/G,CAAC;IAEJ,iBAAiB,CAAC,cAAc,CAAC,CAAC;IAElC,OAAO,cAAc,CAAC,WAAW,CAAC,iBAAiB,CAAC,CAAC,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;QAC5E,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO;YAC7B,EAAE,KAAK,CAAC,GAAG,CAAC;aACX,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;aAC5B,MAAM,CAAC,OAAO,CAAC,CAAC;QACnB,IAAI,OAAO,EAAE,CAAC;YACZ,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;YAChE,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACvB,MAAM,CAAC,KAAK,CACV,yBAAyB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI;oBAC7C,0FAA0F,CAC7F,CAAC;gBACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC;QACH,CAAC;QAED,MAAM,WAAW,GAAY,OAAO,CAAC,eAAe,IAAI,KAAK,CAAC;QAE9D,IAAI,iBAAiB,GAAmB,OAAO,CAAC,gBAAgB,CAAC;QACjE,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;YACpB,MAAM,CAAC,IAAI,CAAC,uCAAuC,CAAC,CAAC;QACvD,CAAC;aAAM,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YAC/B,MAAM,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC;QAC5C,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC,0BAA0B,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAClE,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC,CAAC;QACrE,MAAM,cAAc,GAAG,IAAI,MAAM,CAAC,uBAAuB,CAAC;YACxD,WAAW,EAAE,MAAM,CAAC,kBAAkB,CAAC,OAAO;YAC9C,aAAa,EAAE,MAAM;SACtB,CAAC,CAAC;QACH,MAAM,eAAe,GAAG,IAAI,eAAe,EAAE,CAAC;QAC9C,MAAM,UAAU,GAAG,OAAO,CAAC,aAAa,EAAQ,CAAC;QAEjD,cAAc,CAAC,eAAe,CAAC,aAAa,CAAC,IAAI,EAAE;YACjD,IAAI,EAAE,KAAK,IAAI,EAAE;gBACf,eAAe,CAAC,KAAK,EAAE,CAAC;gBACxB,MAAM,UAAU,CAAC,OAAO,CAAC;YAC3B,CAAC;SACF,CAAC,CAAC;QAEH,8DAA8D;QAC9D,MAAM,aAAa,GAAG,SAAS,CAAC,iBAAiB,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;QACzE,MAAM,aAAa,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;QAE/C,MAAM,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QAExC,IAAI,CAAC;YACH,4EAA4E;YAC5E,MAAM,cAAc,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;YAC7C,MAAM,aAAa,GAAG,cAAc,CAAC,aAAa,CAAC,mBAAmB,CAAC;YACvE,MAAM,8BAA8B,GAAG,IAAI,IAAI,CAC7C,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,cAAc,CAAC,oCAAoC,GAAG,SAAS,CACpF,CAAC;YAEF,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,gCAAgC,EAAE,CAAC;YACvE,KAAK,IAAI,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC3B,MAAM,OAAO,GAAG,aAAa,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;gBAClD,MAAM,oBAAoB,GACxB,CAAC,iBAAiB,IAAI,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,WAAW,IAAI,OAAO,CAAC,sCAAsC,EAAE,CAAC,CAAC;gBAC/G,MAAM,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,qBAAqB,4BAA4B,CAAC,CAAC;gBAC1E,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;oBACpB,MAAM,OAAO,CAAC,OAAO,CAAC;wBACpB,aAAa,EAAE,uBAAuB;wBACtC,cAAc,EAAE,OAAO;wBACvB,oBAAoB;wBACpB,eAAe,EAAE,WAAW;wBAC5B,8BAA8B;wBAC9B,MAAM,EAAE,eAAe,CAAC,MAAM;qBAC/B,CAAC,CAAC;gBACL,CAAC;qBAAM,CAAC;oBACN,MAAM,OAAO,CAAC,OAAO,CAAC;wBACpB,aAAa,EAAE,uBAAuB;wBACtC,oBAAoB;wBACpB,eAAe,EAAE,WAAW;wBAC5B,8BAA8B;wBAC9B,MAAM,EAAE,eAAe,CAAC,MAAM;qBAC/B,CAAC,CAAC;gBACL,CAAC;gBACD,MAAM,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,qBAAqB,mCAAmC,CAAC,CAAC;YACnF,CAAC;QACH,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,CAAC,KAAK,CAAC,oBAAoB,EAAE,CAAC,CAAC,CAAC;YACtC,2CAA2C;YAC3C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;gBAAS,CAAC;YACT,6FAA6F;YAC7F,UAAU,CAAC,OAAO,EAAE,CAAC;YACrB,2CAA2C;YAC3C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -17,7 +17,11 @@ export interface ObservableGauge {
|
|
|
17
17
|
* Set a value provider that provides the value for the gauge at the time of observation.
|
|
18
18
|
* @param valueProvider
|
|
19
19
|
*/
|
|
20
|
-
setValueProvider(valueProvider: () => Promise<number | undefined>): void;
|
|
20
|
+
setValueProvider(valueProvider: () => Promise<number | ObservableGaugeObservation[] | undefined>): void;
|
|
21
|
+
}
|
|
22
|
+
export interface ObservableGaugeObservation {
|
|
23
|
+
value: number;
|
|
24
|
+
attributes?: Record<string, string>;
|
|
21
25
|
}
|
|
22
26
|
export declare enum Precision {
|
|
23
27
|
INT = "int",
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"metrics-interfaces.js","sourceRoot":"","sources":["../../src/metrics/metrics-interfaces.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"metrics-interfaces.js","sourceRoot":"","sources":["../../src/metrics/metrics-interfaces.ts"],"names":[],"mappings":"AA6BA,MAAM,CAAN,IAAY,SAGX;AAHD,WAAY,SAAS;IACnB,wBAAW,CAAA;IACX,8BAAiB,CAAA;AACnB,CAAC,EAHW,SAAS,KAAT,SAAS,QAGpB"}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { ExportResult } from '@opentelemetry/core';
|
|
2
|
+
import { AggregationSelector, AggregationTemporalitySelector, PushMetricExporter, ResourceMetrics } from '@opentelemetry/sdk-metrics';
|
|
3
|
+
/**
|
|
4
|
+
* Decorates a push exporter and removes selected attributes from every metric data point before
|
|
5
|
+
* delegating the export. The input metrics are not mutated, so another reader can export the
|
|
6
|
+
* original attributes.
|
|
7
|
+
*
|
|
8
|
+
* This exporter deliberately does not merge data points when filtering makes their attribute sets
|
|
9
|
+
* identical. Both points are forwarded with the same attributes. There is no generally correct
|
|
10
|
+
* merge operation at this layer: gauges, counters, and histograms have different aggregation
|
|
11
|
+
* semantics. Callers must therefore ensure that an excluded attribute does not distinguish data
|
|
12
|
+
* points for the same metric, resource, and instrumentation scope. Otherwise, the resulting
|
|
13
|
+
* duplicate series identity may be rejected or interpreted unpredictably downstream.
|
|
14
|
+
*
|
|
15
|
+
* Lifecycle methods and aggregation selection are delegated unchanged to the wrapped exporter.
|
|
16
|
+
*/
|
|
17
|
+
export declare class MetricAttributeFilteringExporter implements PushMetricExporter {
|
|
18
|
+
/** The exporter that receives the filtered copy of the metrics. */
|
|
19
|
+
private readonly delegate;
|
|
20
|
+
/** Attribute names to remove from every exported data point. */
|
|
21
|
+
private readonly excludedAttributes;
|
|
22
|
+
readonly selectAggregation?: AggregationSelector;
|
|
23
|
+
readonly selectAggregationTemporality?: AggregationTemporalitySelector;
|
|
24
|
+
constructor(
|
|
25
|
+
/** The exporter that receives the filtered copy of the metrics. */
|
|
26
|
+
delegate: PushMetricExporter,
|
|
27
|
+
/** Attribute names to remove from every exported data point. */
|
|
28
|
+
excludedAttributes: ReadonlySet<string>);
|
|
29
|
+
export(metrics: ResourceMetrics, resultCallback: (result: ExportResult) => void): void;
|
|
30
|
+
forceFlush(): Promise<void>;
|
|
31
|
+
shutdown(): Promise<void>;
|
|
32
|
+
private filterAttributes;
|
|
33
|
+
private filterMetricAttributes;
|
|
34
|
+
private filterDataPoint;
|
|
35
|
+
private filterDataPointAttributes;
|
|
36
|
+
}
|