@powersync/service-core 0.0.0-dev-20260602073133 → 0.0.0-dev-20260803144812

package/CHANGELOG.md

@@ -1,15 +1,94 @@
 # @powersync/service-core
 
-## 0.0.0-dev-20260602073133
+## 0.0.0-dev-20260803144812
+
+### Minor Changes
+
+- 2189250: Add `/sync/checkpoint-request` for client-supplied checkpoint request ids, previously called write checkpoint ids. The route returns the stored `checkpoint_request_id`, storage now treats managed request ids as monotonic per user/client, custom checkpoint request ids continue to use the existing `checkpoint` field for backwards compatibility, and `checkpoint_requested_at` metadata lets compact jobs remove expired request-derived checkpoint records.
+
+  This release includes storage migrations for the checkpoint request metadata. Self-hosters should run migrations as part of the upgrade.
+
+- 922f974: Add experimental support for storing bucket data chunks on S3.
+
+### Patch Changes
+
+- c4860c9: Improve Postgres sync throughput by reading bucket data with ordered, per-bucket index range scans that stop once the requested batch is full.
+- 483415d: Clarify whether too-many-buckets log breakdowns are grouped by sync stream or legacy bucket definition, and include up
+  to 100 entries.
+- aab068b: [MongoDB Storage V3] Add has_clear_op field and simplify checksum queries.
+- 37591e9: Lock replication slots before clearing data, preventing concurrent clearing.
+- Updated dependencies [2189250]
+- Updated dependencies [8daa300]
+- Updated dependencies [df5663b]
+- Updated dependencies [be42e25]
+- Updated dependencies [cb4c627]
+  - @powersync/lib-services-framework@0.0.0-dev-20260803144812
+  - @powersync/service-types@0.0.0-dev-20260803144812
+  - @powersync/service-sync-rules@0.0.0-dev-20260803144812
+  - @powersync/service-rsocket-router@0.0.0-dev-20260803144812
+
+## 1.23.3
+
+### Patch Changes
+
+- ea31f64: Potential fix and improved stack trace for 'evicted' errors.
+- Updated dependencies [ea71bf3]
+- Updated dependencies [edc6ed4]
+  - @powersync/service-sync-rules@0.39.0
+  - @powersync/lib-services-framework@0.9.8
+  - @powersync/service-rsocket-router@0.2.24
+
+## 1.23.2
+
+### Patch Changes
+
+- 71d4a0a: Add logs for checkpoint_complete to assist with debugging. Remove "sync lock" logs.
+- e4f683d: [MongoDB Storage] Add experimental option to allow reading data from secondaries.
+- 71d4a0a: Fix edge case where the service can return incomplete data. This could happen when using bucket priorities, and a checkpoint is invalidated by the compact process while syncing.
+- c2edf86: Add `api.parameters.bucket_count_cache_ttl_minutes` configuration option.
+- df9ab1e: Batch concurrent write checkpoint requests and increase request limits to improve throughput.
+- Updated dependencies [a6ae678]
+- Updated dependencies [c2edf86]
+  - @powersync/service-sync-rules@0.38.1
+  - @powersync/service-types@0.16.1
+  - @powersync/lib-services-framework@0.9.7
+  - @powersync/service-rsocket-router@0.2.23
+
+## 1.23.1
+
+### Patch Changes
+
+- 7e65360: Restructure replication locks for updated sync config.
+
+## 1.23.0
+
+### Minor Changes
+
+- a91a08f: [Experimental] Enable incremental reprocessing for MongoDB source + MongoDB storage. This includes significant changes to the v3 storage format.
+
+### Patch Changes
+
+- Updated dependencies [a91a08f]
+- Updated dependencies [184c39f]
+- Updated dependencies [c3f75df]
+- Updated dependencies [4bd35ea]
+  - @powersync/service-types@0.16.0
+  - @powersync/service-sync-rules@0.38.0
+  - @powersync/lib-services-framework@0.9.6
+  - @powersync/service-rsocket-router@0.2.22
+
+## 1.22.0
 
 ### Minor Changes
 
 - e2bf1ad: [Internal] rework resolveTables to handle multiple SourceTables.
 - 92cc83b: Add the experimental `unstable_sqlite_expression_engine` sync config option to evaluate Sync Streams with SQLite.
 - 15e2466: [MongoDB] Support snapshotting concurrently with streaming in storage v3+.
+- ebeaa3b: [Postgres] Skip storing a redundant copy of replicated rows in `current_data` for tables with `REPLICA IDENTITY FULL`.
 
 ### Patch Changes
 
+- 17fd96b: Fix duration logs for "Processing batch"
 - 6e2a57e: Refactor HydratedSyncConfig to support multiple SyncConfigs.
 - ec6df9f: Add a custom Fastify error handler so uncaught errors honour the negotiated `Content-Encoding` (gzip/zstd) instead of returning a header that doesn't match the body.
 - 99d33d5: Normalize socket addresses to bare hostnames before IP-range validation, so direct-IP literals with any port form are recognized as IPs.
@@ -25,9 +104,9 @@
 - Updated dependencies [92cc83b]
 - Updated dependencies [0aab0f9]
 - Updated dependencies [a94b6c3]
-  - @powersync/service-sync-rules@0.0.0-dev-20260602073133
-  - @powersync/lib-services-framework@0.0.0-dev-20260602073133
-  - @powersync/service-rsocket-router@0.0.0-dev-20260602073133
+  - @powersync/service-sync-rules@0.37.0
+  - @powersync/lib-services-framework@0.9.5
+  - @powersync/service-rsocket-router@0.2.21
 
 ## 1.21.0
 

package/dist/api/RouteAPI.d.ts

@@ -1,6 +1,6 @@
 import { SyncConfig, TablePattern } from '@powersync/service-sync-rules';
 import * as types from '@powersync/service-types';
-import { ParseSyncRulesOptions, SyncRulesBucketStorage } from '../storage/storage-index.js';
+import { ParseSyncConfigOptions, SyncRulesBucketStorage } from '../storage/storage-index.js';
 export interface PatternResult {
     schema: string;
     pattern: string;
@@ -20,7 +20,8 @@ export interface SlotWalBudgetOptions {
     slotName: string;
 }
 /**
- *  Describes all the methods currently required to service the sync API endpoints.
+ * Source-specific capabilities used by admin routes, sync config validation,
+ * checkpointing, diagnostics, and the sync API.
  */
 export interface RouteAPI {
     /**
@@ -33,16 +34,17 @@ export interface RouteAPI {
      */
     getConnectionStatus(): Promise<types.ConnectionStatusV2>;
     /**
-     * Generates replication table information from a given pattern of tables.
+     * Expand sync config table patterns into table metadata from the source connection.
+     *
+     * This is used by validation and diagnostics paths to explain what will be
+     * replicated for the current sync config.
      *
      * @param tablePatterns A set of table patterns which typically come from
      *          the tables listed in sync config definitions.
      *
      * @param sqlSyncRules
-     * @returns A result of all the tables and columns which should be replicated
-     *           based off the input patterns. Certain tests are executed on the
-     *           tables to ensure syncing should function according to the input
-     *           pattern. Debug errors and warnings are reported per table.
+     * @returns Matching tables or collections, their columns or fields where available,
+     *          and per-table warnings or errors for the requested table patterns.
      */
     getDebugTablesInfo(tablePatterns: TablePattern[], sqlSyncRules: SyncConfig): Promise<PatternResult[]>;
     /**
@@ -59,13 +61,26 @@ export interface RouteAPI {
     /**
      * Get the current LSN or equivalent replication HEAD position identifier.
      *
-     * The position is provided to the callback. After the callback returns,
-     * the replication head or a greater one will be streamed on the replication stream.
+     * The position is provided to the callback so the caller can persist its
+     * write-checkpoint mapping before the source adapter forces any required
+     * source-side marker or keepalive. The callback returns whether storage
+     * actually needs a source marker: when `shouldAdvance` is true the adapter
+     * must ensure that the replication stream will observe this position or a
+     * greater one, even when the source is otherwise idle. When it is false, the
+     * adapter may skip the source marker.
+     *
+     * Reading the head and forcing the marker happen within a single source
+     * session/connection where applicable, so the marker is causally ordered
+     * after the head that was handed to the callback.
      */
     createReplicationHead<T>(callback: ReplicationHeadCallback<T>): Promise<T>;
     /**
-     * @returns The schema for tables inside the connected database. This is typically
-     *          used to validate sync config.
+     * @returns The connected source schema in service-friendly table and column
+     *          formats. Strict-schema sources should include actual tables,
+     *          collections, columns, and types. Schemaless sources should return
+     *          the best metadata available and document validation limitations in
+     *          their adapter behavior. This is typically used to validate sync
+     *          config and generate client schemas.
      */
     getConnectionSchema(): Promise<types.DatabaseSchema[]>;
     /**
@@ -78,8 +93,21 @@ export interface RouteAPI {
      */
     shutdown(): Promise<void>;
     /**
-     * Get the default schema (or database) when only a table name is specified in sync config.
+     * Get source-specific sync config parsing defaults, such as the default
+     * schema or database to use when sync config references an unqualified table
+     * name.
+     *
+     * This is used wherever sync config is parsed for this source, including
+     * validation, diagnostics, and sync routing.
+     */
+    getParseSyncRulesOptions(): ParseSyncConfigOptions;
+}
+export interface ReplicationHeadResult<T> {
+    response: T;
+    /**
+     * True when storage needs the source to force a later observable position.
+     * False when storage can prove no marker is needed.
      */
-    getParseSyncRulesOptions(): ParseSyncRulesOptions;
+    shouldAdvance: boolean;
 }
-export type ReplicationHeadCallback<T> = (head: string) => Promise<T>;
+export type ReplicationHeadCallback<T> = (head: string) => Promise<ReplicationHeadResult<T>>;

package/dist/api/diagnostics.d.ts

@@ -12,10 +12,21 @@ export interface DiagnosticsOptions {
      * If false, uses placeholder values for e.g. initial_replication_done.
      */
     live_status: boolean;
+    /**
+     * Whether this is the active sync config or not.
+     */
+    active: boolean;
     /**
      * Check against the source postgres connection.
      */
     check_connection: boolean;
 }
 export declare const DEFAULT_DATASOURCE_ID = "default";
-export declare function getSyncRulesStatus(bucketStorage: storage.BucketStorageFactory, apiHandler: RouteAPI, sync_rules: storage.PersistedSyncRulesContent | null, options: DiagnosticsOptions): Promise<SyncRulesStatus | undefined>;
+export declare function getSyncRulesStatus(apiHandler: RouteAPI, syncConfig: storage.PersistedSyncConfigContent | null, options: DiagnosticsOptions, 
+/**
+ * Storage instance for the replication stream of this config.
+ *
+ * Required to populate live status (snapshot/checkpoint info). The content object
+ * itself is no longer a replication stream, so the caller must resolve this.
+ */
+systemStorage?: storage.SyncRulesBucketStorage): Promise<SyncRulesStatus | undefined>;

package/dist/api/diagnostics.js

@@ -1,9 +1,16 @@
-import { logger } from '@powersync/lib-services-framework';
+import { logger, ServiceAssertionError } from '@powersync/lib-services-framework';
 import { DEFAULT_TAG } from '@powersync/service-sync-rules';
 import { syncConfigYamlErrorToReplicationError } from '../util/errors.js';
 export const DEFAULT_DATASOURCE_ID = 'default';
-export async function getSyncRulesStatus(bucketStorage, apiHandler, sync_rules, options) {
-    if (sync_rules == null) {
+export async function getSyncRulesStatus(apiHandler, syncConfig, options, 
+/**
+ * Storage instance for the replication stream of this config.
+ *
+ * Required to populate live status (snapshot/checkpoint info). The content object
+ * itself is no longer a replication stream, so the caller must resolve this.
+ */
+systemStorage) {
+    if (syncConfig == null) {
         return undefined;
     }
     const include_content = options.include_content ?? false;
@@ -13,12 +20,17 @@ export async function getSyncRulesStatus(bucketStorage, apiHandler, sync_rules,
     let parsed;
     let persisted;
     try {
-        persisted = sync_rules.parsed(apiHandler.getParseSyncRulesOptions());
-        parsed = persisted.syncConfigWithErrors;
+        persisted = syncConfig.parsed(apiHandler.getParseSyncRulesOptions());
+        // A content object represents a single sync config, so its parsed result has exactly one entry.
+        const [singleConfig] = persisted.syncConfigs;
+        if (singleConfig == null) {
+            throw new ServiceAssertionError('Expected one sync config');
+        }
+        parsed = singleConfig;
     }
     catch (e) {
         return {
-            content: include_content ? sync_rules.sync_rules_content : undefined,
+            content: include_content ? syncConfig.sync_rules_content : undefined,
             connections: [],
             errors: [{ level: 'fatal', message: e.message, ts: now }]
         };
@@ -28,8 +40,7 @@ export async function getSyncRulesStatus(bucketStorage, apiHandler, sync_rules,
     // This method can run under some situations if no connection is configured yet.
     // It will return a default tag in such a case. This default tag is not module specific.
     const tag = sourceConfig.tag ?? DEFAULT_TAG;
-    const systemStorage = live_status ? bucketStorage.getInstance(sync_rules) : undefined;
-    const status = await systemStorage?.getStatus();
+    const status = live_status ? await systemStorage?.getStatus() : undefined;
     let replication_lag_bytes = undefined;
     let slot_wal_budget = undefined;
     let tables_flat = [];
@@ -57,10 +68,10 @@ export async function getSyncRulesStatus(bucketStorage, apiHandler, sync_rules,
                 // Ignore
                 logger.warn(`Unable to get replication lag`, e);
             }
-            if (apiHandler.getSlotWalBudget && sync_rules.slot_name) {
+            if (apiHandler.getSlotWalBudget && syncConfig.replicationStreamName) {
                 try {
                     slot_wal_budget = await apiHandler.getSlotWalBudget({
-                        slotName: sync_rules.slot_name
+                        slotName: syncConfig.replicationStreamName
                     });
                 }
                 catch (e) {
@@ -103,11 +114,12 @@ export async function getSyncRulesStatus(bucketStorage, apiHandler, sync_rules,
         });
     }
     const errors = tables_flat.flatMap((info) => info.errors);
-    if (sync_rules.last_fatal_error) {
+    const statusSource = await syncConfig.getSyncConfigStatus();
+    if (statusSource?.last_fatal_error) {
         errors.push({
             level: 'fatal',
-            message: sync_rules.last_fatal_error,
-            ts: sync_rules.last_fatal_error_ts?.toISOString()
+            message: statusSource.last_fatal_error,
+            ts: statusSource.last_fatal_error_ts?.toISOString()
         });
     }
     errors.push(...syncRuleErrors.map((error) => syncConfigYamlErrorToReplicationError(error, now)));
@@ -127,10 +139,10 @@ export async function getSyncRulesStatus(bucketStorage, apiHandler, sync_rules,
             });
         }
     }
-    if (live_status && status?.active) {
+    if (live_status && options.active) {
         // Check replication lag for active replication stream.
         // Right now we exclude mysql, since it we don't have consistent keepalives for it.
-        if (sync_rules.last_checkpoint_ts == null && sync_rules.last_keepalive_ts == null) {
+        if (statusSource?.last_checkpoint_ts == null && statusSource?.last_keepalive_ts == null) {
             errors.push({
                 level: 'warning',
                 message: 'No checkpoint found, cannot calculate replication lag',
@@ -138,7 +150,7 @@ export async function getSyncRulesStatus(bucketStorage, apiHandler, sync_rules,
             });
         }
         else {
-            const lastTime = Math.max(sync_rules.last_checkpoint_ts?.getTime() ?? 0, sync_rules.last_keepalive_ts?.getTime() ?? 0);
+            const lastTime = Math.max(statusSource.last_checkpoint_ts?.getTime() ?? 0, statusSource.last_keepalive_ts?.getTime() ?? 0);
             const lagSeconds = Math.round((Date.now() - lastTime) / 1000);
             // On idle instances, keepalive messages are only persisted every 60 seconds.
             // So we use 5 minutes as a threshold for warnings, and 15 minutes for critical.
@@ -161,17 +173,17 @@ export async function getSyncRulesStatus(bucketStorage, apiHandler, sync_rules,
         }
     }
     return {
-        content: include_content ? sync_rules.sync_rules_content : undefined,
+        content: include_content ? syncConfig.sync_rules_content : undefined,
         connections: [
             {
                 id: sourceConfig.id ?? DEFAULT_DATASOURCE_ID,
                 tag: tag,
-                slot_name: sync_rules.slot_name,
-                initial_replication_done: status?.snapshot_done ?? false,
+                slot_name: syncConfig.replicationStreamName,
+                initial_replication_done: status?.snapshotDone ?? false,
                 // TODO: Rename?
-                last_lsn: status?.checkpoint_lsn ?? undefined,
-                last_checkpoint_ts: sync_rules.last_checkpoint_ts?.toISOString(),
-                last_keepalive_ts: sync_rules.last_keepalive_ts?.toISOString(),
+                last_lsn: status?.resumeLsn ?? undefined,
+                last_checkpoint_ts: statusSource?.last_checkpoint_ts?.toISOString(),
+                last_keepalive_ts: statusSource?.last_keepalive_ts?.toISOString(),
                 replication_lag_bytes: replication_lag_bytes,
                 wal_status: slot_wal_budget?.wal_status,
                 safe_wal_size: slot_wal_budget?.safe_wal_size,

package/dist/api/diagnostics.js.map

@@ -1 +1 @@
-{"version":3,"file":"diagnostics.js","sourceRoot":"","sources":["../../src/api/diagnostics.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAC3D,OAAO,EAAE,WAAW,EAAwC,MAAM,+BAA+B,CAAC;AAIlG,OAAO,EAAE,qCAAqC,EAAE,MAAM,mBAAmB,CAAC;AAsB1E,MAAM,CAAC,MAAM,qBAAqB,GAAG,SAAS,CAAC;AAE/C,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,aAA2C,EAC3C,UAAoB,EACpB,UAAoD,EACpD,OAA2B;IAE3B,IAAI,UAAU,IAAI,IAAI,EAAE,CAAC;QACvB,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,eAAe,GAAG,OAAO,CAAC,eAAe,IAAI,KAAK,CAAC;IACzD,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,KAAK,CAAC;IACjD,MAAM,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,IAAI,KAAK,CAAC;IAC3D,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAErC,IAAI,MAA4B,CAAC;IACjC,IAAI,SAAqC,CAAC;IAC1C,IAAI,CAAC;QACH,SAAS,GAAG,UAAU,CAAC,MAAM,CAAC,UAAU,CAAC,wBAAwB,EAAE,CAAC,CAAC;QACrE,MAAM,GAAG,SAAS,CAAC,oBAAoB,CAAC;IAC1C,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO;YACL,OAAO,EAAE,eAAe,CAAC,CAAC,CAAC,UAAU,CAAC,kBAAkB,CAAC,CAAC,CAAC,SAAS;YACpE,WAAW,EAAE,EAAE;YACf,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC;SAC1D,CAAC;IACJ,CAAC;IAED,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,CAAC;IACzD,MAAM,YAAY,GAAG,MAAM,UAAU,CAAC,eAAe,EAAE,CAAC;IACxD,gFAAgF;IAChF,wFAAwF;IACxF,MAAM,GAAG,GAAG,YAAY,CAAC,GAAG,IAAI,WAAW,CAAC;IAC5C,MAAM,aAAa,GAAG,WAAW,CAAC,CAAC,CAAC,aAAa,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACtF,MAAM,MAAM,GAAG,MAAM,aAAa,EAAE,SAAS,EAAE,CAAC;IAChD,IAAI,qBAAqB,GAAuB,SAAS,CAAC;IAC1D,IAAI,eAAe,GAAkC,SAAS,CAAC;IAE/D,IAAI,WAAW,GAAgB,EAAE,CAAC;IAElC,IAAI,gBAAgB,EAAE,CAAC;QACrB,MAAM,qBAAqB,GAAG,KAAK,CAAC,eAAe,EAAE,CAAC;QACtD,MAAM,eAAe,GAAG,MAAM,UAAU,CAAC,kBAAkB,CAAC,qBAAqB,EAAE,KAAK,CAAC,CAAC;QAC1F,WAAW,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;YAC7C,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACtB,CAAC;iBAAM,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBACvB,OAAO,IAAI,CAAC,MAAM,CAAC;YACrB,CAAC;iBAAM,CAAC;gBACN,OAAO,EAAE,CAAC;YACZ,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,IAAI,aAAa,EAAE,CAAC;YAClB,IAAI,CAAC;gBACH,qBAAqB,GAAG,MAAM,UAAU,CAAC,sBAAsB,CAAC;oBAC9D,aAAa,EAAE,aAAa;iBAC7B,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,SAAS;gBACT,MAAM,CAAC,IAAI,CAAC,+BAA+B,EAAE,CAAC,CAAC,CAAC;YAClD,CAAC;YAED,IAAI,UAAU,CAAC,gBAAgB,IAAI,UAAU,CAAC,SAAS,EAAE,CAAC;gBACxD,IAAI,CAAC;oBACH,eAAe,GAAG,MAAM,UAAU,CAAC,gBAAgB,CAAC;wBAClD,QAAQ,EAAE,UAAU,CAAC,SAAS;qBAC/B,CAAC,CAAC;gBACL,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACX,MAAM,CAAC,IAAI,CAAC,0BAA0B,EAAE,CAAC,CAAC,CAAC;gBAC7C,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;SAAM,CAAC;QACN,MAAM,qBAAqB,GAAG,KAAK,CAAC,eAAe,EAAE,CAAC;QAEtD,WAAW,GAAG,qBAAqB,CAAC,GAAG,CAAC,CAAC,OAAO,EAAa,EAAE;YAC7D,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;gBACvB,OAAO;oBACL,MAAM,EAAE,OAAO,CAAC,MAAM;oBACtB,IAAI,EAAE,OAAO,CAAC,WAAW;oBACzB,OAAO,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS;oBAE9D,YAAY,EAAE,KAAK;oBACnB,iBAAiB,EAAE,KAAK;oBACxB,cAAc,EAAE,EAAE;oBAClB,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,mBAAmB,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC;iBACpE,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,MAAM,MAAM,GAAmB;oBAC7B,aAAa,EAAE,GAAG;oBAClB,MAAM,EAAE,OAAO,CAAC,MAAM;oBACtB,IAAI,EAAE,OAAO,CAAC,YAAY;iBAC3B,CAAC;gBACF,MAAM,QAAQ,GAAG,KAAK,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;gBAC9C,MAAM,cAAc,GAAG,KAAK,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAC;gBAC1D,OAAO;oBACL,MAAM,EAAE,OAAO,CAAC,MAAM;oBACtB,IAAI,EAAE,OAAO,CAAC,IAAI;oBAClB,YAAY,EAAE,QAAQ;oBACtB,iBAAiB,EAAE,cAAc;oBACjC,cAAc,EAAE,EAAE;oBAClB,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,mBAAmB,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC;iBACpE,CAAC;YACJ,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED,MAAM,MAAM,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC1D,IAAI,UAAU,CAAC,gBAAgB,EAAE,CAAC;QAChC,MAAM,CAAC,IAAI,CAAC;YACV,KAAK,EAAE,OAAO;YACd,OAAO,EAAE,UAAU,CAAC,gBAAgB;YACpC,EAAE,EAAE,UAAU,CAAC,mBAAmB,EAAE,WAAW,EAAE;SAClD,CAAC,CAAC;IACL,CAAC;IACD,MAAM,CAAC,IAAI,CAAC,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,qCAAqC,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;IAEjG,IACE,eAAe;QACf,eAAe,CAAC,UAAU,KAAK,MAAM;QACrC,eAAe,CAAC,aAAa,IAAI,IAAI;QACrC,eAAe,CAAC,sBAAsB,IAAI,IAAI;QAC9C,eAAe,CAAC,sBAAsB,GAAG,CAAC,EAC1C,CAAC;QACD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CACxB,CAAC,EACD,IAAI,CAAC,KAAK,CAAC,CAAC,eAAe,CAAC,aAAa,GAAG,eAAe,CAAC,sBAAsB,CAAC,GAAG,GAAG,CAAC,CAC3F,CAAC;QACF,IAAI,SAAS,IAAI,EAAE,EAAE,CAAC;YACpB,MAAM,CAAC,IAAI,CAAC;gBACV,KAAK,EAAE,SAAS;gBAChB,OAAO,EACL,sBAAsB,SAAS,eAAe;oBAC9C,6DAA6D;oBAC7D,qEAAqE;gBACvE,EAAE,EAAE,GAAG;aACR,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,IAAI,WAAW,IAAI,MAAM,EAAE,MAAM,EAAE,CAAC;QAClC,uDAAuD;QACvD,mFAAmF;QACnF,IAAI,UAAU,CAAC,kBAAkB,IAAI,IAAI,IAAI,UAAU,CAAC,iBAAiB,IAAI,IAAI,EAAE,CAAC;YAClF,MAAM,CAAC,IAAI,CAAC;gBACV,KAAK,EAAE,SAAS;gBAChB,OAAO,EAAE,uDAAuD;gBAChE,EAAE,EAAE,GAAG;aACR,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CACvB,UAAU,CAAC,kBAAkB,EAAE,OAAO,EAAE,IAAI,CAAC,EAC7C,UAAU,CAAC,iBAAiB,EAAE,OAAO,EAAE,IAAI,CAAC,CAC7C,CAAC;YACF,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,CAAC,GAAG,IAAI,CAAC,CAAC;YAC9D,6EAA6E;YAC7E,gFAAgF;YAChF,mGAAmG;YACnG,6FAA6F;YAC7F,IAAI,UAAU,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC;gBACzB,MAAM,CAAC,IAAI,CAAC;oBACV,KAAK,EAAE,OAAO;oBACd,OAAO,EAAE,qCAAqC,UAAU,GAAG;oBAC3D,EAAE,EAAE,GAAG;iBACR,CAAC,CAAC;YACL,CAAC;iBAAM,IAAI,UAAU,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC;gBAC/B,MAAM,CAAC,IAAI,CAAC;oBACV,KAAK,EAAE,SAAS;oBAChB,OAAO,EAAE,qCAAqC,UAAU,GAAG;oBAC3D,EAAE,EAAE,GAAG;iBACR,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO;QACL,OAAO,EAAE,eAAe,CAAC,CAAC,CAAC,UAAU,CAAC,kBAAkB,CAAC,CAAC,CAAC,SAAS;QACpE,WAAW,EAAE;YACX;gBACE,EAAE,EAAE,YAAY,CAAC,EAAE,IAAI,qBAAqB;gBAC5C,GAAG,EAAE,GAAG;gBACR,SAAS,EAAE,UAAU,CAAC,SAAS;gBAC/B,wBAAwB,EAAE,MAAM,EAAE,aAAa,IAAI,KAAK;gBACxD,gBAAgB;gBAChB,QAAQ,EAAE,MAAM,EAAE,cAAc,IAAI,SAAS;gBAC7C,kBAAkB,EAAE,UAAU,CAAC,kBAAkB,EAAE,WAAW,EAAE;gBAChE,iBAAiB,EAAE,UAAU,CAAC,iBAAiB,EAAE,WAAW,EAAE;gBAC9D,qBAAqB,EAAE,qBAAqB;gBAC5C,UAAU,EAAE,eAAe,EAAE,UAAU;gBACvC,aAAa,EAAE,eAAe,EAAE,aAAa;gBAC7C,sBAAsB,EAAE,eAAe,EAAE,sBAAsB;gBAC/D,MAAM,EAAE,WAAW;aACpB;SACF;QACD,MAAM,EAAE,WAAW,CAAC,MAAM,CAAC;KAC5B,CAAC;AACJ,CAAC;AAED,SAAS,WAAW,CAAC,MAA0B;IAC7C,IAAI,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC7B,IAAI,MAAM,GAAuB,EAAE,CAAC;IACpC,KAAK,IAAI,KAAK,IAAI,MAAM,EAAE,CAAC;QACzB,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QAClC,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAClB,SAAS;QACX,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACd,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACrB,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}
+{"version":3,"file":"diagnostics.js","sourceRoot":"","sources":["../../src/api/diagnostics.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,qBAAqB,EAAE,MAAM,mCAAmC,CAAC;AAClF,OAAO,EAAE,WAAW,EAAwC,MAAM,+BAA+B,CAAC;AAIlG,OAAO,EAAE,qCAAqC,EAAE,MAAM,mBAAmB,CAAC;AA2B1E,MAAM,CAAC,MAAM,qBAAqB,GAAG,SAAS,CAAC;AAE/C,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,UAAoB,EACpB,UAAqD,EACrD,OAA2B;AAC3B;;;;;GAKG;AACH,aAA8C;IAE9C,IAAI,UAAU,IAAI,IAAI,EAAE,CAAC;QACvB,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,eAAe,GAAG,OAAO,CAAC,eAAe,IAAI,KAAK,CAAC;IACzD,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,KAAK,CAAC;IACjD,MAAM,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,IAAI,KAAK,CAAC;IAC3D,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAErC,IAAI,MAA4B,CAAC;IACjC,IAAI,SAAsC,CAAC;IAC3C,IAAI,CAAC;QACH,SAAS,GAAG,UAAU,CAAC,MAAM,CAAC,UAAU,CAAC,wBAAwB,EAAE,CAAC,CAAC;QACrE,gGAAgG;QAChG,MAAM,CAAC,YAAY,CAAC,GAAG,SAAS,CAAC,WAAW,CAAC;QAC7C,IAAI,YAAY,IAAI,IAAI,EAAE,CAAC;YACzB,MAAM,IAAI,qBAAqB,CAAC,0BAA0B,CAAC,CAAC;QAC9D,CAAC;QACD,MAAM,GAAG,YAAY,CAAC;IACxB,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO;YACL,OAAO,EAAE,eAAe,CAAC,CAAC,CAAC,UAAU,CAAC,kBAAkB,CAAC,CAAC,CAAC,SAAS;YACpE,WAAW,EAAE,EAAE;YACf,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC;SAC1D,CAAC;IACJ,CAAC;IAED,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,CAAC;IACzD,MAAM,YAAY,GAAG,MAAM,UAAU,CAAC,eAAe,EAAE,CAAC;IACxD,gFAAgF;IAChF,wFAAwF;IACxF,MAAM,GAAG,GAAG,YAAY,CAAC,GAAG,IAAI,WAAW,CAAC;IAC5C,MAAM,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC,MAAM,aAAa,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;IAC1E,IAAI,qBAAqB,GAAuB,SAAS,CAAC;IAC1D,IAAI,eAAe,GAAkC,SAAS,CAAC;IAE/D,IAAI,WAAW,GAAgB,EAAE,CAAC;IAElC,IAAI,gBAAgB,EAAE,CAAC;QACrB,MAAM,qBAAqB,GAAG,KAAK,CAAC,eAAe,EAAE,CAAC;QACtD,MAAM,eAAe,GAAG,MAAM,UAAU,CAAC,kBAAkB,CAAC,qBAAqB,EAAE,KAAK,CAAC,CAAC;QAC1F,WAAW,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;YAC7C,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACtB,CAAC;iBAAM,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBACvB,OAAO,IAAI,CAAC,MAAM,CAAC;YACrB,CAAC;iBAAM,CAAC;gBACN,OAAO,EAAE,CAAC;YACZ,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,IAAI,aAAa,EAAE,CAAC;YAClB,IAAI,CAAC;gBACH,qBAAqB,GAAG,MAAM,UAAU,CAAC,sBAAsB,CAAC;oBAC9D,aAAa,EAAE,aAAa;iBAC7B,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,SAAS;gBACT,MAAM,CAAC,IAAI,CAAC,+BAA+B,EAAE,CAAC,CAAC,CAAC;YAClD,CAAC;YAED,IAAI,UAAU,CAAC,gBAAgB,IAAI,UAAU,CAAC,qBAAqB,EAAE,CAAC;gBACpE,IAAI,CAAC;oBACH,eAAe,GAAG,MAAM,UAAU,CAAC,gBAAgB,CAAC;wBAClD,QAAQ,EAAE,UAAU,CAAC,qBAAqB;qBAC3C,CAAC,CAAC;gBACL,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACX,MAAM,CAAC,IAAI,CAAC,0BAA0B,EAAE,CAAC,CAAC,CAAC;gBAC7C,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;SAAM,CAAC;QACN,MAAM,qBAAqB,GAAG,KAAK,CAAC,eAAe,EAAE,CAAC;QAEtD,WAAW,GAAG,qBAAqB,CAAC,GAAG,CAAC,CAAC,OAAO,EAAa,EAAE;YAC7D,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;gBACvB,OAAO;oBACL,MAAM,EAAE,OAAO,CAAC,MAAM;oBACtB,IAAI,EAAE,OAAO,CAAC,WAAW;oBACzB,OAAO,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS;oBAE9D,YAAY,EAAE,KAAK;oBACnB,iBAAiB,EAAE,KAAK;oBACxB,cAAc,EAAE,EAAE;oBAClB,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,mBAAmB,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC;iBACpE,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,MAAM,MAAM,GAAmB;oBAC7B,aAAa,EAAE,GAAG;oBAClB,MAAM,EAAE,OAAO,CAAC,MAAM;oBACtB,IAAI,EAAE,OAAO,CAAC,YAAY;iBAC3B,CAAC;gBACF,MAAM,QAAQ,GAAG,KAAK,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;gBAC9C,MAAM,cAAc,GAAG,KAAK,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAC;gBAC1D,OAAO;oBACL,MAAM,EAAE,OAAO,CAAC,MAAM;oBACtB,IAAI,EAAE,OAAO,CAAC,IAAI;oBAClB,YAAY,EAAE,QAAQ;oBACtB,iBAAiB,EAAE,cAAc;oBACjC,cAAc,EAAE,EAAE;oBAClB,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,mBAAmB,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC;iBACpE,CAAC;YACJ,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED,MAAM,MAAM,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC1D,MAAM,YAAY,GAAG,MAAM,UAAU,CAAC,mBAAmB,EAAE,CAAC;IAE5D,IAAI,YAAY,EAAE,gBAAgB,EAAE,CAAC;QACnC,MAAM,CAAC,IAAI,CAAC;YACV,KAAK,EAAE,OAAO;YACd,OAAO,EAAE,YAAY,CAAC,gBAAgB;YACtC,EAAE,EAAE,YAAY,CAAC,mBAAmB,EAAE,WAAW,EAAE;SACpD,CAAC,CAAC;IACL,CAAC;IACD,MAAM,CAAC,IAAI,CAAC,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,qCAAqC,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;IAEjG,IACE,eAAe;QACf,eAAe,CAAC,UAAU,KAAK,MAAM;QACrC,eAAe,CAAC,aAAa,IAAI,IAAI;QACrC,eAAe,CAAC,sBAAsB,IAAI,IAAI;QAC9C,eAAe,CAAC,sBAAsB,GAAG,CAAC,EAC1C,CAAC;QACD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CACxB,CAAC,EACD,IAAI,CAAC,KAAK,CAAC,CAAC,eAAe,CAAC,aAAa,GAAG,eAAe,CAAC,sBAAsB,CAAC,GAAG,GAAG,CAAC,CAC3F,CAAC;QACF,IAAI,SAAS,IAAI,EAAE,EAAE,CAAC;YACpB,MAAM,CAAC,IAAI,CAAC;gBACV,KAAK,EAAE,SAAS;gBAChB,OAAO,EACL,sBAAsB,SAAS,eAAe;oBAC9C,6DAA6D;oBAC7D,qEAAqE;gBACvE,EAAE,EAAE,GAAG;aACR,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,IAAI,WAAW,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QAClC,uDAAuD;QACvD,mFAAmF;QACnF,IAAI,YAAY,EAAE,kBAAkB,IAAI,IAAI,IAAI,YAAY,EAAE,iBAAiB,IAAI,IAAI,EAAE,CAAC;YACxF,MAAM,CAAC,IAAI,CAAC;gBACV,KAAK,EAAE,SAAS;gBAChB,OAAO,EAAE,uDAAuD;gBAChE,EAAE,EAAE,GAAG;aACR,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CACvB,YAAY,CAAC,kBAAkB,EAAE,OAAO,EAAE,IAAI,CAAC,EAC/C,YAAY,CAAC,iBAAiB,EAAE,OAAO,EAAE,IAAI,CAAC,CAC/C,CAAC;YACF,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,CAAC,GAAG,IAAI,CAAC,CAAC;YAC9D,6EAA6E;YAC7E,gFAAgF;YAChF,mGAAmG;YACnG,6FAA6F;YAC7F,IAAI,UAAU,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC;gBACzB,MAAM,CAAC,IAAI,CAAC;oBACV,KAAK,EAAE,OAAO;oBACd,OAAO,EAAE,qCAAqC,UAAU,GAAG;oBAC3D,EAAE,EAAE,GAAG;iBACR,CAAC,CAAC;YACL,CAAC;iBAAM,IAAI,UAAU,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC;gBAC/B,MAAM,CAAC,IAAI,CAAC;oBACV,KAAK,EAAE,SAAS;oBAChB,OAAO,EAAE,qCAAqC,UAAU,GAAG;oBAC3D,EAAE,EAAE,GAAG;iBACR,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO;QACL,OAAO,EAAE,eAAe,CAAC,CAAC,CAAC,UAAU,CAAC,kBAAkB,CAAC,CAAC,CAAC,SAAS;QACpE,WAAW,EAAE;YACX;gBACE,EAAE,EAAE,YAAY,CAAC,EAAE,IAAI,qBAAqB;gBAC5C,GAAG,EAAE,GAAG;gBACR,SAAS,EAAE,UAAU,CAAC,qBAAqB;gBAC3C,wBAAwB,EAAE,MAAM,EAAE,YAAY,IAAI,KAAK;gBACvD,gBAAgB;gBAChB,QAAQ,EAAE,MAAM,EAAE,SAAS,IAAI,SAAS;gBACxC,kBAAkB,EAAE,YAAY,EAAE,kBAAkB,EAAE,WAAW,EAAE;gBACnE,iBAAiB,EAAE,YAAY,EAAE,iBAAiB,EAAE,WAAW,EAAE;gBACjE,qBAAqB,EAAE,qBAAqB;gBAC5C,UAAU,EAAE,eAAe,EAAE,UAAU;gBACvC,aAAa,EAAE,eAAe,EAAE,aAAa;gBAC7C,sBAAsB,EAAE,eAAe,EAAE,sBAAsB;gBAC/D,MAAM,EAAE,WAAW;aACpB;SACF;QACD,MAAM,EAAE,WAAW,CAAC,MAAM,CAAC;KAC5B,CAAC;AACJ,CAAC;AAED,SAAS,WAAW,CAAC,MAA0B;IAC7C,IAAI,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC7B,IAAI,MAAM,GAAuB,EAAE,CAAC;IACpC,KAAK,IAAI,KAAK,IAAI,MAAM,EAAE,CAAC;QACzB,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QAClC,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAClB,SAAS;QACX,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACd,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACrB,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/entry/commands/compact-action.js

@@ -15,6 +15,7 @@ const HEAP_LIMIT = v8.getHeapStatistics().heap_size_limit;
  * Limit to 1024MB overall.
  */
 const COMPACT_MEMORY_LIMIT_MB = Math.min(HEAP_LIMIT / 1024 / 1024 - 128, 1024);
+const MINUTE_MS = 60 * 1000;
 export function registerCompactAction(program) {
     const compactCommand = program
         .command(COMMAND_NAME)
@@ -66,7 +67,8 @@ export function registerCompactAction(program) {
             // Start the storage engine in order to create the appropriate BucketStorage
             await serviceContext.lifeCycleEngine.start();
             const bucketStorage = serviceContext.storageEngine.activeBucketStorage;
-            const active = await bucketStorage.getActiveStorage();
+            const deleteCheckpointRequestsBefore = new Date(Date.now() - config.api_parameters.checkpoint_request_retention_minutes * MINUTE_MS);
+            const active = (await bucketStorage.getActiveSyncConfig())?.storage;
             if (active == null) {
                 logger.info('No active instance to compact');
                 return;
@@ -77,6 +79,7 @@ export function registerCompactAction(program) {
                     memoryLimitMB: COMPACT_MEMORY_LIMIT_MB,
                     compactBuckets: buckets,
                     compactParameterData: compactParameters ?? false,
+                    deleteCheckpointRequestsBefore,
                     signal: abortController.signal
                 });
             }
@@ -84,6 +87,7 @@ export function registerCompactAction(program) {
                 await active.compact({
                     memoryLimitMB: COMPACT_MEMORY_LIMIT_MB,
                     compactParameterData: compactParameters ?? true,
+                    deleteCheckpointRequestsBefore,
                     signal: abortController.signal
                 });
             }

package/dist/entry/commands/compact-action.js.map

@@ -1 +1 @@
-{"version":3,"file":"compact-action.js","sourceRoot":"","sources":["../../../src/entry/commands/compact-action.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AACtE,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,MAAM,MAAM,8BAA8B,CAAC;AACvD,OAAO,KAAK,KAAK,MAAM,0BAA0B,CAAC;AAElD,OAAO,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AACzC,OAAO,EAAE,oBAAoB,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAE9E,MAAM,YAAY,GAAG,SAAS,CAAC;AAE/B;;GAEG;AACH,MAAM,UAAU,GAAG,EAAE,CAAC,iBAAiB,EAAE,CAAC,eAAe,CAAC;AAE1D;;;;GAIG;AACH,MAAM,uBAAuB,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,GAAG,IAAI,GAAG,IAAI,GAAG,GAAG,EAAE,IAAI,CAAC,CAAC;AAE/E,MAAM,UAAU,qBAAqB,CAAC,OAAgB;IACpD,MAAM,cAAc,GAAG,OAAO;SAC3B,OAAO,CAAC,YAAY,CAAC;SACrB,MAAM,CAAC,yBAAyB,EAAE,6EAA6E,CAAC;SAChH,MAAM,CAAC,yBAAyB,EAAE,6EAA6E,CAAC;SAChH,MAAM,CAAC,wBAAwB,EAAE,yCAAyC,CAAC,CAAC;IAE/E,iBAAiB,CAAC,cAAc,CAAC,CAAC;IAElC,OAAO,cAAc,CAAC,WAAW,CAAC,iBAAiB,CAAC,CAAC,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;QAC5E,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO;YAC7B,EAAE,KAAK,CAAC,GAAG,CAAC;aACX,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;aAC5B,MAAM,CAAC,OAAO,CAAC,CAAC;QACnB,IAAI,OAAO,EAAE,CAAC;YACZ,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;YAChE,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACvB,MAAM,CAAC,KAAK,CACV,yBAAyB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI;oBAC7C,0FAA0F,CAC7F,CAAC;gBACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC;QACH,CAAC;QAED,IAAI,iBAAiB,GAAmB,OAAO,CAAC,gBAAgB,CAAC;QAEjE,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;YACpB,MAAM,CAAC,IAAI,CAAC,uCAAuC,CAAC,CAAC;QACvD,CAAC;aAAM,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YAC/B,MAAM,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC;QAC5C,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC,0BAA0B,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAClE,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC,CAAC;QACrE,MAAM,cAAc,GAAG,IAAI,MAAM,CAAC,uBAAuB,CAAC;YACxD,WAAW,EAAE,MAAM,CAAC,kBAAkB,CAAC,OAAO;YAC9C,aAAa,EAAE,MAAM;SACtB,CAAC,CAAC;QACH,MAAM,eAAe,GAAG,IAAI,eAAe,EAAE,CAAC;QAC9C,MAAM,UAAU,GAAG,OAAO,CAAC,aAAa,EAAQ,CAAC;QAEjD,cAAc,CAAC,eAAe,CAAC,aAAa,CAAC,IAAI,EAAE;YACjD,IAAI,EAAE,KAAK,IAAI,EAAE;gBACf,eAAe,CAAC,KAAK,EAAE,CAAC;gBACxB,MAAM,UAAU,CAAC,OAAO,CAAC;YAC3B,CAAC;SACF,CAAC,CAAC;QAEH,8DAA8D;QAC9D,MAAM,aAAa,GAAG,SAAS,CAAC,iBAAiB,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;QACzE,MAAM,aAAa,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;QAE/C,MAAM,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QAExC,IAAI,CAAC;YACH,4EAA4E;YAC5E,MAAM,cAAc,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;YAC7C,MAAM,aAAa,GAAG,cAAc,CAAC,aAAa,CAAC,mBAAmB,CAAC;YAEvE,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,gBAAgB,EAAE,CAAC;YACtD,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;gBACnB,MAAM,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;gBAC7C,OAAO;YACT,CAAC;YACD,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;gBACpB,MAAM,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;gBACxC,MAAM,MAAM,CAAC,OAAO,CAAC;oBACnB,aAAa,EAAE,uBAAuB;oBACtC,cAAc,EAAE,OAAO;oBACvB,oBAAoB,EAAE,iBAAiB,IAAI,KAAK;oBAChD,MAAM,EAAE,eAAe,CAAC,MAAM;iBAC/B,CAAC,CAAC;YACL,CAAC;iBAAM,CAAC;gBACN,MAAM,MAAM,CAAC,OAAO,CAAC;oBACnB,aAAa,EAAE,uBAAuB;oBACtC,oBAAoB,EAAE,iBAAiB,IAAI,IAAI;oBAC/C,MAAM,EAAE,eAAe,CAAC,MAAM;iBAC/B,CAAC,CAAC;YACL,CAAC;YACD,MAAM,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;QACjD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,CAAC,KAAK,CAAC,oBAAoB,EAAE,CAAC,CAAC,CAAC;YACtC,2CAA2C;YAC3C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;gBAAS,CAAC;YACT,6FAA6F;YAC7F,UAAU,CAAC,OAAO,EAAE,CAAC;YACrB,2CAA2C;YAC3C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC"}
+{"version":3,"file":"compact-action.js","sourceRoot":"","sources":["../../../src/entry/commands/compact-action.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AACtE,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,MAAM,MAAM,8BAA8B,CAAC;AACvD,OAAO,KAAK,KAAK,MAAM,0BAA0B,CAAC;AAElD,OAAO,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AACzC,OAAO,EAAE,oBAAoB,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAE9E,MAAM,YAAY,GAAG,SAAS,CAAC;AAE/B;;GAEG;AACH,MAAM,UAAU,GAAG,EAAE,CAAC,iBAAiB,EAAE,CAAC,eAAe,CAAC;AAE1D;;;;GAIG;AACH,MAAM,uBAAuB,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,GAAG,IAAI,GAAG,IAAI,GAAG,GAAG,EAAE,IAAI,CAAC,CAAC;AAC/E,MAAM,SAAS,GAAG,EAAE,GAAG,IAAI,CAAC;AAE5B,MAAM,UAAU,qBAAqB,CAAC,OAAgB;IACpD,MAAM,cAAc,GAAG,OAAO;SAC3B,OAAO,CAAC,YAAY,CAAC;SACrB,MAAM,CAAC,yBAAyB,EAAE,6EAA6E,CAAC;SAChH,MAAM,CAAC,yBAAyB,EAAE,6EAA6E,CAAC;SAChH,MAAM,CAAC,wBAAwB,EAAE,yCAAyC,CAAC,CAAC;IAE/E,iBAAiB,CAAC,cAAc,CAAC,CAAC;IAElC,OAAO,cAAc,CAAC,WAAW,CAAC,iBAAiB,CAAC,CAAC,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;QAC5E,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO;YAC7B,EAAE,KAAK,CAAC,GAAG,CAAC;aACX,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;aAC5B,MAAM,CAAC,OAAO,CAAC,CAAC;QACnB,IAAI,OAAO,EAAE,CAAC;YACZ,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;YAChE,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACvB,MAAM,CAAC,KAAK,CACV,yBAAyB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI;oBAC7C,0FAA0F,CAC7F,CAAC;gBACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC;QACH,CAAC;QAED,IAAI,iBAAiB,GAAmB,OAAO,CAAC,gBAAgB,CAAC;QAEjE,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;YACpB,MAAM,CAAC,IAAI,CAAC,uCAAuC,CAAC,CAAC;QACvD,CAAC;aAAM,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YAC/B,MAAM,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC;QAC5C,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC,0BAA0B,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAClE,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC,CAAC;QACrE,MAAM,cAAc,GAAG,IAAI,MAAM,CAAC,uBAAuB,CAAC;YACxD,WAAW,EAAE,MAAM,CAAC,kBAAkB,CAAC,OAAO;YAC9C,aAAa,EAAE,MAAM;SACtB,CAAC,CAAC;QACH,MAAM,eAAe,GAAG,IAAI,eAAe,EAAE,CAAC;QAC9C,MAAM,UAAU,GAAG,OAAO,CAAC,aAAa,EAAQ,CAAC;QAEjD,cAAc,CAAC,eAAe,CAAC,aAAa,CAAC,IAAI,EAAE;YACjD,IAAI,EAAE,KAAK,IAAI,EAAE;gBACf,eAAe,CAAC,KAAK,EAAE,CAAC;gBACxB,MAAM,UAAU,CAAC,OAAO,CAAC;YAC3B,CAAC;SACF,CAAC,CAAC;QAEH,8DAA8D;QAC9D,MAAM,aAAa,GAAG,SAAS,CAAC,iBAAiB,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;QACzE,MAAM,aAAa,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;QAE/C,MAAM,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QAExC,IAAI,CAAC;YACH,4EAA4E;YAC5E,MAAM,cAAc,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;YAC7C,MAAM,aAAa,GAAG,cAAc,CAAC,aAAa,CAAC,mBAAmB,CAAC;YACvE,MAAM,8BAA8B,GAAG,IAAI,IAAI,CAC7C,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,cAAc,CAAC,oCAAoC,GAAG,SAAS,CACpF,CAAC;YAEF,MAAM,MAAM,GAAG,CAAC,MAAM,aAAa,CAAC,mBAAmB,EAAE,CAAC,EAAE,OAAO,CAAC;YACpE,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;gBACnB,MAAM,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;gBAC7C,OAAO;YACT,CAAC;YACD,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;gBACpB,MAAM,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;gBACxC,MAAM,MAAM,CAAC,OAAO,CAAC;oBACnB,aAAa,EAAE,uBAAuB;oBACtC,cAAc,EAAE,OAAO;oBACvB,oBAAoB,EAAE,iBAAiB,IAAI,KAAK;oBAChD,8BAA8B;oBAC9B,MAAM,EAAE,eAAe,CAAC,MAAM;iBAC/B,CAAC,CAAC;YACL,CAAC;iBAAM,CAAC;gBACN,MAAM,MAAM,CAAC,OAAO,CAAC;oBACnB,aAAa,EAAE,uBAAuB;oBACtC,oBAAoB,EAAE,iBAAiB,IAAI,IAAI;oBAC/C,8BAA8B;oBAC9B,MAAM,EAAE,eAAe,CAAC,MAAM;iBAC/B,CAAC,CAAC;YACL,CAAC;YACD,MAAM,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;QACjD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,CAAC,KAAK,CAAC,oBAAoB,EAAE,CAAC,CAAC,CAAC;YACtC,2CAA2C;YAC3C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;gBAAS,CAAC;YACT,6FAA6F;YAC7F,UAAU,CAAC,OAAO,EAAE,CAAC;YACrB,2CAA2C;YAC3C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/modules/AbstractModule.d.ts

@@ -1,11 +1,15 @@
 import winston from 'winston';
-import { PersistedSyncRulesContent } from '../storage/storage-index.js';
+import { PersistedReplicationStream } from '../storage/storage-index.js';
 import { ServiceContextContainer } from '../system/ServiceContext.js';
 export interface TearDownOptions {
     /**
      *  If required, tear down any configuration/state for the specific replication stream
      */
-    syncRules?: PersistedSyncRulesContent[];
+    replicationStreams?: PersistedReplicationStream[];
+    /**
+     * @deprecated Use replicationStreams.
+     */
+    syncRules?: PersistedReplicationStream[];
 }
 export interface AbstractModuleOptions {
     name: string;

package/dist/modules/AbstractModule.js.map

@@ -1 +1 @@
-{"version":3,"file":"AbstractModule.js","sourceRoot":"","sources":["../../src/modules/AbstractModule.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAgB3D,MAAM,OAAgB,cAAc;IAGF;IAFtB,MAAM,CAAiB;IAEjC,YAAgC,OAA8B;QAA9B,YAAO,GAAP,OAAO,CAAuB;QAC5D,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,UAAU,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;IACjE,CAAC;IAYD,IAAW,IAAI;QACb,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;IAC3B,CAAC;CACF"}
+{"version":3,"file":"AbstractModule.js","sourceRoot":"","sources":["../../src/modules/AbstractModule.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAoB3D,MAAM,OAAgB,cAAc;IAGF;IAFtB,MAAM,CAAiB;IAEjC,YAAgC,OAA8B;QAA9B,YAAO,GAAP,OAAO,CAAuB;QAC5D,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,UAAU,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;IACjE,CAAC;IAYD,IAAW,IAAI;QACb,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;IAC3B,CAAC;CACF"}

package/dist/replication/AbstractReplicator.d.ts

@@ -28,19 +28,18 @@ export interface AbstractReplicatorOptions {
 export declare abstract class AbstractReplicator<T extends AbstractReplicationJob = AbstractReplicationJob> {
     private options;
     protected logger: winston.Logger;
-    private lockAlerted;
+    private lastReplicationStreamInfoLogs;
     /**
-     *  Map of replication jobs by replication stream id. Usually there is only one running job, but there could be two when
-     *  transitioning to a new replication stream.
+     *  Map of replication jobs by replication stream job id. Usually there is only one running job, but there could be two
+     *  when transitioning to a new replication stream.
      */
     private replicationJobs;
     /**
-     * Map of replciation stream ids to promises that are clearing the replication stream.
+     * Map of replication stream ids to promises that are clearing the replication stream.
      *
-     * We primarily do this to keep track of what we're currently clearing, but don't currently
-     * use the Promise value.
+     * We primarily do this to keep track of what we're currently clearing.
      */
-    private clearingJobs;
+    protected readonly clearingJobs: Map<number, Promise<void>>;
     /**
      * Used for replication lag computation.
      */
@@ -48,6 +47,11 @@ export declare abstract class AbstractReplicator<T extends AbstractReplicationJo
     private lastPing;
     private abortController;
     protected constructor(options: AbstractReplicatorOptions);
+    /**
+     * Create the source-specific replication job for a locked storage stream.
+     *
+     * The job owns the per-stream snapshot and streaming work for this attempt.
+     */
     abstract createJob(options: CreateJobOptions): T;
     /**
      * Clean up any configuration or state for the specified replication stream on the datasource.
@@ -64,8 +68,23 @@ export declare abstract class AbstractReplicator<T extends AbstractReplicationJo
     stop(): Promise<void>;
     private runLoop;
     private refresh;
+    private logReplicationStreamInfoOnce;
+    private clearReplicationStreamInfoLog;
+    /**
+     * When we load sync config from the filesystem/config source, we ignore any config loaded by other processes.
+     * The idea is that if a different process loads updated config, that process should process it, not this one.
+     *
+     * This specifically helps for cases with rolling deploys.
+     *
+     * This does not apply when sync config is loaded from the database/API, instead of in the config.
+     */
+    private shouldHandleReplicationStream;
     protected createJobId(syncRuleId: number): string;
+    protected terminateStoppedReplicationStream(replicationStream: storage.PersistedReplicationStream, syncRuleStorage: storage.SyncRulesBucketStorage): Promise<void>;
     protected terminateSyncRules(syncRuleStorage: storage.SyncRulesBucketStorage): Promise<void>;
+    /**
+     * Check source connectivity using the replicator's decoded connection config.
+     */
     abstract testConnection(): Promise<ConnectionTestResult>;
     /**
      * Measure replication lag in milliseconds.