@powersync/service-core 0.4.1 → 0.5.0

package/CHANGELOG.md

@@ -1,5 +1,21 @@
 # @powersync/service-core
 
+## 0.5.0
+
+### Minor Changes
+
+- 1c1a3bf: Implement a compact command
+
+## 0.4.2
+
+### Patch Changes
+
+- bdbf95c: Log user_id and sync stats for each connection
+- Updated dependencies [876f4a0]
+- Updated dependencies [9bff878]
+  - @powersync/service-sync-rules@0.18.1
+  - @powersync/service-rsocket-router@0.0.10
+
 ## 0.4.1
 
 ### Patch Changes

package/dist/entry/cli-entry.js

@@ -1,7 +1,7 @@
 import { Command } from 'commander';
 import { registerMigrationAction } from './commands/migrate-action.js';
 import { registerTearDownAction } from './commands/teardown-action.js';
-import { registerStartAction } from './entry-index.js';
+import { registerCompactAction, registerStartAction } from './entry-index.js';
 import { logger } from '@powersync/lib-services-framework';
 /**
  * Generates a Commander program which serves as the entry point
@@ -14,6 +14,7 @@ export function generateEntryProgram(startHandlers) {
     entryProgram.name('powersync-runner').description('CLI to initiate a PowerSync service runner');
     registerTearDownAction(entryProgram);
     registerMigrationAction(entryProgram);
+    registerCompactAction(entryProgram);
     if (startHandlers) {
         registerStartAction(entryProgram, startHandlers);
     }

package/dist/entry/cli-entry.js.map

@@ -1 +1 @@
-{"version":3,"file":"cli-entry.js","sourceRoot":"","sources":["../../src/entry/cli-entry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAGpC,OAAO,EAAE,uBAAuB,EAAE,MAAM,8BAA8B,CAAC;AACvE,OAAO,EAAE,sBAAsB,EAAE,MAAM,+BAA+B,CAAC;AACvE,OAAO,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC;AACvD,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAE3D;;;;;GAKG;AACH,MAAM,UAAU,oBAAoB,CAAC,aAAyD;IAC5F,MAAM,YAAY,GAAG,IAAI,OAAO,EAAE,CAAC;IACnC,YAAY,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,WAAW,CAAC,4CAA4C,CAAC,CAAC;IAEhG,sBAAsB,CAAC,YAAY,CAAC,CAAC;IACrC,uBAAuB,CAAC,YAAY,CAAC,CAAC;IAEtC,IAAI,aAAa,EAAE;QACjB,mBAAmB,CAAC,YAAY,EAAE,aAAa,CAAC,CAAC;KAClD;IAED,OAAO;QACL,OAAO,EAAE,YAAY;QACrB;;WAEG;QACH,OAAO,EAAE,KAAK,UAAU,UAAU;YAChC,IAAI;gBACF,MAAM,YAAY,CAAC,UAAU,EAAE,CAAC;aACjC;YAAC,OAAO,CAAC,EAAE;gBACV,MAAM,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC;gBAC/B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;aACjB;QACH,CAAC;KACF,CAAC;AACJ,CAAC"}
+{"version":3,"file":"cli-entry.js","sourceRoot":"","sources":["../../src/entry/cli-entry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAGpC,OAAO,EAAE,uBAAuB,EAAE,MAAM,8BAA8B,CAAC;AACvE,OAAO,EAAE,sBAAsB,EAAE,MAAM,+BAA+B,CAAC;AACvE,OAAO,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC;AAC9E,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAE3D;;;;;GAKG;AACH,MAAM,UAAU,oBAAoB,CAAC,aAAyD;IAC5F,MAAM,YAAY,GAAG,IAAI,OAAO,EAAE,CAAC;IACnC,YAAY,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,WAAW,CAAC,4CAA4C,CAAC,CAAC;IAEhG,sBAAsB,CAAC,YAAY,CAAC,CAAC;IACrC,uBAAuB,CAAC,YAAY,CAAC,CAAC;IACtC,qBAAqB,CAAC,YAAY,CAAC,CAAC;IAEpC,IAAI,aAAa,EAAE;QACjB,mBAAmB,CAAC,YAAY,EAAE,aAAa,CAAC,CAAC;KAClD;IAED,OAAO;QACL,OAAO,EAAE,YAAY;QACrB;;WAEG;QACH,OAAO,EAAE,KAAK,UAAU,UAAU;YAChC,IAAI;gBACF,MAAM,YAAY,CAAC,UAAU,EAAE,CAAC;aACjC;YAAC,OAAO,CAAC,EAAE;gBACV,MAAM,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC;gBAC/B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;aACjB;QACH,CAAC;KACF,CAAC;AACJ,CAAC"}

package/dist/entry/commands/compact-action.d.ts

@@ -0,0 +1,2 @@
+import { Command } from 'commander';
+export declare function registerCompactAction(program: Command): Command;

package/dist/entry/commands/compact-action.js

@@ -0,0 +1,49 @@
+import { logger } from '@powersync/lib-services-framework';
+import * as v8 from 'v8';
+import { mongo } from '../../db/db-index.js';
+import { MongoBucketStorage, PowerSyncMongo } from '../../storage/storage-index.js';
+import { loadConfig } from '../../util/config.js';
+import { extractRunnerOptions, wrapConfigCommand } from './config-command.js';
+const COMMAND_NAME = 'compact';
+/**
+ * Approximately max-old-space-size + 64MB.
+ */
+const HEAP_LIMIT = v8.getHeapStatistics().heap_size_limit;
+/**
+ * Subtract 128MB for process overhead.
+ *
+ * Limit to 1024MB overall.
+ */
+const COMPACT_MEMORY_LIMIT_MB = Math.min(HEAP_LIMIT / 1024 / 1024 - 128, 1024);
+export function registerCompactAction(program) {
+    const compactCommand = program.command(COMMAND_NAME);
+    wrapConfigCommand(compactCommand);
+    return compactCommand.description('Compact storage').action(async (options) => {
+        const runnerConfig = extractRunnerOptions(options);
+        const config = await loadConfig(runnerConfig);
+        const { storage } = config;
+        const client = mongo.createMongoClient(storage);
+        await client.connect();
+        try {
+            const psdb = new PowerSyncMongo(client);
+            const bucketStorage = new MongoBucketStorage(psdb, { slot_name_prefix: config.slot_name_prefix });
+            const active = await bucketStorage.getActiveSyncRules();
+            if (active == null) {
+                logger.info('No active instance to compact');
+                return;
+            }
+            const p = bucketStorage.getInstance(active);
+            await p.compact({ memoryLimitMB: COMPACT_MEMORY_LIMIT_MB });
+            logger.info('done');
+        }
+        catch (e) {
+            logger.error(`Failed to compact: ${e.toString()}`);
+            process.exit(1);
+        }
+        finally {
+            await client.close();
+            process.exit(0);
+        }
+    });
+}
+//# sourceMappingURL=compact-action.js.map

package/dist/entry/commands/compact-action.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"compact-action.js","sourceRoot":"","sources":["../../../src/entry/commands/compact-action.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAC3D,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAC7C,OAAO,EAAE,kBAAkB,EAAE,cAAc,EAAE,MAAM,gCAAgC,CAAC;AACpF,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAClD,OAAO,EAAE,oBAAoB,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAE9E,MAAM,YAAY,GAAG,SAAS,CAAC;AAE/B;;GAEG;AACH,MAAM,UAAU,GAAG,EAAE,CAAC,iBAAiB,EAAE,CAAC,eAAe,CAAC;AAE1D;;;;GAIG;AACH,MAAM,uBAAuB,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,GAAG,IAAI,GAAG,IAAI,GAAG,GAAG,EAAE,IAAI,CAAC,CAAC;AAE/E,MAAM,UAAU,qBAAqB,CAAC,OAAgB;IACpD,MAAM,cAAc,GAAG,OAAO,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;IAErD,iBAAiB,CAAC,cAAc,CAAC,CAAC;IAElC,OAAO,cAAc,CAAC,WAAW,CAAC,iBAAiB,CAAC,CAAC,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;QAC5E,MAAM,YAAY,GAAG,oBAAoB,CAAC,OAAO,CAAC,CAAC;QAEnD,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,YAAY,CAAC,CAAC;QAC9C,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,CAAC;QAC3B,MAAM,MAAM,GAAG,KAAK,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;QAChD,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;QACvB,IAAI;YACF,MAAM,IAAI,GAAG,IAAI,cAAc,CAAC,MAAM,CAAC,CAAC;YACxC,MAAM,aAAa,GAAG,IAAI,kBAAkB,CAAC,IAAI,EAAE,EAAE,gBAAgB,EAAE,MAAM,CAAC,gBAAgB,EAAE,CAAC,CAAC;YAClG,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,kBAAkB,EAAE,CAAC;YACxD,IAAI,MAAM,IAAI,IAAI,EAAE;gBAClB,MAAM,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;gBAC7C,OAAO;aACR;YACD,MAAM,CAAC,GAAG,aAAa,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;YAC5C,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,aAAa,EAAE,uBAAuB,EAAE,CAAC,CAAC;YAC5D,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;SACrB;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YACnD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;gBAAS;YACR,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;YACrB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;IACH,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/entry/entry-index.d.ts

@@ -3,3 +3,4 @@ export * from './commands/config-command.js';
 export * from './commands/migrate-action.js';
 export * from './commands/start-action.js';
 export * from './commands/teardown-action.js';
+export * from './commands/compact-action.js';

package/dist/entry/entry-index.js

@@ -3,4 +3,5 @@ export * from './commands/config-command.js';
 export * from './commands/migrate-action.js';
 export * from './commands/start-action.js';
 export * from './commands/teardown-action.js';
+export * from './commands/compact-action.js';
 //# sourceMappingURL=entry-index.js.map

package/dist/entry/entry-index.js.map

@@ -1 +1 @@
-{"version":3,"file":"entry-index.js","sourceRoot":"","sources":["../../src/entry/entry-index.ts"],"names":[],"mappings":"AAAA,cAAc,gBAAgB,CAAC;AAC/B,cAAc,8BAA8B,CAAC;AAC7C,cAAc,8BAA8B,CAAC;AAC7C,cAAc,4BAA4B,CAAC;AAC3C,cAAc,+BAA+B,CAAC"}
+{"version":3,"file":"entry-index.js","sourceRoot":"","sources":["../../src/entry/entry-index.ts"],"names":[],"mappings":"AAAA,cAAc,gBAAgB,CAAC;AAC/B,cAAc,8BAA8B,CAAC;AAC7C,cAAc,8BAA8B,CAAC;AAC7C,cAAc,4BAA4B,CAAC;AAC3C,cAAc,+BAA+B,CAAC;AAC9C,cAAc,8BAA8B,CAAC"}

package/dist/routes/endpoints/socket-route.js

@@ -5,6 +5,7 @@ import { Metrics } from '../../metrics/Metrics.js';
 import { streamResponse } from '../../sync/sync.js';
 import * as util from '../../util/util-index.js';
 import { SyncRoutes } from './sync-stream.js';
+import { RequestTracker } from '../../sync/RequestTracker.js';
 export const syncStreamReactive = (router) => router.reactiveStream(SyncRoutes.STREAM, {
     authorize: ({ context }) => {
         return {
@@ -51,6 +52,7 @@ export const syncStreamReactive = (router) => router.reactiveStream(SyncRoutes.S
             observer.triggerCancel();
         });
         Metrics.getInstance().concurrent_connections.add(1);
+        const tracker = new RequestTracker();
         try {
             for await (const data of streamResponse({
                 storage,
@@ -64,6 +66,7 @@ export const syncStreamReactive = (router) => router.reactiveStream(SyncRoutes.S
                     // RSocket handles keepalive events by default
                     keep_alive: false
                 },
+                tracker,
                 signal: controller.signal
             })) {
                 if (data == null) {
@@ -79,7 +82,7 @@ export const syncStreamReactive = (router) => router.reactiveStream(SyncRoutes.S
                     const serialized = serialize(data);
                     responder.onNext({ data: serialized }, false);
                     requestedN--;
-                    Metrics.getInstance().data_synced_bytes.add(serialized.length);
+                    tracker.addDataSynced(serialized.length);
                 }
                 if (requestedN <= 0) {
                     await new Promise((resolve) => {
@@ -112,6 +115,11 @@ export const syncStreamReactive = (router) => router.reactiveStream(SyncRoutes.S
             responder.onComplete();
             removeStopHandler();
             disposer();
+            logger.info(`Sync stream complete`, {
+                user_id: syncParams.user_id,
+                operations_synced: tracker.operationsSynced,
+                data_synced_bytes: tracker.dataSyncedBytes
+            });
             Metrics.getInstance().concurrent_connections.add(-1);
         }
     }

package/dist/routes/endpoints/socket-route.js.map

@@ -1 +1 @@
-{"version":3,"file":"socket-route.js","sourceRoot":"","sources":["../../../src/routes/endpoints/socket-route.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAC3E,OAAO,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AAClE,OAAO,EAAE,SAAS,EAAE,MAAM,MAAM,CAAC;AAEjC,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AACnD,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,KAAK,IAAI,MAAM,0BAA0B,CAAC;AAEjD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAE9C,MAAM,CAAC,MAAM,kBAAkB,GAAyB,CAAC,MAAM,EAAE,EAAE,CACjE,MAAM,CAAC,cAAc,CAAiC,UAAU,CAAC,MAAM,EAAE;IACvE,SAAS,EAAE,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE;QACzB,OAAO;YACL,UAAU,EAAE,CAAC,CAAC,OAAO,CAAC,aAAa;YACnC,MAAM,EAAE,CAAC,yBAAyB,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,YAAY,IAAI,EAAE,CAAC;SACvE,CAAC;IACJ,CAAC;IACD,SAAS,EAAE,MAAM,CAAC,sBAAsB,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC;IAC9F,OAAO,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,EAAE,EAAE,EAAE;QACpE,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;QAE3B,IAAI,MAAM,CAAC,MAAM,EAAE;YACjB,SAAS,CAAC,OAAO,CACf,IAAI,MAAM,CAAC,YAAY,CAAC;gBACtB,MAAM,EAAE,GAAG;gBACX,IAAI,EAAE,qBAAqB;gBAC3B,WAAW,EAAE,iCAAiC;aAC/C,CAAC,CACH,CAAC;YACF,SAAS,CAAC,UAAU,EAAE,CAAC;YACvB,OAAO;SACR;QAED,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QAEzC,MAAM,UAAU,GAAG,IAAI,iBAAiB,CAAC,OAAO,CAAC,aAAc,EAAE,MAAM,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC;QAE1F,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;QAC/B,0CAA0C;QAC1C,MAAM,EAAE,GAAG,MAAM,OAAO,CAAC,mBAAmB,EAAE,CAAC;QAC/C,IAAI,CAAC,EAAE,CAAC,YAAY,EAAE,EAAE;YACtB,SAAS,CAAC,OAAO,CACf,IAAI,MAAM,CAAC,YAAY,CAAC;gBACtB,MAAM,EAAE,GAAG;gBACX,IAAI,EAAE,eAAe;gBACrB,WAAW,EAAE,yBAAyB;aACvC,CAAC,CACH,CAAC;YACF,SAAS,CAAC,UAAU,EAAE,CAAC;YACvB,OAAO;SACR;QAED,IAAI,UAAU,GAAG,QAAQ,CAAC;QAC1B,MAAM,QAAQ,GAAG,QAAQ,CAAC,gBAAgB,CAAC;YACzC,OAAO,CAAC,CAAC;gBACP,UAAU,IAAI,CAAC,CAAC;YAClB,CAAC;YACD,MAAM,EAAE,GAAG,EAAE;gBACX,UAAU,CAAC,KAAK,EAAE,CAAC;YACrB,CAAC;SACF,CAAC,CAAC;QAEH,MAAM,iBAAiB,GAAG,MAAM,CAAC,cAAc,CAAC,GAAG,EAAE;YACnD,QAAQ,CAAC,aAAa,EAAE,CAAC;QAC3B,CAAC,CAAC,CAAC;QAEH,OAAO,CAAC,WAAW,EAAE,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACpD,IAAI;YACF,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,cAAc,CAAC;gBACtC,OAAO;gBACP,MAAM,EAAE;oBACN,GAAG,MAAM;oBACT,WAAW,EAAE,IAAI,CAAC,8BAA8B;iBACjD;gBACD,UAAU;gBACV,KAAK,EAAE,OAAQ,CAAC,aAAc;gBAC9B,kBAAkB,EAAE;oBAClB,8CAA8C;oBAC9C,UAAU,EAAE,KAAK;iBAClB;gBACD,MAAM,EAAE,UAAU,CAAC,MAAM;aAC1B,CAAC,EAAE;gBACF,IAAI,IAAI,IAAI,IAAI,EAAE;oBAChB,4CAA4C;oBAC5C,SAAS;iBACV;qBAAM,IAAI,OAAO,IAAI,IAAI,QAAQ,EAAE;oBAClC,2CAA2C;oBAC3C,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,EAAE,CAAC,CAAC;iBACpD;gBAED;oBACE,+CAA+C;oBAC/C,MAAM,UAAU,GAAG,SAAS,CAAC,IAAI,CAAW,CAAC;oBAC7C,SAAS,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE,KAAK,CAAC,CAAC;oBAC9C,UAAU,EAAE,CAAC;oBACb,OAAO,CAAC,WAAW,EAAE,CAAC,iBAAiB,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;iBAChE;gBAED,IAAI,UAAU,IAAI,CAAC,EAAE;oBACnB,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;wBAClC,MAAM,CAAC,GAAG,QAAQ,CAAC,gBAAgB,CAAC;4BAClC,OAAO;gCACL,IAAI,UAAU,GAAG,CAAC,EAAE;oCAClB,iEAAiE;oCACjE,OAAO,EAAE,CAAC;oCACV,CAAC,EAAE,CAAC;iCACL;4BACH,CAAC;4BACD,MAAM,EAAE,GAAG,EAAE;gCACX,8CAA8C;gCAC9C,OAAO,EAAE,CAAC;gCACV,CAAC,EAAE,CAAC;4BACN,CAAC;yBACF,CAAC,CAAC;oBACL,CAAC,CAAC,CAAC;iBACJ;aACF;SACF;QAAC,OAAO,EAAE,EAAE;YACX,kDAAkD;YAClD,4CAA4C;YAC5C,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,mBAAmB,CAAC,EAAE,CAAC,CAAC;YACjD,MAAM,CAAC,KAAK,CAAC,mBAAmB,EAAE,KAAK,CAAC,CAAC;YACzC,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;SAC1B;gBAAS;YACR,SAAS,CAAC,UAAU,EAAE,CAAC;YACvB,iBAAiB,EAAE,CAAC;YACpB,QAAQ,EAAE,CAAC;YACX,OAAO,CAAC,WAAW,EAAE,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;SACtD;IACH,CAAC;CACF,CAAC,CAAC"}
+{"version":3,"file":"socket-route.js","sourceRoot":"","sources":["../../../src/routes/endpoints/socket-route.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAC3E,OAAO,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AAClE,OAAO,EAAE,SAAS,EAAE,MAAM,MAAM,CAAC;AAEjC,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AACnD,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,KAAK,IAAI,MAAM,0BAA0B,CAAC;AAEjD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAE9D,MAAM,CAAC,MAAM,kBAAkB,GAAyB,CAAC,MAAM,EAAE,EAAE,CACjE,MAAM,CAAC,cAAc,CAAiC,UAAU,CAAC,MAAM,EAAE;IACvE,SAAS,EAAE,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE;QACzB,OAAO;YACL,UAAU,EAAE,CAAC,CAAC,OAAO,CAAC,aAAa;YACnC,MAAM,EAAE,CAAC,yBAAyB,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,YAAY,IAAI,EAAE,CAAC;SACvE,CAAC;IACJ,CAAC;IACD,SAAS,EAAE,MAAM,CAAC,sBAAsB,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC;IAC9F,OAAO,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,EAAE,EAAE,EAAE;QACpE,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;QAE3B,IAAI,MAAM,CAAC,MAAM,EAAE;YACjB,SAAS,CAAC,OAAO,CACf,IAAI,MAAM,CAAC,YAAY,CAAC;gBACtB,MAAM,EAAE,GAAG;gBACX,IAAI,EAAE,qBAAqB;gBAC3B,WAAW,EAAE,iCAAiC;aAC/C,CAAC,CACH,CAAC;YACF,SAAS,CAAC,UAAU,EAAE,CAAC;YACvB,OAAO;SACR;QAED,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QAEzC,MAAM,UAAU,GAAG,IAAI,iBAAiB,CAAC,OAAO,CAAC,aAAc,EAAE,MAAM,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC;QAE1F,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;QAC/B,0CAA0C;QAC1C,MAAM,EAAE,GAAG,MAAM,OAAO,CAAC,mBAAmB,EAAE,CAAC;QAC/C,IAAI,CAAC,EAAE,CAAC,YAAY,EAAE,EAAE;YACtB,SAAS,CAAC,OAAO,CACf,IAAI,MAAM,CAAC,YAAY,CAAC;gBACtB,MAAM,EAAE,GAAG;gBACX,IAAI,EAAE,eAAe;gBACrB,WAAW,EAAE,yBAAyB;aACvC,CAAC,CACH,CAAC;YACF,SAAS,CAAC,UAAU,EAAE,CAAC;YACvB,OAAO;SACR;QAED,IAAI,UAAU,GAAG,QAAQ,CAAC;QAC1B,MAAM,QAAQ,GAAG,QAAQ,CAAC,gBAAgB,CAAC;YACzC,OAAO,CAAC,CAAC;gBACP,UAAU,IAAI,CAAC,CAAC;YAClB,CAAC;YACD,MAAM,EAAE,GAAG,EAAE;gBACX,UAAU,CAAC,KAAK,EAAE,CAAC;YACrB,CAAC;SACF,CAAC,CAAC;QAEH,MAAM,iBAAiB,GAAG,MAAM,CAAC,cAAc,CAAC,GAAG,EAAE;YACnD,QAAQ,CAAC,aAAa,EAAE,CAAC;QAC3B,CAAC,CAAC,CAAC;QAEH,OAAO,CAAC,WAAW,EAAE,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACpD,MAAM,OAAO,GAAG,IAAI,cAAc,EAAE,CAAC;QACrC,IAAI;YACF,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,cAAc,CAAC;gBACtC,OAAO;gBACP,MAAM,EAAE;oBACN,GAAG,MAAM;oBACT,WAAW,EAAE,IAAI,CAAC,8BAA8B;iBACjD;gBACD,UAAU;gBACV,KAAK,EAAE,OAAQ,CAAC,aAAc;gBAC9B,kBAAkB,EAAE;oBAClB,8CAA8C;oBAC9C,UAAU,EAAE,KAAK;iBAClB;gBACD,OAAO;gBACP,MAAM,EAAE,UAAU,CAAC,MAAM;aAC1B,CAAC,EAAE;gBACF,IAAI,IAAI,IAAI,IAAI,EAAE;oBAChB,4CAA4C;oBAC5C,SAAS;iBACV;qBAAM,IAAI,OAAO,IAAI,IAAI,QAAQ,EAAE;oBAClC,2CAA2C;oBAC3C,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,EAAE,CAAC,CAAC;iBACpD;gBAED;oBACE,+CAA+C;oBAC/C,MAAM,UAAU,GAAG,SAAS,CAAC,IAAI,CAAW,CAAC;oBAC7C,SAAS,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE,KAAK,CAAC,CAAC;oBAC9C,UAAU,EAAE,CAAC;oBACb,OAAO,CAAC,aAAa,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;iBAC1C;gBAED,IAAI,UAAU,IAAI,CAAC,EAAE;oBACnB,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;wBAClC,MAAM,CAAC,GAAG,QAAQ,CAAC,gBAAgB,CAAC;4BAClC,OAAO;gCACL,IAAI,UAAU,GAAG,CAAC,EAAE;oCAClB,iEAAiE;oCACjE,OAAO,EAAE,CAAC;oCACV,CAAC,EAAE,CAAC;iCACL;4BACH,CAAC;4BACD,MAAM,EAAE,GAAG,EAAE;gCACX,8CAA8C;gCAC9C,OAAO,EAAE,CAAC;gCACV,CAAC,EAAE,CAAC;4BACN,CAAC;yBACF,CAAC,CAAC;oBACL,CAAC,CAAC,CAAC;iBACJ;aACF;SACF;QAAC,OAAO,EAAE,EAAE;YACX,kDAAkD;YAClD,4CAA4C;YAC5C,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,mBAAmB,CAAC,EAAE,CAAC,CAAC;YACjD,MAAM,CAAC,KAAK,CAAC,mBAAmB,EAAE,KAAK,CAAC,CAAC;YACzC,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;SAC1B;gBAAS;YACR,SAAS,CAAC,UAAU,EAAE,CAAC;YACvB,iBAAiB,EAAE,CAAC;YACpB,QAAQ,EAAE,CAAC;YACX,MAAM,CAAC,IAAI,CAAC,sBAAsB,EAAE;gBAClC,OAAO,EAAE,UAAU,CAAC,OAAO;gBAC3B,iBAAiB,EAAE,OAAO,CAAC,gBAAgB;gBAC3C,iBAAiB,EAAE,OAAO,CAAC,eAAe;aAC3C,CAAC,CAAC;YACH,OAAO,CAAC,WAAW,EAAE,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;SACtD;IACH,CAAC;CACF,CAAC,CAAC"}

package/dist/routes/endpoints/sync-stream.js

@@ -6,6 +6,7 @@ import * as util from '../../util/util-index.js';
 import { Metrics } from '../../metrics/Metrics.js';
 import { authUser } from '../auth.js';
 import { routeDefinition } from '../router.js';
+import { RequestTracker } from '../../sync/RequestTracker.js';
 export var SyncRoutes;
 (function (SyncRoutes) {
     SyncRoutes["STREAM"] = "/sync/stream";
@@ -37,6 +38,7 @@ export const syncStreamed = routeDefinition({
             });
         }
         const controller = new AbortController();
+        const tracker = new RequestTracker();
         try {
             Metrics.getInstance().concurrent_connections.add(1);
             const stream = Readable.from(sync.transformToBytesTracked(sync.ndjson(sync.streamResponse({
@@ -44,8 +46,9 @@ export const syncStreamed = routeDefinition({
                 params,
                 syncParams,
                 token: payload.context.token_payload,
+                tracker,
                 signal: controller.signal
-            }))), { objectMode: false, highWaterMark: 16 * 1024 });
+            })), tracker), { objectMode: false, highWaterMark: 16 * 1024 });
             const deregister = system.addStopHandler(() => {
                 // This error is not currently propagated to the client
                 controller.abort();
@@ -70,6 +73,11 @@ export const syncStreamed = routeDefinition({
                 afterSend: async () => {
                     controller.abort();
                     Metrics.getInstance().concurrent_connections.add(-1);
+                    logger.info(`Sync stream complete`, {
+                        user_id: syncParams.user_id,
+                        operations_synced: tracker.operationsSynced,
+                        data_synced_bytes: tracker.dataSyncedBytes
+                    });
                 }
             });
         }

package/dist/routes/endpoints/sync-stream.js.map

@@ -1 +1 @@
-{"version":3,"file":"sync-stream.js","sourceRoot":"","sources":["../../../src/routes/endpoints/sync-stream.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AACnF,OAAO,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AAClE,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAElC,OAAO,KAAK,IAAI,MAAM,0BAA0B,CAAC;AACjD,OAAO,KAAK,IAAI,MAAM,0BAA0B,CAAC;AAEjD,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AACnD,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAE/C,MAAM,CAAN,IAAY,UAEX;AAFD,WAAY,UAAU;IACpB,qCAAuB,CAAA;AACzB,CAAC,EAFW,UAAU,KAAV,UAAU,QAErB;AAED,MAAM,CAAC,MAAM,YAAY,GAAG,eAAe,CAAC;IAC1C,IAAI,EAAE,UAAU,CAAC,MAAM;IACvB,MAAM,EAAE,MAAM,CAAC,UAAU,CAAC,IAAI;IAC9B,SAAS,EAAE,QAAQ;IACnB,SAAS,EAAE,MAAM,CAAC,sBAAsB,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC;IAC9F,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;QACzB,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC;QAEtC,IAAI,MAAM,CAAC,MAAM,EAAE;YACjB,MAAM,IAAI,MAAM,CAAC,YAAY,CAAC;gBAC5B,MAAM,EAAE,GAAG;gBACX,IAAI,EAAE,qBAAqB;gBAC3B,WAAW,EAAE,iCAAiC;aAC/C,CAAC,CAAC;SACJ;QAED,MAAM,MAAM,GAA8B,OAAO,CAAC,MAAM,CAAC;QACzD,MAAM,UAAU,GAAG,IAAI,iBAAiB,CAAC,OAAO,CAAC,OAAO,CAAC,aAAc,EAAE,OAAO,CAAC,MAAM,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC;QAE1G,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;QAC/B,0CAA0C;QAC1C,MAAM,EAAE,GAAG,MAAM,OAAO,CAAC,mBAAmB,EAAE,CAAC;QAC/C,IAAI,CAAC,EAAE,CAAC,YAAY,EAAE,EAAE;YACtB,MAAM,IAAI,MAAM,CAAC,YAAY,CAAC;gBAC5B,MAAM,EAAE,GAAG;gBACX,IAAI,EAAE,eAAe;gBACrB,WAAW,EAAE,yBAAyB;aACvC,CAAC,CAAC;SACJ;QACD,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,IAAI;YACF,OAAO,CAAC,WAAW,EAAE,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACpD,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAC1B,IAAI,CAAC,uBAAuB,CAC1B,IAAI,CAAC,MAAM,CACT,IAAI,CAAC,cAAc,CAAC;gBAClB,OAAO;gBACP,MAAM;gBACN,UAAU;gBACV,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,aAAc;gBACrC,MAAM,EAAE,UAAU,CAAC,MAAM;aAC1B,CAAC,CACH,CACF,EACD,EAAE,UAAU,EAAE,KAAK,EAAE,aAAa,EAAE,EAAE,GAAG,IAAI,EAAE,CAChD,CAAC;YAEF,MAAM,UAAU,GAAG,MAAM,CAAC,cAAc,CAAC,GAAG,EAAE;gBAC5C,uDAAuD;gBACvD,UAAU,CAAC,KAAK,EAAE,CAAC;gBACnB,MAAM,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YACH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;gBACtB,UAAU,EAAE,CAAC;YACf,CAAC,CAAC,CAAC;YAEH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;gBAC3B,UAAU,CAAC,KAAK,EAAE,CAAC;gBACnB,oDAAoD;gBACpD,IAAI,KAAK,CAAC,OAAO,IAAI,sBAAsB,EAAE;oBAC3C,MAAM,CAAC,KAAK,CAAC,+BAA+B,EAAE,KAAK,CAAC,CAAC;iBACtD;YACH,CAAC,CAAC,CAAC;YAEH,OAAO,IAAI,MAAM,CAAC,cAAc,CAAC;gBAC/B,MAAM,EAAE,GAAG;gBACX,OAAO,EAAE;oBACP,cAAc,EAAE,sBAAsB;iBACvC;gBACD,IAAI,EAAE,MAAM;gBACZ,SAAS,EAAE,KAAK,IAAI,EAAE;oBACpB,UAAU,CAAC,KAAK,EAAE,CAAC;oBACnB,OAAO,CAAC,WAAW,EAAE,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;gBACvD,CAAC;aACF,CAAC,CAAC;SACJ;QAAC,OAAO,EAAE,EAAE;YACX,UAAU,CAAC,KAAK,EAAE,CAAC;YACnB,OAAO,CAAC,WAAW,EAAE,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;SACtD;IACH,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,YAAY,CAAC,CAAC"}
+{"version":3,"file":"sync-stream.js","sourceRoot":"","sources":["../../../src/routes/endpoints/sync-stream.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AACnF,OAAO,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AAClE,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAElC,OAAO,KAAK,IAAI,MAAM,0BAA0B,CAAC;AACjD,OAAO,KAAK,IAAI,MAAM,0BAA0B,CAAC;AAEjD,OAAO,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AACnD,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAC/C,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAE9D,MAAM,CAAN,IAAY,UAEX;AAFD,WAAY,UAAU;IACpB,qCAAuB,CAAA;AACzB,CAAC,EAFW,UAAU,KAAV,UAAU,QAErB;AAED,MAAM,CAAC,MAAM,YAAY,GAAG,eAAe,CAAC;IAC1C,IAAI,EAAE,UAAU,CAAC,MAAM;IACvB,MAAM,EAAE,MAAM,CAAC,UAAU,CAAC,IAAI;IAC9B,SAAS,EAAE,QAAQ;IACnB,SAAS,EAAE,MAAM,CAAC,sBAAsB,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC;IAC9F,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;QACzB,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC;QAEtC,IAAI,MAAM,CAAC,MAAM,EAAE;YACjB,MAAM,IAAI,MAAM,CAAC,YAAY,CAAC;gBAC5B,MAAM,EAAE,GAAG;gBACX,IAAI,EAAE,qBAAqB;gBAC3B,WAAW,EAAE,iCAAiC;aAC/C,CAAC,CAAC;SACJ;QAED,MAAM,MAAM,GAA8B,OAAO,CAAC,MAAM,CAAC;QACzD,MAAM,UAAU,GAAG,IAAI,iBAAiB,CAAC,OAAO,CAAC,OAAO,CAAC,aAAc,EAAE,OAAO,CAAC,MAAM,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC;QAE1G,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;QAC/B,0CAA0C;QAC1C,MAAM,EAAE,GAAG,MAAM,OAAO,CAAC,mBAAmB,EAAE,CAAC;QAC/C,IAAI,CAAC,EAAE,CAAC,YAAY,EAAE,EAAE;YACtB,MAAM,IAAI,MAAM,CAAC,YAAY,CAAC;gBAC5B,MAAM,EAAE,GAAG;gBACX,IAAI,EAAE,eAAe;gBACrB,WAAW,EAAE,yBAAyB;aACvC,CAAC,CAAC;SACJ;QACD,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,OAAO,GAAG,IAAI,cAAc,EAAE,CAAC;QACrC,IAAI;YACF,OAAO,CAAC,WAAW,EAAE,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACpD,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAC1B,IAAI,CAAC,uBAAuB,CAC1B,IAAI,CAAC,MAAM,CACT,IAAI,CAAC,cAAc,CAAC;gBAClB,OAAO;gBACP,MAAM;gBACN,UAAU;gBACV,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,aAAc;gBACrC,OAAO;gBACP,MAAM,EAAE,UAAU,CAAC,MAAM;aAC1B,CAAC,CACH,EACD,OAAO,CACR,EACD,EAAE,UAAU,EAAE,KAAK,EAAE,aAAa,EAAE,EAAE,GAAG,IAAI,EAAE,CAChD,CAAC;YAEF,MAAM,UAAU,GAAG,MAAM,CAAC,cAAc,CAAC,GAAG,EAAE;gBAC5C,uDAAuD;gBACvD,UAAU,CAAC,KAAK,EAAE,CAAC;gBACnB,MAAM,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YACH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;gBACtB,UAAU,EAAE,CAAC;YACf,CAAC,CAAC,CAAC;YAEH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;gBAC3B,UAAU,CAAC,KAAK,EAAE,CAAC;gBACnB,oDAAoD;gBACpD,IAAI,KAAK,CAAC,OAAO,IAAI,sBAAsB,EAAE;oBAC3C,MAAM,CAAC,KAAK,CAAC,+BAA+B,EAAE,KAAK,CAAC,CAAC;iBACtD;YACH,CAAC,CAAC,CAAC;YAEH,OAAO,IAAI,MAAM,CAAC,cAAc,CAAC;gBAC/B,MAAM,EAAE,GAAG;gBACX,OAAO,EAAE;oBACP,cAAc,EAAE,sBAAsB;iBACvC;gBACD,IAAI,EAAE,MAAM;gBACZ,SAAS,EAAE,KAAK,IAAI,EAAE;oBACpB,UAAU,CAAC,KAAK,EAAE,CAAC;oBACnB,OAAO,CAAC,WAAW,EAAE,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;oBACrD,MAAM,CAAC,IAAI,CAAC,sBAAsB,EAAE;wBAClC,OAAO,EAAE,UAAU,CAAC,OAAO;wBAC3B,iBAAiB,EAAE,OAAO,CAAC,gBAAgB;wBAC3C,iBAAiB,EAAE,OAAO,CAAC,eAAe;qBAC3C,CAAC,CAAC;gBACL,CAAC;aACF,CAAC,CAAC;SACJ;QAAC,OAAO,EAAE,EAAE;YACX,UAAU,CAAC,KAAK,EAAE,CAAC;YACnB,OAAO,CAAC,WAAW,EAAE,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;SACtD;IACH,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,YAAY,CAAC,CAAC"}

package/dist/storage/BucketStorage.d.ts

@@ -175,7 +175,7 @@ export interface SyncRulesBucketStorage {
      * @param dataBuckets current bucket states
      * @param options batch size options
      */
-    getBucketDataBatch(checkpoint: util.OpId, dataBuckets: Map<string, string>, options?: BucketDataBatchOptions): AsyncIterable<util.SyncBucketData>;
+    getBucketDataBatch(checkpoint: util.OpId, dataBuckets: Map<string, string>, options?: BucketDataBatchOptions): AsyncIterable<SyncBucketDataBatch>;
     /**
      * Compute checksums for a given list of buckets.
      *
@@ -206,6 +206,7 @@ export interface SyncRulesBucketStorage {
      * Errors are cleared on commit.
      */
     reportError(e: any): Promise<void>;
+    compact(options?: CompactOptions): Promise<void>;
 }
 export interface SyncRuleStatus {
     checkpoint_lsn: string | null;
@@ -307,4 +308,27 @@ export interface SaveDelete {
     before: SqliteRow;
     after?: undefined;
 }
+export interface SyncBucketDataBatch {
+    batch: util.SyncBucketData;
+    targetOp: bigint | null;
+}
 export declare function mergeToast(record: ToastableSqliteRow, persisted: ToastableSqliteRow): ToastableSqliteRow;
+export interface CompactOptions {
+    /**
+     * Heap memory limit for the compact process.
+     *
+     * Add around 64MB to this to determine the "--max-old-space-size" argument.
+     * Add another 80MB to get RSS usage / memory limits.
+     */
+    memoryLimitMB?: number;
+    /**
+     * If specified, ignore any operations newer than this when compacting.
+     *
+     * This is primarily for tests, where we want to test compacting at a specific
+     * point.
+     *
+     * This can also be used to create a "safe buffer" of recent operations that should
+     * not be compacted, to avoid invalidating checkpoints in use.
+     */
+    maxOpId?: bigint;
+}

package/dist/storage/BucketStorage.js.map

@@ -1 +1 @@
-{"version":3,"file":"BucketStorage.js","sourceRoot":"","sources":["../../src/storage/BucketStorage.ts"],"names":[],"mappings":"AA+JA,MAAM,OAAO,yBAAyB;IAGpC,YAA4B,EAAU,EAAkB,UAAwB,EAAE,cAA6B;QAAnF,OAAE,GAAF,EAAE,CAAQ;QAAkB,eAAU,GAAV,UAAU,CAAc;QAC9E,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;IACvC,CAAC;IAED,IAAI,SAAS;QACX,OAAO,aAAa,IAAI,CAAC,EAAE,EAAE,CAAC;IAChC,CAAC;CACF;AAYD,MAAM,CAAC,MAAM,4BAA4B,GAAG,IAAI,CAAC;AACjD,MAAM,CAAC,MAAM,kCAAkC,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;AAgNlE,MAAM,UAAU,UAAU,CAAC,MAA0B,EAAE,SAA6B;IAClF,MAAM,SAAS,GAAuB,EAAE,CAAC;IACzC,KAAK,IAAI,GAAG,IAAI,MAAM,EAAE;QACtB,IAAI,OAAO,MAAM,CAAC,GAAG,CAAC,IAAI,WAAW,EAAE;YACrC,SAAS,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;SACjC;aAAM;YACL,SAAS,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;SAC9B;KACF;IACD,OAAO,SAAS,CAAC;AACnB,CAAC"}
+{"version":3,"file":"BucketStorage.js","sourceRoot":"","sources":["../../src/storage/BucketStorage.ts"],"names":[],"mappings":"AA+JA,MAAM,OAAO,yBAAyB;IAGpC,YAA4B,EAAU,EAAkB,UAAwB,EAAE,cAA6B;QAAnF,OAAE,GAAF,EAAE,CAAQ;QAAkB,eAAU,GAAV,UAAU,CAAc;QAC9E,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;IACvC,CAAC;IAED,IAAI,SAAS;QACX,OAAO,aAAa,IAAI,CAAC,EAAE,EAAE,CAAC;IAChC,CAAC;CACF;AAYD,MAAM,CAAC,MAAM,4BAA4B,GAAG,IAAI,CAAC;AACjD,MAAM,CAAC,MAAM,kCAAkC,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;AAuNlE,MAAM,UAAU,UAAU,CAAC,MAA0B,EAAE,SAA6B;IAClF,MAAM,SAAS,GAAuB,EAAE,CAAC;IACzC,KAAK,IAAI,GAAG,IAAI,MAAM,EAAE;QACtB,IAAI,OAAO,MAAM,CAAC,GAAG,CAAC,IAAI,WAAW,EAAE;YACrC,SAAS,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;SACjC;aAAM;YACL,SAAS,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;SAC9B;KACF;IACD,OAAO,SAAS,CAAC;AACnB,CAAC"}

package/dist/storage/mongo/MongoCompactor.d.ts

@@ -0,0 +1,38 @@
+import { PowerSyncMongo } from './db.js';
+import { CompactOptions } from '../BucketStorage.js';
+/**
+ * Additional options, primarily for testing.
+ */
+export interface MongoCompactOptions extends CompactOptions {
+    /** Minimum of 2 */
+    clearBatchLimit?: number;
+    /** Minimum of 1 */
+    moveBatchLimit?: number;
+    /** Minimum of 1 */
+    moveBatchQueryLimit?: number;
+}
+export declare class MongoCompactor {
+    private db;
+    private group_id;
+    private updates;
+    private idLimitBytes;
+    private moveBatchLimit;
+    private moveBatchQueryLimit;
+    private clearBatchLimit;
+    private maxOpId;
+    constructor(db: PowerSyncMongo, group_id: number, options?: MongoCompactOptions);
+    /**
+     * Compact buckets by converting operations into MOVE and/or CLEAR operations.
+     *
+     * See /docs/compacting-operations.md for details.
+     */
+    compact(): Promise<void>;
+    private flush;
+    /**
+     * Perform a CLEAR compact for a bucket.
+     *
+     * @param bucket bucket name
+     * @param op op_id of the last non-PUT operation, which will be converted to CLEAR.
+     */
+    private clearBucket;
+}

package/dist/storage/mongo/MongoCompactor.js

@@ -0,0 +1,278 @@
+import { logger } from '@powersync/lib-services-framework';
+import { MaxKey, MinKey } from 'mongodb';
+import { addChecksums } from '../../util/utils.js';
+const DEFAULT_CLEAR_BATCH_LIMIT = 5000;
+const DEFAULT_MOVE_BATCH_LIMIT = 2000;
+const DEFAULT_MOVE_BATCH_QUERY_LIMIT = 10000;
+/** This default is primarily for tests. */
+const DEFAULT_MEMORY_LIMIT_MB = 64;
+export class MongoCompactor {
+    constructor(db, group_id, options) {
+        this.db = db;
+        this.group_id = group_id;
+        this.updates = [];
+        this.idLimitBytes = (options?.memoryLimitMB ?? DEFAULT_MEMORY_LIMIT_MB) * 1024 * 1024;
+        this.moveBatchLimit = options?.moveBatchLimit ?? DEFAULT_MOVE_BATCH_LIMIT;
+        this.moveBatchQueryLimit = options?.moveBatchQueryLimit ?? DEFAULT_MOVE_BATCH_QUERY_LIMIT;
+        this.clearBatchLimit = options?.clearBatchLimit ?? DEFAULT_CLEAR_BATCH_LIMIT;
+        this.maxOpId = options?.maxOpId;
+    }
+    /**
+     * Compact buckets by converting operations into MOVE and/or CLEAR operations.
+     *
+     * See /docs/compacting-operations.md for details.
+     */
+    async compact() {
+        const idLimitBytes = this.idLimitBytes;
+        let currentState = null;
+        // Constant lower bound
+        const lowerBound = {
+            g: this.group_id,
+            b: new MinKey(),
+            o: new MinKey()
+        };
+        // Upper bound is adjusted for each batch
+        let upperBound = {
+            g: this.group_id,
+            b: new MaxKey(),
+            o: new MaxKey()
+        };
+        while (true) {
+            // Query one batch at a time, to avoid cursor timeouts
+            const batch = await this.db.bucket_data
+                .find({
+                _id: {
+                    $gte: lowerBound,
+                    $lt: upperBound
+                }
+            }, {
+                projection: {
+                    _id: 1,
+                    op: 1,
+                    table: 1,
+                    row_id: 1,
+                    source_table: 1,
+                    source_key: 1
+                },
+                limit: this.moveBatchQueryLimit,
+                sort: { _id: -1 },
+                singleBatch: true
+            })
+                .toArray();
+            if (batch.length == 0) {
+                // We've reached the end
+                break;
+            }
+            // Set upperBound for the next batch
+            upperBound = batch[batch.length - 1]._id;
+            for (let doc of batch) {
+                if (currentState == null || doc._id.b != currentState.bucket) {
+                    if (currentState != null && currentState.lastNotPut != null && currentState.opsSincePut >= 1) {
+                        // Important to flush before clearBucket()
+                        await this.flush();
+                        logger.info(`Inserting CLEAR at ${this.group_id}:${currentState.bucket}:${currentState.lastNotPut} to remove ${currentState.opsSincePut} operations`);
+                        const bucket = currentState.bucket;
+                        const clearOp = currentState.lastNotPut;
+                        // Free memory before clearing bucket
+                        currentState = null;
+                        await this.clearBucket(bucket, clearOp);
+                    }
+                    currentState = {
+                        bucket: doc._id.b,
+                        seen: new Map(),
+                        trackingSize: 0,
+                        lastNotPut: null,
+                        opsSincePut: 0
+                    };
+                }
+                if (this.maxOpId != null && doc._id.o > this.maxOpId) {
+                    continue;
+                }
+                let isPersistentPut = doc.op == 'PUT';
+                if (doc.op == 'REMOVE' || doc.op == 'PUT') {
+                    const key = `${doc.table}/${doc.row_id}/${doc.source_table}/${doc.source_key?.toHexString()}`;
+                    const targetOp = currentState.seen.get(key);
+                    if (targetOp) {
+                        // Will convert to MOVE, so don't count as PUT
+                        isPersistentPut = false;
+                        this.updates.push({
+                            updateOne: {
+                                filter: {
+                                    _id: doc._id
+                                },
+                                update: {
+                                    $set: {
+                                        op: 'MOVE',
+                                        target_op: targetOp
+                                    },
+                                    $unset: {
+                                        source_table: 1,
+                                        source_key: 1,
+                                        table: 1,
+                                        row_id: 1,
+                                        data: 1
+                                    }
+                                }
+                            }
+                        });
+                    }
+                    else {
+                        if (currentState.trackingSize >= idLimitBytes) {
+                            // Reached memory limit.
+                            // Keep the highest seen values in this case.
+                        }
+                        else {
+                            // flatstr reduces the memory usage by flattening the string
+                            currentState.seen.set(flatstr(key), doc._id.o);
+                            // length + 16 for the string
+                            // 24 for the bigint
+                            // 50 for map overhead
+                            // 50 for additional overhead
+                            currentState.trackingSize += key.length + 140;
+                        }
+                    }
+                }
+                if (isPersistentPut) {
+                    currentState.lastNotPut = null;
+                    currentState.opsSincePut = 0;
+                }
+                else if (doc.op != 'CLEAR') {
+                    if (currentState.lastNotPut == null) {
+                        currentState.lastNotPut = doc._id.o;
+                    }
+                    currentState.opsSincePut += 1;
+                }
+                if (this.updates.length >= this.moveBatchLimit) {
+                    await this.flush();
+                }
+            }
+        }
+        await this.flush();
+        currentState?.seen.clear();
+        if (currentState?.lastNotPut != null && currentState?.opsSincePut > 1) {
+            logger.info(`Inserting CLEAR at ${this.group_id}:${currentState.bucket}:${currentState.lastNotPut} to remove ${currentState.opsSincePut} operations`);
+            const bucket = currentState.bucket;
+            const clearOp = currentState.lastNotPut;
+            // Free memory before clearing bucket
+            currentState = null;
+            await this.clearBucket(bucket, clearOp);
+        }
+    }
+    async flush() {
+        if (this.updates.length > 0) {
+            logger.info(`Compacting ${this.updates.length} ops`);
+            await this.db.bucket_data.bulkWrite(this.updates, {
+                // Order is not important.
+                // Since checksums are not affected, these operations can happen in any order,
+                // and it's fine if the operations are partially applied.
+                // Each individual operation is atomic.
+                ordered: false
+            });
+            this.updates = [];
+        }
+    }
+    /**
+     * Perform a CLEAR compact for a bucket.
+     *
+     * @param bucket bucket name
+     * @param op op_id of the last non-PUT operation, which will be converted to CLEAR.
+     */
+    async clearBucket(bucket, op) {
+        const opFilter = {
+            _id: {
+                $gte: {
+                    g: this.group_id,
+                    b: bucket,
+                    o: new MinKey()
+                },
+                $lte: {
+                    g: this.group_id,
+                    b: bucket,
+                    o: op
+                }
+            }
+        };
+        const session = this.db.client.startSession();
+        try {
+            let done = false;
+            while (!done) {
+                // Do the CLEAR operation in batches, with each batch a separate transaction.
+                // The state after each batch is fully consistent.
+                // We need a transaction per batch to make sure checksums stay consistent.
+                await session.withTransaction(async () => {
+                    const query = this.db.bucket_data.find(opFilter, {
+                        session,
+                        sort: { _id: 1 },
+                        projection: {
+                            _id: 1,
+                            op: 1,
+                            checksum: 1,
+                            target_op: 1
+                        },
+                        limit: this.clearBatchLimit
+                    });
+                    let checksum = 0;
+                    let lastOpId = null;
+                    let targetOp = null;
+                    let gotAnOp = false;
+                    for await (let op of query.stream()) {
+                        if (op.op == 'MOVE' || op.op == 'REMOVE' || op.op == 'CLEAR') {
+                            checksum = addChecksums(checksum, op.checksum);
+                            lastOpId = op._id;
+                            if (op.op != 'CLEAR') {
+                                gotAnOp = true;
+                            }
+                            if (op.target_op != null) {
+                                if (targetOp == null || op.target_op > targetOp) {
+                                    targetOp = op.target_op;
+                                }
+                            }
+                        }
+                        else {
+                            throw new Error(`Unexpected ${op.op} operation at ${op._id.g}:${op._id.b}:${op._id.o}`);
+                        }
+                    }
+                    if (!gotAnOp) {
+                        done = true;
+                        return;
+                    }
+                    logger.info(`Flushing CLEAR at ${lastOpId?.o}`);
+                    await this.db.bucket_data.deleteMany({
+                        _id: {
+                            $gte: {
+                                g: this.group_id,
+                                b: bucket,
+                                o: new MinKey()
+                            },
+                            $lte: lastOpId
+                        }
+                    }, { session });
+                    await this.db.bucket_data.insertOne({
+                        _id: lastOpId,
+                        op: 'CLEAR',
+                        checksum: checksum,
+                        data: null,
+                        target_op: targetOp
+                    }, { session });
+                }, {
+                    writeConcern: { w: 'majority' },
+                    readConcern: { level: 'snapshot' }
+                });
+            }
+        }
+        finally {
+            await session.endSession();
+        }
+    }
+}
+/**
+ * Flattens string to reduce memory usage (around 320 bytes -> 120 bytes),
+ * at the cost of some upfront CPU usage.
+ *
+ * From: https://github.com/davidmarkclements/flatstr/issues/8
+ */
+function flatstr(s) {
+    s.match(/\n/g);
+    return s;
+}
+//# sourceMappingURL=MongoCompactor.js.map

package/dist/storage/mongo/MongoCompactor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"MongoCompactor.js","sourceRoot":"","sources":["../../../src/storage/mongo/MongoCompactor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAC3D,OAAO,EAAyB,MAAM,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AAChE,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAwCnD,MAAM,yBAAyB,GAAG,IAAI,CAAC;AACvC,MAAM,wBAAwB,GAAG,IAAI,CAAC;AACtC,MAAM,8BAA8B,GAAG,KAAM,CAAC;AAE9C,2CAA2C;AAC3C,MAAM,uBAAuB,GAAG,EAAE,CAAC;AAEnC,MAAM,OAAO,cAAc;IASzB,YAAoB,EAAkB,EAAU,QAAgB,EAAE,OAA6B;QAA3E,OAAE,GAAF,EAAE,CAAgB;QAAU,aAAQ,GAAR,QAAQ,CAAQ;QARxD,YAAO,GAAgD,EAAE,CAAC;QAShE,IAAI,CAAC,YAAY,GAAG,CAAC,OAAO,EAAE,aAAa,IAAI,uBAAuB,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;QACtF,IAAI,CAAC,cAAc,GAAG,OAAO,EAAE,cAAc,IAAI,wBAAwB,CAAC;QAC1E,IAAI,CAAC,mBAAmB,GAAG,OAAO,EAAE,mBAAmB,IAAI,8BAA8B,CAAC;QAC1F,IAAI,CAAC,eAAe,GAAG,OAAO,EAAE,eAAe,IAAI,yBAAyB,CAAC;QAC7E,IAAI,CAAC,OAAO,GAAG,OAAO,EAAE,OAAO,CAAC;IAClC,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,OAAO;QACX,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC;QAEvC,IAAI,YAAY,GAA8B,IAAI,CAAC;QAEnD,uBAAuB;QACvB,MAAM,UAAU,GAAkB;YAChC,CAAC,EAAE,IAAI,CAAC,QAAQ;YAChB,CAAC,EAAE,IAAI,MAAM,EAAS;YACtB,CAAC,EAAE,IAAI,MAAM,EAAS;SACvB,CAAC;QAEF,yCAAyC;QACzC,IAAI,UAAU,GAAkB;YAC9B,CAAC,EAAE,IAAI,CAAC,QAAQ;YAChB,CAAC,EAAE,IAAI,MAAM,EAAS;YACtB,CAAC,EAAE,IAAI,MAAM,EAAS;SACvB,CAAC;QAEF,OAAO,IAAI,EAAE;YACX,sDAAsD;YACtD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW;iBACpC,IAAI,CACH;gBACE,GAAG,EAAE;oBACH,IAAI,EAAE,UAAU;oBAChB,GAAG,EAAE,UAAU;iBAChB;aACF,EACD;gBACE,UAAU,EAAE;oBACV,GAAG,EAAE,CAAC;oBACN,EAAE,EAAE,CAAC;oBACL,KAAK,EAAE,CAAC;oBACR,MAAM,EAAE,CAAC;oBACT,YAAY,EAAE,CAAC;oBACf,UAAU,EAAE,CAAC;iBACd;gBACD,KAAK,EAAE,IAAI,CAAC,mBAAmB;gBAC/B,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE;gBACjB,WAAW,EAAE,IAAI;aAClB,CACF;iBACA,OAAO,EAAE,CAAC;YAEb,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE;gBACrB,wBAAwB;gBACxB,MAAM;aACP;YAED,oCAAoC;YACpC,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;YAEzC,KAAK,IAAI,GAAG,IAAI,KAAK,EAAE;gBACrB,IAAI,YAAY,IAAI,IAAI,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,YAAY,CAAC,MAAM,EAAE;oBAC5D,IAAI,YAAY,IAAI,IAAI,IAAI,YAAY,CAAC,UAAU,IAAI,IAAI,IAAI,YAAY,CAAC,WAAW,IAAI,CAAC,EAAE;wBAC5F,0CAA0C;wBAC1C,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;wBACnB,MAAM,CAAC,IAAI,CACT,sBAAsB,IAAI,CAAC,QAAQ,IAAI,YAAY,CAAC,MAAM,IAAI,YAAY,CAAC,UAAU,cAAc,YAAY,CAAC,WAAW,aAAa,CACzI,CAAC;wBAEF,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC;wBACnC,MAAM,OAAO,GAAG,YAAY,CAAC,UAAU,CAAC;wBACxC,qCAAqC;wBACrC,YAAY,GAAG,IAAI,CAAC;wBACpB,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;qBACzC;oBACD,YAAY,GAAG;wBACb,MAAM,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;wBACjB,IAAI,EAAE,IAAI,GAAG,EAAE;wBACf,YAAY,EAAE,CAAC;wBACf,UAAU,EAAE,IAAI;wBAChB,WAAW,EAAE,CAAC;qBACf,CAAC;iBACH;gBAED,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,OAAO,EAAE;oBACpD,SAAS;iBACV;gBAED,IAAI,eAAe,GAAG,GAAG,CAAC,EAAE,IAAI,KAAK,CAAC;gBAEtC,IAAI,GAAG,CAAC,EAAE,IAAI,QAAQ,IAAI,GAAG,CAAC,EAAE,IAAI,KAAK,EAAE;oBACzC,MAAM,GAAG,GAAG,GAAG,GAAG,CAAC,KAAK,IAAI,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,YAAY,IAAI,GAAG,CAAC,UAAU,EAAE,WAAW,EAAE,EAAE,CAAC;oBAC9F,MAAM,QAAQ,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;oBAC5C,IAAI,QAAQ,EAAE;wBACZ,8CAA8C;wBAC9C,eAAe,GAAG,KAAK,CAAC;wBAExB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;4BAChB,SAAS,EAAE;gCACT,MAAM,EAAE;oCACN,GAAG,EAAE,GAAG,CAAC,GAAG;iCACb;gCACD,MAAM,EAAE;oCACN,IAAI,EAAE;wCACJ,EAAE,EAAE,MAAM;wCACV,SAAS,EAAE,QAAQ;qCACpB;oCACD,MAAM,EAAE;wCACN,YAAY,EAAE,CAAC;wCACf,UAAU,EAAE,CAAC;wCACb,KAAK,EAAE,CAAC;wCACR,MAAM,EAAE,CAAC;wCACT,IAAI,EAAE,CAAC;qCACR;iCACF;6BACF;yBACF,CAAC,CAAC;qBACJ;yBAAM;wBACL,IAAI,YAAY,CAAC,YAAY,IAAI,YAAY,EAAE;4BAC7C,wBAAwB;4BACxB,6CAA6C;yBAC9C;6BAAM;4BACL,4DAA4D;4BAC5D,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;4BAC/C,6BAA6B;4BAC7B,oBAAoB;4BACpB,sBAAsB;4BACtB,6BAA6B;4BAC7B,YAAY,CAAC,YAAY,IAAI,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC;yBAC/C;qBACF;iBACF;gBAED,IAAI,eAAe,EAAE;oBACnB,YAAY,CAAC,UAAU,GAAG,IAAI,CAAC;oBAC/B,YAAY,CAAC,WAAW,GAAG,CAAC,CAAC;iBAC9B;qBAAM,IAAI,GAAG,CAAC,EAAE,IAAI,OAAO,EAAE;oBAC5B,IAAI,YAAY,CAAC,UAAU,IAAI,IAAI,EAAE;wBACnC,YAAY,CAAC,UAAU,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;qBACrC;oBACD,YAAY,CAAC,WAAW,IAAI,CAAC,CAAC;iBAC/B;gBAED,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,cAAc,EAAE;oBAC9C,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;iBACpB;aACF;SACF;QAED,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACnB,YAAY,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC;QAC3B,IAAI,YAAY,EAAE,UAAU,IAAI,IAAI,IAAI,YAAY,EAAE,WAAW,GAAG,CAAC,EAAE;YACrE,MAAM,CAAC,IAAI,CACT,sBAAsB,IAAI,CAAC,QAAQ,IAAI,YAAY,CAAC,MAAM,IAAI,YAAY,CAAC,UAAU,cAAc,YAAY,CAAC,WAAW,aAAa,CACzI,CAAC;YACF,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC;YACnC,MAAM,OAAO,GAAG,YAAY,CAAC,UAAU,CAAC;YACxC,qCAAqC;YACrC,YAAY,GAAG,IAAI,CAAC;YACpB,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;SACzC;IACH,CAAC;IAEO,KAAK,CAAC,KAAK;QACjB,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE;YAC3B,MAAM,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,OAAO,CAAC,MAAM,MAAM,CAAC,CAAC;YACrD,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE;gBAChD,0BAA0B;gBAC1B,8EAA8E;gBAC9E,yDAAyD;gBACzD,uCAAuC;gBACvC,OAAO,EAAE,KAAK;aACf,CAAC,CAAC;YACH,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;SACnB;IACH,CAAC;IAED;;;;;OAKG;IACK,KAAK,CAAC,WAAW,CAAC,MAAc,EAAE,EAAU;QAClD,MAAM,QAAQ,GAAG;YACf,GAAG,EAAE;gBACH,IAAI,EAAE;oBACJ,CAAC,EAAE,IAAI,CAAC,QAAQ;oBAChB,CAAC,EAAE,MAAM;oBACT,CAAC,EAAE,IAAI,MAAM,EAAS;iBACvB;gBACD,IAAI,EAAE;oBACJ,CAAC,EAAE,IAAI,CAAC,QAAQ;oBAChB,CAAC,EAAE,MAAM;oBACT,CAAC,EAAE,EAAE;iBACN;aACF;SACF,CAAC;QAEF,MAAM,OAAO,GAAG,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;QAC9C,IAAI;YACF,IAAI,IAAI,GAAG,KAAK,CAAC;YACjB,OAAO,CAAC,IAAI,EAAE;gBACZ,6EAA6E;gBAC7E,kDAAkD;gBAClD,0EAA0E;gBAC1E,MAAM,OAAO,CAAC,eAAe,CAC3B,KAAK,IAAI,EAAE;oBACT,MAAM,KAAK,GAAG,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,QAAQ,EAAE;wBAC/C,OAAO;wBACP,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE;wBAChB,UAAU,EAAE;4BACV,GAAG,EAAE,CAAC;4BACN,EAAE,EAAE,CAAC;4BACL,QAAQ,EAAE,CAAC;4BACX,SAAS,EAAE,CAAC;yBACb;wBACD,KAAK,EAAE,IAAI,CAAC,eAAe;qBAC5B,CAAC,CAAC;oBACH,IAAI,QAAQ,GAAG,CAAC,CAAC;oBACjB,IAAI,QAAQ,GAAyB,IAAI,CAAC;oBAC1C,IAAI,QAAQ,GAAkB,IAAI,CAAC;oBACnC,IAAI,OAAO,GAAG,KAAK,CAAC;oBACpB,IAAI,KAAK,EAAE,IAAI,EAAE,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE;wBACnC,IAAI,EAAE,CAAC,EAAE,IAAI,MAAM,IAAI,EAAE,CAAC,EAAE,IAAI,QAAQ,IAAI,EAAE,CAAC,EAAE,IAAI,OAAO,EAAE;4BAC5D,QAAQ,GAAG,YAAY,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC;4BAC/C,QAAQ,GAAG,EAAE,CAAC,GAAG,CAAC;4BAClB,IAAI,EAAE,CAAC,EAAE,IAAI,OAAO,EAAE;gCACpB,OAAO,GAAG,IAAI,CAAC;6BAChB;4BACD,IAAI,EAAE,CAAC,SAAS,IAAI,IAAI,EAAE;gCACxB,IAAI,QAAQ,IAAI,IAAI,IAAI,EAAE,CAAC,SAAS,GAAG,QAAQ,EAAE;oCAC/C,QAAQ,GAAG,EAAE,CAAC,SAAS,CAAC;iCACzB;6BACF;yBACF;6BAAM;4BACL,MAAM,IAAI,KAAK,CAAC,cAAc,EAAE,CAAC,EAAE,iBAAiB,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;yBACzF;qBACF;oBACD,IAAI,CAAC,OAAO,EAAE;wBACZ,IAAI,GAAG,IAAI,CAAC;wBACZ,OAAO;qBACR;oBAED,MAAM,CAAC,IAAI,CAAC,qBAAqB,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;oBAChD,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,UAAU,CAClC;wBACE,GAAG,EAAE;4BACH,IAAI,EAAE;gCACJ,CAAC,EAAE,IAAI,CAAC,QAAQ;gCAChB,CAAC,EAAE,MAAM;gCACT,CAAC,EAAE,IAAI,MAAM,EAAS;6BACvB;4BACD,IAAI,EAAE,QAAS;yBAChB;qBACF,EACD,EAAE,OAAO,EAAE,CACZ,CAAC;oBAEF,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,SAAS,CACjC;wBACE,GAAG,EAAE,QAAS;wBACd,EAAE,EAAE,OAAO;wBACX,QAAQ,EAAE,QAAQ;wBAClB,IAAI,EAAE,IAAI;wBACV,SAAS,EAAE,QAAQ;qBACpB,EACD,EAAE,OAAO,EAAE,CACZ,CAAC;gBACJ,CAAC,EACD;oBACE,YAAY,EAAE,EAAE,CAAC,EAAE,UAAU,EAAE;oBAC/B,WAAW,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE;iBACnC,CACF,CAAC;aACH;SACF;gBAAS;YACR,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;SAC5B;IACH,CAAC;CACF;AAED;;;;;GAKG;AACH,SAAS,OAAO,CAAC,CAAS;IACxB,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACf,OAAO,CAAC,CAAC;AACX,CAAC"}