@powersync/service-core 0.0.0-dev-20240709124106 → 0.0.0-dev-20240725112650

package/dist/sync/sync.js

@@ -3,7 +3,6 @@ import { Semaphore } from 'async-mutex';
 import { AbortError } from 'ix/aborterror.js';
 import * as util from '../util/util-index.js';
 import { logger } from '@powersync/lib-services-framework';
-import { Metrics } from '../metrics/Metrics.js';
 import { mergeAsyncIterables } from './merge.js';
 import { tokenStream } from './util.js';
 /**
@@ -12,7 +11,7 @@ import { tokenStream } from './util.js';
 const MAX_ACTIVE_CONNECTIONS = 10;
 const syncSemaphore = new Semaphore(MAX_ACTIVE_CONNECTIONS);
 export async function* streamResponse(options) {
-    const { storage, params, syncParams, token, tokenStreamOptions, signal } = options;
+    const { storage, params, syncParams, token, tokenStreamOptions, tracker, signal } = options;
     // We also need to be able to abort, so we create our own controller.
     const controller = new AbortController();
     if (signal) {
@@ -24,7 +23,7 @@ export async function* streamResponse(options) {
         }
     }
     const ki = tokenStream(token, controller.signal, tokenStreamOptions);
-    const stream = streamResponseInner(storage, params, syncParams, controller.signal);
+    const stream = streamResponseInner(storage, params, syncParams, tracker, controller.signal);
     // Merge the two streams, and abort as soon as one of the streams end.
     const merged = mergeAsyncIterables([stream, ki], controller.signal);
     try {
@@ -44,7 +43,7 @@ export async function* streamResponse(options) {
         controller.abort();
     }
 }
-async function* streamResponseInner(storage, params, syncParams, signal) {
+async function* streamResponseInner(storage, params, syncParams, tracker, signal) {
     // Bucket state of bucket id -> op_id.
     // This starts with the state from the client. May contain buckets that the user do not have access to (anymore).
     let dataBuckets = new Map();
@@ -73,6 +72,11 @@ async function* streamResponseInner(storage, params, syncParams, signal) {
             parameters: syncParams
         });
         if (allBuckets.length > 1000) {
+            logger.error(`Too many buckets`, {
+                checkpoint,
+                user_id: syncParams.user_id,
+                buckets: allBuckets.length
+            });
             // TODO: Limit number of buckets even before we get to this point
             throw new Error(`Too many buckets: ${allBuckets.length}`);
         }
@@ -94,11 +98,18 @@ async function* streamResponseInner(storage, params, syncParams, signal) {
                 continue;
             }
             bucketsToFetch = diff.updatedBuckets.map((c) => c.bucket);
-            let message = `Updated checkpoint: ${checkpoint} | write: ${writeCheckpoint} | `;
+            let message = `Updated checkpoint: ${checkpoint} | `;
+            message += `write: ${writeCheckpoint} | `;
             message += `buckets: ${allBuckets.length} | `;
             message += `updated: ${limitedBuckets(diff.updatedBuckets, 20)} | `;
-            message += `removed: ${limitedBuckets(diff.removedBuckets, 20)} | `;
-            logger.info(message);
+            message += `removed: ${limitedBuckets(diff.removedBuckets, 20)}`;
+            logger.info(message, {
+                checkpoint,
+                user_id: syncParams.user_id,
+                buckets: allBuckets.length,
+                updated: diff.updatedBuckets.length,
+                removed: diff.removedBuckets.length
+            });
             const checksum_line = {
                 checkpoint_diff: {
                     last_op_id: checkpoint,
@@ -112,7 +123,7 @@ async function* streamResponseInner(storage, params, syncParams, signal) {
         else {
             let message = `New checkpoint: ${checkpoint} | write: ${writeCheckpoint} | `;
             message += `buckets: ${allBuckets.length} ${limitedBuckets(allBuckets, 20)}`;
-            logger.info(message);
+            logger.info(message, { checkpoint, user_id: syncParams.user_id, buckets: allBuckets.length });
             bucketsToFetch = allBuckets;
             const checksum_line = {
                 checkpoint: {
@@ -127,7 +138,16 @@ async function* streamResponseInner(storage, params, syncParams, signal) {
         lastWriteCheckpoint = writeCheckpoint;
         // This incrementally updates dataBuckets with each individual bucket position.
         // At the end of this, we can be sure that all buckets have data up to the checkpoint.
-        yield* bucketDataInBatches({ storage, checkpoint, bucketsToFetch, dataBuckets, raw_data, binary_data, signal });
+        yield* bucketDataInBatches({
+            storage,
+            checkpoint,
+            bucketsToFetch,
+            dataBuckets,
+            raw_data,
+            binary_data,
+            signal,
+            tracker
+        });
         await new Promise((resolve) => setTimeout(resolve, 10));
     }
 }
@@ -168,7 +188,9 @@ async function* bucketDataInBatches(request) {
  * Extracted as a separate internal function just to avoid memory leaks.
  */
 async function* bucketDataBatch(request) {
-    const { storage, checkpoint, bucketsToFetch, dataBuckets, raw_data, binary_data, signal } = request;
+    const { storage, checkpoint, bucketsToFetch, dataBuckets, raw_data, binary_data, tracker, signal } = request;
+    const checkpointOp = BigInt(checkpoint);
+    let checkpointInvalidated = false;
     const [_, release] = await syncSemaphore.acquire();
     try {
         // Optimization: Only fetch buckets for which the checksums have changed since the last checkpoint
@@ -176,13 +198,16 @@ async function* bucketDataBatch(request) {
         const filteredBuckets = new Map(bucketsToFetch.map((bucket) => [bucket, dataBuckets.get(bucket)]));
         const data = storage.getBucketDataBatch(checkpoint, filteredBuckets);
         let has_more = false;
-        for await (let r of data) {
+        for await (let { batch: r, targetOp } of data) {
             if (signal.aborted) {
                 return;
             }
             if (r.has_more) {
                 has_more = true;
             }
+            if (targetOp != null && targetOp > checkpointOp) {
+                checkpointInvalidated = true;
+            }
             if (r.data.length == 0) {
                 continue;
             }
@@ -213,16 +238,24 @@ async function* bucketDataBatch(request) {
                 // iterator memory in case if large data sent.
                 yield { data: null, done: false };
             }
-            Metrics.getInstance().operations_synced_total.add(r.data.length);
+            tracker.addOperationsSynced(r.data.length);
             dataBuckets.set(r.bucket, r.next_after);
         }
         if (!has_more) {
-            const line = {
-                checkpoint_complete: {
-                    last_op_id: checkpoint
-                }
-            };
-            yield { data: line, done: true };
+            if (checkpointInvalidated) {
+                // Checkpoint invalidated by a CLEAR or MOVE op.
+                // Don't send the checkpoint_complete line in this case.
+                // More data should be available immediately for a new checkpoint.
+                yield { data: null, done: true };
+            }
+            else {
+                const line = {
+                    checkpoint_complete: {
+                        last_op_id: checkpoint
+                    }
+                };
+                yield { data: line, done: true };
+            }
         }
     }
     finally {

package/dist/sync/sync.js.map

@@ -1 +1 @@
-{"version":3,"file":"sync.js","sourceRoot":"","sources":["../../src/sync/sync.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAEpE,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAI9C,OAAO,KAAK,IAAI,MAAM,uBAAuB,CAAC;AAE9C,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAC3D,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAChD,OAAO,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC;AACjD,OAAO,EAAsB,WAAW,EAAE,MAAM,WAAW,CAAC;AAE5D;;GAEG;AACH,MAAM,sBAAsB,GAAG,EAAE,CAAC;AAClC,MAAM,aAAa,GAAG,IAAI,SAAS,CAAC,sBAAsB,CAAC,CAAC;AAc5D,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,cAAc,CACnC,OAA6B;IAE7B,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,kBAAkB,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IACnF,qEAAqE;IACrE,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,IAAI,MAAM,EAAE;QACV,MAAM,CAAC,gBAAgB,CACrB,OAAO,EACP,GAAG,EAAE;YACH,UAAU,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC,EACD,EAAE,IAAI,EAAE,IAAI,EAAE,CACf,CAAC;QACF,IAAI,MAAM,CAAC,OAAO,EAAE;YAClB,UAAU,CAAC,KAAK,EAAE,CAAC;SACpB;KACF;IACD,MAAM,EAAE,GAAG,WAAW,CAAC,KAAK,EAAE,UAAU,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAC;IACrE,MAAM,MAAM,GAAG,mBAAmB,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;IACnF,sEAAsE;IACtE,MAAM,MAAM,GAAG,mBAAmB,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;IAEpE,IAAI;QACF,KAAK,CAAC,CAAC,MAAM,CAAC;KACf;IAAC,OAAO,CAAC,EAAE;QACV,IAAI,CAAC,YAAY,UAAU,EAAE;YAC3B,OAAO;SACR;aAAM;YACL,MAAM,CAAC,CAAC;SACT;KACF;YAAS;QACR,iFAAiF;QACjF,qBAAqB;QACrB,UAAU,CAAC,KAAK,EAAE,CAAC;KACpB;AACH,CAAC;AAED,KAAK,SAAS,CAAC,CAAC,mBAAmB,CACjC,OAAqC,EACrC,MAAiC,EACjC,UAA6B,EAC7B,MAAmB;IAEnB,sCAAsC;IACtC,iHAAiH;IACjH,IAAI,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAC;IAE5C,IAAI,aAAa,GAA4B,IAAI,CAAC;IAClD,IAAI,mBAAmB,GAAkB,IAAI,CAAC;IAE9C,MAAM,EAAE,QAAQ,EAAE,WAAW,EAAE,GAAG,MAAM,CAAC;IAEzC,IAAI,MAAM,CAAC,OAAO,EAAE;QAClB,KAAK,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,MAAM,CAAC,OAAO,EAAE;YACjD,WAAW,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;SAC9B;KACF;IAED,MAAM,MAAM,GAAG,OAAO,CAAC,oBAAoB,CAAC,UAAU,CAAC,gBAAgB,CAAC,OAAiB,EAAE,MAAM,CAAC,CAAC;IACnG,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,MAAM,EAAE;QAC/B,MAAM,EAAE,IAAI,EAAE,eAAe,EAAE,GAAG,IAAI,CAAC;QACvC,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QAEnC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAC9C,IAAI,OAAO,IAAI,IAAI,EAAE;YACnB,2EAA2E;YAC3E,SAAS;SACV;QACD,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;QAEtC,MAAM,UAAU,GAAG,MAAM,UAAU,CAAC,cAAc,CAAC;YACjD,gBAAgB,CAAC,OAAO;gBACtB,OAAO,OAAO,CAAC,gBAAgB,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;YACvD,CAAC;YACD,UAAU,EAAE,UAAU;SACvB,CAAC,CAAC;QAEH,IAAI,UAAU,CAAC,MAAM,GAAG,IAAI,EAAE;YAC5B,iEAAiE;YACjE,MAAM,IAAI,KAAK,CAAC,qBAAqB,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;SAC3D;QAED,IAAI,cAAc,GAAG,IAAI,GAAG,EAAkB,CAAC;QAC/C,KAAK,IAAI,MAAM,IAAI,UAAU,EAAE;YAC7B,cAAc,CAAC,GAAG,CAAC,MAAM,EAAE,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,CAAC;SAC5D;QACD,WAAW,GAAG,cAAc,CAAC;QAE7B,MAAM,UAAU,GAAG,CAAC,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC;QAC3C,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,YAAY,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;QACvE,mEAAmE;QACnE,IAAI,cAAwB,CAAC;QAE7B,IAAI,aAAa,EAAE;YACjB,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,CAAC,aAAa,EAAE,WAAW,CAAC,CAAC;YAE5D,IACE,mBAAmB,IAAI,eAAe;gBACtC,IAAI,CAAC,cAAc,CAAC,MAAM,IAAI,CAAC;gBAC/B,IAAI,CAAC,cAAc,CAAC,MAAM,IAAI,CAAC,EAC/B;gBACA,iDAAiD;gBACjD,SAAS;aACV;YACD,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YAE1D,IAAI,OAAO,GAAG,uBAAuB,UAAU,aAAa,eAAe,KAAK,CAAC;YACjF,OAAO,IAAI,YAAY,UAAU,CAAC,MAAM,KAAK,CAAC;YAC9C,OAAO,IAAI,YAAY,cAAc,CAAC,IAAI,CAAC,cAAc,EAAE,EAAE,CAAC,KAAK,CAAC;YACpE,OAAO,IAAI,YAAY,cAAc,CAAC,IAAI,CAAC,cAAc,EAAE,EAAE,CAAC,KAAK,CAAC;YACpE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAErB,MAAM,aAAa,GAAqC;gBACtD,eAAe,EAAE;oBACf,UAAU,EAAE,UAAU;oBACtB,gBAAgB,EAAE,eAAe,CAAC,CAAC,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,SAAS;oBACvE,eAAe,EAAE,IAAI,CAAC,cAAc;oBACpC,eAAe,EAAE,IAAI,CAAC,cAAc;iBACrC;aACF,CAAC;YAEF,MAAM,aAAa,CAAC;SACrB;aAAM;YACL,IAAI,OAAO,GAAG,mBAAmB,UAAU,aAAa,eAAe,KAAK,CAAC;YAC7E,OAAO,IAAI,YAAY,UAAU,CAAC,MAAM,IAAI,cAAc,CAAC,UAAU,EAAE,EAAE,CAAC,EAAE,CAAC;YAC7E,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACrB,cAAc,GAAG,UAAU,CAAC;YAC5B,MAAM,aAAa,GAAiC;gBAClD,UAAU,EAAE;oBACV,UAAU,EAAE,UAAU;oBACtB,gBAAgB,EAAE,eAAe,CAAC,CAAC,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,SAAS;oBACvE,OAAO,EAAE,CAAC,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC;iBACnC;aACF,CAAC;YACF,MAAM,aAAa,CAAC;SACrB;QACD,aAAa,GAAG,WAAW,CAAC;QAC5B,mBAAmB,GAAG,eAAe,CAAC;QAEtC,+EAA+E;QAC/E,sFAAsF;QACtF,KAAK,CAAC,CAAC,mBAAmB,CAAC,EAAE,OAAO,EAAE,UAAU,EAAE,cAAc,EAAE,WAAW,EAAE,QAAQ,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,CAAC;QAEhH,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;KACzD;AACH,CAAC;AAaD,KAAK,SAAS,CAAC,CAAC,mBAAmB,CAAC,OAA0B;IAC5D,IAAI,MAAM,GAAG,KAAK,CAAC;IACnB,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,MAAM,EAAE;QACzC,wEAAwE;QACxE,6EAA6E;QAC7E,4GAA4G;QAC5G,gBAAgB;QAChB,gBAAgB;QAChB,qBAAqB;QACrB,MAAM;QACN,WAAW;QACX,IAAI;QACJ,MAAM,IAAI,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;QACtC,IAAI;YACF,OAAO,IAAI,EAAE;gBACX,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;gBACpD,IAAI,QAAQ,EAAE;oBACZ,MAAM;iBACP;qBAAM;oBACL,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,KAAK,CAAC;oBAC7B,MAAM,IAAI,CAAC;oBACX,IAAI,IAAI,EAAE;wBACR,MAAM,GAAG,IAAI,CAAC;qBACf;iBACF;aACF;SACF;gBAAS;YACR,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC;SACrB;KACF;AACH,CAAC;AAED;;GAEG;AACH,KAAK,SAAS,CAAC,CAAC,eAAe,CAAC,OAA0B;IACxD,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,cAAc,EAAE,WAAW,EAAE,QAAQ,EAAE,WAAW,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAEpG,MAAM,CAAC,CAAC,EAAE,OAAO,CAAC,GAAG,MAAM,aAAa,CAAC,OAAO,EAAE,CAAC;IACnD,IAAI;QACF,kGAAkG;QAClG,iDAAiD;QACjD,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,WAAW,CAAC,GAAG,CAAC,MAAM,CAAE,CAAC,CAAC,CAAC,CAAC;QACpG,MAAM,IAAI,GAAG,OAAO,CAAC,kBAAkB,CAAC,UAAU,EAAE,eAAe,CAAC,CAAC;QAErE,IAAI,QAAQ,GAAG,KAAK,CAAC;QAErB,IAAI,KAAK,EAAE,IAAI,CAAC,IAAI,IAAI,EAAE;YACxB,IAAI,MAAM,CAAC,OAAO,EAAE;gBAClB,OAAO;aACR;YACD,IAAI,CAAC,CAAC,QAAQ,EAAE;gBACd,QAAQ,GAAG,IAAI,CAAC;aACjB;YACD,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,EAAE;gBACtB,SAAS;aACV;YACD,MAAM,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;YAE7C,IAAI,SAAc,CAAC;YACnB,IAAI,WAAW,EAAE;gBACf,wEAAwE;gBACxE,SAAS,GAAG,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;aACzB;iBAAM,IAAI,QAAQ,EAAE;gBACnB,uEAAuE;gBACvE,MAAM,QAAQ,GAA2B;oBACvC,IAAI,EAAE,CAAC;iBACR,CAAC;gBACF,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;aACtC;iBAAM;gBACL,8EAA8E;gBAC9E,4BAA4B;gBAC5B,MAAM,QAAQ,GAA2B;oBACvC,IAAI,EAAE,uBAAuB,CAAC,CAAC,CAAC;iBACjC,CAAC;gBACF,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;aACzC;YACD,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;YACvC,IAAI,SAAS,CAAC,MAAM,GAAG,KAAM,EAAE;gBAC7B,0EAA0E;gBAC1E,8CAA8C;gBAC9C,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;aACnC;YACD,OAAO,CAAC,WAAW,EAAE,CAAC,uBAAuB,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAEjE,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC;SACzC;QAED,IAAI,CAAC,QAAQ,EAAE;YACb,MAAM,IAAI,GAAyC;gBACjD,mBAAmB,EAAE;oBACnB,UAAU,EAAE,UAAU;iBACvB;aACF,CAAC;YACF,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;SAClC;KACF;YAAS;QACR,OAAO,EAAE,CAAC;KACX;AACH,CAAC;AAED,SAAS,uBAAuB,CAAC,UAA+B;IAC9D,OAAO;QACL,GAAG,UAAU;QACb,IAAI,EAAE,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;YAClC,OAAO;gBACL,GAAG,KAAK;gBACR,IAAI,EAAE,KAAK,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,aAAa,CAAC,KAAK,CAAC,IAAc,CAAC;gBACzE,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC;aACjC,CAAC;QACJ,CAAC,CAAC;KACH,CAAC;AACJ,CAAC;AAED,SAAS,cAAc,CAAC,OAAyC,EAAE,KAAa;IAC9E,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QAC1B,IAAI,OAAO,CAAC,IAAI,QAAQ,EAAE;YACxB,OAAO,CAAC,CAAC,MAAM,CAAC;SACjB;aAAM;YACL,OAAO,CAAC,CAAC;SACV;IACH,CAAC,CAAC,CAAC;IACH,IAAI,OAAO,CAAC,MAAM,IAAI,KAAK,EAAE;QAC3B,OAAO,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;KAChC;IACD,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IACxC,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC;AACzC,CAAC"}
+{"version":3,"file":"sync.js","sourceRoot":"","sources":["../../src/sync/sync.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAEpE,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAI9C,OAAO,KAAK,IAAI,MAAM,uBAAuB,CAAC;AAE9C,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAE3D,OAAO,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC;AACjD,OAAO,EAAsB,WAAW,EAAE,MAAM,WAAW,CAAC;AAG5D;;GAEG;AACH,MAAM,sBAAsB,GAAG,EAAE,CAAC;AAClC,MAAM,aAAa,GAAG,IAAI,SAAS,CAAC,sBAAsB,CAAC,CAAC;AAgB5D,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,cAAc,CACnC,OAA6B;IAE7B,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,kBAAkB,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAC5F,qEAAqE;IACrE,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,IAAI,MAAM,EAAE;QACV,MAAM,CAAC,gBAAgB,CACrB,OAAO,EACP,GAAG,EAAE;YACH,UAAU,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC,EACD,EAAE,IAAI,EAAE,IAAI,EAAE,CACf,CAAC;QACF,IAAI,MAAM,CAAC,OAAO,EAAE;YAClB,UAAU,CAAC,KAAK,EAAE,CAAC;SACpB;KACF;IACD,MAAM,EAAE,GAAG,WAAW,CAAC,KAAK,EAAE,UAAU,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAC;IACrE,MAAM,MAAM,GAAG,mBAAmB,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;IAC5F,sEAAsE;IACtE,MAAM,MAAM,GAAG,mBAAmB,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;IAEpE,IAAI;QACF,KAAK,CAAC,CAAC,MAAM,CAAC;KACf;IAAC,OAAO,CAAC,EAAE;QACV,IAAI,CAAC,YAAY,UAAU,EAAE;YAC3B,OAAO;SACR;aAAM;YACL,MAAM,CAAC,CAAC;SACT;KACF;YAAS;QACR,iFAAiF;QACjF,qBAAqB;QACrB,UAAU,CAAC,KAAK,EAAE,CAAC;KACpB;AACH,CAAC;AAED,KAAK,SAAS,CAAC,CAAC,mBAAmB,CACjC,OAAqC,EACrC,MAAiC,EACjC,UAA6B,EAC7B,OAAuB,EACvB,MAAmB;IAEnB,sCAAsC;IACtC,iHAAiH;IACjH,IAAI,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAC;IAE5C,IAAI,aAAa,GAA4B,IAAI,CAAC;IAClD,IAAI,mBAAmB,GAAkB,IAAI,CAAC;IAE9C,MAAM,EAAE,QAAQ,EAAE,WAAW,EAAE,GAAG,MAAM,CAAC;IAEzC,IAAI,MAAM,CAAC,OAAO,EAAE;QAClB,KAAK,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,MAAM,CAAC,OAAO,EAAE;YACjD,WAAW,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;SAC9B;KACF;IAED,MAAM,MAAM,GAAG,OAAO,CAAC,oBAAoB,CAAC,UAAU,CAAC,gBAAgB,CAAC,OAAiB,EAAE,MAAM,CAAC,CAAC;IACnG,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,MAAM,EAAE;QAC/B,MAAM,EAAE,IAAI,EAAE,eAAe,EAAE,GAAG,IAAI,CAAC;QACvC,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QAEnC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAC9C,IAAI,OAAO,IAAI,IAAI,EAAE;YACnB,2EAA2E;YAC3E,SAAS;SACV;QACD,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;QAEtC,MAAM,UAAU,GAAG,MAAM,UAAU,CAAC,cAAc,CAAC;YACjD,gBAAgB,CAAC,OAAO;gBACtB,OAAO,OAAO,CAAC,gBAAgB,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;YACvD,CAAC;YACD,UAAU,EAAE,UAAU;SACvB,CAAC,CAAC;QAEH,IAAI,UAAU,CAAC,MAAM,GAAG,IAAI,EAAE;YAC5B,MAAM,CAAC,KAAK,CAAC,kBAAkB,EAAE;gBAC/B,UAAU;gBACV,OAAO,EAAE,UAAU,CAAC,OAAO;gBAC3B,OAAO,EAAE,UAAU,CAAC,MAAM;aAC3B,CAAC,CAAC;YACH,iEAAiE;YACjE,MAAM,IAAI,KAAK,CAAC,qBAAqB,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;SAC3D;QAED,IAAI,cAAc,GAAG,IAAI,GAAG,EAAkB,CAAC;QAC/C,KAAK,IAAI,MAAM,IAAI,UAAU,EAAE;YAC7B,cAAc,CAAC,GAAG,CAAC,MAAM,EAAE,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,CAAC;SAC5D;QACD,WAAW,GAAG,cAAc,CAAC;QAE7B,MAAM,UAAU,GAAG,CAAC,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC;QAC3C,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,YAAY,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;QACvE,mEAAmE;QACnE,IAAI,cAAwB,CAAC;QAE7B,IAAI,aAAa,EAAE;YACjB,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,CAAC,aAAa,EAAE,WAAW,CAAC,CAAC;YAE5D,IACE,mBAAmB,IAAI,eAAe;gBACtC,IAAI,CAAC,cAAc,CAAC,MAAM,IAAI,CAAC;gBAC/B,IAAI,CAAC,cAAc,CAAC,MAAM,IAAI,CAAC,EAC/B;gBACA,iDAAiD;gBACjD,SAAS;aACV;YACD,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YAE1D,IAAI,OAAO,GAAG,uBAAuB,UAAU,KAAK,CAAC;YACrD,OAAO,IAAI,UAAU,eAAe,KAAK,CAAC;YAC1C,OAAO,IAAI,YAAY,UAAU,CAAC,MAAM,KAAK,CAAC;YAC9C,OAAO,IAAI,YAAY,cAAc,CAAC,IAAI,CAAC,cAAc,EAAE,EAAE,CAAC,KAAK,CAAC;YACpE,OAAO,IAAI,YAAY,cAAc,CAAC,IAAI,CAAC,cAAc,EAAE,EAAE,CAAC,EAAE,CAAC;YACjE,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE;gBACnB,UAAU;gBACV,OAAO,EAAE,UAAU,CAAC,OAAO;gBAC3B,OAAO,EAAE,UAAU,CAAC,MAAM;gBAC1B,OAAO,EAAE,IAAI,CAAC,cAAc,CAAC,MAAM;gBACnC,OAAO,EAAE,IAAI,CAAC,cAAc,CAAC,MAAM;aACpC,CAAC,CAAC;YAEH,MAAM,aAAa,GAAqC;gBACtD,eAAe,EAAE;oBACf,UAAU,EAAE,UAAU;oBACtB,gBAAgB,EAAE,eAAe,CAAC,CAAC,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,SAAS;oBACvE,eAAe,EAAE,IAAI,CAAC,cAAc;oBACpC,eAAe,EAAE,IAAI,CAAC,cAAc;iBACrC;aACF,CAAC;YAEF,MAAM,aAAa,CAAC;SACrB;aAAM;YACL,IAAI,OAAO,GAAG,mBAAmB,UAAU,aAAa,eAAe,KAAK,CAAC;YAC7E,OAAO,IAAI,YAAY,UAAU,CAAC,MAAM,IAAI,cAAc,CAAC,UAAU,EAAE,EAAE,CAAC,EAAE,CAAC;YAC7E,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,UAAU,EAAE,OAAO,EAAE,UAAU,CAAC,OAAO,EAAE,OAAO,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;YAC9F,cAAc,GAAG,UAAU,CAAC;YAC5B,MAAM,aAAa,GAAiC;gBAClD,UAAU,EAAE;oBACV,UAAU,EAAE,UAAU;oBACtB,gBAAgB,EAAE,eAAe,CAAC,CAAC,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,SAAS;oBACvE,OAAO,EAAE,CAAC,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC;iBACnC;aACF,CAAC;YACF,MAAM,aAAa,CAAC;SACrB;QACD,aAAa,GAAG,WAAW,CAAC;QAC5B,mBAAmB,GAAG,eAAe,CAAC;QAEtC,+EAA+E;QAC/E,sFAAsF;QACtF,KAAK,CAAC,CAAC,mBAAmB,CAAC;YACzB,OAAO;YACP,UAAU;YACV,cAAc;YACd,WAAW;YACX,QAAQ;YACR,WAAW;YACX,MAAM;YACN,OAAO;SACR,CAAC,CAAC;QAEH,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;KACzD;AACH,CAAC;AAcD,KAAK,SAAS,CAAC,CAAC,mBAAmB,CAAC,OAA0B;IAC5D,IAAI,MAAM,GAAG,KAAK,CAAC;IACnB,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,MAAM,EAAE;QACzC,wEAAwE;QACxE,6EAA6E;QAC7E,4GAA4G;QAC5G,gBAAgB;QAChB,gBAAgB;QAChB,qBAAqB;QACrB,MAAM;QACN,WAAW;QACX,IAAI;QACJ,MAAM,IAAI,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;QACtC,IAAI;YACF,OAAO,IAAI,EAAE;gBACX,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;gBACpD,IAAI,QAAQ,EAAE;oBACZ,MAAM;iBACP;qBAAM;oBACL,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,KAAK,CAAC;oBAC7B,MAAM,IAAI,CAAC;oBACX,IAAI,IAAI,EAAE;wBACR,MAAM,GAAG,IAAI,CAAC;qBACf;iBACF;aACF;SACF;gBAAS;YACR,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC;SACrB;KACF;AACH,CAAC;AAOD;;GAEG;AACH,KAAK,SAAS,CAAC,CAAC,eAAe,CAAC,OAA0B;IACxD,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,cAAc,EAAE,WAAW,EAAE,QAAQ,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAE7G,MAAM,YAAY,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;IACxC,IAAI,qBAAqB,GAAG,KAAK,CAAC;IAElC,MAAM,CAAC,CAAC,EAAE,OAAO,CAAC,GAAG,MAAM,aAAa,CAAC,OAAO,EAAE,CAAC;IACnD,IAAI;QACF,kGAAkG;QAClG,iDAAiD;QACjD,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,WAAW,CAAC,GAAG,CAAC,MAAM,CAAE,CAAC,CAAC,CAAC,CAAC;QACpG,MAAM,IAAI,GAAG,OAAO,CAAC,kBAAkB,CAAC,UAAU,EAAE,eAAe,CAAC,CAAC;QAErE,IAAI,QAAQ,GAAG,KAAK,CAAC;QAErB,IAAI,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,IAAI,IAAI,EAAE;YAC7C,IAAI,MAAM,CAAC,OAAO,EAAE;gBAClB,OAAO;aACR;YACD,IAAI,CAAC,CAAC,QAAQ,EAAE;gBACd,QAAQ,GAAG,IAAI,CAAC;aACjB;YACD,IAAI,QAAQ,IAAI,IAAI,IAAI,QAAQ,GAAG,YAAY,EAAE;gBAC/C,qBAAqB,GAAG,IAAI,CAAC;aAC9B;YACD,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,EAAE;gBACtB,SAAS;aACV;YACD,MAAM,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;YAE7C,IAAI,SAAc,CAAC;YACnB,IAAI,WAAW,EAAE;gBACf,wEAAwE;gBACxE,SAAS,GAAG,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;aACzB;iBAAM,IAAI,QAAQ,EAAE;gBACnB,uEAAuE;gBACvE,MAAM,QAAQ,GAA2B;oBACvC,IAAI,EAAE,CAAC;iBACR,CAAC;gBACF,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;aACtC;iBAAM;gBACL,8EAA8E;gBAC9E,4BAA4B;gBAC5B,MAAM,QAAQ,GAA2B;oBACvC,IAAI,EAAE,uBAAuB,CAAC,CAAC,CAAC;iBACjC,CAAC;gBACF,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;aACzC;YACD,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;YACvC,IAAI,SAAS,CAAC,MAAM,GAAG,KAAM,EAAE;gBAC7B,0EAA0E;gBAC1E,8CAA8C;gBAC9C,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;aACnC;YACD,OAAO,CAAC,mBAAmB,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAE3C,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC;SACzC;QAED,IAAI,CAAC,QAAQ,EAAE;YACb,IAAI,qBAAqB,EAAE;gBACzB,gDAAgD;gBAChD,wDAAwD;gBACxD,kEAAkE;gBAClE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;aAClC;iBAAM;gBACL,MAAM,IAAI,GAAyC;oBACjD,mBAAmB,EAAE;wBACnB,UAAU,EAAE,UAAU;qBACvB;iBACF,CAAC;gBACF,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;aAClC;SACF;KACF;YAAS;QACR,OAAO,EAAE,CAAC;KACX;AACH,CAAC;AAED,SAAS,uBAAuB,CAAC,UAA+B;IAC9D,OAAO;QACL,GAAG,UAAU;QACb,IAAI,EAAE,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;YAClC,OAAO;gBACL,GAAG,KAAK;gBACR,IAAI,EAAE,KAAK,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,aAAa,CAAC,KAAK,CAAC,IAAc,CAAC;gBACzE,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC;aACjC,CAAC;QACJ,CAAC,CAAC;KACH,CAAC;AACJ,CAAC;AAED,SAAS,cAAc,CAAC,OAAyC,EAAE,KAAa;IAC9E,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QAC1B,IAAI,OAAO,CAAC,IAAI,QAAQ,EAAE;YACxB,OAAO,CAAC,CAAC,MAAM,CAAC;SACjB;aAAM;YACL,OAAO,CAAC,CAAC;SACV;IACH,CAAC,CAAC,CAAC;IACH,IAAI,OAAO,CAAC,MAAM,IAAI,KAAK,EAAE;QAC3B,OAAO,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;KAChC;IACD,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IACxC,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC;AACzC,CAAC"}

package/dist/sync/util.d.ts

@@ -1,6 +1,7 @@
 /// <reference types="node" resolution-mode="require"/>
 /// <reference types="node" resolution-mode="require"/>
 import * as util from '../util/util-index.js';
+import { RequestTracker } from './RequestTracker.js';
 export type TokenStreamOptions = {
     /**
      * Adds periodic keepalive events
@@ -23,4 +24,4 @@ export declare function tokenStream(token: {
     exp: number;
 }, signal: AbortSignal, options?: Partial<TokenStreamOptions>): AsyncGenerator<util.StreamingSyncKeepalive>;
 export declare function ndjson(iterator: AsyncIterable<string | null | Record<string, any>>): AsyncGenerator<string>;
-export declare function transformToBytesTracked(iterator: AsyncIterable<string>): AsyncGenerator<Buffer>;
+export declare function transformToBytesTracked(iterator: AsyncIterable<string>, tracker: RequestTracker): AsyncGenerator<Buffer>;

package/dist/sync/util.js

@@ -1,5 +1,4 @@
 import * as timers from 'timers/promises';
-import { Metrics } from '../metrics/Metrics.js';
 const KEEPALIVE_INTERVAL = 20000;
 const DEFAULT_TOKEN_STREAM_OPTIONS = {
     keep_alive: true,
@@ -63,10 +62,10 @@ export async function* ndjson(iterator) {
         }
     }
 }
-export async function* transformToBytesTracked(iterator) {
+export async function* transformToBytesTracked(iterator, tracker) {
     for await (let data of iterator) {
         const encoded = Buffer.from(data, 'utf8');
-        Metrics.getInstance().data_synced_bytes.add(encoded.length);
+        tracker.addDataSynced(encoded.length);
         yield encoded;
     }
 }

package/dist/sync/util.js.map

@@ -1 +1 @@
-{"version":3,"file":"util.js","sourceRoot":"","sources":["../../src/sync/util.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,MAAM,iBAAiB,CAAC;AAG1C,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAahD,MAAM,kBAAkB,GAAG,KAAM,CAAC;AAElC,MAAM,4BAA4B,GAAuB;IACvD,UAAU,EAAE,IAAI;IAChB,qBAAqB,EAAE,KAAM;CAC9B,CAAC;AAEF;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,WAAW,CAChC,KAAsB,EACtB,MAAmB,EACnB,OAAqC;IAErC,MAAM,gBAAgB,GAAuB;QAC3C,GAAG,4BAA4B;QAC/B,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC;KACnB,CAAC;IAEF,MAAM,EAAE,UAAU,EAAE,qBAAqB,EAAE,GAAG,gBAAgB,CAAC;IAE/D,8CAA8C;IAC9C,+DAA+D;IAC/D,+BAA+B;IAC/B,MAAM,UAAU,GAAG,KAAK,CAAC,GAAG,GAAG,IAAI,CAAC;IACpC,MAAM,iBAAiB,GAAG,UAAU,GAAG,qBAAqB,CAAC;IAE7D,IAAI,iBAAiB,GAAG,IAAI,CAAC;IAE7B,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE;QACtB,MAAM,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC;QAC/E,IAAI,iBAAiB,IAAI,gBAAgB,GAAG,CAAC,EAAE;YAC7C,iBAAiB,GAAG,KAAK,CAAC;SAC3B;aAAM;YACL,MAAM,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,CAAC;YAC7C,IAAI,gBAAgB,IAAI,CAAC,EAAE;gBACzB,OAAO;aACR;SACF;QAED,MAAM,gBAAgB,GAAG,kBAAkB,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC,CAAC;QAE1E,4CAA4C;QAC5C,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;QAC9D,MAAM,oBAAoB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,iBAAiB,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;QAC7E,6CAA6C;QAC7C,MAAM,qBAAqB,GAAG,oBAAoB,IAAI,CAAC,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,YAAY,CAAC;QAE9F,MAAM,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,qBAAqB,EAAE,gBAAgB,CAAC,CAAC,CAAC,CAAC,qBAAqB,CAAC;QACrG,MAAM,MAAM,CAAC,UAAU,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE;YAC1D,oBAAoB;QACtB,CAAC,CAAC,CAAC;KACJ;AACH,CAAC;AAED,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,MAAM,CAAC,QAA4D;IACxF,IAAI,KAAK,EAAE,IAAI,IAAI,IAAI,QAAQ,EAAE;QAC/B,IAAI,IAAI,IAAI,IAAI,EAAE;YAChB,uCAAuC;YACvC,SAAS;SACV;aAAM,IAAI,OAAO,IAAI,IAAI,QAAQ,EAAE;YAClC,uBAAuB;YACvB,MAAM,IAAI,GAAG,IAAI,CAAC;SACnB;aAAM;YACL,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;SACnC;KACF;AACH,CAAC;AAED,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,uBAAuB,CAAC,QAA+B;IAC5E,IAAI,KAAK,EAAE,IAAI,IAAI,IAAI,QAAQ,EAAE;QAC/B,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAC1C,OAAO,CAAC,WAAW,EAAE,CAAC,iBAAiB,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC5D,MAAM,OAAO,CAAC;KACf;AACH,CAAC"}
+{"version":3,"file":"util.js","sourceRoot":"","sources":["../../src/sync/util.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,MAAM,iBAAiB,CAAC;AAiB1C,MAAM,kBAAkB,GAAG,KAAM,CAAC;AAElC,MAAM,4BAA4B,GAAuB;IACvD,UAAU,EAAE,IAAI;IAChB,qBAAqB,EAAE,KAAM;CAC9B,CAAC;AAEF;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,WAAW,CAChC,KAAsB,EACtB,MAAmB,EACnB,OAAqC;IAErC,MAAM,gBAAgB,GAAuB;QAC3C,GAAG,4BAA4B;QAC/B,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC;KACnB,CAAC;IAEF,MAAM,EAAE,UAAU,EAAE,qBAAqB,EAAE,GAAG,gBAAgB,CAAC;IAE/D,8CAA8C;IAC9C,+DAA+D;IAC/D,+BAA+B;IAC/B,MAAM,UAAU,GAAG,KAAK,CAAC,GAAG,GAAG,IAAI,CAAC;IACpC,MAAM,iBAAiB,GAAG,UAAU,GAAG,qBAAqB,CAAC;IAE7D,IAAI,iBAAiB,GAAG,IAAI,CAAC;IAE7B,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE;QACtB,MAAM,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC;QAC/E,IAAI,iBAAiB,IAAI,gBAAgB,GAAG,CAAC,EAAE;YAC7C,iBAAiB,GAAG,KAAK,CAAC;SAC3B;aAAM;YACL,MAAM,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,CAAC;YAC7C,IAAI,gBAAgB,IAAI,CAAC,EAAE;gBACzB,OAAO;aACR;SACF;QAED,MAAM,gBAAgB,GAAG,kBAAkB,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC,CAAC;QAE1E,4CAA4C;QAC5C,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;QAC9D,MAAM,oBAAoB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,iBAAiB,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;QAC7E,6CAA6C;QAC7C,MAAM,qBAAqB,GAAG,oBAAoB,IAAI,CAAC,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,YAAY,CAAC;QAE9F,MAAM,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,qBAAqB,EAAE,gBAAgB,CAAC,CAAC,CAAC,CAAC,qBAAqB,CAAC;QACrG,MAAM,MAAM,CAAC,UAAU,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE;YAC1D,oBAAoB;QACtB,CAAC,CAAC,CAAC;KACJ;AACH,CAAC;AAED,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,MAAM,CAAC,QAA4D;IACxF,IAAI,KAAK,EAAE,IAAI,IAAI,IAAI,QAAQ,EAAE;QAC/B,IAAI,IAAI,IAAI,IAAI,EAAE;YAChB,uCAAuC;YACvC,SAAS;SACV;aAAM,IAAI,OAAO,IAAI,IAAI,QAAQ,EAAE;YAClC,uBAAuB;YACvB,MAAM,IAAI,GAAG,IAAI,CAAC;SACnB;aAAM;YACL,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;SACnC;KACF;AACH,CAAC;AAED,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,uBAAuB,CAC5C,QAA+B,EAC/B,OAAuB;IAEvB,IAAI,KAAK,EAAE,IAAI,IAAI,IAAI,QAAQ,EAAE;QAC/B,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAC1C,OAAO,CAAC,aAAa,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QACtC,MAAM,OAAO,CAAC;KACf;AACH,CAAC"}

package/package.json

@@ -5,7 +5,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "0.0.0-dev-20240709124106",
+  "version": "0.0.0-dev-20240725112650",
   "main": "dist/index.js",
   "license": "FSL-1.1-Apache-2.0",
   "type": "module",
@@ -33,12 +33,12 @@
     "uuid": "^9.0.1",
     "winston": "^3.13.0",
     "yaml": "^2.3.2",
-    "@powersync/lib-services-framework": "0.1.0",
+    "@powersync/lib-services-framework": "0.1.1",
+    "@powersync/service-jsonbig": "0.17.10",
+    "@powersync/service-rsocket-router": "0.0.10",
+    "@powersync/service-sync-rules": "0.18.1",
     "@powersync/service-jpgwire": "0.17.13",
-    "@powersync/service-sync-rules": "0.0.0-dev-20240709124106",
-    "@powersync/service-types": "0.1.0",
-    "@powersync/service-rsocket-router": "0.0.8",
-    "@powersync/service-jsonbig": "0.17.10"
+    "@powersync/service-types": "0.1.0"
   },
   "devDependencies": {
     "@types/async": "^3.2.24",

package/src/entry/cli-entry.ts

@@ -3,7 +3,7 @@ import { Command } from 'commander';
 import * as utils from '../util/util-index.js';
 import { registerMigrationAction } from './commands/migrate-action.js';
 import { registerTearDownAction } from './commands/teardown-action.js';
-import { registerStartAction } from './entry-index.js';
+import { registerCompactAction, registerStartAction } from './entry-index.js';
 import { logger } from '@powersync/lib-services-framework';
 
 /**
@@ -18,6 +18,7 @@ export function generateEntryProgram(startHandlers?: Record<utils.ServiceRunner,
 
   registerTearDownAction(entryProgram);
   registerMigrationAction(entryProgram);
+  registerCompactAction(entryProgram);
 
   if (startHandlers) {
     registerStartAction(entryProgram, startHandlers);

package/src/entry/commands/compact-action.ts

@@ -0,0 +1,54 @@
+import { Command } from 'commander';
+
+import { logger } from '@powersync/lib-services-framework';
+import * as v8 from 'v8';
+import { createPowerSyncMongo, MongoBucketStorage } from '../../storage/storage-index.js';
+import { loadConfig } from '../../util/config.js';
+import { extractRunnerOptions, wrapConfigCommand } from './config-command.js';
+
+const COMMAND_NAME = 'compact';
+
+/**
+ * Approximately max-old-space-size + 64MB.
+ */
+const HEAP_LIMIT = v8.getHeapStatistics().heap_size_limit;
+
+/**
+ * Subtract 128MB for process overhead.
+ *
+ * Limit to 1024MB overall.
+ */
+const COMPACT_MEMORY_LIMIT_MB = Math.min(HEAP_LIMIT / 1024 / 1024 - 128, 1024);
+
+export function registerCompactAction(program: Command) {
+  const compactCommand = program.command(COMMAND_NAME);
+
+  wrapConfigCommand(compactCommand);
+
+  return compactCommand.description('Compact storage').action(async (options) => {
+    const runnerConfig = extractRunnerOptions(options);
+
+    const config = await loadConfig(runnerConfig);
+    const { storage } = config;
+    const psdb = createPowerSyncMongo(storage);
+    const client = psdb.client;
+    await client.connect();
+    try {
+      const bucketStorage = new MongoBucketStorage(psdb, { slot_name_prefix: config.slot_name_prefix });
+      const active = await bucketStorage.getActiveSyncRules();
+      if (active == null) {
+        logger.info('No active instance to compact');
+        return;
+      }
+      const p = bucketStorage.getInstance(active);
+      await p.compact({ memoryLimitMB: COMPACT_MEMORY_LIMIT_MB });
+      logger.info('done');
+    } catch (e) {
+      logger.error(`Failed to compact: ${e.toString()}`);
+      process.exit(1);
+    } finally {
+      await client.close();
+      process.exit(0);
+    }
+  });
+}

package/src/entry/entry-index.ts

@@ -3,3 +3,4 @@ export * from './commands/config-command.js';
 export * from './commands/migrate-action.js';
 export * from './commands/start-action.js';
 export * from './commands/teardown-action.js';
+export * from './commands/compact-action.js';

package/src/routes/configure-fastify.ts

@@ -0,0 +1,102 @@
+import type fastify from 'fastify';
+import { registerFastifyRoutes } from './route-register.js';
+
+import * as system from '../system/system-index.js';
+
+import { ADMIN_ROUTES } from './endpoints/admin.js';
+import { CHECKPOINT_ROUTES } from './endpoints/checkpointing.js';
+import { DEV_ROUTES } from './endpoints/dev.js';
+import { SYNC_RULES_ROUTES } from './endpoints/sync-rules.js';
+import { SYNC_STREAM_ROUTES } from './endpoints/sync-stream.js';
+import { createRequestQueueHook, CreateRequestQueueParams } from './hooks.js';
+import { RouteDefinition } from './router.js';
+
+/**
+ * A list of route definitions to be registered as endpoints.
+ * Supplied concurrency limits will be applied to the grouped routes.
+ */
+export type RouteRegistrationOptions = {
+  routes: RouteDefinition[];
+  queueOptions: CreateRequestQueueParams;
+};
+
+/**
+ * HTTP routes separated by API and Sync stream categories.
+ * This allows for separate concurrency limits.
+ */
+export type RouteDefinitions = {
+  api?: Partial<RouteRegistrationOptions>;
+  syncStream?: Partial<RouteRegistrationOptions>;
+};
+
+export type FastifyServerConfig = {
+  system: system.CorePowerSyncSystem;
+  routes?: RouteDefinitions;
+};
+
+export const DEFAULT_ROUTE_OPTIONS = {
+  api: {
+    routes: [...ADMIN_ROUTES, ...CHECKPOINT_ROUTES, ...DEV_ROUTES, ...SYNC_RULES_ROUTES],
+    queueOptions: {
+      concurrency: 10,
+      max_queue_depth: 20
+    }
+  },
+  syncStream: {
+    routes: [...SYNC_STREAM_ROUTES],
+    queueOptions: {
+      concurrency: 200,
+      max_queue_depth: 0
+    }
+  }
+};
+
+/**
+ * Registers default routes on a Fastify server. Consumers can optionally configure
+ * concurrency queue limits or override routes.
+ */
+export function configureFastifyServer(server: fastify.FastifyInstance, options: FastifyServerConfig) {
+  const { system, routes = DEFAULT_ROUTE_OPTIONS } = options;
+  /**
+   * Fastify creates an encapsulated context for each `.register` call.
+   * Creating a separate context here to separate the concurrency limits for Admin APIs
+   * and Sync Streaming routes.
+   * https://github.com/fastify/fastify/blob/main/docs/Reference/Encapsulation.md
+   */
+  server.register(async function (childContext) {
+    registerFastifyRoutes(
+      childContext,
+      async () => {
+        return {
+          user_id: undefined,
+          system: system
+        };
+      },
+      routes.api?.routes ?? DEFAULT_ROUTE_OPTIONS.api.routes
+    );
+    // Limit the active concurrent requests
+    childContext.addHook(
+      'onRequest',
+      createRequestQueueHook(routes.api?.queueOptions ?? DEFAULT_ROUTE_OPTIONS.api.queueOptions)
+    );
+  });
+
+  // Create a separate context for concurrency queueing
+  server.register(async function (childContext) {
+    registerFastifyRoutes(
+      childContext,
+      async () => {
+        return {
+          user_id: undefined,
+          system: system
+        };
+      },
+      routes.syncStream?.routes ?? DEFAULT_ROUTE_OPTIONS.syncStream.routes
+    );
+    // Limit the active concurrent requests
+    childContext.addHook(
+      'onRequest',
+      createRequestQueueHook(routes.syncStream?.queueOptions ?? DEFAULT_ROUTE_OPTIONS.syncStream.queueOptions)
+    );
+  });
+}

package/src/routes/configure-rsocket.ts

@@ -0,0 +1,59 @@
+import { deserialize } from 'bson';
+import * as http from 'http';
+
+import { errors, logger } from '@powersync/lib-services-framework';
+import { ReactiveSocketRouter, RSocketRequestMeta } from '@powersync/service-rsocket-router';
+
+import { CorePowerSyncSystem } from '../system/CorePowerSyncSystem.js';
+import { generateContext, getTokenFromHeader } from './auth.js';
+import { syncStreamReactive } from './endpoints/socket-route.js';
+import { RSocketContextMeta, SocketRouteGenerator } from './router-socket.js';
+import { Context } from './router.js';
+
+export type RSockerRouterConfig = {
+  system: CorePowerSyncSystem;
+  server: http.Server;
+  routeGenerators?: SocketRouteGenerator[];
+};
+
+export const DEFAULT_SOCKET_ROUTES = [syncStreamReactive];
+
+export function configureRSocket(router: ReactiveSocketRouter<Context>, options: RSockerRouterConfig) {
+  const { routeGenerators = DEFAULT_SOCKET_ROUTES, server, system } = options;
+
+  router.applyWebSocketEndpoints(server, {
+    contextProvider: async (data: Buffer) => {
+      const { token } = RSocketContextMeta.decode(deserialize(data) as any);
+
+      if (!token) {
+        throw new errors.AuthorizationError('No token provided');
+      }
+
+      try {
+        const extracted_token = getTokenFromHeader(token);
+        if (extracted_token != null) {
+          const { context, errors: token_errors } = await generateContext(system, extracted_token);
+          if (context?.token_payload == null) {
+            throw new errors.AuthorizationError(token_errors ?? 'Authentication required');
+          }
+          return {
+            token,
+            ...context,
+            token_errors: token_errors,
+            system
+          };
+        } else {
+          throw new errors.AuthorizationError('No token provided');
+        }
+      } catch (ex) {
+        logger.error(ex);
+        throw ex;
+      }
+    },
+    endpoints: routeGenerators.map((generator) => generator(router)),
+    metaDecoder: async (meta: Buffer) => {
+      return RSocketRequestMeta.decode(deserialize(meta) as any);
+    },
+    payloadDecoder: async (rawData?: Buffer) => rawData && deserialize(rawData)
+  });
+}

package/src/routes/endpoints/socket-route.ts

@@ -3,19 +3,13 @@ import { RequestParameters } from '@powersync/service-sync-rules';
 import { serialize } from 'bson';
 
 import { Metrics } from '../../metrics/Metrics.js';
-import { streamResponse } from '../../sync/sync.js';
+import * as sync from '../../sync/sync-index.js';
 import * as util from '../../util/util-index.js';
 import { SocketRouteGenerator } from '../router-socket.js';
 import { SyncRoutes } from './sync-stream.js';
 
 export const syncStreamReactive: SocketRouteGenerator = (router) =>
   router.reactiveStream<util.StreamingSyncRequest, any>(SyncRoutes.STREAM, {
-    authorize: ({ context }) => {
-      return {
-        authorized: !!context.token_payload,
-        errors: ['Authentication required'].concat(context.token_errors ?? [])
-      };
-    },
     validator: schema.createTsCodecValidator(util.StreamingSyncRequest, { allowAdditional: true }),
     handler: async ({ context, params, responder, observer, initialN }) => {
       const { system } = context;
@@ -66,8 +60,9 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
       });
 
       Metrics.getInstance().concurrent_connections.add(1);
+      const tracker = new sync.RequestTracker();
       try {
-        for await (const data of streamResponse({
+        for await (const data of sync.streamResponse({
           storage,
           params: {
             ...params,
@@ -79,6 +74,7 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
             // RSocket handles keepalive events by default
             keep_alive: false
           },
+          tracker,
           signal: controller.signal
         })) {
           if (data == null) {
@@ -94,7 +90,7 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
             const serialized = serialize(data) as Buffer;
             responder.onNext({ data: serialized }, false);
             requestedN--;
-            Metrics.getInstance().data_synced_bytes.add(serialized.length);
+            tracker.addDataSynced(serialized.length);
           }
 
           if (requestedN <= 0) {
@@ -126,6 +122,11 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
         responder.onComplete();
         removeStopHandler();
         disposer();
+        logger.info(`Sync stream complete`, {
+          user_id: syncParams.user_id,
+          operations_synced: tracker.operationsSynced,
+          data_synced_bytes: tracker.dataSyncedBytes
+        });
         Metrics.getInstance().concurrent_connections.add(-1);
       }
     }

package/src/routes/endpoints/sync-stream.ts

@@ -8,6 +8,7 @@ import * as util from '../../util/util-index.js';
 import { Metrics } from '../../metrics/Metrics.js';
 import { authUser } from '../auth.js';
 import { routeDefinition } from '../router.js';
+import { RequestTracker } from '../../sync/RequestTracker.js';
 
 export enum SyncRoutes {
   STREAM = '/sync/stream'
@@ -43,6 +44,7 @@ export const syncStreamed = routeDefinition({
       });
     }
     const controller = new AbortController();
+    const tracker = new RequestTracker();
     try {
       Metrics.getInstance().concurrent_connections.add(1);
       const stream = Readable.from(
@@ -53,9 +55,11 @@ export const syncStreamed = routeDefinition({
               params,
               syncParams,
               token: payload.context.token_payload!,
+              tracker,
               signal: controller.signal
             })
-          )
+          ),
+          tracker
         ),
         { objectMode: false, highWaterMark: 16 * 1024 }
       );
@@ -86,6 +90,11 @@ export const syncStreamed = routeDefinition({
         afterSend: async () => {
           controller.abort();
           Metrics.getInstance().concurrent_connections.add(-1);
+          logger.info(`Sync stream complete`, {
+            user_id: syncParams.user_id,
+            operations_synced: tracker.operationsSynced,
+            data_synced_bytes: tracker.dataSyncedBytes
+          });
         }
       });
     } catch (ex) {

package/src/routes/route-register.ts

@@ -1,6 +1,6 @@
-import fastify from 'fastify';
+import type fastify from 'fastify';
 
-import { errors, router, HTTPMethod, logger } from '@powersync/lib-services-framework';
+import { errors, HTTPMethod, logger, router } from '@powersync/lib-services-framework';
 import { Context, ContextProvider, RequestEndpoint, RequestEndpointHandlerPayload } from './router.js';
 
 export type FastifyEndpoint<I, O, C> = RequestEndpoint<I, O, C> & {
@@ -63,6 +63,7 @@ export function registerFastifyRoutes(
             }
           } catch (ex) {
             const journeyError = errors.JourneyError.isJourneyError(ex) ? ex : new errors.InternalServerError(ex);
+            logger.error(`Request failed`, journeyError);
 
             response = new router.RouterResponse({
               status: journeyError.errorData.status || 500,

package/src/routes/router-socket.ts

@@ -1,13 +1,13 @@
+import { IReactiveStream, ReactiveSocketRouter } from '@powersync/service-rsocket-router';
 import * as t from 'ts-codec';
-import { ReactiveSocketRouter, IReactiveStream } from '@powersync/service-rsocket-router';
 
 import { Context } from './router.js';
 
-export const RSocketContextMeta = t.object({
-  token: t.string
-});
-
 /**
  * Creates a socket route handler given a router instance
  */
 export type SocketRouteGenerator = (router: ReactiveSocketRouter<Context>) => IReactiveStream;
+
+export const RSocketContextMeta = t.object({
+  token: t.string
+});

package/src/routes/router.ts

@@ -36,6 +36,8 @@ export type RequestEndpointHandlerPayload<
   request: Request;
 };
 
+export type RouteDefinition<I = any, O = any> = RequestEndpoint<I, O>;
+
 /**
  * Helper function for making generics work well when defining routes
  */