@powersync/service-core 0.10.1 → 0.12.0

package/dist/util/utils.js

@@ -116,4 +116,60 @@ export function checkpointUserId(user_id, client_id) {
     }
     return `${user_id}/${client_id}`;
 }
+/**
+ * Reduce a bucket to the final state as stored on the client.
+ *
+ * This keeps the final state for each row as a PUT operation.
+ *
+ * All other operations are replaced with a single CLEAR operation,
+ * summing their checksums, and using a 0 as an op_id.
+ *
+ * This is the function $r(B)$, as described in /docs/bucket-properties.md.
+ *
+ * Used for tests.
+ */
+export function reduceBucket(operations) {
+    let rowState = new Map();
+    let otherChecksum = 0;
+    for (let op of operations) {
+        const key = rowKey(op);
+        if (op.op == 'PUT') {
+            const existing = rowState.get(key);
+            if (existing) {
+                otherChecksum = addChecksums(otherChecksum, existing.checksum);
+            }
+            rowState.set(key, op);
+        }
+        else if (op.op == 'REMOVE') {
+            const existing = rowState.get(key);
+            if (existing) {
+                otherChecksum = addChecksums(otherChecksum, existing.checksum);
+            }
+            rowState.delete(key);
+            otherChecksum = addChecksums(otherChecksum, op.checksum);
+        }
+        else if (op.op == 'CLEAR') {
+            rowState.clear();
+            otherChecksum = op.checksum;
+        }
+        else if (op.op == 'MOVE') {
+            otherChecksum = addChecksums(otherChecksum, op.checksum);
+        }
+        else {
+            throw new Error(`Unknown operation ${op.op}`);
+        }
+    }
+    const puts = [...rowState.values()].sort((a, b) => {
+        return Number(BigInt(a.op_id) - BigInt(b.op_id));
+    });
+    let finalState = [
+        // Special operation to indiciate the checksum remainder
+        { op_id: '0', op: 'CLEAR', checksum: otherChecksum },
+        ...puts
+    ];
+    return finalState;
+}
+function rowKey(entry) {
+    return `${entry.object_type}/${entry.object_id}/${entry.subkey}`;
+}
 //# sourceMappingURL=utils.js.map

package/dist/util/utils.js.map

@@ -1 +1 @@
-{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/util/utils.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAS7B,MAAM,CAAC,MAAM,YAAY,GAAG,sCAAsC,CAAC;AAEnE,MAAM,UAAU,gBAAgB,CAAC,UAAkB;IACjD,OAAO,IAAI,UAAU,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC;AACrE,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,IAAY,EAAE,EAAU,EAAE,IAAY;IAC7D,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IACzC,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC;IACzC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;IAC7B,OAAO,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,SAAiB;IAC1C,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IACzC,IAAI,CAAC,MAAM,CAAC,UAAU,SAAS,EAAE,CAAC,CAAC;IACnC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;IAC7B,OAAO,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,EAAU;IACxC,6EAA6E;IAC7E,6CAA6C;IAC7C,IAAI,OAAO,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,yBAAyB,EAAE,KAAK,OAAO,EAAE,GAAG,CAAC,CAAC;IAChE,CAAC;IACD,OAAO,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;AACzB,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,QAAqB,EAAE,OAAoB;IACvE,mBAAmB;IACnB,MAAM,cAAc,GAAG,IAAI,GAAG,EAA0B,CAAC;IAEzD,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAS,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;IAElD,KAAK,IAAI,QAAQ,IAAI,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;QACtC,MAAM,CAAC,GAAG,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACxC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;YACd,QAAQ;YACR,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAChD,CAAC;aAAM,CAAC;YACN,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YACjC,IAAI,QAAQ,CAAC,QAAQ,IAAI,CAAC,CAAC,QAAQ,IAAI,QAAQ,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;gBACjE,UAAU;gBACV,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YAChD,CAAC;iBAAM,CAAC;gBACN,YAAY;YACd,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO;QACL,cAAc,EAAE,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC;QAC5C,cAAc,EAAE,CAAC,GAAG,QAAQ,CAAC;KAC9B,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,CAAS,EAAE,CAAS;IAC/C,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,UAAU,CAAC;AAC9B,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,CAAiB,EAAE,CAAyB;IAC7E,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QACd,OAAO,CAAC,CAAC;IACX,CAAC;SAAM,IAAI,CAAC,CAAC,cAAc,EAAE,CAAC;QAC5B,OAAO;YACL,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,KAAK,EAAE,CAAC,CAAC,YAAY;YACrB,QAAQ,EAAE,CAAC,CAAC,eAAe;SAC5B,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,OAAO;YACL,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,KAAK,EAAE,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,YAAY;YAC/B,QAAQ,EAAE,YAAY,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,eAAe,CAAC;SACtD,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,qBAAqB,CAC5B,KAAoC,EACpC,OAAmC;IAEnC,IAAI,MAAM,GAAwB,EAAE,CAAC;IACrC,KAAK,IAAI,MAAM,IAAI,OAAO,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;QACzB,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC;IAC7B,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,0BAA0B,CACxC,KAAoC,EACpC,OAAmC;IAEnC,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QACxB,gDAAgD;QAChD,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;IAClC,CAAC;IACD,MAAM,WAAW,GAAG,qBAAqB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAE1D,OAAO,cAAc,CAAC,WAAW,CAAC,CAAC;AACrC,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,GAAyB;IACtD,+EAA+E;IAC/E,4EAA4E;IAC5E,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IACjC,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAChC,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC;AAC5D,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,GAAkC;IACjE,KAAK,IAAI,GAAG,IAAI,GAAG,EAAE,CAAC;QACpB,IAAI,OAAO,GAAG,CAAC,GAAG,CAAC,IAAI,WAAW,EAAE,CAAC;YACnC,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,GAAkC;IAC9D,OAAO,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,OAA2B,EAAE,SAA6B;IACzF,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;QACpB,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;IACzC,CAAC;IACD,IAAI,SAAS,IAAI,IAAI,EAAE,CAAC;QACtB,OAAO,OAAO,CAAC;IACjB,CAAC;IACD,OAAO,GAAG,OAAO,IAAI,SAAS,EAAE,CAAC;AACnC,CAAC"}
+{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/util/utils.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAS7B,MAAM,CAAC,MAAM,YAAY,GAAG,sCAAsC,CAAC;AAEnE,MAAM,UAAU,gBAAgB,CAAC,UAAkB;IACjD,OAAO,IAAI,UAAU,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC;AACrE,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,IAAY,EAAE,EAAU,EAAE,IAAY;IAC7D,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IACzC,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC;IACzC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;IAC7B,OAAO,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,SAAiB;IAC1C,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IACzC,IAAI,CAAC,MAAM,CAAC,UAAU,SAAS,EAAE,CAAC,CAAC;IACnC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;IAC7B,OAAO,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,EAAU;IACxC,6EAA6E;IAC7E,6CAA6C;IAC7C,IAAI,OAAO,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,yBAAyB,EAAE,KAAK,OAAO,EAAE,GAAG,CAAC,CAAC;IAChE,CAAC;IACD,OAAO,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;AACzB,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,QAAqB,EAAE,OAAoB;IACvE,mBAAmB;IACnB,MAAM,cAAc,GAAG,IAAI,GAAG,EAA0B,CAAC;IAEzD,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAS,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;IAElD,KAAK,IAAI,QAAQ,IAAI,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;QACtC,MAAM,CAAC,GAAG,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACxC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;YACd,QAAQ;YACR,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAChD,CAAC;aAAM,CAAC;YACN,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YACjC,IAAI,QAAQ,CAAC,QAAQ,IAAI,CAAC,CAAC,QAAQ,IAAI,QAAQ,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;gBACjE,UAAU;gBACV,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YAChD,CAAC;iBAAM,CAAC;gBACN,YAAY;YACd,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO;QACL,cAAc,EAAE,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC;QAC5C,cAAc,EAAE,CAAC,GAAG,QAAQ,CAAC;KAC9B,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,CAAS,EAAE,CAAS;IAC/C,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,UAAU,CAAC;AAC9B,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,CAAiB,EAAE,CAAyB;IAC7E,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QACd,OAAO,CAAC,CAAC;IACX,CAAC;SAAM,IAAI,CAAC,CAAC,cAAc,EAAE,CAAC;QAC5B,OAAO;YACL,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,KAAK,EAAE,CAAC,CAAC,YAAY;YACrB,QAAQ,EAAE,CAAC,CAAC,eAAe;SAC5B,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,OAAO;YACL,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,KAAK,EAAE,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,YAAY;YAC/B,QAAQ,EAAE,YAAY,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,eAAe,CAAC;SACtD,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,qBAAqB,CAC5B,KAAoC,EACpC,OAAmC;IAEnC,IAAI,MAAM,GAAwB,EAAE,CAAC;IACrC,KAAK,IAAI,MAAM,IAAI,OAAO,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;QACzB,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC;IAC7B,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,0BAA0B,CACxC,KAAoC,EACpC,OAAmC;IAEnC,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QACxB,gDAAgD;QAChD,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;IAClC,CAAC;IACD,MAAM,WAAW,GAAG,qBAAqB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAE1D,OAAO,cAAc,CAAC,WAAW,CAAC,CAAC;AACrC,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,GAAyB;IACtD,+EAA+E;IAC/E,4EAA4E;IAC5E,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IACjC,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAChC,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC;AAC5D,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,GAAkC;IACjE,KAAK,IAAI,GAAG,IAAI,GAAG,EAAE,CAAC;QACpB,IAAI,OAAO,GAAG,CAAC,GAAG,CAAC,IAAI,WAAW,EAAE,CAAC;YACnC,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,GAAkC;IAC9D,OAAO,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,OAA2B,EAAE,SAA6B;IACzF,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;QACpB,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;IACzC,CAAC;IACD,IAAI,SAAS,IAAI,IAAI,EAAE,CAAC;QACtB,OAAO,OAAO,CAAC;IACjB,CAAC;IACD,OAAO,GAAG,OAAO,IAAI,SAAS,EAAE,CAAC;AACnC,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,YAAY,CAAC,UAAwB;IACnD,IAAI,QAAQ,GAAG,IAAI,GAAG,EAAsB,CAAC;IAC7C,IAAI,aAAa,GAAG,CAAC,CAAC;IAEtB,KAAK,IAAI,EAAE,IAAI,UAAU,EAAE,CAAC;QAC1B,MAAM,GAAG,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC;QACvB,IAAI,EAAE,CAAC,EAAE,IAAI,KAAK,EAAE,CAAC;YACnB,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACnC,IAAI,QAAQ,EAAE,CAAC;gBACb,aAAa,GAAG,YAAY,CAAC,aAAa,EAAE,QAAQ,CAAC,QAAkB,CAAC,CAAC;YAC3E,CAAC;YACD,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACxB,CAAC;aAAM,IAAI,EAAE,CAAC,EAAE,IAAI,QAAQ,EAAE,CAAC;YAC7B,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACnC,IAAI,QAAQ,EAAE,CAAC;gBACb,aAAa,GAAG,YAAY,CAAC,aAAa,EAAE,QAAQ,CAAC,QAAkB,CAAC,CAAC;YAC3E,CAAC;YACD,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACrB,aAAa,GAAG,YAAY,CAAC,aAAa,EAAE,EAAE,CAAC,QAAkB,CAAC,CAAC;QACrE,CAAC;aAAM,IAAI,EAAE,CAAC,EAAE,IAAI,OAAO,EAAE,CAAC;YAC5B,QAAQ,CAAC,KAAK,EAAE,CAAC;YACjB,aAAa,GAAG,EAAE,CAAC,QAAkB,CAAC;QACxC,CAAC;aAAM,IAAI,EAAE,CAAC,EAAE,IAAI,MAAM,EAAE,CAAC;YAC3B,aAAa,GAAG,YAAY,CAAC,aAAa,EAAE,EAAE,CAAC,QAAkB,CAAC,CAAC;QACrE,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,KAAK,CAAC,qBAAqB,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;IAED,MAAM,IAAI,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QAChD,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;IAEH,IAAI,UAAU,GAAiB;QAC7B,wDAAwD;QACxD,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE;QACpD,GAAG,IAAI;KACR,CAAC;IAEF,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,SAAS,MAAM,CAAC,KAAiB;IAC/B,OAAO,GAAG,KAAK,CAAC,WAAW,IAAI,KAAK,CAAC,SAAS,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;AACnE,CAAC"}

package/package.json

@@ -5,7 +5,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "0.10.1",
+  "version": "0.12.0",
   "main": "dist/index.js",
   "license": "FSL-1.1-Apache-2.0",
   "type": "module",
@@ -36,8 +36,8 @@
     "@powersync/lib-services-framework": "0.2.0",
     "@powersync/service-jsonbig": "0.17.10",
     "@powersync/service-rsocket-router": "0.0.14",
-    "@powersync/service-sync-rules": "0.21.0",
-    "@powersync/service-types": "0.4.0"
+    "@powersync/service-sync-rules": "0.22.0",
+    "@powersync/service-types": "0.5.0"
   },
   "devDependencies": {
     "@types/async": "^3.2.24",

package/src/auth/KeySpec.ts

@@ -2,7 +2,8 @@ import * as jose from 'jose';
 
 export const HS_ALGORITHMS = ['HS256', 'HS384', 'HS512'];
 export const RSA_ALGORITHMS = ['RS256', 'RS384', 'RS512'];
-export const SUPPORTED_ALGORITHMS = [...HS_ALGORITHMS, ...RSA_ALGORITHMS];
+export const OKP_ALGORITHMS = ['EdDSA'];
+export const SUPPORTED_ALGORITHMS = [...HS_ALGORITHMS, ...RSA_ALGORITHMS, ...OKP_ALGORITHMS];
 
 export interface KeyOptions {
   /**
@@ -38,17 +39,19 @@ export class KeySpec {
     return this.source.kid;
   }
 
-  matchesAlgorithm(jwtAlg: string) {
+  matchesAlgorithm(jwtAlg: string): boolean {
     if (this.source.alg) {
-      return jwtAlg == this.source.alg;
-    } else if (this.source.kty == 'RSA') {
+      return jwtAlg === this.source.alg;
+    } else if (this.source.kty === 'RSA') {
       return RSA_ALGORITHMS.includes(jwtAlg);
-    } else if (this.source.kty == 'oct') {
+    } else if (this.source.kty === 'oct') {
       return HS_ALGORITHMS.includes(jwtAlg);
-    } else {
-      // We don't support 'ec' yet
-      return false;
+    } else if (this.source.kty === 'OKP') {
+      return OKP_ALGORITHMS.includes(jwtAlg);
     }
+    
+    // 'EC' is unsupported
+    return false;
   }
 
   async isValidSignature(token: string): Promise<boolean> {
@@ -56,7 +59,7 @@ export class KeySpec {
       await jose.compactVerify(token, this.key);
       return true;
     } catch (e) {
-      if (e.code == 'ERR_JWS_SIGNATURE_VERIFICATION_FAILED') {
+      if (e.code === 'ERR_JWS_SIGNATURE_VERIFICATION_FAILED') {
         return false;
       } else {
         // Token format error most likely

package/src/auth/RemoteJWKSCollector.ts

@@ -73,8 +73,8 @@ export class RemoteJWKSCollector implements KeyCollector {
 
     let keys: KeySpec[] = [];
     for (let keyData of data.keys) {
-      if (keyData.kty != 'RSA') {
-        // Only RSA public keys supported here
+      if (keyData.kty != 'RSA' && keyData.kty != 'OKP') {
+        // HS (oct) keys not allowed because they are symmetric
         continue;
       }
 

package/src/storage/BucketStorage.ts

@@ -211,6 +211,13 @@ export interface StartBatchOptions extends ParseSyncRulesOptions {
    * database, for example from MongoDB.
    */
   storeCurrentData: boolean;
+
+  /**
+   * Set to true for initial replication.
+   *
+   * This will avoid creating new operations for rows previously replicated.
+   */
+  skipExistingRows?: boolean;
 }
 
 export interface SyncRulesBucketStorageListener extends DisposableListener {

package/src/storage/MongoBucketStorage.ts

@@ -199,6 +199,7 @@ export class MongoBucketStorage
         last_checkpoint: null,
         last_checkpoint_lsn: null,
         no_checkpoint_before: null,
+        keepalive_op: null,
         snapshot_done: false,
         state: SyncRuleState.PROCESSING,
         slot_name: slot_name,

package/src/storage/mongo/MongoBucketBatch.ts

@@ -9,6 +9,7 @@ import {
   BucketStorageBatch,
   FlushedResult,
   mergeToast,
+  SaveOperationTag,
   SaveOptions
 } from '../BucketStorage.js';
 import { SourceTable } from '../SourceTable.js';
@@ -20,6 +21,7 @@ import { batchCreateCustomWriteCheckpoints } from './MongoWriteCheckpointAPI.js'
 import { cacheKey, OperationBatch, RecordOperation } from './OperationBatch.js';
 import { PersistedBatch } from './PersistedBatch.js';
 import { BSON_DESERIALIZE_OPTIONS, idPrefixFilter, replicaIdEquals, serializeLookup } from './util.js';
+import * as timers from 'node:timers/promises';
 
 /**
  * 15MB
@@ -39,8 +41,13 @@ export interface MongoBucketBatchOptions {
   groupId: number;
   slotName: string;
   lastCheckpointLsn: string | null;
+  keepaliveOp: string | null;
   noCheckpointBeforeLsn: string;
   storeCurrentData: boolean;
+  /**
+   * Set to true for initial replication.
+   */
+  skipExistingRows: boolean;
 }
 
 export class MongoBucketBatch extends DisposableObserver<BucketBatchStorageListener> implements BucketStorageBatch {
@@ -53,6 +60,7 @@ export class MongoBucketBatch extends DisposableObserver<BucketBatchStorageListe
 
   private readonly slot_name: string;
   private readonly storeCurrentData: boolean;
+  private readonly skipExistingRows: boolean;
 
   private batch: OperationBatch | null = null;
   private write_checkpoint_batch: CustomWriteCheckpointOptions[] = [];
@@ -86,7 +94,12 @@ export class MongoBucketBatch extends DisposableObserver<BucketBatchStorageListe
     this.slot_name = options.slotName;
     this.sync_rules = options.syncRules;
     this.storeCurrentData = options.storeCurrentData;
+    this.skipExistingRows = options.skipExistingRows;
     this.batch = new OperationBatch();
+
+    if (options.keepaliveOp) {
+      this.persisted_op = BigInt(options.keepaliveOp);
+    }
   }
 
   addCustomWriteCheckpoint(checkpoint: BatchedCustomWriteCheckpointOptions): void {
@@ -148,10 +161,13 @@ export class MongoBucketBatch extends DisposableObserver<BucketBatchStorageListe
     op_seq: MongoIdSequence
   ): Promise<OperationBatch | null> {
     let sizes: Map<string, number> | undefined = undefined;
-    if (this.storeCurrentData) {
+    if (this.storeCurrentData && !this.skipExistingRows) {
       // We skip this step if we don't store current_data, since the sizes will
       // always be small in that case.
 
+      // With skipExistingRows, we don't load the full documents into memory,
+      // so we can also skip the size lookup step.
+
       // Find sizes of current_data documents, to assist in intelligent batching without
       // exceeding memory limits.
       //
@@ -204,11 +220,13 @@ export class MongoBucketBatch extends DisposableObserver<BucketBatchStorageListe
         return { g: this.group_id, t: r.record.sourceTable.id, k: r.beforeId };
       });
       let current_data_lookup = new Map<string, CurrentDataDocument>();
+      // With skipExistingRows, we only need to know whether or not the row exists.
+      const projection = this.skipExistingRows ? { _id: 1 } : undefined;
       const cursor = this.db.current_data.find(
         {
           _id: { $in: lookups }
         },
-        { session }
+        { session, projection }
       );
       for await (let doc of cursor.stream()) {
         current_data_lookup.set(cacheKey(doc._id.t, doc._id.k), doc);
@@ -273,7 +291,21 @@ export class MongoBucketBatch extends DisposableObserver<BucketBatchStorageListe
 
     const before_key: SourceKey = { g: this.group_id, t: record.sourceTable.id, k: beforeId };
 
-    if (record.tag == 'update') {
+    if (this.skipExistingRows) {
+      if (record.tag == SaveOperationTag.INSERT) {
+        if (current_data != null) {
+          // Initial replication, and we already have the record.
+          // This may be a different version of the record, but streaming replication
+          // will take care of that.
+          // Skip the insert here.
+          return null;
+        }
+      } else {
+        throw new Error(`${record.tag} not supported with skipExistingRows: true`);
+      }
+    }
+
+    if (record.tag == SaveOperationTag.UPDATE) {
       const result = current_data;
       if (result == null) {
         // Not an error if we re-apply a transaction
@@ -293,7 +325,7 @@ export class MongoBucketBatch extends DisposableObserver<BucketBatchStorageListe
           after = mergeToast(after!, data);
         }
       }
-    } else if (record.tag == 'delete') {
+    } else if (record.tag == SaveOperationTag.DELETE) {
       const result = current_data;
       if (result == null) {
         // Not an error if we re-apply a transaction
@@ -494,7 +526,7 @@ export class MongoBucketBatch extends DisposableObserver<BucketBatchStorageListe
             } else {
               logger.warn('Transaction error', e as Error);
             }
-            await new Promise((resolve) => setTimeout(resolve, Math.random() * 50));
+            await timers.setTimeout(Math.random() * 50);
             throw e;
           }
         },
@@ -587,7 +619,28 @@ export class MongoBucketBatch extends DisposableObserver<BucketBatchStorageListe
       return false;
     }
     if (lsn < this.no_checkpoint_before_lsn) {
-      logger.info(`Waiting until ${this.no_checkpoint_before_lsn} before creating checkpoint, currently at ${lsn}`);
+      logger.info(
+        `Waiting until ${this.no_checkpoint_before_lsn} before creating checkpoint, currently at ${lsn}. Persisted op: ${this.persisted_op}`
+      );
+
+      // Edge case: During initial replication, we have a no_checkpoint_before_lsn set,
+      // and don't actually commit the snapshot.
+      // The first commit can happen from an implicit keepalive message.
+      // That needs the persisted_op to get an accurate checkpoint, so
+      // we persist that in keepalive_op.
+
+      await this.db.sync_rules.updateOne(
+        {
+          _id: this.group_id
+        },
+        {
+          $set: {
+            keepalive_op: this.persisted_op == null ? null : String(this.persisted_op)
+          }
+        },
+        { session: this.session }
+      );
+
       return false;
     }
 
@@ -597,7 +650,8 @@ export class MongoBucketBatch extends DisposableObserver<BucketBatchStorageListe
       last_checkpoint_ts: now,
       last_keepalive_ts: now,
       snapshot_done: true,
-      last_fatal_error: null
+      last_fatal_error: null,
+      keepalive_op: null
     };
 
     if (this.persisted_op != null) {
@@ -631,6 +685,7 @@ export class MongoBucketBatch extends DisposableObserver<BucketBatchStorageListe
     if (this.persisted_op != null) {
       // The commit may have been skipped due to "no_checkpoint_before_lsn".
       // Apply it now if relevant
+      logger.info(`Commit due to keepalive at ${lsn} / ${this.persisted_op}`);
       return await this.commit(lsn);
     }
 
@@ -684,9 +739,8 @@ export class MongoBucketBatch extends DisposableObserver<BucketBatchStorageListe
 
     if (this.batch.shouldFlush()) {
       const r = await this.flush();
-      // HACK: Give other streams a chance to also flush
-      const t = 150;
-      await new Promise((resolve) => setTimeout(resolve, t));
+      // HACK: Give other streams a  chance to also flush
+      await timers.setTimeout(5);
       return r;
     }
     return null;

package/src/storage/mongo/MongoSyncBucketStorage.ts

@@ -137,7 +137,7 @@ export class MongoSyncBucketStorage
       {
         _id: this.group_id
       },
-      { projection: { last_checkpoint_lsn: 1, no_checkpoint_before: 1 } }
+      { projection: { last_checkpoint_lsn: 1, no_checkpoint_before: 1, keepalive_op: 1 } }
     );
     const checkpoint_lsn = doc?.last_checkpoint_lsn ?? null;
 
@@ -148,7 +148,9 @@ export class MongoSyncBucketStorage
       slotName: this.slot_name,
       lastCheckpointLsn: checkpoint_lsn,
       noCheckpointBeforeLsn: doc?.no_checkpoint_before ?? options.zeroLSN,
-      storeCurrentData: options.storeCurrentData
+      keepaliveOp: doc?.keepalive_op ?? null,
+      storeCurrentData: options.storeCurrentData,
+      skipExistingRows: options.skipExistingRows ?? false
     });
     this.iterateListeners((cb) => cb.batchStarted?.(batch));
 

package/src/storage/mongo/models.ts

@@ -135,6 +135,14 @@ export interface SyncRuleDocument {
    */
   no_checkpoint_before: string | null;
 
+  /**
+   * Goes together with no_checkpoint_before.
+   *
+   * If a keepalive is triggered that creates the checkpoint > no_checkpoint_before,
+   * then the checkpoint must be equal to this keepalive_op.
+   */
+  keepalive_op: string | null;
+
   slot_name: string | null;
 
   /**

package/src/util/utils.ts

@@ -2,7 +2,7 @@ import * as sync_rules from '@powersync/service-sync-rules';
 import * as bson from 'bson';
 import crypto from 'crypto';
 import * as uuid from 'uuid';
-import { BucketChecksum, OpId } from './protocol-types.js';
+import { BucketChecksum, OpId, OplogEntry } from './protocol-types.js';
 
 import * as storage from '../storage/storage-index.js';
 
@@ -144,3 +144,61 @@ export function checkpointUserId(user_id: string | undefined, client_id: string
   }
   return `${user_id}/${client_id}`;
 }
+
+/**
+ * Reduce a bucket to the final state as stored on the client.
+ *
+ * This keeps the final state for each row as a PUT operation.
+ *
+ * All other operations are replaced with a single CLEAR operation,
+ * summing their checksums, and using a 0 as an op_id.
+ *
+ * This is the function $r(B)$, as described in /docs/bucket-properties.md.
+ *
+ * Used for tests.
+ */
+export function reduceBucket(operations: OplogEntry[]) {
+  let rowState = new Map<string, OplogEntry>();
+  let otherChecksum = 0;
+
+  for (let op of operations) {
+    const key = rowKey(op);
+    if (op.op == 'PUT') {
+      const existing = rowState.get(key);
+      if (existing) {
+        otherChecksum = addChecksums(otherChecksum, existing.checksum as number);
+      }
+      rowState.set(key, op);
+    } else if (op.op == 'REMOVE') {
+      const existing = rowState.get(key);
+      if (existing) {
+        otherChecksum = addChecksums(otherChecksum, existing.checksum as number);
+      }
+      rowState.delete(key);
+      otherChecksum = addChecksums(otherChecksum, op.checksum as number);
+    } else if (op.op == 'CLEAR') {
+      rowState.clear();
+      otherChecksum = op.checksum as number;
+    } else if (op.op == 'MOVE') {
+      otherChecksum = addChecksums(otherChecksum, op.checksum as number);
+    } else {
+      throw new Error(`Unknown operation ${op.op}`);
+    }
+  }
+
+  const puts = [...rowState.values()].sort((a, b) => {
+    return Number(BigInt(a.op_id) - BigInt(b.op_id));
+  });
+
+  let finalState: OplogEntry[] = [
+    // Special operation to indiciate the checksum remainder
+    { op_id: '0', op: 'CLEAR', checksum: otherChecksum },
+    ...puts
+  ];
+
+  return finalState;
+}
+
+function rowKey(entry: OplogEntry) {
+  return `${entry.object_type}/${entry.object_id}/${entry.subkey}`;
+}

package/test/src/auth.test.ts

@@ -1,13 +1,14 @@
-import { CachedKeyCollector } from '@/auth/CachedKeyCollector.js';
-import { KeyResult } from '@/auth/KeyCollector.js';
-import { KeySpec } from '@/auth/KeySpec.js';
-import { KeyStore } from '@/auth/KeyStore.js';
-import { RemoteJWKSCollector } from '@/auth/RemoteJWKSCollector.js';
-import { StaticKeyCollector } from '@/auth/StaticKeyCollector.js';
-import * as jose from 'jose';
 import { describe, expect, test } from 'vitest';
-
-const publicKey: jose.JWK = {
+import { StaticKeyCollector } from '../../src/auth/StaticKeyCollector.js';
+import * as jose from 'jose';
+import { KeyStore } from '../../src/auth/KeyStore.js';
+import { KeySpec } from '../../src/auth/KeySpec.js';
+import { RemoteJWKSCollector } from '../../src/auth/RemoteJWKSCollector.js';
+import { KeyResult } from '../../src/auth/KeyCollector.js';
+import { CachedKeyCollector } from '../../src/auth/CachedKeyCollector.js';
+import { JwtPayload } from '@/index.js';
+
+const publicKeyRSA: jose.JWK = {
   use: 'sig',
   kty: 'RSA',
   e: 'AQAB',
@@ -29,6 +30,16 @@ const sharedKey2: jose.JWK = {
   k: Buffer.from('mysecret2', 'utf-8').toString('base64url')
 };
 
+const privateKeyEdDSA: jose.JWK = {
+  use: 'sig',
+  kty: 'OKP',
+  crv: 'Ed25519',
+  kid: 'k2',
+  x: 'nfaqgxakPaiiEdAtRGrubgh_SQ1mr6gAUx3--N-ehvo',
+  d: 'wweBqMbTrME6oChSEMYAOyYzxsGisQb-C1t0XMjb_Ng',
+  alg: 'EdDSA'
+};
+
 describe('JWT Auth', () => {
   test('KeyStore basics', async () => {
     const keys = await StaticKeyCollector.importKeys([sharedKey]);
@@ -86,20 +97,20 @@ describe('JWT Auth', () => {
   });
 
   test('Algorithm validation', async () => {
-    const keys = await StaticKeyCollector.importKeys([publicKey]);
+    const keys = await StaticKeyCollector.importKeys([publicKeyRSA]);
     const store = new KeyStore(keys);
 
     // Bad attempt at signing token with rsa public key
     const spoofedKey: jose.JWK = {
       kty: 'oct',
-      kid: publicKey.kid!,
+      kid: publicKeyRSA.kid!,
       alg: 'HS256',
-      k: publicKey.n!
+      k: publicKeyRSA.n!
     };
     const signKey = (await jose.importJWK(spoofedKey)) as jose.KeyLike;
 
     const signedJwt = await new jose.SignJWT({})
-      .setProtectedHeader({ alg: 'HS256', kid: publicKey.kid! })
+      .setProtectedHeader({ alg: 'HS256', kid: publicKeyRSA.kid! })
       .setSubject('f1')
       .setIssuedAt()
       .setIssuer('tester')
@@ -116,7 +127,7 @@ describe('JWT Auth', () => {
   });
 
   test('key selection for key with kid', async () => {
-    const keys = await StaticKeyCollector.importKeys([publicKey, sharedKey, sharedKey2]);
+    const keys = await StaticKeyCollector.importKeys([publicKeyRSA, sharedKey, sharedKey2]);
     const store = new KeyStore(keys);
     const signKey = (await jose.importJWK(sharedKey)) as jose.KeyLike;
     const signKey2 = (await jose.importJWK(sharedKey2)) as jose.KeyLike;
@@ -296,30 +307,30 @@ describe('JWT Auth', () => {
 
     currentResponse = Promise.resolve({
       errors: [],
-      keys: [await KeySpec.importKey(publicKey)]
+      keys: [await KeySpec.importKey(publicKeyRSA)]
     });
 
     let key = (await cached.getKeys()).keys[0];
-    expect(key.kid).toEqual(publicKey.kid!);
+    expect(key.kid).toEqual(publicKeyRSA.kid!);
 
     currentResponse = undefined as any;
 
     key = (await cached.getKeys()).keys[0];
-    expect(key.kid).toEqual(publicKey.kid!);
+    expect(key.kid).toEqual(publicKeyRSA.kid!);
 
     cached.addTimeForTests(301_000);
     currentResponse = Promise.reject('refresh failed');
 
     // Uses the promise, refreshes in the background
     let response = await cached.getKeys();
-    expect(response.keys[0].kid).toEqual(publicKey.kid!);
+    expect(response.keys[0].kid).toEqual(publicKeyRSA.kid!);
     expect(response.errors).toEqual([]);
 
     // Wait for refresh to finish
     await cached.addTimeForTests(0);
     response = await cached.getKeys();
     // Still have the cached key, but also have the error
-    expect(response.keys[0].kid).toEqual(publicKey.kid!);
+    expect(response.keys[0].kid).toEqual(publicKeyRSA.kid!);
     expect(response.errors[0].message).toMatch('Failed to fetch');
 
     await cached.addTimeForTests(3601_000);
@@ -331,12 +342,34 @@ describe('JWT Auth', () => {
 
     currentResponse = Promise.resolve({
       errors: [],
-      keys: [await KeySpec.importKey(publicKey)]
+      keys: [await KeySpec.importKey(publicKeyRSA)]
     });
 
     // After a delay, we can refresh again
     await cached.addTimeForTests(30_000);
     key = (await cached.getKeys()).keys[0];
-    expect(key.kid).toEqual(publicKey.kid!);
+    expect(key.kid).toEqual(publicKeyRSA.kid!);
+  });
+
+  test('signing with EdDSA', async () => {
+    const keys = await StaticKeyCollector.importKeys([privateKeyEdDSA]);
+    const store = new KeyStore(keys);
+    const signKey = (await jose.importJWK(privateKeyEdDSA)) as jose.KeyLike;
+
+    const signedJwt = await new jose.SignJWT({ claim: 'test-claim' })
+      .setProtectedHeader({ alg: 'EdDSA', kid: 'k2' })
+      .setSubject('f1')
+      .setIssuedAt()
+      .setIssuer('tester')
+      .setAudience('tests')
+      .setExpirationTime('5m')
+      .sign(signKey);
+
+    const verified = (await store.verifyJwt(signedJwt, {
+      defaultAudiences: ['tests'],
+      maxAge: '6m'
+    })) as JwtPayload & { claim: string };
+
+    expect(verified.claim).toEqual('test-claim');
   });
 });

package/test/src/bucket_validation.test.ts

@@ -1,6 +1,7 @@
 import { OplogEntry } from '@/util/protocol-types.js';
 import { describe, expect, test } from 'vitest';
-import { reduceBucket, validateBucket } from './bucket_validation.js';
+import { validateBucket } from './bucket_validation.js';
+import { reduceBucket } from '@/index.js';
 
 // This tests the reduceBucket function.
 // While this function is not used directly in the service implementation,