@powersync/service-core 0.8.8 → 0.10.0

package/dist/util/utils.js

@@ -1,7 +1,10 @@
+import * as bson from 'bson';
 import crypto from 'crypto';
-import { logger } from '@powersync/lib-services-framework';
-import { pgwireRows } from '@powersync/service-jpgwire';
-import { retriedQuery } from './pgwire_utils.js';
+import * as uuid from 'uuid';
+export const ID_NAMESPACE = 'a396dd91-09fc-4017-a28d-3df722f651e9';
+export function escapeIdentifier(identifier) {
+    return `"${identifier.replace(/"/g, '""').replace(/\./g, '"."')}"`;
+}
 export function hashData(type, id, data) {
     const hash = crypto.createHash('sha256');
     hash.update(`put.${type}.${id}.${data}`);
@@ -70,31 +73,39 @@ export function addBucketChecksums(a, b) {
         };
     }
 }
-export async function getClientCheckpoint(db, bucketStorage, options) {
-    const start = Date.now();
-    const [{ lsn }] = pgwireRows(await db.query(`SELECT pg_logical_emit_message(false, 'powersync', 'ping') as lsn`));
-    // This old API needs a persisted checkpoint id.
-    // Since we don't use LSNs anymore, the only way to get that is to wait.
-    const timeout = options?.timeout ?? 50000;
-    logger.info(`Waiting for LSN checkpoint: ${lsn}`);
-    while (Date.now() - start < timeout) {
-        const cp = await bucketStorage.getActiveCheckpoint();
-        if (!cp.hasSyncRules()) {
-            throw new Error('No sync rules available');
-        }
-        if (cp.lsn >= lsn) {
-            logger.info(`Got write checkpoint: ${lsn} : ${cp.checkpoint}`);
-            return cp.checkpoint;
+function getRawReplicaIdentity(tuple, columns) {
+    let result = {};
+    for (let column of columns) {
+        const name = column.name;
+        result[name] = tuple[name];
+    }
+    return result;
+}
+export function getUuidReplicaIdentityBson(tuple, columns) {
+    if (columns.length == 0) {
+        // REPLICA IDENTITY NOTHING - generate random id
+        return new bson.UUID(uuid.v4());
+    }
+    const rawIdentity = getRawReplicaIdentity(tuple, columns);
+    return uuidForRowBson(rawIdentity);
+}
+export function uuidForRowBson(row) {
+    // Important: This must not change, since it will affect how ids are generated.
+    // Use BSON so that it's a well-defined format without encoding ambiguities.
+    const repr = bson.serialize(row);
+    const buffer = Buffer.alloc(16);
+    return new bson.UUID(uuid.v5(repr, ID_NAMESPACE, buffer));
+}
+export function hasToastedValues(row) {
+    for (let key in row) {
+        if (typeof row[key] == 'undefined') {
+            return true;
         }
-        await new Promise((resolve) => setTimeout(resolve, 30));
     }
-    throw new Error('Timeout while waiting for checkpoint');
+    return false;
 }
-export async function createWriteCheckpoint(db, bucketStorage, user_id) {
-    const [{ lsn }] = pgwireRows(await retriedQuery(db, `SELECT pg_logical_emit_message(false, 'powersync', 'ping') as lsn`));
-    const id = await bucketStorage.createWriteCheckpoint(user_id, { '1': lsn });
-    logger.info(`Write checkpoint 2: ${JSON.stringify({ lsn, id: String(id) })}`);
-    return id;
+export function isCompleteRow(row) {
+    return !hasToastedValues(row);
 }
 export function checkpointUserId(user_id, client_id) {
     if (user_id == null) {

package/dist/util/utils.js.map

@@ -1 +1 @@
-{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/util/utils.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAE3D,OAAO,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AAGxD,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAKjD,MAAM,UAAU,QAAQ,CAAC,IAAY,EAAE,EAAU,EAAE,IAAY;IAC7D,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IACzC,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC;IACzC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;IAC7B,OAAO,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,SAAiB;IAC1C,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IACzC,IAAI,CAAC,MAAM,CAAC,UAAU,SAAS,EAAE,CAAC,CAAC;IACnC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;IAC7B,OAAO,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,EAAU;IACxC,6EAA6E;IAC7E,6CAA6C;IAC7C,IAAI,OAAO,EAAE,IAAI,QAAQ,EAAE;QACzB,MAAM,IAAI,KAAK,CAAC,yBAAyB,EAAE,KAAK,OAAO,EAAE,GAAG,CAAC,CAAC;KAC/D;IACD,OAAO,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;AACzB,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,QAAqB,EAAE,OAAoB;IACvE,mBAAmB;IACnB,MAAM,cAAc,GAAG,IAAI,GAAG,EAA0B,CAAC;IAEzD,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAS,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;IAElD,KAAK,IAAI,QAAQ,IAAI,OAAO,CAAC,MAAM,EAAE,EAAE;QACrC,MAAM,CAAC,GAAG,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACxC,IAAI,CAAC,IAAI,IAAI,EAAE;YACb,QAAQ;YACR,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;SAC/C;aAAM;YACL,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YACjC,IAAI,QAAQ,CAAC,QAAQ,IAAI,CAAC,CAAC,QAAQ,IAAI,QAAQ,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,EAAE;gBAChE,UAAU;gBACV,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;aAC/C;iBAAM;gBACL,YAAY;aACb;SACF;KACF;IAED,OAAO;QACL,cAAc,EAAE,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC;QAC5C,cAAc,EAAE,CAAC,GAAG,QAAQ,CAAC;KAC9B,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,CAAS,EAAE,CAAS;IAC/C,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,UAAU,CAAC;AAC9B,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,CAAiB,EAAE,CAAyB;IAC7E,IAAI,CAAC,IAAI,IAAI,EAAE;QACb,OAAO,CAAC,CAAC;KACV;SAAM,IAAI,CAAC,CAAC,cAAc,EAAE;QAC3B,OAAO;YACL,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,KAAK,EAAE,CAAC,CAAC,YAAY;YACrB,QAAQ,EAAE,CAAC,CAAC,eAAe;SAC5B,CAAC;KACH;SAAM;QACL,OAAO;YACL,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,KAAK,EAAE,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,YAAY;YAC/B,QAAQ,EAAE,YAAY,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,eAAe,CAAC;SACtD,CAAC;KACH;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,EAAmB,EACnB,aAA2C,EAC3C,OAA8B;IAE9B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAEzB,MAAM,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,mEAAmE,CAAC,CAAC,CAAC;IAElH,gDAAgD;IAChD,wEAAwE;IAExE,MAAM,OAAO,GAAG,OAAO,EAAE,OAAO,IAAI,KAAM,CAAC;IAE3C,MAAM,CAAC,IAAI,CAAC,+BAA+B,GAAG,EAAE,CAAC,CAAC;IAClD,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,GAAG,OAAO,EAAE;QACnC,MAAM,EAAE,GAAG,MAAM,aAAa,CAAC,mBAAmB,EAAE,CAAC;QACrD,IAAI,CAAC,EAAE,CAAC,YAAY,EAAE,EAAE;YACtB,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;SAC5C;QACD,IAAI,EAAE,CAAC,GAAG,IAAI,GAAG,EAAE;YACjB,MAAM,CAAC,IAAI,CAAC,yBAAyB,GAAG,MAAM,EAAE,CAAC,UAAU,EAAE,CAAC,CAAC;YAC/D,OAAO,EAAE,CAAC,UAAU,CAAC;SACtB;QAED,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;KACzD;IAED,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;AAC1D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACzC,EAAmB,EACnB,aAA2C,EAC3C,OAAe;IAEf,MAAM,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,UAAU,CAC1B,MAAM,YAAY,CAAC,EAAE,EAAE,mEAAmE,CAAC,CAC5F,CAAC;IAEF,MAAM,EAAE,GAAG,MAAM,aAAa,CAAC,qBAAqB,CAAC,OAAO,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;IAC5E,MAAM,CAAC,IAAI,CAAC,uBAAuB,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;IAC9E,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,OAA2B,EAAE,SAA6B;IACzF,IAAI,OAAO,IAAI,IAAI,EAAE;QACnB,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;KACxC;IACD,IAAI,SAAS,IAAI,IAAI,EAAE;QACrB,OAAO,OAAO,CAAC;KAChB;IACD,OAAO,GAAG,OAAO,IAAI,SAAS,EAAE,CAAC;AACnC,CAAC"}
+{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/util/utils.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAS7B,MAAM,CAAC,MAAM,YAAY,GAAG,sCAAsC,CAAC;AAEnE,MAAM,UAAU,gBAAgB,CAAC,UAAkB;IACjD,OAAO,IAAI,UAAU,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC;AACrE,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,IAAY,EAAE,EAAU,EAAE,IAAY;IAC7D,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IACzC,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC;IACzC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;IAC7B,OAAO,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,SAAiB;IAC1C,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IACzC,IAAI,CAAC,MAAM,CAAC,UAAU,SAAS,EAAE,CAAC,CAAC;IACnC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;IAC7B,OAAO,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,EAAU;IACxC,6EAA6E;IAC7E,6CAA6C;IAC7C,IAAI,OAAO,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,yBAAyB,EAAE,KAAK,OAAO,EAAE,GAAG,CAAC,CAAC;IAChE,CAAC;IACD,OAAO,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;AACzB,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,QAAqB,EAAE,OAAoB;IACvE,mBAAmB;IACnB,MAAM,cAAc,GAAG,IAAI,GAAG,EAA0B,CAAC;IAEzD,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAS,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;IAElD,KAAK,IAAI,QAAQ,IAAI,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;QACtC,MAAM,CAAC,GAAG,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACxC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;YACd,QAAQ;YACR,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAChD,CAAC;aAAM,CAAC;YACN,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YACjC,IAAI,QAAQ,CAAC,QAAQ,IAAI,CAAC,CAAC,QAAQ,IAAI,QAAQ,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;gBACjE,UAAU;gBACV,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YAChD,CAAC;iBAAM,CAAC;gBACN,YAAY;YACd,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO;QACL,cAAc,EAAE,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC;QAC5C,cAAc,EAAE,CAAC,GAAG,QAAQ,CAAC;KAC9B,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,CAAS,EAAE,CAAS;IAC/C,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,UAAU,CAAC;AAC9B,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,CAAiB,EAAE,CAAyB;IAC7E,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QACd,OAAO,CAAC,CAAC;IACX,CAAC;SAAM,IAAI,CAAC,CAAC,cAAc,EAAE,CAAC;QAC5B,OAAO;YACL,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,KAAK,EAAE,CAAC,CAAC,YAAY;YACrB,QAAQ,EAAE,CAAC,CAAC,eAAe;SAC5B,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,OAAO;YACL,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,KAAK,EAAE,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,YAAY;YAC/B,QAAQ,EAAE,YAAY,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,eAAe,CAAC;SACtD,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,qBAAqB,CAC5B,KAAoC,EACpC,OAAmC;IAEnC,IAAI,MAAM,GAAwB,EAAE,CAAC;IACrC,KAAK,IAAI,MAAM,IAAI,OAAO,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;QACzB,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC;IAC7B,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,0BAA0B,CACxC,KAAoC,EACpC,OAAmC;IAEnC,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QACxB,gDAAgD;QAChD,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;IAClC,CAAC;IACD,MAAM,WAAW,GAAG,qBAAqB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAE1D,OAAO,cAAc,CAAC,WAAW,CAAC,CAAC;AACrC,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,GAAyB;IACtD,+EAA+E;IAC/E,4EAA4E;IAC5E,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IACjC,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAChC,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC;AAC5D,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,GAAkC;IACjE,KAAK,IAAI,GAAG,IAAI,GAAG,EAAE,CAAC;QACpB,IAAI,OAAO,GAAG,CAAC,GAAG,CAAC,IAAI,WAAW,EAAE,CAAC;YACnC,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,GAAkC;IAC9D,OAAO,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,OAA2B,EAAE,SAA6B;IACzF,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;QACpB,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;IACzC,CAAC;IACD,IAAI,SAAS,IAAI,IAAI,EAAE,CAAC;QACtB,OAAO,OAAO,CAAC;IACjB,CAAC;IACD,OAAO,GAAG,OAAO,IAAI,SAAS,EAAE,CAAC;AACnC,CAAC"}

package/package.json

@@ -5,7 +5,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "0.8.8",
+  "version": "0.10.0",
   "main": "dist/index.js",
   "license": "FSL-1.1-Apache-2.0",
   "type": "module",
@@ -28,32 +28,28 @@
     "lru-cache": "^10.2.2",
     "mongodb": "^6.7.0",
     "node-fetch": "^3.3.2",
-    "pgwire": "github:kagis/pgwire#f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87",
     "ts-codec": "^1.2.2",
+    "uri-js": "^4.4.1",
     "uuid": "^9.0.1",
     "winston": "^3.13.0",
     "yaml": "^2.3.2",
-    "@powersync/lib-services-framework": "0.1.1",
+    "@powersync/lib-services-framework": "0.2.0",
     "@powersync/service-jsonbig": "0.17.10",
-    "@powersync/service-jpgwire": "0.17.14",
-    "@powersync/service-rsocket-router": "0.0.13",
-    "@powersync/service-sync-rules": "0.20.0",
-    "@powersync/service-types": "0.2.0"
+    "@powersync/service-rsocket-router": "0.0.14",
+    "@powersync/service-sync-rules": "0.21.0",
+    "@powersync/service-types": "0.4.0"
   },
   "devDependencies": {
     "@types/async": "^3.2.24",
     "@types/lodash": "^4.17.5",
     "@types/uuid": "^9.0.4",
     "fastify": "4.23.2",
-    "fastify-plugin": "^4.5.1",
-    "typescript": "^5.2.2",
-    "vite-tsconfig-paths": "^4.3.2",
-    "vitest": "^0.34.6"
+    "fastify-plugin": "^4.5.1"
   },
   "scripts": {
     "build": "tsc -b",
     "build:tests": "tsc -b test/tsconfig.json",
-    "test": "vitest --no-threads",
+    "test": "vitest",
     "clean": "rm -rf ./dist && tsc -b --clean"
   }
 }

package/src/api/RouteAPI.ts

@@ -0,0 +1,78 @@
+import { SqlSyncRules, TablePattern } from '@powersync/service-sync-rules';
+import * as types from '@powersync/service-types';
+import { ParseSyncRulesOptions, SyncRulesBucketStorage } from '../storage/BucketStorage.js';
+
+export interface PatternResult {
+  schema: string;
+  pattern: string;
+  wildcard: boolean;
+  tables?: types.TableInfo[];
+  table?: types.TableInfo;
+}
+
+export interface ReplicationLagOptions {
+  bucketStorage: SyncRulesBucketStorage;
+}
+
+/**
+ *  Describes all the methods currently required to service the sync API endpoints.
+ */
+export interface RouteAPI {
+  /**
+   * @returns basic identification of the connection
+   */
+  getSourceConfig(): Promise<types.configFile.ResolvedDataSourceConfig>;
+
+  /**
+   * Checks the current connection status of the data source.
+   * This is usually some test query to verify the source can be reached.
+   */
+  getConnectionStatus(): Promise<types.ConnectionStatusV2>;
+
+  /**
+   * Generates replication table information from a given pattern of tables.
+   *
+   * @param tablePatterns A set of table patterns which typically come from
+   *          the tables listed in sync rules definitions.
+   *
+   * @param sqlSyncRules
+   * @returns A result of all the tables and columns which should be replicated
+   *           based off the input patterns. Certain tests are executed on the
+   *           tables to ensure syncing should function according to the input
+   *           pattern. Debug errors and warnings are reported per table.
+   */
+  getDebugTablesInfo(tablePatterns: TablePattern[], sqlSyncRules: SqlSyncRules): Promise<PatternResult[]>;
+
+  /**
+   * @returns The replication lag: that is the amount of data which has not been
+   *          replicated yet, in bytes.
+   */
+  getReplicationLag(options: ReplicationLagOptions): Promise<number | undefined>;
+
+  /**
+   * Get the current LSN or equivalent replication HEAD position identifier
+   */
+  getReplicationHead(): Promise<string>;
+
+  /**
+   * @returns The schema for tables inside the connected database. This is typically
+   *          used to validate sync rules.
+   */
+  getConnectionSchema(): Promise<types.DatabaseSchema[]>;
+
+  /**
+   * Executes a query and return the result from the data source. This is currently used in the
+   * admin API which is exposed in Collide.
+   */
+  executeQuery(query: string, params: any[]): Promise<types.internal_routes.ExecuteSqlResponse>;
+
+  /**
+   * Close any resources that need graceful termination.
+   */
+  shutdown(): Promise<void>;
+
+  /**
+   * Get the default schema (or database) when only a table name is specified in sync rules.
+   */
+  getParseSyncRulesOptions(): ParseSyncRulesOptions;
+}

package/src/api/api-index.ts

@@ -1,2 +1,3 @@
 export * from './diagnostics.js';
+export * from './RouteAPI.js';
 export * from './schema.js';

package/src/api/diagnostics.ts

@@ -1,51 +1,9 @@
+import { logger } from '@powersync/lib-services-framework';
 import { DEFAULT_TAG, SourceTableInterface, SqlSyncRules } from '@powersync/service-sync-rules';
-import { pgwireRows } from '@powersync/service-jpgwire';
-import { ConnectionStatus, SyncRulesStatus, TableInfo, baseUri } from '@powersync/service-types';
+import { SyncRulesStatus, TableInfo } from '@powersync/service-types';
 
-import * as replication from '../replication/replication-index.js';
 import * as storage from '../storage/storage-index.js';
-import * as util from '../util/util-index.js';
-
-import { CorePowerSyncSystem } from '../system/CorePowerSyncSystem.js';
-import { logger } from '@powersync/lib-services-framework';
-
-export async function getConnectionStatus(system: CorePowerSyncSystem): Promise<ConnectionStatus | null> {
-  if (system.pgwire_pool == null) {
-    return null;
-  }
-
-  const pool = system.requirePgPool();
-
-  const base = {
-    id: system.config.connection!.id,
-    postgres_uri: baseUri(system.config.connection!)
-  };
-  try {
-    await util.retriedQuery(pool, `SELECT 'PowerSync connection test'`);
-  } catch (e) {
-    return {
-      ...base,
-      connected: false,
-      errors: [{ level: 'fatal', message: e.message }]
-    };
-  }
-
-  try {
-    await replication.checkSourceConfiguration(pool);
-  } catch (e) {
-    return {
-      ...base,
-      connected: true,
-      errors: [{ level: 'fatal', message: e.message }]
-    };
-  }
-
-  return {
-    ...base,
-    connected: true,
-    errors: []
-  };
-}
+import { RouteAPI } from './RouteAPI.js';
 
 export interface DiagnosticsOptions {
   /**
@@ -66,9 +24,12 @@ export interface DiagnosticsOptions {
   check_connection: boolean;
 }
 
+export const DEFAULT_DATASOURCE_ID = 'default';
+
 export async function getSyncRulesStatus(
+  bucketStorage: storage.BucketStorageFactory,
+  apiHandler: RouteAPI,
   sync_rules: storage.PersistedSyncRulesContent | null,
-  system: CorePowerSyncSystem,
   options: DiagnosticsOptions
 ): Promise<SyncRulesStatus | undefined> {
   if (sync_rules == null) {
@@ -82,7 +43,7 @@ export async function getSyncRulesStatus(
   let rules: SqlSyncRules;
   let persisted: storage.PersistedSyncRules;
   try {
-    persisted = sync_rules.parsed();
+    persisted = sync_rules.parsed(apiHandler.getParseSyncRulesOptions());
     rules = persisted.sync_rules;
   } catch (e) {
     return {
@@ -92,21 +53,19 @@ export async function getSyncRulesStatus(
     };
   }
 
-  const systemStorage = live_status ? await system.storage.getInstance(persisted) : undefined;
+  const sourceConfig = await apiHandler.getSourceConfig();
+  // This method can run under some situations if no connection is configured yet.
+  // It will return a default tag in such a case. This default tag is not module specific.
+  const tag = sourceConfig.tag ?? DEFAULT_TAG;
+  using systemStorage = live_status ? bucketStorage.getInstance(sync_rules) : undefined;
   const status = await systemStorage?.getStatus();
   let replication_lag_bytes: number | undefined = undefined;
 
   let tables_flat: TableInfo[] = [];
 
   if (check_connection) {
-    const pool = system.requirePgPool();
-
     const source_table_patterns = rules.getSourceTables();
-    const wc = new replication.WalConnection({
-      db: pool,
-      sync_rules: rules
-    });
-    const resolved_tables = await wc.getDebugTablesInfo(source_table_patterns);
+    const resolved_tables = await apiHandler.getDebugTablesInfo(source_table_patterns, rules);
     tables_flat = resolved_tables.flatMap((info) => {
       if (info.table) {
         return [info.table];
@@ -119,19 +78,9 @@ export async function getSyncRulesStatus(
 
     if (systemStorage) {
       try {
-        const results = await util.retriedQuery(pool, {
-          statement: `SELECT
-      slot_name,
-      confirmed_flush_lsn, 
-      pg_current_wal_lsn(), 
-      (pg_current_wal_lsn() - confirmed_flush_lsn) AS lsn_distance
-    FROM pg_replication_slots WHERE slot_name = $1 LIMIT 1;`,
-          params: [{ type: 'varchar', value: systemStorage!.slot_name }]
+        replication_lag_bytes = await apiHandler.getReplicationLag({
+          bucketStorage: systemStorage
         });
-        const [row] = pgwireRows(results);
-        if (row) {
-          replication_lag_bytes = Number(row.lsn_distance);
-        }
       } catch (e) {
         // Ignore
         logger.warn(`Unable to get replication lag`, e);
@@ -139,7 +88,6 @@ export async function getSyncRulesStatus(
     }
   } else {
     const source_table_patterns = rules.getSourceTables();
-    const tag = system.config.connection!.tag ?? DEFAULT_TAG;
 
     tables_flat = source_table_patterns.map((pattern): TableInfo => {
       if (pattern.isWildcard) {
@@ -190,8 +138,8 @@ export async function getSyncRulesStatus(
     content: include_content ? sync_rules.sync_rules_content : undefined,
     connections: [
       {
-        id: system.config.connection!.id,
-        tag: system.config.connection!.tag ?? DEFAULT_TAG,
+        id: sourceConfig.id ?? DEFAULT_DATASOURCE_ID,
+        tag: tag,
         slot_name: sync_rules.slot_name,
         initial_replication_done: status?.snapshot_done ?? false,
         // TODO: Rename?

package/src/api/schema.ts

@@ -1,99 +1,27 @@
-import type * as pgwire from '@powersync/service-jpgwire';
-import { pgwireRows } from '@powersync/service-jpgwire';
-import { DatabaseSchema, internal_routes } from '@powersync/service-types';
+import { internal_routes } from '@powersync/service-types';
 
-import * as util from '../util/util-index.js';
-import { CorePowerSyncSystem } from '../system/CorePowerSyncSystem.js';
+import * as api from '../api/api-index.js';
 
-export async function getConnectionsSchema(system: CorePowerSyncSystem): Promise<internal_routes.GetSchemaResponse> {
-  if (system.config.connection == null) {
-    return { connections: [] };
+export async function getConnectionsSchema(api: api.RouteAPI): Promise<internal_routes.GetSchemaResponse> {
+  if (!api) {
+    return {
+      connections: [],
+      defaultConnectionTag: 'default',
+      defaultSchema: ''
+    };
   }
-  const schemas = await getConnectionSchema(system.requirePgPool());
+
+  const baseConfig = await api.getSourceConfig();
+
   return {
     connections: [
       {
-        schemas,
-        tag: system.config.connection!.tag,
-        id: system.config.connection!.id
+        id: baseConfig.id,
+        tag: baseConfig.tag,
+        schemas: await api.getConnectionSchema()
       }
-    ]
+    ],
+    defaultConnectionTag: baseConfig.tag!,
+    defaultSchema: api.getParseSyncRulesOptions().defaultSchema
   };
 }
-
-export async function getConnectionSchema(db: pgwire.PgClient): Promise<DatabaseSchema[]> {
-  // https://github.com/Borvik/vscode-postgres/blob/88ec5ed061a0c9bced6c5d4ec122d0759c3f3247/src/language/server.ts
-  const results = await util.retriedQuery(
-    db,
-    `SELECT
-  tbl.schemaname,
-  tbl.tablename,
-  tbl.quoted_name,
-  json_agg(a ORDER BY attnum) as columns
-FROM
-  (
-    SELECT
-      n.nspname as schemaname,
-      c.relname as tablename,
-      (quote_ident(n.nspname) || '.' || quote_ident(c.relname)) as quoted_name
-    FROM
-      pg_catalog.pg_class c
-      JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
-    WHERE
-      c.relkind = 'r'
-      AND n.nspname not in ('information_schema', 'pg_catalog', 'pg_toast')
-      AND n.nspname not like 'pg_temp_%'
-      AND n.nspname not like 'pg_toast_temp_%'
-      AND c.relnatts > 0
-      AND has_schema_privilege(n.oid, 'USAGE') = true
-      AND has_table_privilege(quote_ident(n.nspname) || '.' || quote_ident(c.relname), 'SELECT, INSERT, UPDATE, DELETE, TRUNCATE, REFERENCES, TRIGGER') = true
-  ) as tbl
-  LEFT JOIN (
-    SELECT
-      attrelid,
-      attname,
-      format_type(atttypid, atttypmod) as data_type,
-      (SELECT typname FROM pg_catalog.pg_type WHERE oid = atttypid) as pg_type,
-      attnum,
-      attisdropped
-    FROM
-      pg_attribute
-  ) as a ON (
-    a.attrelid = tbl.quoted_name::regclass
-    AND a.attnum > 0
-    AND NOT a.attisdropped
-    AND has_column_privilege(tbl.quoted_name, a.attname, 'SELECT, INSERT, UPDATE, REFERENCES')
-  )
-GROUP BY schemaname, tablename, quoted_name`
-  );
-  const rows = pgwireRows(results);
-
-  let schemas: Record<string, any> = {};
-
-  for (let row of rows) {
-    const schema = (schemas[row.schemaname] ??= {
-      name: row.schemaname,
-      tables: []
-    });
-    const table = {
-      name: row.tablename,
-      columns: [] as any[]
-    };
-    schema.tables.push(table);
-
-    const columnInfo = JSON.parse(row.columns);
-    for (let column of columnInfo) {
-      let pg_type = column.pg_type as string;
-      if (pg_type.startsWith('_')) {
-        pg_type = `${pg_type.substring(1)}[]`;
-      }
-      table.columns.push({
-        name: column.attname,
-        type: column.data_type,
-        pg_type: pg_type
-      });
-    }
-  }
-
-  return Object.values(schemas);
-}

package/src/auth/KeyStore.ts

@@ -1,9 +1,9 @@
+import { logger } from '@powersync/lib-services-framework';
 import * as jose from 'jose';
 import secs from '../util/secs.js';
-import { KeyOptions, KeySpec, SUPPORTED_ALGORITHMS } from './KeySpec.js';
-import { KeyCollector } from './KeyCollector.js';
 import { JwtPayload } from './JwtPayload.js';
-import { logger } from '@powersync/lib-services-framework';
+import { KeyCollector } from './KeyCollector.js';
+import { KeyOptions, KeySpec, SUPPORTED_ALGORITHMS } from './KeySpec.js';
 
 /**
  * KeyStore to get keys and verify tokens.
@@ -32,10 +32,13 @@ import { logger } from '@powersync/lib-services-framework';
  * If we have a matching kid, we can generally get a detailed error (e.g. signature verification failed, invalid algorithm, etc).
  * If we don't have a matching kid, we'll generally just get an error "Could not find an appropriate key...".
  */
-export class KeyStore {
-  private collector: KeyCollector;
+export class KeyStore<Collector extends KeyCollector = KeyCollector> {
+  /**
+   * @internal
+   */
+  collector: Collector;
 
-  constructor(collector: KeyCollector) {
+  constructor(collector: Collector) {
     this.collector = collector;
   }
 

package/src/auth/RemoteJWKSCollector.ts

@@ -22,7 +22,10 @@ export type RemoteJWKSCollectorOptions = {
 export class RemoteJWKSCollector implements KeyCollector {
   private url: URL;
 
-  constructor(url: string, protected options?: RemoteJWKSCollectorOptions) {
+  constructor(
+    url: string,
+    protected options?: RemoteJWKSCollectorOptions
+  ) {
     try {
       this.url = new URL(url);
     } catch (e) {

package/src/auth/StaticSupabaseKeyCollector.ts

@@ -0,0 +1,31 @@
+import * as jose from 'jose';
+import { KeySpec, KeyOptions } from './KeySpec.js';
+import { KeyCollector, KeyResult } from './KeyCollector.js';
+
+const SUPABASE_KEY_OPTIONS: KeyOptions = {
+  requiresAudience: ['authenticated'],
+  maxLifetimeSeconds: 86400 * 7 + 1200 // 1 week + 20 minutes margin
+};
+
+/**
+ * Set of static keys for Supabase.
+ *
+ * Same as StaticKeyCollector, but with some configuration tweaks for Supabase.
+ *
+ * Similar to SupabaseKeyCollector, but using hardcoded keys instead of fetching
+ * from the database.
+ *
+ * A key can be added both with and without a kid, in case wildcard matching is desired.
+ */
+export class StaticSupabaseKeyCollector implements KeyCollector {
+  static async importKeys(keys: jose.JWK[]) {
+    const parsedKeys = await Promise.all(keys.map((key) => KeySpec.importKey(key, SUPABASE_KEY_OPTIONS)));
+    return new StaticSupabaseKeyCollector(parsedKeys);
+  }
+
+  constructor(private keys: KeySpec[]) {}
+
+  async getKeys(): Promise<KeyResult> {
+    return { keys: this.keys, errors: [] };
+  }
+}

package/src/auth/auth-index.ts

@@ -7,4 +7,4 @@ export * from './KeyStore.js';
 export * from './LeakyBucket.js';
 export * from './RemoteJWKSCollector.js';
 export * from './StaticKeyCollector.js';
-export * from './SupabaseKeyCollector.js';
+export * from './StaticSupabaseKeyCollector.js';

package/src/db/mongo.ts

@@ -2,6 +2,7 @@ import * as mongo from 'mongodb';
 import * as timers from 'timers/promises';
 
 import { configFile } from '@powersync/service-types';
+import { normalizeMongoConfig } from '../storage/storage-index.js';
 
 /**
  * Time for new connection to timeout.
@@ -30,10 +31,11 @@ export const MONGO_OPERATION_TIMEOUT_MS = 30_000;
 export const MONGO_CLEAR_OPERATION_TIMEOUT_MS = 5_000;
 
 export function createMongoClient(config: configFile.PowerSyncConfig['storage']) {
-  return new mongo.MongoClient(config.uri, {
+  const normalized = normalizeMongoConfig(config);
+  return new mongo.MongoClient(normalized.uri, {
     auth: {
-      username: config.username,
-      password: config.password
+      username: normalized.username,
+      password: normalized.password
     },
     // Time for connection to timeout
     connectTimeoutMS: MONGO_CONNECT_TIMEOUT_MS,

package/src/entry/cli-entry.ts

@@ -1,10 +1,11 @@
 import { Command } from 'commander';
 
+import { logger } from '@powersync/lib-services-framework';
 import * as utils from '../util/util-index.js';
+import { registerCompactAction } from './commands/compact-action.js';
 import { registerMigrationAction } from './commands/migrate-action.js';
+import { registerStartAction } from './commands/start-action.js';
 import { registerTearDownAction } from './commands/teardown-action.js';
-import { registerCompactAction, registerStartAction } from './entry-index.js';
-import { logger } from '@powersync/lib-services-framework';
 
 /**
  * Generates a Commander program which serves as the entry point