@powersync/service-core 0.8.7 → 0.9.0

package/dist/util/utils.js.map

@@ -1 +1 @@
-{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/util/utils.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAE3D,OAAO,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AAGxD,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAKjD,MAAM,UAAU,QAAQ,CAAC,IAAY,EAAE,EAAU,EAAE,IAAY;IAC7D,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IACzC,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC;IACzC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;IAC7B,OAAO,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,SAAiB;IAC1C,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IACzC,IAAI,CAAC,MAAM,CAAC,UAAU,SAAS,EAAE,CAAC,CAAC;IACnC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;IAC7B,OAAO,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,EAAU;IACxC,6EAA6E;IAC7E,6CAA6C;IAC7C,IAAI,OAAO,EAAE,IAAI,QAAQ,EAAE;QACzB,MAAM,IAAI,KAAK,CAAC,yBAAyB,EAAE,KAAK,OAAO,EAAE,GAAG,CAAC,CAAC;KAC/D;IACD,OAAO,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;AACzB,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,QAAqB,EAAE,OAAoB;IACvE,mBAAmB;IACnB,MAAM,cAAc,GAAG,IAAI,GAAG,EAA0B,CAAC;IAEzD,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAS,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;IAElD,KAAK,IAAI,QAAQ,IAAI,OAAO,CAAC,MAAM,EAAE,EAAE;QACrC,MAAM,CAAC,GAAG,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACxC,IAAI,CAAC,IAAI,IAAI,EAAE;YACb,QAAQ;YACR,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;SAC/C;aAAM;YACL,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YACjC,IAAI,QAAQ,CAAC,QAAQ,IAAI,CAAC,CAAC,QAAQ,IAAI,QAAQ,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,EAAE;gBAChE,UAAU;gBACV,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;aAC/C;iBAAM;gBACL,YAAY;aACb;SACF;KACF;IAED,OAAO;QACL,cAAc,EAAE,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC;QAC5C,cAAc,EAAE,CAAC,GAAG,QAAQ,CAAC;KAC9B,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,CAAS,EAAE,CAAS;IAC/C,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,UAAU,CAAC;AAC9B,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,CAAiB,EAAE,CAAyB;IAC7E,IAAI,CAAC,IAAI,IAAI,EAAE;QACb,OAAO,CAAC,CAAC;KACV;SAAM,IAAI,CAAC,CAAC,cAAc,EAAE;QAC3B,OAAO;YACL,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,KAAK,EAAE,CAAC,CAAC,YAAY;YACrB,QAAQ,EAAE,CAAC,CAAC,eAAe;SAC5B,CAAC;KACH;SAAM;QACL,OAAO;YACL,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,KAAK,EAAE,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,YAAY;YAC/B,QAAQ,EAAE,YAAY,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,eAAe,CAAC;SACtD,CAAC;KACH;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,EAAmB,EACnB,aAA2C,EAC3C,OAA8B;IAE9B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAEzB,MAAM,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,mEAAmE,CAAC,CAAC,CAAC;IAElH,gDAAgD;IAChD,wEAAwE;IAExE,MAAM,OAAO,GAAG,OAAO,EAAE,OAAO,IAAI,KAAM,CAAC;IAE3C,MAAM,CAAC,IAAI,CAAC,+BAA+B,GAAG,EAAE,CAAC,CAAC;IAClD,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,GAAG,OAAO,EAAE;QACnC,MAAM,EAAE,GAAG,MAAM,aAAa,CAAC,mBAAmB,EAAE,CAAC;QACrD,IAAI,CAAC,EAAE,CAAC,YAAY,EAAE,EAAE;YACtB,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;SAC5C;QACD,IAAI,EAAE,CAAC,GAAG,IAAI,GAAG,EAAE;YACjB,MAAM,CAAC,IAAI,CAAC,yBAAyB,GAAG,MAAM,EAAE,CAAC,UAAU,EAAE,CAAC,CAAC;YAC/D,OAAO,EAAE,CAAC,UAAU,CAAC;SACtB;QAED,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;KACzD;IAED,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;AAC1D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACzC,EAAmB,EACnB,aAA2C,EAC3C,OAAe;IAEf,MAAM,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,UAAU,CAC1B,MAAM,YAAY,CAAC,EAAE,EAAE,mEAAmE,CAAC,CAC5F,CAAC;IAEF,MAAM,EAAE,GAAG,MAAM,aAAa,CAAC,qBAAqB,CAAC,OAAO,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;IAC5E,MAAM,CAAC,IAAI,CAAC,uBAAuB,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;IAC9E,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,OAA2B,EAAE,SAA6B;IACzF,IAAI,OAAO,IAAI,IAAI,EAAE;QACnB,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;KACxC;IACD,IAAI,SAAS,IAAI,IAAI,EAAE;QACrB,OAAO,OAAO,CAAC;KAChB;IACD,OAAO,GAAG,OAAO,IAAI,SAAS,EAAE,CAAC;AACnC,CAAC"}
+{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/util/utils.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAS7B,MAAM,CAAC,MAAM,YAAY,GAAG,sCAAsC,CAAC;AAEnE,MAAM,UAAU,gBAAgB,CAAC,UAAkB;IACjD,OAAO,IAAI,UAAU,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC;AACrE,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,IAAY,EAAE,EAAU,EAAE,IAAY;IAC7D,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IACzC,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC;IACzC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;IAC7B,OAAO,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,SAAiB;IAC1C,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IACzC,IAAI,CAAC,MAAM,CAAC,UAAU,SAAS,EAAE,CAAC,CAAC;IACnC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;IAC7B,OAAO,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,EAAU;IACxC,6EAA6E;IAC7E,6CAA6C;IAC7C,IAAI,OAAO,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,yBAAyB,EAAE,KAAK,OAAO,EAAE,GAAG,CAAC,CAAC;IAChE,CAAC;IACD,OAAO,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;AACzB,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,QAAqB,EAAE,OAAoB;IACvE,mBAAmB;IACnB,MAAM,cAAc,GAAG,IAAI,GAAG,EAA0B,CAAC;IAEzD,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAS,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;IAElD,KAAK,IAAI,QAAQ,IAAI,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;QACtC,MAAM,CAAC,GAAG,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACxC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;YACd,QAAQ;YACR,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAChD,CAAC;aAAM,CAAC;YACN,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YACjC,IAAI,QAAQ,CAAC,QAAQ,IAAI,CAAC,CAAC,QAAQ,IAAI,QAAQ,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;gBACjE,UAAU;gBACV,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YAChD,CAAC;iBAAM,CAAC;gBACN,YAAY;YACd,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO;QACL,cAAc,EAAE,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC;QAC5C,cAAc,EAAE,CAAC,GAAG,QAAQ,CAAC;KAC9B,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,CAAS,EAAE,CAAS;IAC/C,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,UAAU,CAAC;AAC9B,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,CAAiB,EAAE,CAAyB;IAC7E,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QACd,OAAO,CAAC,CAAC;IACX,CAAC;SAAM,IAAI,CAAC,CAAC,cAAc,EAAE,CAAC;QAC5B,OAAO;YACL,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,KAAK,EAAE,CAAC,CAAC,YAAY;YACrB,QAAQ,EAAE,CAAC,CAAC,eAAe;SAC5B,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,OAAO;YACL,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,KAAK,EAAE,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,YAAY;YAC/B,QAAQ,EAAE,YAAY,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,eAAe,CAAC;SACtD,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,qBAAqB,CAC5B,KAAoC,EACpC,OAAmC;IAEnC,IAAI,MAAM,GAAwB,EAAE,CAAC;IACrC,KAAK,IAAI,MAAM,IAAI,OAAO,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;QACzB,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC;IAC7B,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,0BAA0B,CACxC,KAAoC,EACpC,OAAmC;IAEnC,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QACxB,gDAAgD;QAChD,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;IAClC,CAAC;IACD,MAAM,WAAW,GAAG,qBAAqB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAE1D,OAAO,cAAc,CAAC,WAAW,CAAC,CAAC;AACrC,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,GAAyB;IACtD,+EAA+E;IAC/E,4EAA4E;IAC5E,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IACjC,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAChC,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC;AAC5D,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,GAAkC;IACjE,KAAK,IAAI,GAAG,IAAI,GAAG,EAAE,CAAC;QACpB,IAAI,OAAO,GAAG,CAAC,GAAG,CAAC,IAAI,WAAW,EAAE,CAAC;YACnC,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,GAAkC;IAC9D,OAAO,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,OAA2B,EAAE,SAA6B;IACzF,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;QACpB,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;IACzC,CAAC;IACD,IAAI,SAAS,IAAI,IAAI,EAAE,CAAC;QACtB,OAAO,OAAO,CAAC;IACjB,CAAC;IACD,OAAO,GAAG,OAAO,IAAI,SAAS,EAAE,CAAC;AACnC,CAAC"}

package/package.json

@@ -5,7 +5,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "0.8.7",
+  "version": "0.9.0",
   "main": "dist/index.js",
   "license": "FSL-1.1-Apache-2.0",
   "type": "module",
@@ -28,32 +28,28 @@
     "lru-cache": "^10.2.2",
     "mongodb": "^6.7.0",
     "node-fetch": "^3.3.2",
-    "pgwire": "github:kagis/pgwire#f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87",
     "ts-codec": "^1.2.2",
+    "uri-js": "^4.4.1",
     "uuid": "^9.0.1",
     "winston": "^3.13.0",
     "yaml": "^2.3.2",
-    "@powersync/lib-services-framework": "0.1.1",
+    "@powersync/lib-services-framework": "0.2.0",
     "@powersync/service-jsonbig": "0.17.10",
-    "@powersync/service-jpgwire": "0.17.14",
-    "@powersync/service-rsocket-router": "0.0.13",
-    "@powersync/service-sync-rules": "0.20.0",
-    "@powersync/service-types": "0.2.0"
+    "@powersync/service-rsocket-router": "0.0.14",
+    "@powersync/service-sync-rules": "0.21.0",
+    "@powersync/service-types": "0.3.0"
   },
   "devDependencies": {
     "@types/async": "^3.2.24",
     "@types/lodash": "^4.17.5",
     "@types/uuid": "^9.0.4",
     "fastify": "4.23.2",
-    "fastify-plugin": "^4.5.1",
-    "typescript": "^5.2.2",
-    "vite-tsconfig-paths": "^4.3.2",
-    "vitest": "^0.34.6"
+    "fastify-plugin": "^4.5.1"
   },
   "scripts": {
     "build": "tsc -b",
     "build:tests": "tsc -b test/tsconfig.json",
-    "test": "vitest --no-threads",
+    "test": "vitest",
     "clean": "rm -rf ./dist && tsc -b --clean"
   }
 }

package/src/api/RouteAPI.ts

@@ -0,0 +1,78 @@
+import { SqlSyncRules, TablePattern } from '@powersync/service-sync-rules';
+import * as types from '@powersync/service-types';
+import { ParseSyncRulesOptions, SyncRulesBucketStorage } from '../storage/BucketStorage.js';
+
+export interface PatternResult {
+  schema: string;
+  pattern: string;
+  wildcard: boolean;
+  tables?: types.TableInfo[];
+  table?: types.TableInfo;
+}
+
+export interface ReplicationLagOptions {
+  bucketStorage: SyncRulesBucketStorage;
+}
+
+/**
+ *  Describes all the methods currently required to service the sync API endpoints.
+ */
+export interface RouteAPI {
+  /**
+   * @returns basic identification of the connection
+   */
+  getSourceConfig(): Promise<types.configFile.ResolvedDataSourceConfig>;
+
+  /**
+   * Checks the current connection status of the data source.
+   * This is usually some test query to verify the source can be reached.
+   */
+  getConnectionStatus(): Promise<types.ConnectionStatusV2>;
+
+  /**
+   * Generates replication table information from a given pattern of tables.
+   *
+   * @param tablePatterns A set of table patterns which typically come from
+   *          the tables listed in sync rules definitions.
+   *
+   * @param sqlSyncRules
+   * @returns A result of all the tables and columns which should be replicated
+   *           based off the input patterns. Certain tests are executed on the
+   *           tables to ensure syncing should function according to the input
+   *           pattern. Debug errors and warnings are reported per table.
+   */
+  getDebugTablesInfo(tablePatterns: TablePattern[], sqlSyncRules: SqlSyncRules): Promise<PatternResult[]>;
+
+  /**
+   * @returns The replication lag: that is the amount of data which has not been
+   *          replicated yet, in bytes.
+   */
+  getReplicationLag(options: ReplicationLagOptions): Promise<number | undefined>;
+
+  /**
+   * Get the current LSN or equivalent replication HEAD position identifier
+   */
+  getReplicationHead(): Promise<string>;
+
+  /**
+   * @returns The schema for tables inside the connected database. This is typically
+   *          used to validate sync rules.
+   */
+  getConnectionSchema(): Promise<types.DatabaseSchema[]>;
+
+  /**
+   * Executes a query and return the result from the data source. This is currently used in the
+   * admin API which is exposed in Collide.
+   */
+  executeQuery(query: string, params: any[]): Promise<types.internal_routes.ExecuteSqlResponse>;
+
+  /**
+   * Close any resources that need graceful termination.
+   */
+  shutdown(): Promise<void>;
+
+  /**
+   * Get the default schema (or database) when only a table name is specified in sync rules.
+   */
+  getParseSyncRulesOptions(): ParseSyncRulesOptions;
+}

package/src/api/api-index.ts

@@ -1,2 +1,3 @@
 export * from './diagnostics.js';
+export * from './RouteAPI.js';
 export * from './schema.js';

package/src/api/diagnostics.ts

@@ -1,51 +1,9 @@
+import { logger } from '@powersync/lib-services-framework';
 import { DEFAULT_TAG, SourceTableInterface, SqlSyncRules } from '@powersync/service-sync-rules';
-import { pgwireRows } from '@powersync/service-jpgwire';
-import { ConnectionStatus, SyncRulesStatus, TableInfo, baseUri } from '@powersync/service-types';
+import { SyncRulesStatus, TableInfo } from '@powersync/service-types';
 
-import * as replication from '../replication/replication-index.js';
 import * as storage from '../storage/storage-index.js';
-import * as util from '../util/util-index.js';
-
-import { CorePowerSyncSystem } from '../system/CorePowerSyncSystem.js';
-import { logger } from '@powersync/lib-services-framework';
-
-export async function getConnectionStatus(system: CorePowerSyncSystem): Promise<ConnectionStatus | null> {
-  if (system.pgwire_pool == null) {
-    return null;
-  }
-
-  const pool = system.requirePgPool();
-
-  const base = {
-    id: system.config.connection!.id,
-    postgres_uri: baseUri(system.config.connection!)
-  };
-  try {
-    await util.retriedQuery(pool, `SELECT 'PowerSync connection test'`);
-  } catch (e) {
-    return {
-      ...base,
-      connected: false,
-      errors: [{ level: 'fatal', message: e.message }]
-    };
-  }
-
-  try {
-    await replication.checkSourceConfiguration(pool);
-  } catch (e) {
-    return {
-      ...base,
-      connected: true,
-      errors: [{ level: 'fatal', message: e.message }]
-    };
-  }
-
-  return {
-    ...base,
-    connected: true,
-    errors: []
-  };
-}
+import { RouteAPI } from './RouteAPI.js';
 
 export interface DiagnosticsOptions {
   /**
@@ -66,9 +24,12 @@ export interface DiagnosticsOptions {
   check_connection: boolean;
 }
 
+export const DEFAULT_DATASOURCE_ID = 'default';
+
 export async function getSyncRulesStatus(
+  bucketStorage: storage.BucketStorageFactory,
+  apiHandler: RouteAPI,
   sync_rules: storage.PersistedSyncRulesContent | null,
-  system: CorePowerSyncSystem,
   options: DiagnosticsOptions
 ): Promise<SyncRulesStatus | undefined> {
   if (sync_rules == null) {
@@ -82,7 +43,7 @@ export async function getSyncRulesStatus(
   let rules: SqlSyncRules;
   let persisted: storage.PersistedSyncRules;
   try {
-    persisted = sync_rules.parsed();
+    persisted = sync_rules.parsed(apiHandler.getParseSyncRulesOptions());
     rules = persisted.sync_rules;
   } catch (e) {
     return {
@@ -92,21 +53,19 @@ export async function getSyncRulesStatus(
     };
   }
 
-  const systemStorage = live_status ? await system.storage.getInstance(persisted) : undefined;
+  const sourceConfig = await apiHandler.getSourceConfig();
+  // This method can run under some situations if no connection is configured yet.
+  // It will return a default tag in such a case. This default tag is not module specific.
+  const tag = sourceConfig.tag ?? DEFAULT_TAG;
+  using systemStorage = live_status ? bucketStorage.getInstance(sync_rules) : undefined;
   const status = await systemStorage?.getStatus();
   let replication_lag_bytes: number | undefined = undefined;
 
   let tables_flat: TableInfo[] = [];
 
   if (check_connection) {
-    const pool = system.requirePgPool();
-
     const source_table_patterns = rules.getSourceTables();
-    const wc = new replication.WalConnection({
-      db: pool,
-      sync_rules: rules
-    });
-    const resolved_tables = await wc.getDebugTablesInfo(source_table_patterns);
+    const resolved_tables = await apiHandler.getDebugTablesInfo(source_table_patterns, rules);
     tables_flat = resolved_tables.flatMap((info) => {
       if (info.table) {
         return [info.table];
@@ -119,19 +78,9 @@ export async function getSyncRulesStatus(
 
     if (systemStorage) {
       try {
-        const results = await util.retriedQuery(pool, {
-          statement: `SELECT
-      slot_name,
-      confirmed_flush_lsn, 
-      pg_current_wal_lsn(), 
-      (pg_current_wal_lsn() - confirmed_flush_lsn) AS lsn_distance
-    FROM pg_replication_slots WHERE slot_name = $1 LIMIT 1;`,
-          params: [{ type: 'varchar', value: systemStorage!.slot_name }]
+        replication_lag_bytes = await apiHandler.getReplicationLag({
+          bucketStorage: systemStorage
         });
-        const [row] = pgwireRows(results);
-        if (row) {
-          replication_lag_bytes = Number(row.lsn_distance);
-        }
       } catch (e) {
         // Ignore
         logger.warn(`Unable to get replication lag`, e);
@@ -139,7 +88,6 @@ export async function getSyncRulesStatus(
     }
   } else {
     const source_table_patterns = rules.getSourceTables();
-    const tag = system.config.connection!.tag ?? DEFAULT_TAG;
 
     tables_flat = source_table_patterns.map((pattern): TableInfo => {
       if (pattern.isWildcard) {
@@ -190,8 +138,8 @@ export async function getSyncRulesStatus(
     content: include_content ? sync_rules.sync_rules_content : undefined,
     connections: [
       {
-        id: system.config.connection!.id,
-        tag: system.config.connection!.tag ?? DEFAULT_TAG,
+        id: sourceConfig.id ?? DEFAULT_DATASOURCE_ID,
+        tag: tag,
         slot_name: sync_rules.slot_name,
         initial_replication_done: status?.snapshot_done ?? false,
         // TODO: Rename?

package/src/api/schema.ts

@@ -1,99 +1,27 @@
-import type * as pgwire from '@powersync/service-jpgwire';
-import { pgwireRows } from '@powersync/service-jpgwire';
-import { DatabaseSchema, internal_routes } from '@powersync/service-types';
+import { internal_routes } from '@powersync/service-types';
 
-import * as util from '../util/util-index.js';
-import { CorePowerSyncSystem } from '../system/CorePowerSyncSystem.js';
+import * as api from '../api/api-index.js';
 
-export async function getConnectionsSchema(system: CorePowerSyncSystem): Promise<internal_routes.GetSchemaResponse> {
-  if (system.config.connection == null) {
-    return { connections: [] };
+export async function getConnectionsSchema(api: api.RouteAPI): Promise<internal_routes.GetSchemaResponse> {
+  if (!api) {
+    return {
+      connections: [],
+      defaultConnectionTag: 'default',
+      defaultSchema: ''
+    };
   }
-  const schemas = await getConnectionSchema(system.requirePgPool());
+
+  const baseConfig = await api.getSourceConfig();
+
   return {
     connections: [
       {
-        schemas,
-        tag: system.config.connection!.tag,
-        id: system.config.connection!.id
+        id: baseConfig.id,
+        tag: baseConfig.tag,
+        schemas: await api.getConnectionSchema()
       }
-    ]
+    ],
+    defaultConnectionTag: baseConfig.tag!,
+    defaultSchema: api.getParseSyncRulesOptions().defaultSchema
   };
 }
-
-export async function getConnectionSchema(db: pgwire.PgClient): Promise<DatabaseSchema[]> {
-  // https://github.com/Borvik/vscode-postgres/blob/88ec5ed061a0c9bced6c5d4ec122d0759c3f3247/src/language/server.ts
-  const results = await util.retriedQuery(
-    db,
-    `SELECT
-  tbl.schemaname,
-  tbl.tablename,
-  tbl.quoted_name,
-  json_agg(a ORDER BY attnum) as columns
-FROM
-  (
-    SELECT
-      n.nspname as schemaname,
-      c.relname as tablename,
-      (quote_ident(n.nspname) || '.' || quote_ident(c.relname)) as quoted_name
-    FROM
-      pg_catalog.pg_class c
-      JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
-    WHERE
-      c.relkind = 'r'
-      AND n.nspname not in ('information_schema', 'pg_catalog', 'pg_toast')
-      AND n.nspname not like 'pg_temp_%'
-      AND n.nspname not like 'pg_toast_temp_%'
-      AND c.relnatts > 0
-      AND has_schema_privilege(n.oid, 'USAGE') = true
-      AND has_table_privilege(quote_ident(n.nspname) || '.' || quote_ident(c.relname), 'SELECT, INSERT, UPDATE, DELETE, TRUNCATE, REFERENCES, TRIGGER') = true
-  ) as tbl
-  LEFT JOIN (
-    SELECT
-      attrelid,
-      attname,
-      format_type(atttypid, atttypmod) as data_type,
-      (SELECT typname FROM pg_catalog.pg_type WHERE oid = atttypid) as pg_type,
-      attnum,
-      attisdropped
-    FROM
-      pg_attribute
-  ) as a ON (
-    a.attrelid = tbl.quoted_name::regclass
-    AND a.attnum > 0
-    AND NOT a.attisdropped
-    AND has_column_privilege(tbl.quoted_name, a.attname, 'SELECT, INSERT, UPDATE, REFERENCES')
-  )
-GROUP BY schemaname, tablename, quoted_name`
-  );
-  const rows = pgwireRows(results);
-
-  let schemas: Record<string, any> = {};
-
-  for (let row of rows) {
-    const schema = (schemas[row.schemaname] ??= {
-      name: row.schemaname,
-      tables: []
-    });
-    const table = {
-      name: row.tablename,
-      columns: [] as any[]
-    };
-    schema.tables.push(table);
-
-    const columnInfo = JSON.parse(row.columns);
-    for (let column of columnInfo) {
-      let pg_type = column.pg_type as string;
-      if (pg_type.startsWith('_')) {
-        pg_type = `${pg_type.substring(1)}[]`;
-      }
-      table.columns.push({
-        name: column.attname,
-        type: column.data_type,
-        pg_type: pg_type
-      });
-    }
-  }
-
-  return Object.values(schemas);
-}

package/src/auth/KeyStore.ts

@@ -1,9 +1,9 @@
+import { logger } from '@powersync/lib-services-framework';
 import * as jose from 'jose';
 import secs from '../util/secs.js';
-import { KeyOptions, KeySpec, SUPPORTED_ALGORITHMS } from './KeySpec.js';
-import { KeyCollector } from './KeyCollector.js';
 import { JwtPayload } from './JwtPayload.js';
-import { logger } from '@powersync/lib-services-framework';
+import { KeyCollector } from './KeyCollector.js';
+import { KeyOptions, KeySpec, SUPPORTED_ALGORITHMS } from './KeySpec.js';
 
 /**
  * KeyStore to get keys and verify tokens.
@@ -32,10 +32,13 @@ import { logger } from '@powersync/lib-services-framework';
  * If we have a matching kid, we can generally get a detailed error (e.g. signature verification failed, invalid algorithm, etc).
  * If we don't have a matching kid, we'll generally just get an error "Could not find an appropriate key...".
  */
-export class KeyStore {
-  private collector: KeyCollector;
+export class KeyStore<Collector extends KeyCollector = KeyCollector> {
+  /**
+   * @internal
+   */
+  collector: Collector;
 
-  constructor(collector: KeyCollector) {
+  constructor(collector: Collector) {
     this.collector = collector;
   }
 

package/src/auth/RemoteJWKSCollector.ts

@@ -22,7 +22,10 @@ export type RemoteJWKSCollectorOptions = {
 export class RemoteJWKSCollector implements KeyCollector {
   private url: URL;
 
-  constructor(url: string, protected options?: RemoteJWKSCollectorOptions) {
+  constructor(
+    url: string,
+    protected options?: RemoteJWKSCollectorOptions
+  ) {
     try {
       this.url = new URL(url);
     } catch (e) {

package/src/auth/auth-index.ts

@@ -7,4 +7,3 @@ export * from './KeyStore.js';
 export * from './LeakyBucket.js';
 export * from './RemoteJWKSCollector.js';
 export * from './StaticKeyCollector.js';
-export * from './SupabaseKeyCollector.js';

package/src/db/mongo.ts

@@ -2,6 +2,7 @@ import * as mongo from 'mongodb';
 import * as timers from 'timers/promises';
 
 import { configFile } from '@powersync/service-types';
+import { normalizeMongoConfig } from '../storage/storage-index.js';
 
 /**
  * Time for new connection to timeout.
@@ -30,10 +31,11 @@ export const MONGO_OPERATION_TIMEOUT_MS = 30_000;
 export const MONGO_CLEAR_OPERATION_TIMEOUT_MS = 5_000;
 
 export function createMongoClient(config: configFile.PowerSyncConfig['storage']) {
-  return new mongo.MongoClient(config.uri, {
+  const normalized = normalizeMongoConfig(config);
+  return new mongo.MongoClient(normalized.uri, {
     auth: {
-      username: config.username,
-      password: config.password
+      username: normalized.username,
+      password: normalized.password
     },
     // Time for connection to timeout
     connectTimeoutMS: MONGO_CONNECT_TIMEOUT_MS,

package/src/entry/cli-entry.ts

@@ -1,10 +1,11 @@
 import { Command } from 'commander';
 
+import { logger } from '@powersync/lib-services-framework';
 import * as utils from '../util/util-index.js';
+import { registerCompactAction } from './commands/compact-action.js';
 import { registerMigrationAction } from './commands/migrate-action.js';
+import { registerStartAction } from './commands/start-action.js';
 import { registerTearDownAction } from './commands/teardown-action.js';
-import { registerCompactAction, registerStartAction } from './entry-index.js';
-import { logger } from '@powersync/lib-services-framework';
 
 /**
  * Generates a Commander program which serves as the entry point

package/src/entry/commands/compact-action.ts

@@ -2,8 +2,8 @@ import { Command } from 'commander';
 
 import { logger } from '@powersync/lib-services-framework';
 import * as v8 from 'v8';
-import { createPowerSyncMongo, MongoBucketStorage } from '../../storage/storage-index.js';
-import { loadConfig } from '../../util/config.js';
+import * as storage from '../../storage/storage-index.js';
+import * as utils from '../../util/util-index.js';
 import { extractRunnerOptions, wrapConfigCommand } from './config-command.js';
 
 const COMMAND_NAME = 'compact';
@@ -21,28 +21,40 @@ const HEAP_LIMIT = v8.getHeapStatistics().heap_size_limit;
 const COMPACT_MEMORY_LIMIT_MB = Math.min(HEAP_LIMIT / 1024 / 1024 - 128, 1024);
 
 export function registerCompactAction(program: Command) {
-  const compactCommand = program.command(COMMAND_NAME);
+  const compactCommand = program
+    .command(COMMAND_NAME)
+    .option(`-b, --buckets [buckets]`, 'Bucket or bucket definition name (optional, comma-separate multiple names)');
 
   wrapConfigCommand(compactCommand);
 
   return compactCommand.description('Compact storage').action(async (options) => {
+    const buckets = options.buckets?.split(',');
+    if (buckets != null) {
+      logger.info('Compacting storage for all buckets...');
+    } else {
+      logger.info(`Compacting storage for ${buckets.join(', ')}...`);
+    }
     const runnerConfig = extractRunnerOptions(options);
-
-    const config = await loadConfig(runnerConfig);
-    const { storage } = config;
-    const psdb = createPowerSyncMongo(storage);
+    const configuration = await utils.loadConfig(runnerConfig);
+    logger.info('Successfully loaded configuration...');
+    const { storage: storageConfig } = configuration;
+    logger.info('Connecting to storage...');
+    const psdb = storage.createPowerSyncMongo(storageConfig);
     const client = psdb.client;
     await client.connect();
     try {
-      const bucketStorage = new MongoBucketStorage(psdb, { slot_name_prefix: config.slot_name_prefix });
-      const active = await bucketStorage.getActiveSyncRules();
+      const bucketStorage = new storage.MongoBucketStorage(psdb, {
+        slot_name_prefix: configuration.slot_name_prefix
+      });
+      const active = await bucketStorage.getActiveSyncRulesContent();
       if (active == null) {
         logger.info('No active instance to compact');
         return;
       }
-      const p = bucketStorage.getInstance(active);
-      await p.compact({ memoryLimitMB: COMPACT_MEMORY_LIMIT_MB });
-      logger.info('done');
+      using p = bucketStorage.getInstance(active);
+      logger.info('Performing compaction...');
+      await p.compact({ memoryLimitMB: COMPACT_MEMORY_LIMIT_MB, compactBuckets: buckets });
+      logger.info('Successfully compacted storage.');
     } catch (e) {
       logger.error(`Failed to compact: ${e.toString()}`);
       process.exit(1);

package/src/entry/commands/migrate-action.ts

@@ -1,9 +1,8 @@
+import { logger } from '@powersync/lib-services-framework';
 import { Command } from 'commander';
 
+import * as migrations from '../../migrations/migrations-index.js';
 import { extractRunnerOptions, wrapConfigCommand } from './config-command.js';
-import { migrate } from '../../migrations/migrations.js';
-import { Direction } from '../../migrations/definitions.js';
-import { logger } from '@powersync/lib-services-framework';
 
 const COMMAND_NAME = 'migrate';
 
@@ -15,13 +14,11 @@ export function registerMigrationAction(program: Command) {
   return migrationCommand
     .description('Run migrations')
     .argument('<direction>', 'Migration direction. `up` or `down`')
-    .action(async (direction: Direction, options) => {
-      const runnerConfig = extractRunnerOptions(options);
-
+    .action(async (direction: migrations.Direction, options) => {
       try {
-        await migrate({
+        await migrations.migrate({
           direction,
-          runner_config: runnerConfig
+          runner_config: extractRunnerOptions(options)
         });
 
         process.exit(0);

package/src/entry/commands/teardown-action.ts

@@ -1,7 +1,7 @@
 import { Command } from 'commander';
 
-import { extractRunnerOptions, wrapConfigCommand } from './config-command.js';
 import { teardown } from '../../runner/teardown.js';
+import { extractRunnerOptions, wrapConfigCommand } from './config-command.js';
 
 const COMMAND_NAME = 'teardown';
 
@@ -12,7 +12,7 @@ export function registerTearDownAction(program: Command) {
 
   return teardownCommand
     .argument('[ack]', 'Type `TEARDOWN` to confirm teardown should occur')
-    .description('Terminate all replicating sync rules, deleting the replication slots')
+    .description('Terminate all replicating sync rules, clear remote configuration and remove all data')
     .action(async (ack, options) => {
       if (ack !== 'TEARDOWN') {
         throw new Error('TEARDOWN was not acknowledged.');

package/src/index.ts

@@ -18,8 +18,11 @@ export * as framework from '@powersync/lib-services-framework';
 export * from './metrics/Metrics.js';
 export * as metrics from './metrics/Metrics.js';
 
-export * from './migrations/migrations.js';
 export * as migrations from './migrations/migrations-index.js';
+export * from './migrations/migrations.js';
+
+export * from './modules/modules-index.js';
+export * as modules from './modules/modules-index.js';
 
 export * from './replication/replication-index.js';
 export * as replication from './replication/replication-index.js';
@@ -33,7 +36,7 @@ export * as storage from './storage/storage-index.js';
 export * from './sync/sync-index.js';
 export * as sync from './sync/sync-index.js';
 
-export * from './system/CorePowerSyncSystem.js';
+export * from './system/system-index.js';
 export * as system from './system/system-index.js';
 
 export * from './util/util-index.js';