@powersync/service-module-postgres-storage 0.0.0-dev-20250827091123 → 0.0.0-dev-20250828134335

package/src/storage/PostgresStorageProvider.ts

@@ -5,9 +5,8 @@ import { storage } from '@powersync/service-core';
 import { isPostgresStorageConfig, normalizePostgresStorageConfig, PostgresStorageConfig } from '../types/types.js';
 import { dropTables } from '../utils/db.js';
 import { PostgresBucketStorageFactory } from './PostgresBucketStorageFactory.js';
-import { PostgresReportStorageFactory } from './PostgresReportStorageFactory.js';
 
-export class PostgresStorageProvider implements storage.StorageProvider {
+export class PostgresStorageProvider implements storage.BucketStorageProvider {
   get type() {
     return lib_postgres.POSTGRES_CONNECTION_TYPE;
   }
@@ -29,23 +28,13 @@ export class PostgresStorageProvider implements storage.StorageProvider {
       config: normalizedConfig,
       slot_name_prefix: options.resolvedConfig.slot_name_prefix
     });
-
-    const reportStorageFactory = new PostgresReportStorageFactory({
-      config: normalizedConfig
-    });
-
     return {
-      reportStorage: reportStorageFactory,
       storage: storageFactory,
-      shutDown: async () => {
-        await storageFactory.db[Symbol.asyncDispose]();
-        await reportStorageFactory.db[Symbol.asyncDispose]();
-      },
+      shutDown: async () => storageFactory.db[Symbol.asyncDispose](),
       tearDown: async () => {
         logger.info(`Tearing down Postgres storage: ${normalizedConfig.database}...`);
         await dropTables(storageFactory.db);
         await storageFactory.db[Symbol.asyncDispose]();
-        await reportStorageFactory.db[Symbol.asyncDispose]();
         return true;
       }
     } satisfies storage.ActiveStorage;

package/src/storage/PostgresSyncRulesStorage.ts

@@ -1,13 +1,16 @@
 import * as lib_postgres from '@powersync/lib-service-postgres';
 import {
   BroadcastIterable,
+  BucketChecksum,
   CHECKPOINT_INVALIDATE_ALL,
   CheckpointChanges,
+  CompactOptions,
   GetCheckpointChangesOptions,
   InternalOpId,
   internalToExternalOpId,
   LastValueSink,
   maxLsn,
+  PartialChecksum,
   ReplicationCheckpoint,
   storage,
   utils,
@@ -109,6 +112,10 @@ export class PostgresSyncRulesStorage
     return new PostgresCompactor(this.db, this.group_id, options).compact();
   }
 
+  async populatePersistentChecksumCache(options: Pick<CompactOptions, 'signal' | 'maxOpId'>): Promise<void> {
+    // no-op - checksum cache is not implemented for Postgres yet
+  }
+
   lastWriteCheckpoint(filters: storage.SyncStorageLastWriteCheckpointFilters): Promise<bigint | null> {
     return this.writeCheckpointAPI.lastWriteCheckpoint({
       ...filters,
@@ -571,6 +578,10 @@ export class PostgresSyncRulesStorage
     return this.checksumCache.getChecksumMap(checkpoint, buckets);
   }
 
+  clearChecksumCache() {
+    this.checksumCache.clear();
+  }
+
   async terminate(options?: storage.TerminateOptions) {
     if (!options || options?.clearStorage) {
       await this.clear(options);
@@ -692,16 +703,24 @@ export class PostgresSyncRulesStorage
         b.bucket_name;
     `.rows<{ bucket: string; checksum_total: bigint; total: bigint; has_clear_op: number }>();
 
-    return new Map<string, storage.PartialChecksum>(
+    return new Map<string, storage.PartialOrFullChecksum>(
       results.map((doc) => {
+        const checksum = Number(BigInt(doc.checksum_total) & 0xffffffffn) & 0xffffffff;
+
         return [
           doc.bucket,
-          {
-            bucket: doc.bucket,
-            partialCount: Number(doc.total),
-            partialChecksum: Number(BigInt(doc.checksum_total) & 0xffffffffn) & 0xffffffff,
-            isFullChecksum: doc.has_clear_op == 1
-          } satisfies storage.PartialChecksum
+          doc.has_clear_op == 1
+            ? ({
+                // full checksum
+                bucket: doc.bucket,
+                count: Number(doc.total),
+                checksum
+              } satisfies BucketChecksum)
+            : ({
+                bucket: doc.bucket,
+                partialCount: Number(doc.total),
+                partialChecksum: checksum
+              } satisfies PartialChecksum)
         ];
       })
     );

package/src/{utils/test-utils.ts → storage/PostgresTestStorageFactoryGenerator.ts}

@@ -1,8 +1,7 @@
 import { framework, PowerSyncMigrationManager, ServiceContext, TestStorageOptions } from '@powersync/service-core';
 import { PostgresMigrationAgent } from '../migrations/PostgresMigrationAgent.js';
 import { normalizePostgresStorageConfig, PostgresStorageConfigDecoded } from '../types/types.js';
-import { PostgresReportStorageFactory } from '../storage/PostgresReportStorageFactory.js';
-import { PostgresBucketStorageFactory } from '../storage/PostgresBucketStorageFactory.js';
+import { PostgresBucketStorageFactory } from './PostgresBucketStorageFactory.js';
 
 export type PostgresTestStorageOptions = {
   url: string;
@@ -13,7 +12,7 @@ export type PostgresTestStorageOptions = {
   migrationAgent?: (config: PostgresStorageConfigDecoded) => PostgresMigrationAgent;
 };
 
-export function postgresTestSetup(factoryOptions: PostgresTestStorageOptions) {
+export const postgresTestSetup = (factoryOptions: PostgresTestStorageOptions) => {
   const BASE_CONFIG = {
     type: 'postgresql' as const,
     uri: factoryOptions.url,
@@ -49,21 +48,6 @@ export function postgresTestSetup(factoryOptions: PostgresTestStorageOptions) {
   };
 
   return {
-    reportFactory: async (options?: TestStorageOptions) => {
-      try {
-        if (!options?.doNotClear) {
-          await migrate(framework.migrations.Direction.Up);
-        }
-
-        return new PostgresReportStorageFactory({
-          config: TEST_CONNECTION_OPTIONS
-        });
-      } catch (ex) {
-        // Vitest does not display these errors nicely when using the `await using` syntx
-        console.error(ex, ex.cause);
-        throw ex;
-      }
-    },
     factory: async (options?: TestStorageOptions) => {
       try {
         if (!options?.doNotClear) {
@@ -82,8 +66,8 @@ export function postgresTestSetup(factoryOptions: PostgresTestStorageOptions) {
     },
     migrate
   };
-}
+};
 
-export function postgresTestStorageFactoryGenerator(factoryOptions: PostgresTestStorageOptions) {
+export const PostgresTestStorageFactoryGenerator = (factoryOptions: PostgresTestStorageOptions) => {
   return postgresTestSetup(factoryOptions).factory;
-}
+};

package/src/storage/batch/PostgresBucketBatch.ts

@@ -687,7 +687,8 @@ export class PostgresBucketBatch
     // We store bytea colums for source keys
     const beforeId = operation.beforeId;
     const afterId = operation.afterId;
-    let after = record.after;
+    let sourceAfter = record.after;
+    let after = sourceAfter && this.sync_rules.applyRowContext(sourceAfter);
     const sourceTable = record.sourceTable;
 
     let existingBuckets: CurrentBucket[] = [];
@@ -825,7 +826,8 @@ export class PostgresBucketBatch
       if (sourceTable.syncData) {
         const { results: evaluated, errors: syncErrors } = this.sync_rules.evaluateRowWithErrors({
           record: after,
-          sourceTable
+          sourceTable,
+          bucketIdTransformer: sync_rules.SqlSyncRules.versionedBucketIdTransformer(`${this.group_id}`)
         });
 
         for (const error of syncErrors) {

package/src/storage/storage-index.ts

@@ -2,3 +2,4 @@ export * from './PostgresBucketStorageFactory.js';
 export * from './PostgresCompactor.js';
 export * from './PostgresStorageProvider.js';
 export * from './PostgresSyncRulesStorage.js';
+export * from './PostgresTestStorageFactoryGenerator.js';

package/src/types/models/models-index.ts

@@ -8,4 +8,3 @@ export * from './Migration.js';
 export * from './SourceTable.js';
 export * from './SyncRules.js';
 export * from './WriteCheckpoint.js';
-export * from './SdkReporting.js';

package/src/utils/utils-index.ts

@@ -2,4 +2,3 @@ export * from './bson.js';
 export * from './bucket-data.js';
 export * from './db.js';
 export * from './ts-codec.js';
-export * as test_utils from './test-utils.js';

package/test/src/__snapshots__/storage_sync.test.ts.snap

@@ -104,6 +104,116 @@ exports[`sync - postgres > compacting data - invalidate checkpoint 2`] = `
 ]
 `;
 
+exports[`sync - postgres > encodes sync rules id in buckes for streams 1`] = `
+[
+  {
+    "checkpoint": {
+      "buckets": [
+        {
+          "bucket": "1#test|0[]",
+          "checksum": 920318466,
+          "count": 1,
+          "priority": 3,
+          "subscriptions": [
+            {
+              "default": 0,
+            },
+          ],
+        },
+      ],
+      "last_op_id": "1",
+      "streams": [
+        {
+          "errors": [],
+          "is_default": true,
+          "name": "test",
+        },
+      ],
+      "write_checkpoint": undefined,
+    },
+  },
+  {
+    "data": {
+      "after": "0",
+      "bucket": "1#test|0[]",
+      "data": [
+        {
+          "checksum": 920318466,
+          "data": "{"id":"t1","description":"Test 1"}",
+          "object_id": "t1",
+          "object_type": "test",
+          "op": "PUT",
+          "op_id": "1",
+          "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1",
+        },
+      ],
+      "has_more": false,
+      "next_after": "1",
+    },
+  },
+  {
+    "checkpoint_complete": {
+      "last_op_id": "1",
+    },
+  },
+]
+`;
+
+exports[`sync - postgres > encodes sync rules id in buckes for streams 2`] = `
+[
+  {
+    "checkpoint": {
+      "buckets": [
+        {
+          "bucket": "2#test|0[]",
+          "checksum": 920318466,
+          "count": 1,
+          "priority": 3,
+          "subscriptions": [
+            {
+              "default": 0,
+            },
+          ],
+        },
+      ],
+      "last_op_id": "2",
+      "streams": [
+        {
+          "errors": [],
+          "is_default": true,
+          "name": "test",
+        },
+      ],
+      "write_checkpoint": undefined,
+    },
+  },
+  {
+    "data": {
+      "after": "0",
+      "bucket": "2#test|0[]",
+      "data": [
+        {
+          "checksum": 920318466,
+          "data": "{"id":"t1","description":"Test 1"}",
+          "object_id": "t1",
+          "object_type": "test",
+          "op": "PUT",
+          "op_id": "2",
+          "subkey": "02d285ac-4f96-5124-8fba-c6d1df992dd1",
+        },
+      ],
+      "has_more": false,
+      "next_after": "2",
+    },
+  },
+  {
+    "checkpoint_complete": {
+      "last_op_id": "2",
+    },
+  },
+]
+`;
+
 exports[`sync - postgres > expired token 1`] = `
 [
   {

package/test/src/util.ts

@@ -1,8 +1,12 @@
 import path from 'path';
 import { fileURLToPath } from 'url';
-import { normalizePostgresStorageConfig, PostgresMigrationAgent } from '../../src/index.js';
+import { normalizePostgresStorageConfig } from '../../src//types/types.js';
+import { PostgresMigrationAgent } from '../../src/migrations/PostgresMigrationAgent.js';
+import {
+  postgresTestSetup,
+  PostgresTestStorageFactoryGenerator
+} from '../../src/storage/PostgresTestStorageFactoryGenerator.js';
 import { env } from './env.js';
-import { postgresTestSetup } from '../../src/utils/test-utils.js';
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
@@ -33,4 +37,3 @@ export const POSTGRES_STORAGE_SETUP = postgresTestSetup({
 });
 
 export const POSTGRES_STORAGE_FACTORY = POSTGRES_STORAGE_SETUP.factory;
-export const POSTGRES_REPORT_STORAGE_FACTORY = POSTGRES_STORAGE_SETUP.reportFactory;

package/dist/@types/migrations/scripts/1756282360128-connection-reporting.d.ts

@@ -1,3 +0,0 @@
-import { migrations } from '@powersync/service-core';
-export declare const up: migrations.PowerSyncMigrationFunction;
-export declare const down: migrations.PowerSyncMigrationFunction;

package/dist/@types/storage/PostgresReportStorageFactory.d.ts

@@ -1,24 +0,0 @@
-import { storage } from '@powersync/service-core';
-import * as pg_wire from '@powersync/service-jpgwire';
-import { event_types } from '@powersync/service-types';
-import * as lib_postgres from '@powersync/lib-service-postgres';
-import { NormalizedPostgresStorageConfig } from '../types/types.js';
-export type PostgresReportStorageOptions = {
-    config: NormalizedPostgresStorageConfig;
-};
-export declare class PostgresReportStorageFactory implements storage.ReportStorage {
-    protected options: PostgresReportStorageOptions;
-    readonly db: lib_postgres.DatabaseClient;
-    constructor(options: PostgresReportStorageOptions);
-    private parseJsDate;
-    private mapListCurrentConnectionsResponse;
-    private listConnectionsQuery;
-    private updateTableFilter;
-    reportClientConnection(data: event_types.ClientConnectionBucketData): Promise<void>;
-    reportClientDisconnection(data: event_types.ClientDisconnectionEventData): Promise<void>;
-    getConnectedClients(): Promise<event_types.ClientConnectionReportResponse>;
-    getClientConnectionReports(data: event_types.ClientConnectionReportRequest): Promise<event_types.ClientConnectionReportResponse>;
-    deleteOldConnectionData(data: event_types.DeleteOldConnectionData): Promise<void>;
-    [Symbol.asyncDispose](): Promise<void>;
-    prepareStatements(connection: pg_wire.PgConnection): Promise<void>;
-}

package/dist/@types/types/models/SdkReporting.d.ts

@@ -1,21 +0,0 @@
-import * as t from 'ts-codec';
-export declare const Sdks: t.ObjectCodec<{
-    sdk: t.IdentityCodec<t.CodecType.String>;
-    clients: t.IdentityCodec<t.CodecType.Number>;
-    users: t.IdentityCodec<t.CodecType.Number>;
-}>;
-export type Sdks = t.Encoded<typeof Sdks>;
-export declare const SdkReporting: t.ObjectCodec<{
-    users: t.Codec<bigint, string | number, string, t.CodecProps>;
-    sdks: t.Union<t.Codec<{
-        data: {
-            sdk: string;
-            clients: number;
-            users: number;
-        }[];
-    } | undefined, {
-        data: string;
-    } | undefined, string, t.CodecProps>, t.Codec<null, null, t.CodecType.Null, t.CodecProps>>;
-}>;
-export type SdkReporting = t.Encoded<typeof SdkReporting>;
-export type SdkReportingDecoded = t.Decoded<typeof SdkReporting>;

package/dist/migrations/scripts/1756282360128-connection-reporting.js

@@ -1,107 +0,0 @@
-var __addDisposableResource = (this && this.__addDisposableResource) || function (env, value, async) {
-    if (value !== null && value !== void 0) {
-        if (typeof value !== "object" && typeof value !== "function") throw new TypeError("Object expected.");
-        var dispose, inner;
-        if (async) {
-            if (!Symbol.asyncDispose) throw new TypeError("Symbol.asyncDispose is not defined.");
-            dispose = value[Symbol.asyncDispose];
-        }
-        if (dispose === void 0) {
-            if (!Symbol.dispose) throw new TypeError("Symbol.dispose is not defined.");
-            dispose = value[Symbol.dispose];
-            if (async) inner = dispose;
-        }
-        if (typeof dispose !== "function") throw new TypeError("Object not disposable.");
-        if (inner) dispose = function() { try { inner.call(this); } catch (e) { return Promise.reject(e); } };
-        env.stack.push({ value: value, dispose: dispose, async: async });
-    }
-    else if (async) {
-        env.stack.push({ async: true });
-    }
-    return value;
-};
-var __disposeResources = (this && this.__disposeResources) || (function (SuppressedError) {
-    return function (env) {
-        function fail(e) {
-            env.error = env.hasError ? new SuppressedError(e, env.error, "An error was suppressed during disposal.") : e;
-            env.hasError = true;
-        }
-        var r, s = 0;
-        function next() {
-            while (r = env.stack.pop()) {
-                try {
-                    if (!r.async && s === 1) return s = 0, env.stack.push(r), Promise.resolve().then(next);
-                    if (r.dispose) {
-                        var result = r.dispose.call(r.value);
-                        if (r.async) return s |= 2, Promise.resolve(result).then(next, function(e) { fail(e); return next(); });
-                    }
-                    else s |= 1;
-                }
-                catch (e) {
-                    fail(e);
-                }
-            }
-            if (s === 1) return env.hasError ? Promise.reject(env.error) : Promise.resolve();
-            if (env.hasError) throw env.error;
-        }
-        return next();
-    };
-})(typeof SuppressedError === "function" ? SuppressedError : function (error, suppressed, message) {
-    var e = new Error(message);
-    return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
-});
-import { openMigrationDB } from '../migration-utils.js';
-export const up = async (context) => {
-    const env_1 = { stack: [], error: void 0, hasError: false };
-    try {
-        const { service_context: { configuration } } = context;
-        const client = __addDisposableResource(env_1, openMigrationDB(configuration.storage), true);
-        await client.transaction(async (db) => {
-            await db.sql `
-      CREATE TABLE IF NOT EXISTS connection_report_events (
-        id TEXT PRIMARY KEY,
-        user_agent TEXT NOT NULL,
-        client_id TEXT NOT NULL,
-        user_id TEXT NOT NULL,
-        sdk TEXT NOT NULL,
-        jwt_exp TIMESTAMP WITH TIME ZONE,
-        connected_at TIMESTAMP WITH TIME ZONE NOT NULL,
-        disconnected_at TIMESTAMP WITH TIME ZONE
-      )
-    `.execute();
-            await db.sql `
-      CREATE INDEX IF NOT EXISTS sdk_list_index ON connection_report_events (connected_at, jwt_exp, disconnected_at)
-    `.execute();
-            await db.sql `CREATE INDEX IF NOT EXISTS sdk_user_id_index ON connection_report_events (user_id)`.execute();
-            await db.sql `CREATE INDEX IF NOT EXISTS sdk_client_id_index ON connection_report_events (client_id)`.execute();
-            await db.sql `CREATE INDEX IF NOT EXISTS sdk_index ON connection_report_events (sdk)`.execute();
-        });
-    }
-    catch (e_1) {
-        env_1.error = e_1;
-        env_1.hasError = true;
-    }
-    finally {
-        const result_1 = __disposeResources(env_1);
-        if (result_1)
-            await result_1;
-    }
-};
-export const down = async (context) => {
-    const env_2 = { stack: [], error: void 0, hasError: false };
-    try {
-        const { service_context: { configuration } } = context;
-        const client = __addDisposableResource(env_2, openMigrationDB(configuration.storage), true);
-        await client.sql `DROP TABLE IF EXISTS connection_report_events`.execute();
-    }
-    catch (e_2) {
-        env_2.error = e_2;
-        env_2.hasError = true;
-    }
-    finally {
-        const result_2 = __disposeResources(env_2);
-        if (result_2)
-            await result_2;
-    }
-};
-//# sourceMappingURL=1756282360128-connection-reporting.js.map

package/dist/migrations/scripts/1756282360128-connection-reporting.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"1756282360128-connection-reporting.js","sourceRoot":"","sources":["../../../src/migrations/scripts/1756282360128-connection-reporting.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AACA,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAExD,MAAM,CAAC,MAAM,EAAE,GAA0C,KAAK,EAAE,OAAO,EAAE,EAAE;;;QACzE,MAAM,EACJ,eAAe,EAAE,EAAE,aAAa,EAAE,EACnC,GAAG,OAAO,CAAC;QACZ,MAAY,MAAM,kCAAG,eAAe,CAAC,aAAa,CAAC,OAAO,CAAC,OAAA,CAAC;QAC5D,MAAM,MAAM,CAAC,WAAW,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE;YACpC,MAAM,EAAE,CAAC,GAAG,CAAA;;;;;;;;;;;KAWX,CAAC,OAAO,EAAE,CAAC;YAEZ,MAAM,EAAE,CAAC,GAAG,CAAA;;KAEX,CAAC,OAAO,EAAE,CAAC;YAEZ,MAAM,EAAE,CAAC,GAAG,CAAA,oFAAoF,CAAC,OAAO,EAAE,CAAC;YAE3G,MAAM,EAAE,CAAC,GAAG,CAAA,wFAAwF,CAAC,OAAO,EAAE,CAAC;YAE/G,MAAM,EAAE,CAAC,GAAG,CAAA,wEAAwE,CAAC,OAAO,EAAE,CAAC;QACjG,CAAC,CAAC,CAAC;;;;;;;;;;;CACJ,CAAC;AAEF,MAAM,CAAC,MAAM,IAAI,GAA0C,KAAK,EAAE,OAAO,EAAE,EAAE;;;QAC3E,MAAM,EACJ,eAAe,EAAE,EAAE,aAAa,EAAE,EACnC,GAAG,OAAO,CAAC;QACZ,MAAY,MAAM,kCAAG,eAAe,CAAC,aAAa,CAAC,OAAO,CAAC,OAAA,CAAC;QAC5D,MAAM,MAAM,CAAC,GAAG,CAAA,+CAA+C,CAAC,OAAO,EAAE,CAAC;;;;;;;;;;;CAC3E,CAAC"}