@powersync/service-module-postgres-storage 0.16.3 → 0.18.0

package/dist/types/models/WriteCheckpoint.js

@@ -1,13 +1,16 @@
+import { framework } from '@powersync/service-core';
 import * as t from 'ts-codec';
 import { bigint, jsonb } from '../codecs.js';
 export const WriteCheckpoint = t.object({
     user_id: t.string,
     lsns: jsonb(t.record(t.string)),
-    write_checkpoint: bigint
+    write_checkpoint: bigint,
+    checkpoint_requested_at: t.Null.or(framework.codecs.date)
 });
 export const CustomWriteCheckpoint = t.object({
     user_id: t.string,
     write_checkpoint: bigint,
-    sync_rules_id: bigint
+    sync_rules_id: bigint,
+    checkpoint_requested_at: t.Null.or(framework.codecs.date)
 });
 //# sourceMappingURL=WriteCheckpoint.js.map

package/dist/types/models/WriteCheckpoint.js.map

@@ -1 +1 @@
-{"version":3,"file":"WriteCheckpoint.js","sourceRoot":"","sources":["../../../src/types/models/WriteCheckpoint.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC;AAC9B,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,cAAc,CAAC;AAE7C,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC;IACtC,OAAO,EAAE,CAAC,CAAC,MAAM;IACjB,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IAC/B,gBAAgB,EAAE,MAAM;CACzB,CAAC,CAAC;AAKH,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5C,OAAO,EAAE,CAAC,CAAC,MAAM;IACjB,gBAAgB,EAAE,MAAM;IACxB,aAAa,EAAE,MAAM;CACtB,CAAC,CAAC"}
+{"version":3,"file":"WriteCheckpoint.js","sourceRoot":"","sources":["../../../src/types/models/WriteCheckpoint.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AACpD,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC;AAC9B,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,cAAc,CAAC;AAE7C,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC;IACtC,OAAO,EAAE,CAAC,CAAC,MAAM;IACjB,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IAC/B,gBAAgB,EAAE,MAAM;IACxB,uBAAuB,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC;CAC1D,CAAC,CAAC;AAKH,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5C,OAAO,EAAE,CAAC,CAAC,MAAM;IACjB,gBAAgB,EAAE,MAAM;IACxB,aAAa,EAAE,MAAM;IACrB,uBAAuB,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC;CAC1D,CAAC,CAAC"}

package/dist/utils/checkpoints.d.ts

@@ -0,0 +1,9 @@
+import * as lib_postgres from '@powersync/lib-service-postgres';
+import { models } from '../types/types.js';
+/**
+ * Gets the latest active checkpoint document.
+ * This is mainly exported for mocking in tests.
+ */
+export declare function getActiveCheckpointDocument({ db }: {
+    db: lib_postgres.DatabaseClient;
+}): Promise<models.ActiveCheckpointDecoded | null>;

package/dist/utils/checkpoints.js

@@ -0,0 +1,26 @@
+import { storage } from '@powersync/service-core';
+import { models } from '../types/types.js';
+/**
+ * Gets the latest active checkpoint document.
+ * This is mainly exported for mocking in tests.
+ */
+export async function getActiveCheckpointDocument({ db }) {
+    return db.sql `
+    SELECT
+      id,
+      last_checkpoint,
+      last_checkpoint_lsn
+    FROM
+      sync_rules
+    WHERE
+      state = ${{ value: storage.SyncRuleState.ACTIVE, type: 'varchar' }}
+      OR state = ${{ value: storage.SyncRuleState.ERRORED, type: 'varchar' }}
+    ORDER BY
+      id DESC
+    LIMIT
+      1
+  `
+        .decoded(models.ActiveCheckpoint)
+        .first();
+}
+//# sourceMappingURL=checkpoints.js.map

package/dist/utils/checkpoints.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"checkpoints.js","sourceRoot":"","sources":["../../src/utils/checkpoints.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAClD,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAE3C;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAAC,EAChD,EAAE,EAGH;IACC,OAAO,EAAE,CAAC,GAAG,CAAA;;;;;;;;gBAQC,EAAE,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE;mBACrD,EAAE,KAAK,EAAE,OAAO,CAAC,aAAa,CAAC,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE;;;;;GAKzE;SACE,OAAO,CAAC,MAAM,CAAC,gBAAgB,CAAC;SAChC,KAAK,EAAE,CAAC;AACb,CAAC"}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@powersync/service-module-postgres-storage",
   "repository": "https://github.com/powersync-ja/powersync-service",
-  "version": "0.16.3",
+  "version": "0.18.0",
   "license": "FSL-1.1-ALv2",
   "type": "module",
   "publishConfig": {
@@ -16,17 +16,17 @@
     "lru-cache": "^10.2.2",
     "ts-codec": "^1.3.0",
     "uuid": "^14.0.0",
-    "@powersync/lib-service-postgres": "0.5.2",
-    "@powersync/lib-services-framework": "0.9.8",
-    "@powersync/service-core": "1.23.3",
-    "@powersync/service-types": "0.16.1",
-    "@powersync/service-jpgwire": "0.21.21",
+    "@powersync/lib-service-postgres": "0.5.3",
+    "@powersync/lib-services-framework": "0.10.0",
+    "@powersync/service-core": "1.25.0",
+    "@powersync/service-types": "0.17.0",
+    "@powersync/service-jpgwire": "0.21.22",
     "@powersync/service-jsonbig": "0.17.13",
-    "@powersync/service-sync-rules": "0.39.0"
+    "@powersync/service-sync-rules": "0.40.0"
   },
   "devDependencies": {
     "typescript": "~6.0.3",
-    "@powersync/service-core-tests": "0.17.2"
+    "@powersync/service-core-tests": "0.18.0"
   },
   "scripts": {
     "build": "tsc -b",

package/src/migrations/scripts/1782950400000-checkpoint-requested-at.ts

@@ -0,0 +1,51 @@
+import { migrations } from '@powersync/service-core';
+
+import { openMigrationDB } from '../migration-utils.js';
+
+export const up: migrations.PowerSyncMigrationFunction = async (context) => {
+  const {
+    service_context: { configuration }
+  } = context;
+  await using client = openMigrationDB(configuration.storage);
+
+  await client.transaction(async (db) => {
+    await db.sql`
+      ALTER TABLE write_checkpoints
+      ADD COLUMN checkpoint_requested_at TIMESTAMP WITH TIME ZONE
+    `.execute();
+    await db.sql`
+      ALTER TABLE custom_write_checkpoints
+      ADD COLUMN checkpoint_requested_at TIMESTAMP WITH TIME ZONE
+    `.execute();
+    await db.sql`
+      CREATE INDEX write_checkpoints_checkpoint_requested_at ON write_checkpoints (checkpoint_requested_at)
+      WHERE
+        checkpoint_requested_at IS NOT NULL
+    `.execute();
+    await db.sql`
+      CREATE INDEX custom_write_checkpoints_checkpoint_requested_at ON custom_write_checkpoints (checkpoint_requested_at)
+      WHERE
+        checkpoint_requested_at IS NOT NULL
+    `.execute();
+  });
+};
+
+export const down: migrations.PowerSyncMigrationFunction = async (context) => {
+  const {
+    service_context: { configuration }
+  } = context;
+  await using client = openMigrationDB(configuration.storage);
+
+  await client.transaction(async (db) => {
+    // Also drops the write_checkpoints_checkpoint_requested_at index.
+    await db.sql`
+      ALTER TABLE write_checkpoints
+      DROP COLUMN checkpoint_requested_at
+    `.execute();
+    // Also drops the custom_write_checkpoints_checkpoint_requested_at index.
+    await db.sql`
+      ALTER TABLE custom_write_checkpoints
+      DROP COLUMN checkpoint_requested_at
+    `.execute();
+  });
+};

package/src/migrations/scripts/1784900000000-source-metadata.ts

@@ -0,0 +1,31 @@
+import { migrations } from '@powersync/service-core';
+
+import { openMigrationDB } from '../migration-utils.js';
+
+export const up: migrations.PowerSyncMigrationFunction = async (context) => {
+  const {
+    service_context: { configuration }
+  } = context;
+  await using client = openMigrationDB(configuration.storage);
+
+  await client.transaction(async (db) => {
+    await db.sql`
+      ALTER TABLE source_tables
+      ADD COLUMN source_metadata JSONB
+    `.execute();
+  });
+};
+
+export const down: migrations.PowerSyncMigrationFunction = async (context) => {
+  const {
+    service_context: { configuration }
+  } = context;
+  await using client = openMigrationDB(configuration.storage);
+
+  await client.transaction(async (db) => {
+    await db.sql`
+      ALTER TABLE source_tables
+      DROP COLUMN source_metadata
+    `.execute();
+  });
+};

package/src/storage/PostgresBucketStorageFactory.ts

@@ -8,7 +8,6 @@ import { models, NormalizedPostgresStorageConfig } from '../types/types.js';
 
 import { getStorageApplicationName } from '../utils/application-name.js';
 import { NOTIFICATION_CHANNEL, STORAGE_SCHEMA_NAME } from '../utils/db.js';
-import { notifySyncRulesUpdate } from './batch/PostgresBucketBatch.js';
 import { PostgresSyncRulesStorage } from './PostgresSyncRulesStorage.js';
 import { PostgresPersistedReplicationStream } from './sync-rules/PostgresPersistedSyncConfigContent.js';
 
@@ -220,8 +219,6 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
         .decoded(models.SyncRules)
         .first();
 
-      await notifySyncRulesUpdate(this.db, newSyncRulesRow!);
-
       return new PostgresPersistedReplicationStream(this.db, newSyncRulesRow!);
     });
   }

package/src/storage/PostgresCompactor.ts

@@ -53,6 +53,7 @@ export class PostgresCompactor {
   private clearBatchLimit: number;
   private maxOpId: InternalOpId;
   private buckets: string[] | undefined;
+  private deleteCheckpointRequestsBefore: Date | undefined;
 
   constructor(
     private db: lib_postgres.DatabaseClient,
@@ -65,6 +66,7 @@ export class PostgresCompactor {
     this.clearBatchLimit = options.clearBatchLimit ?? DEFAULT_CLEAR_BATCH_LIMIT;
     this.maxOpId = options.maxOpId ?? 0n;
     this.buckets = options.compactBuckets;
+    this.deleteCheckpointRequestsBefore = options.deleteCheckpointRequestsBefore;
   }
 
   /**
@@ -73,6 +75,8 @@ export class PostgresCompactor {
    * See /docs/storage/compacting-operations.md for details.
    */
   async compact() {
+    await this.deleteOldCheckpointRequests();
+
     if (this.buckets) {
       for (let bucket of this.buckets) {
         await this.compactSingleBucket(bucket);
@@ -82,6 +86,26 @@ export class PostgresCompactor {
     }
   }
 
+  private async deleteOldCheckpointRequests() {
+    if (this.deleteCheckpointRequestsBefore == null) {
+      return;
+    }
+
+    await this.db.sql`
+      DELETE FROM write_checkpoints
+      WHERE
+        checkpoint_requested_at IS NOT NULL
+        AND checkpoint_requested_at < ${{ type: 1184, value: this.deleteCheckpointRequestsBefore.toISOString() }}
+    `.execute();
+
+    await this.db.sql`
+      DELETE FROM custom_write_checkpoints
+      WHERE
+        checkpoint_requested_at IS NOT NULL
+        AND checkpoint_requested_at < ${{ type: 1184, value: this.deleteCheckpointRequestsBefore.toISOString() }}
+    `.execute();
+  }
+
   private async compactAllBuckets() {
     const DISCOVERY_BATCH_SIZE = 200;
     let lastBucket = '';