@powersync/service-core 0.8.7 → 0.9.0

package/src/storage/mongo/models.ts

@@ -1,5 +1,14 @@
-import * as bson from 'bson';
 import { SqliteJsonValue } from '@powersync/service-sync-rules';
+import * as bson from 'bson';
+
+/**
+ * Replica id uniquely identifying a row on the source database.
+ *
+ * Can be any value serializable to BSON.
+ *
+ * If the value is an entire document, the data serialized to a v5 UUID may be a good choice here.
+ */
+export type ReplicaId = bson.UUID | bson.Document | any;
 
 export interface SourceKey {
   /** group_id */
@@ -7,7 +16,7 @@ export interface SourceKey {
   /** source table id */
   t: bson.ObjectId;
   /** source key */
-  k: bson.UUID;
+  k: ReplicaId;
 }
 
 export interface BucketDataKey {
@@ -43,7 +52,7 @@ export interface BucketDataDocument {
   _id: BucketDataKey;
   op: OpType;
   source_table?: bson.ObjectId;
-  source_key?: bson.UUID;
+  source_key?: ReplicaId;
   table?: string;
   row_id?: string;
   checksum: number;
@@ -57,11 +66,11 @@ export interface SourceTableDocument {
   _id: bson.ObjectId;
   group_id: number;
   connection_id: number;
-  relation_id: number | undefined;
+  relation_id: number | string | undefined;
   schema_name: string;
   table_name: string;
   replica_id_columns: string[] | null;
-  replica_id_columns2: { name: string; type_oid: number }[] | undefined;
+  replica_id_columns2: { name: string; type_oid?: number; type?: string }[] | undefined;
   snapshot_done: boolean | undefined;
 }
 
@@ -150,6 +159,13 @@ export interface SyncRuleDocument {
   content: string;
 }
 
+export interface CustomWriteCheckpointDocument {
+  _id: bson.ObjectId;
+  user_id: string;
+  checkpoint: bigint;
+  sync_rules_id: number;
+}
+
 export interface WriteCheckpointDocument {
   _id: bson.ObjectId;
   user_id: string;

package/src/storage/mongo/util.ts

@@ -2,9 +2,10 @@ import { SqliteJsonValue } from '@powersync/service-sync-rules';
 import * as bson from 'bson';
 import * as crypto from 'crypto';
 import * as mongo from 'mongodb';
+import * as uuid from 'uuid';
 import { OplogEntry } from '../../util/protocol-types.js';
-import { timestampToOpId } from '../../util/utils.js';
-import { BucketDataDocument } from './models.js';
+import { ID_NAMESPACE, timestampToOpId } from '../../util/utils.js';
+import { BucketDataDocument, ReplicaId } from './models.js';
 
 /**
  * Lookup serialization must be number-agnostic. I.e. normalize numbers, instead of preserving numbers.
@@ -98,7 +99,7 @@ export function mapOpEntry(row: BucketDataDocument): OplogEntry {
       object_type: row.table,
       object_id: row.row_id,
       checksum: Number(row.checksum),
-      subkey: `${row.source_table}/${row.source_key!.toHexString()}`,
+      subkey: replicaIdToSubkey(row.source_table!, row.source_key!),
       data: row.data
     };
   } else {
@@ -111,3 +112,47 @@ export function mapOpEntry(row: BucketDataDocument): OplogEntry {
     };
   }
 }
+
+/**
+ * Returns true if two ReplicaId values are the same (serializes to the same BSON value).
+ */
+export function replicaIdEquals(a: ReplicaId, b: ReplicaId) {
+  if (a === b) {
+    return true;
+  } else if (typeof a == 'string' && typeof b == 'string') {
+    return a == b;
+  } else if (isUUID(a) && isUUID(b)) {
+    return a.equals(b);
+  } else if (a == null && b == null) {
+    return true;
+  } else if (a != null || b != null) {
+    return false;
+  } else {
+    // There are many possible primitive values, this covers them all
+    return (bson.serialize({ id: a }) as Buffer).equals(bson.serialize({ id: b }));
+  }
+}
+
+export function replicaIdToSubkey(table: bson.ObjectId, id: ReplicaId): string {
+  if (isUUID(id)) {
+    // Special case for UUID for backwards-compatiblity
+    return `${table.toHexString()}/${id.toHexString()}`;
+  } else {
+    // Hashed UUID from the table and id
+    const repr = bson.serialize({ table, id });
+    return uuid.v5(repr, ID_NAMESPACE);
+  }
+}
+
+/**
+ * True if this is a bson.UUID.
+ *
+ * Works even with multiple copies of the bson package.
+ */
+export function isUUID(value: any): value is bson.UUID {
+  if (value == null || typeof value != 'object') {
+    return false;
+  }
+  const uuid = value as bson.UUID;
+  return uuid._bsontype == 'Binary' && uuid.sub_type == bson.Binary.SUBTYPE_UUID;
+}

package/src/storage/storage-index.ts

@@ -1,15 +1,21 @@
-export * from './SourceTable.js';
-export * from './MongoBucketStorage.js';
 export * from './BucketStorage.js';
+export * from './MongoBucketStorage.js';
+export * from './ReplicationEventPayload.js';
+export * from './SourceEntity.js';
+export * from './SourceTable.js';
+export * from './StorageEngine.js';
 
+export * from './mongo/config.js';
 export * from './mongo/db.js';
 export * from './mongo/models.js';
 export * from './mongo/MongoBucketBatch.js';
 export * from './mongo/MongoIdSequence.js';
 export * from './mongo/MongoPersistedSyncRules.js';
 export * from './mongo/MongoPersistedSyncRulesContent.js';
+export * from './mongo/MongoStorageProvider.js';
 export * from './mongo/MongoSyncBucketStorage.js';
 export * from './mongo/MongoSyncRulesLock.js';
 export * from './mongo/OperationBatch.js';
 export * from './mongo/PersistedBatch.js';
 export * from './mongo/util.js';
+export * from './WriteCheckpointAPI.js';

package/src/sync/sync.ts

@@ -1,6 +1,7 @@
 import { JSONBig, JsonContainer } from '@powersync/service-jsonbig';
 import { RequestParameters } from '@powersync/service-sync-rules';
 import { Semaphore, withTimeout } from 'async-mutex';
+
 import { AbortError } from 'ix/aborterror.js';
 
 import * as auth from '../auth/auth-index.js';
@@ -35,6 +36,7 @@ export interface SyncStreamParameters {
   params: util.StreamingSyncRequest;
   syncParams: RequestParameters;
   token: auth.JwtPayload;
+  parseOptions: storage.ParseSyncRulesOptions;
   /**
    * If this signal is aborted, the stream response ends as soon as possible, without error.
    */
@@ -47,7 +49,7 @@ export interface SyncStreamParameters {
 export async function* streamResponse(
   options: SyncStreamParameters
 ): AsyncIterable<util.StreamingSyncLine | string | null> {
-  const { storage, params, syncParams, token, tokenStreamOptions, tracker, signal } = options;
+  const { storage, params, syncParams, token, tokenStreamOptions, tracker, signal, parseOptions } = options;
   // We also need to be able to abort, so we create our own controller.
   const controller = new AbortController();
   if (signal) {
@@ -63,7 +65,7 @@ export async function* streamResponse(
     }
   }
   const ki = tokenStream(token, controller.signal, tokenStreamOptions);
-  const stream = streamResponseInner(storage, params, syncParams, tracker, controller.signal);
+  const stream = streamResponseInner(storage, params, syncParams, tracker, parseOptions, controller.signal);
   // Merge the two streams, and abort as soon as one of the streams end.
   const merged = mergeAsyncIterables([stream, ki], controller.signal);
 
@@ -87,6 +89,7 @@ async function* streamResponseInner(
   params: util.StreamingSyncRequest,
   syncParams: RequestParameters,
   tracker: RequestTracker,
+  parseOptions: storage.ParseSyncRulesOptions,
   signal: AbortSignal
 ): AsyncGenerator<util.StreamingSyncLine | string | null> {
   // Bucket state of bucket id -> op_id.
@@ -115,9 +118,9 @@ async function* streamResponseInner(
       // Sync rules deleted in the meantime - try again with the next checkpoint.
       continue;
     }
-    const sync_rules = storage.sync_rules;
+    const syncRules = storage.getParsedSyncRules(parseOptions);
 
-    const allBuckets = await sync_rules.queryBucketIds({
+    const allBuckets = await syncRules.queryBucketIds({
       getParameterSets(lookups) {
         return storage.getParameterSets(checkpoint, lookups);
       },

package/src/sync/util.ts

@@ -1,7 +1,6 @@
 import * as timers from 'timers/promises';
 
 import * as util from '../util/util-index.js';
-import { Metrics } from '../metrics/Metrics.js';
 import { RequestTracker } from './RequestTracker.js';
 
 export type TokenStreamOptions = {

package/src/system/ServiceContext.ts

@@ -0,0 +1,68 @@
+import { LifeCycledSystem, ServiceIdentifier, container } from '@powersync/lib-services-framework';
+
+import * as metrics from '../metrics/Metrics.js';
+import * as replication from '../replication/replication-index.js';
+import * as routes from '../routes/routes-index.js';
+import * as storage from '../storage/storage-index.js';
+import * as utils from '../util/util-index.js';
+
+export interface ServiceContext {
+  configuration: utils.ResolvedPowerSyncConfig;
+  lifeCycleEngine: LifeCycledSystem;
+  metrics: metrics.Metrics | null;
+  replicationEngine: replication.ReplicationEngine | null;
+  routerEngine: routes.RouterEngine | null;
+  storageEngine: storage.StorageEngine;
+}
+
+/**
+ * Context which allows for registering and getting implementations
+ * of various service engines.
+ * This controls registering, initializing and the lifecycle of various services.
+ */
+export class ServiceContextContainer implements ServiceContext {
+  lifeCycleEngine: LifeCycledSystem;
+  storageEngine: storage.StorageEngine;
+
+  constructor(public configuration: utils.ResolvedPowerSyncConfig) {
+    this.lifeCycleEngine = new LifeCycledSystem();
+
+    this.storageEngine = new storage.StorageEngine({
+      configuration
+    });
+    this.lifeCycleEngine.withLifecycle(this.storageEngine, {
+      start: (storageEngine) => storageEngine.start(),
+      stop: (storageEngine) => storageEngine.shutDown()
+    });
+
+    // Mongo storage is available as an option by default TODO: Consider moving this to a Mongo Storage Module
+    this.storageEngine.registerProvider(new storage.MongoStorageProvider());
+  }
+
+  get replicationEngine(): replication.ReplicationEngine | null {
+    return container.getOptional(replication.ReplicationEngine);
+  }
+
+  get routerEngine(): routes.RouterEngine | null {
+    return container.getOptional(routes.RouterEngine);
+  }
+
+  get metrics(): metrics.Metrics | null {
+    return container.getOptional(metrics.Metrics);
+  }
+
+  /**
+   * Allows for registering core and generic implementations of services/helpers.
+   * This uses the framework container under the hood.
+   */
+  register<T>(identifier: ServiceIdentifier<T>, implementation: T) {
+    container.register(identifier, implementation);
+  }
+
+  /**
+   * Gets the implementation of an identifiable service.
+   */
+  get<T>(identifier: ServiceIdentifier<T>) {
+    return container.getImplementation(identifier);
+  }
+}

package/src/system/system-index.ts

@@ -1 +1 @@
-export * from './CorePowerSyncSystem.js';
+export * from './ServiceContext.js';

package/src/util/config/compound-config-collector.ts

@@ -1,17 +1,15 @@
-import { configFile, normalizeConnection } from '@powersync/service-types';
-import { ConfigCollector } from './collectors/config-collector.js';
-import { ResolvedConnection, ResolvedPowerSyncConfig, RunnerConfig, SyncRulesConfig } from './types.js';
+import { logger } from '@powersync/lib-services-framework';
+import { configFile } from '@powersync/service-types';
 import * as auth from '../../auth/auth-index.js';
-import { SyncRulesCollector } from './sync-rules/sync-collector.js';
+import { ConfigCollector } from './collectors/config-collector.js';
 import { Base64ConfigCollector } from './collectors/impl/base64-config-collector.js';
+import { FallbackConfigCollector } from './collectors/impl/fallback-config-collector.js';
 import { FileSystemConfigCollector } from './collectors/impl/filesystem-config-collector.js';
 import { Base64SyncRulesCollector } from './sync-rules/impl/base64-sync-rules-collector.js';
-import { InlineSyncRulesCollector } from './sync-rules/impl/inline-sync-rules-collector.js';
 import { FileSystemSyncRulesCollector } from './sync-rules/impl/filesystem-sync-rules-collector.js';
-import { FallbackConfigCollector } from './collectors/impl/fallback-config-collector.js';
-import { logger } from '@powersync/lib-services-framework';
-
-const POWERSYNC_DEV_KID = 'powersync-dev';
+import { InlineSyncRulesCollector } from './sync-rules/impl/inline-sync-rules-collector.js';
+import { SyncRulesCollector } from './sync-rules/sync-collector.js';
+import { ResolvedPowerSyncConfig, RunnerConfig, SyncRulesConfig } from './types.js';
 
 export type CompoundConfigCollectorOptions = {
   /**
@@ -28,6 +26,17 @@ export type CompoundConfigCollectorOptions = {
   syncRulesCollectors: SyncRulesCollector[];
 };
 
+export type ConfigCollectedEvent = {
+  base_config: configFile.PowerSyncConfig;
+  resolved_config: ResolvedPowerSyncConfig;
+};
+
+export type ConfigCollectorListener = {
+  configCollected?: (event: ConfigCollectedEvent) => Promise<void>;
+};
+
+const POWERSYNC_DEV_KID = 'powersync-dev';
+
 const DEFAULT_COLLECTOR_OPTIONS: CompoundConfigCollectorOptions = {
   configCollectors: [new Base64ConfigCollector(), new FileSystemConfigCollector(), new FallbackConfigCollector()],
   syncRulesCollectors: [
@@ -43,24 +52,14 @@ export class CompoundConfigCollector {
   /**
    * Collects and resolves base config
    */
-  async collectConfig(runner_config: RunnerConfig = {}): Promise<ResolvedPowerSyncConfig> {
-    const baseConfig = await this.collectBaseConfig(runner_config);
+  async collectConfig(runnerConfig: RunnerConfig = {}): Promise<ResolvedPowerSyncConfig> {
+    const baseConfig = await this.collectBaseConfig(runnerConfig);
 
-    const connections = baseConfig.replication?.connections ?? [];
-    if (connections.length > 1) {
-      throw new Error('Only a single replication connection is supported currently');
+    const dataSources = baseConfig.replication?.connections ?? [];
+    if (dataSources.length > 1) {
+      throw new Error('Only a single replication data source is supported currently');
     }
 
-    const mapped = connections.map((c) => {
-      const conf: ResolvedConnection = {
-        type: 'postgresql' as const,
-        ...normalizeConnection(c),
-        debug_api: c.debug_api ?? false
-      };
-
-      return conf;
-    });
-
     const collectors = new auth.CompoundKeyCollector();
     const keyStore = new auth.KeyStore(collectors);
 
@@ -69,10 +68,6 @@ export class CompoundConfigCollector {
 
     collectors.add(staticCollector);
 
-    if (baseConfig.client_auth?.supabase && mapped.length > 0) {
-      collectors.add(new auth.CachedKeyCollector(new auth.SupabaseKeyCollector(mapped[0])));
-    }
-
     let jwks_uris = baseConfig.client_auth?.jwks_uri ?? [];
     if (typeof jwks_uris == 'string') {
       jwks_uris = [jwks_uris];
@@ -93,12 +88,13 @@ export class CompoundConfigCollector {
       devKey = await auth.KeySpec.importKey(baseDevKey);
     }
 
-    const sync_rules = await this.collectSyncRules(baseConfig, runner_config);
+    const sync_rules = await this.collectSyncRules(baseConfig, runnerConfig);
 
     let jwt_audiences: string[] = baseConfig.client_auth?.audience ?? [];
 
     let config: ResolvedPowerSyncConfig = {
-      connection: mapped[0],
+      base_config: baseConfig,
+      connections: baseConfig.replication?.connections || [],
       storage: baseConfig.storage,
       client_keystore: keyStore,
       // Dev tokens only use the static keys, no external key sources
@@ -124,8 +120,12 @@ export class CompoundConfigCollector {
         internal_service_endpoint:
           baseConfig.telemetry?.internal_service_endpoint ?? 'https://pulse.journeyapps.com/v1/metrics'
       },
-      slot_name_prefix: connections[0]?.slot_name_prefix ?? 'powersync_'
+      // TODO maybe move this out of the connection or something
+      // slot_name_prefix: connections[0]?.slot_name_prefix ?? 'powersync_'
+      slot_name_prefix: 'powersync_',
+      parameters: baseConfig.parameters ?? {}
     };
+
     return config;
   }
 

package/src/util/config/sync-rules/sync-rules-provider.ts

@@ -0,0 +1,18 @@
+import { SyncRulesConfig } from '../types.js';
+import fs from 'fs/promises';
+
+export interface SyncRulesProvider {
+  get(): Promise<string | undefined>;
+}
+
+export class ConfigurationFileSyncRulesProvider implements SyncRulesProvider {
+  constructor(private config: SyncRulesConfig) {}
+
+  async get(): Promise<string | undefined> {
+    if (this.config.content) {
+      return this.config.content;
+    } else if (this.config.path) {
+      return await fs.readFile(this.config.path, 'utf-8');
+    }
+  }
+}

package/src/util/config/types.ts

@@ -1,4 +1,6 @@
-import { NormalizedPostgresConnection, configFile } from '@powersync/service-types';
+import { configFile } from '@powersync/service-types';
+import { PowerSyncConfig } from '@powersync/service-types/src/config/PowerSyncConfig.js';
+import { CompoundKeyCollector } from '../../auth/CompoundKeyCollector.js';
 import { KeySpec } from '../../auth/KeySpec.js';
 import { KeyStore } from '../../auth/KeyStore.js';
 
@@ -20,8 +22,6 @@ export type MigrationContext = {
 
 export type Runner = (config: RunnerConfig) => Promise<void>;
 
-export type ResolvedConnection = configFile.PostgresConnection & NormalizedPostgresConnection;
-
 export type SyncRulesConfig = {
   present: boolean;
   content?: string;
@@ -29,7 +29,8 @@ export type SyncRulesConfig = {
 };
 
 export type ResolvedPowerSyncConfig = {
-  connection?: ResolvedConnection;
+  base_config: PowerSyncConfig;
+  connections?: configFile.DataSourceConfig[];
   storage: configFile.StorageConfig;
   dev: {
     demo_auth: boolean;
@@ -41,7 +42,7 @@ export type ResolvedPowerSyncConfig = {
      */
     dev_key?: KeySpec;
   };
-  client_keystore: KeyStore;
+  client_keystore: KeyStore<CompoundKeyCollector>;
   /**
    * Keystore for development tokens.
    */
@@ -63,4 +64,5 @@ export type ResolvedPowerSyncConfig = {
 
   /** Prefix for postgres replication slot names. May eventually be connection-specific. */
   slot_name_prefix: string;
+  parameters: Record<string, number | string | boolean | null>;
 };

package/src/util/config.ts

@@ -1,31 +1,14 @@
 import * as fs from 'fs/promises';
-import { baseUri } from '@powersync/service-types';
 
-import { ResolvedConnection, ResolvedPowerSyncConfig, RunnerConfig } from './config/types.js';
-import { CompoundConfigCollector } from './config/compound-config-collector.js';
+import { container } from '@powersync/lib-services-framework';
+import { ResolvedPowerSyncConfig, RunnerConfig } from './config/types.js';
+import { CompoundConfigCollector } from './util-index.js';
 
 /**
- * Build a single URI from full postgres credentials.
+ * Loads the resolved config using the registered config collector
  */
-export function buildDemoPgUri(options: ResolvedConnection): string {
-  if (!options.debug_api) {
-    throw new Error('Not supported');
-  }
-
-  const uri = new URL(baseUri(options));
-  uri.username = options.username;
-  uri.password = options.password;
-  if (options.sslmode != 'disable') {
-    // verify-full is tricky to actually use on a client, since they won't have the cert
-    // Just use "require" by default
-    // uri.searchParams.set('sslmode', options.sslmode);
-    uri.searchParams.set('sslmode', 'require');
-  }
-  return uri.toString();
-}
-
-export function loadConfig(runnerConfig: RunnerConfig = {}) {
-  const collector = new CompoundConfigCollector();
+export async function loadConfig(runnerConfig: RunnerConfig) {
+  const collector = container.getImplementation(CompoundConfigCollector);
   return collector.collectConfig(runnerConfig);
 }
 

package/src/util/util-index.ts

@@ -1,25 +1,22 @@
 export * from './alerting.js';
 export * from './env.js';
 export * from './memory-tracking.js';
-export * from './migration_lib.js';
 export * from './Mutex.js';
-export * from './PgManager.js';
-export * from './pgwire_utils.js';
-export * from './populate_test_data.js';
 export * from './protocol-types.js';
 export * from './secs.js';
 export * from './utils.js';
 
 export * from './config.js';
-export * from './config/types.js';
 export * from './config/compound-config-collector.js';
+export * from './config/types.js';
 
 export * from './config/collectors/config-collector.js';
 export * from './config/collectors/impl/base64-config-collector.js';
 export * from './config/collectors/impl/fallback-config-collector.js';
 export * from './config/collectors/impl/filesystem-config-collector.js';
 
-export * from './config/sync-rules/sync-collector.js';
 export * from './config/sync-rules/impl/base64-sync-rules-collector.js';
 export * from './config/sync-rules/impl/filesystem-sync-rules-collector.js';
 export * from './config/sync-rules/impl/inline-sync-rules-collector.js';
+export * from './config/sync-rules/sync-collector.js';
+export * from './config/sync-rules/sync-rules-provider.js';

package/src/util/utils.ts

@@ -1,15 +1,21 @@
+import * as sync_rules from '@powersync/service-sync-rules';
+import * as bson from 'bson';
 import crypto from 'crypto';
+import * as uuid from 'uuid';
+import { BucketChecksum, OpId } from './protocol-types.js';
 
-import { logger } from '@powersync/lib-services-framework';
-import * as pgwire from '@powersync/service-jpgwire';
-import { pgwireRows } from '@powersync/service-jpgwire';
-import { PartialChecksum } from '../storage/ChecksumCache.js';
 import * as storage from '../storage/storage-index.js';
-import { retriedQuery } from './pgwire_utils.js';
-import { BucketChecksum, OpId } from './protocol-types.js';
+
+import { PartialChecksum } from '../storage/ChecksumCache.js';
 
 export type ChecksumMap = Map<string, BucketChecksum>;
 
+export const ID_NAMESPACE = 'a396dd91-09fc-4017-a28d-3df722f651e9';
+
+export function escapeIdentifier(identifier: string) {
+  return `"${identifier.replace(/"/g, '""').replace(/\./g, '"."')}"`;
+}
+
 export function hashData(type: string, id: string, data: string): number {
   const hash = crypto.createHash('sha256');
   hash.update(`put.${type}.${id}.${data}`);
@@ -83,49 +89,50 @@ export function addBucketChecksums(a: BucketChecksum, b: PartialChecksum | null)
   }
 }
 
-export async function getClientCheckpoint(
-  db: pgwire.PgClient,
-  bucketStorage: storage.BucketStorageFactory,
-  options?: { timeout?: number }
-): Promise<OpId> {
-  const start = Date.now();
+function getRawReplicaIdentity(
+  tuple: sync_rules.ToastableSqliteRow,
+  columns: storage.ColumnDescriptor[]
+): Record<string, any> {
+  let result: Record<string, any> = {};
+  for (let column of columns) {
+    const name = column.name;
+    result[name] = tuple[name];
+  }
+  return result;
+}
 
-  const [{ lsn }] = pgwireRows(await db.query(`SELECT pg_logical_emit_message(false, 'powersync', 'ping') as lsn`));
+export function getUuidReplicaIdentityBson(
+  tuple: sync_rules.ToastableSqliteRow,
+  columns: storage.ColumnDescriptor[]
+): bson.UUID {
+  if (columns.length == 0) {
+    // REPLICA IDENTITY NOTHING - generate random id
+    return new bson.UUID(uuid.v4());
+  }
+  const rawIdentity = getRawReplicaIdentity(tuple, columns);
 
-  // This old API needs a persisted checkpoint id.
-  // Since we don't use LSNs anymore, the only way to get that is to wait.
+  return uuidForRowBson(rawIdentity);
+}
 
-  const timeout = options?.timeout ?? 50_000;
+export function uuidForRowBson(row: sync_rules.SqliteRow): bson.UUID {
+  // Important: This must not change, since it will affect how ids are generated.
+  // Use BSON so that it's a well-defined format without encoding ambiguities.
+  const repr = bson.serialize(row);
+  const buffer = Buffer.alloc(16);
+  return new bson.UUID(uuid.v5(repr, ID_NAMESPACE, buffer));
+}
 
-  logger.info(`Waiting for LSN checkpoint: ${lsn}`);
-  while (Date.now() - start < timeout) {
-    const cp = await bucketStorage.getActiveCheckpoint();
-    if (!cp.hasSyncRules()) {
-      throw new Error('No sync rules available');
-    }
-    if (cp.lsn >= lsn) {
-      logger.info(`Got write checkpoint: ${lsn} : ${cp.checkpoint}`);
-      return cp.checkpoint;
+export function hasToastedValues(row: sync_rules.ToastableSqliteRow) {
+  for (let key in row) {
+    if (typeof row[key] == 'undefined') {
+      return true;
     }
-
-    await new Promise((resolve) => setTimeout(resolve, 30));
   }
-
-  throw new Error('Timeout while waiting for checkpoint');
+  return false;
 }
 
-export async function createWriteCheckpoint(
-  db: pgwire.PgClient,
-  bucketStorage: storage.BucketStorageFactory,
-  user_id: string
-): Promise<bigint> {
-  const [{ lsn }] = pgwireRows(
-    await retriedQuery(db, `SELECT pg_logical_emit_message(false, 'powersync', 'ping') as lsn`)
-  );
-
-  const id = await bucketStorage.createWriteCheckpoint(user_id, { '1': lsn });
-  logger.info(`Write checkpoint 2: ${JSON.stringify({ lsn, id: String(id) })}`);
-  return id;
+export function isCompleteRow(row: sync_rules.ToastableSqliteRow): row is sync_rules.SqliteRow {
+  return !hasToastedValues(row);
 }
 
 export function checkpointUserId(user_id: string | undefined, client_id: string | undefined) {