@powersync/service-core 0.13.0 → 0.14.0

package/src/migrations/ensure-automatic-migrations.ts

@@ -0,0 +1,15 @@
+import * as framework from '@powersync/lib-services-framework';
+import * as system from '../system/system-index.js';
+
+export const ensureAutomaticMigrations = async (options: { serviceContext: system.ServiceContext }) => {
+  const { serviceContext } = options;
+  if (serviceContext.configuration.migrations?.disable_auto_migration) {
+    return;
+  }
+  await serviceContext.migrations.migrate({
+    direction: framework.migrations.Direction.Up,
+    migrationContext: {
+      service_context: serviceContext
+    }
+  });
+};

package/src/migrations/migrations-index.ts

@@ -1,3 +1,2 @@
-export * from './definitions.js';
-export * from './executor.js';
-export * from './migrations.js';
+export * from './ensure-automatic-migrations.js';
+export * from './PowerSyncMigrationManager.js';

package/src/storage/BucketStorage.ts

@@ -8,13 +8,49 @@ import {
   SqliteRow,
   ToastableSqliteRow
 } from '@powersync/service-sync-rules';
+import { BSON } from 'bson';
 import * as util from '../util/util-index.js';
 import { ReplicationEventPayload } from './ReplicationEventPayload.js';
 import { SourceEntityDescriptor } from './SourceEntity.js';
 import { SourceTable } from './SourceTable.js';
-import { BatchedCustomWriteCheckpointOptions, ReplicaId } from './storage-index.js';
+import { BatchedCustomWriteCheckpointOptions } from './storage-index.js';
 import { SyncStorageWriteCheckpointAPI } from './WriteCheckpointAPI.js';
 
+/**
+ * Replica id uniquely identifying a row on the source database.
+ *
+ * Can be any value serializable to BSON.
+ *
+ * If the value is an entire document, the data serialized to a v5 UUID may be a good choice here.
+ */
+export type ReplicaId = BSON.UUID | BSON.Document | any;
+
+export enum SyncRuleState {
+  /**
+   * New sync rules - needs to be processed (initial replication).
+   *
+   * While multiple sets of sync rules _can_ be in PROCESSING,
+   * it's generally pointless, so we only keep one in that state.
+   */
+  PROCESSING = 'PROCESSING',
+
+  /**
+   * Sync rule processing is done, and can be used for sync.
+   *
+   * Only one set of sync rules should be in ACTIVE state.
+   */
+  ACTIVE = 'ACTIVE',
+  /**
+   * This state is used when the sync rules has been replaced,
+   * and replication is or should be stopped.
+   */
+  STOP = 'STOP',
+  /**
+   * After sync rules have been stopped, the data needs to be
+   * deleted. Once deleted, the state is TERMINATED.
+   */
+  TERMINATED = 'TERMINATED'
+}
 export interface BucketStorageFactoryListener extends DisposableListener {
   syncStorageCreated: (storage: SyncRulesBucketStorage) => void;
   replicationEvent: (event: ReplicationEventPayload) => void;
@@ -501,3 +537,19 @@ export interface TerminateOptions {
    */
   clearStorage: boolean;
 }
+
+/**
+ * Helper for tests.
+ * This is not in the `service-core-tests` package in order for storage modules
+ * to provide relevant factories without requiring `service-core-tests` as a direct dependency.
+ */
+export interface TestStorageOptions {
+  /**
+   * By default, collections are only cleared/
+   * Setting this to true will drop the collections completely.
+   */
+  dropAll?: boolean;
+
+  doNotClear?: boolean;
+}
+export type TestStorageFactory = (options?: TestStorageOptions) => Promise<BucketStorageFactory>;

package/src/storage/bson.ts

@@ -0,0 +1,78 @@
+import * as bson from 'bson';
+
+import { SqliteJsonValue } from '@powersync/service-sync-rules';
+import { ReplicaId } from './BucketStorage.js';
+
+export const BSON_DESERIALIZE_OPTIONS: bson.DeserializeOptions = {
+  // use bigint instead of Long
+  useBigInt64: true
+};
+
+/**
+ * Lookup serialization must be number-agnostic. I.e. normalize numbers, instead of preserving numbers.
+ * @param lookup
+ */
+export const serializeLookupBuffer = (lookup: SqliteJsonValue[]): Buffer => {
+  const normalized = lookup.map((value) => {
+    if (typeof value == 'number' && Number.isInteger(value)) {
+      return BigInt(value);
+    } else {
+      return value;
+    }
+  });
+  return bson.serialize({ l: normalized }) as Buffer;
+};
+
+export const serializeLookup = (lookup: SqliteJsonValue[]) => {
+  return new bson.Binary(serializeLookupBuffer(lookup));
+};
+
+/**
+ * True if this is a bson.UUID.
+ *
+ * Works even with multiple copies of the bson package.
+ */
+export const isUUID = (value: any): value is bson.UUID => {
+  if (value == null || typeof value != 'object') {
+    return false;
+  }
+  const uuid = value as bson.UUID;
+  return uuid._bsontype == 'Binary' && uuid.sub_type == bson.Binary.SUBTYPE_UUID;
+};
+
+export const serializeReplicaId = (id: ReplicaId): Buffer => {
+  return bson.serialize({ id }) as Buffer;
+};
+
+export const deserializeReplicaId = (id: Buffer): ReplicaId => {
+  const deserialized = deserializeBson(id);
+  return deserialized.id;
+};
+
+export const deserializeBson = (buffer: Buffer) => {
+  return bson.deserialize(buffer, BSON_DESERIALIZE_OPTIONS);
+};
+
+export const serializeBson = (document: any): Buffer => {
+  return bson.serialize(document) as Buffer;
+};
+
+/**
+ * Returns true if two ReplicaId values are the same (serializes to the same BSON value).
+ */
+export const replicaIdEquals = (a: ReplicaId, b: ReplicaId) => {
+  if (a === b) {
+    return true;
+  } else if (typeof a == 'string' && typeof b == 'string') {
+    return a == b;
+  } else if (isUUID(a) && isUUID(b)) {
+    return a.equals(b);
+  } else if (a == null && b == null) {
+    return true;
+  } else if ((b == null && a != null) || (a == null && b != null)) {
+    return false;
+  } else {
+    // There are many possible primitive values, this covers them all
+    return serializeReplicaId(a).equals(serializeReplicaId(b) as ArrayBuffer as Uint8Array);
+  }
+};

package/src/storage/storage-index.ts

@@ -1,21 +1,9 @@
+export * from './bson.js';
 export * from './BucketStorage.js';
-export * from './MongoBucketStorage.js';
+export * from './ChecksumCache.js';
 export * from './ReplicationEventPayload.js';
 export * from './SourceEntity.js';
 export * from './SourceTable.js';
 export * from './StorageEngine.js';
-
-export * from './mongo/config.js';
-export * from './mongo/db.js';
-export * from './mongo/models.js';
-export * from './mongo/MongoBucketBatch.js';
-export * from './mongo/MongoIdSequence.js';
-export * from './mongo/MongoPersistedSyncRules.js';
-export * from './mongo/MongoPersistedSyncRulesContent.js';
-export * from './mongo/MongoStorageProvider.js';
-export * from './mongo/MongoSyncBucketStorage.js';
-export * from './mongo/MongoSyncRulesLock.js';
-export * from './mongo/OperationBatch.js';
-export * from './mongo/PersistedBatch.js';
-export * from './mongo/util.js';
+export * from './StorageProvider.js';
 export * from './WriteCheckpointAPI.js';

package/src/sync/sync.ts

@@ -318,7 +318,9 @@ async function* bucketDataBatch(request: BucketDataRequest): AsyncGenerator<Buck
         // Send the object as is, will most likely be encoded as a BSON document
         send_data = { data: r };
       } else if (raw_data) {
-        // Data is a raw string - we can use the more efficient JSON.stringify.
+        /**
+         * Data is a raw string - we can use the more efficient JSON.stringify.
+         */
         const response: util.StreamingSyncData = {
           data: r
         };

package/src/system/ServiceContext.ts

@@ -1,6 +1,8 @@
-import { LifeCycledSystem, ServiceIdentifier, container } from '@powersync/lib-services-framework';
+import { LifeCycledSystem, MigrationManager, ServiceIdentifier, container } from '@powersync/lib-services-framework';
 
+import { framework } from '../index.js';
 import * as metrics from '../metrics/Metrics.js';
+import { PowerSyncMigrationManager } from '../migrations/PowerSyncMigrationManager.js';
 import * as replication from '../replication/replication-index.js';
 import * as routes from '../routes/routes-index.js';
 import * as storage from '../storage/storage-index.js';
@@ -13,6 +15,7 @@ export interface ServiceContext {
   replicationEngine: replication.ReplicationEngine | null;
   routerEngine: routes.RouterEngine | null;
   storageEngine: storage.StorageEngine;
+  migrations: PowerSyncMigrationManager;
 }
 
 /**
@@ -30,13 +33,19 @@ export class ServiceContextContainer implements ServiceContext {
     this.storageEngine = new storage.StorageEngine({
       configuration
     });
+
+    const migrationManager = new MigrationManager();
+    container.register(framework.ContainerImplementation.MIGRATION_MANAGER, migrationManager);
+
+    this.lifeCycleEngine.withLifecycle(migrationManager, {
+      // Migrations should be executed before the system starts
+      start: () => migrationManager[Symbol.asyncDispose]()
+    });
+
     this.lifeCycleEngine.withLifecycle(this.storageEngine, {
       start: (storageEngine) => storageEngine.start(),
       stop: (storageEngine) => storageEngine.shutDown()
     });
-
-    // Mongo storage is available as an option by default TODO: Consider moving this to a Mongo Storage Module
-    this.storageEngine.registerProvider(new storage.MongoStorageProvider());
   }
 
   get replicationEngine(): replication.ReplicationEngine | null {
@@ -51,6 +60,10 @@ export class ServiceContextContainer implements ServiceContext {
     return container.getOptional(metrics.Metrics);
   }
 
+  get migrations(): PowerSyncMigrationManager {
+    return container.getImplementation(framework.ContainerImplementation.MIGRATION_MANAGER);
+  }
+
   /**
    * Allows for registering core and generic implementations of services/helpers.
    * This uses the framework container under the hood.

package/src/util/config/types.ts

@@ -30,8 +30,8 @@ export type SyncRulesConfig = {
 
 export type ResolvedPowerSyncConfig = {
   base_config: PowerSyncConfig;
-  connections?: configFile.DataSourceConfig[];
-  storage: configFile.StorageConfig;
+  connections?: configFile.GenericDataSourceConfig[];
+  storage: configFile.GenericStorageConfig;
   dev: {
     demo_auth: boolean;
     demo_password?: string;

package/src/util/utils.ts

@@ -131,7 +131,16 @@ export function hasToastedValues(row: sync_rules.ToastableSqliteRow) {
   return false;
 }
 
-export function isCompleteRow(row: sync_rules.ToastableSqliteRow): row is sync_rules.SqliteRow {
+/**
+ * Returns true if we have a complete row.
+ *
+ * If we don't store data, we assume we always have a complete row.
+ */
+export function isCompleteRow(storeData: boolean, row: sync_rules.ToastableSqliteRow): row is sync_rules.SqliteRow {
+  if (!storeData) {
+    // Assume the row is complete - no need to check
+    return true;
+  }
   return !hasToastedValues(row);
 }
 
@@ -199,6 +208,17 @@ export function reduceBucket(operations: OplogEntry[]) {
   return finalState;
 }
 
+/**
+ * Flattens string to reduce memory usage (around 320 bytes -> 120 bytes),
+ * at the cost of some upfront CPU usage.
+ *
+ * From: https://github.com/davidmarkclements/flatstr/issues/8
+ */
+export function flatstr(s: string) {
+  s.match(/\n/g);
+  return s;
+}
+
 function rowKey(entry: OplogEntry) {
   return `${entry.object_type}/${entry.object_id}/${entry.subkey}`;
 }

package/test/src/env.ts

@@ -1,6 +1,5 @@
 import { utils } from '@powersync/lib-services-framework';
 
 export const env = utils.collectEnvironmentVariables({
-  MONGO_TEST_URL: utils.type.string.default('mongodb://localhost:27017/powersync_test'),
   CI: utils.type.boolean.default('false')
 });