@powersync/service-core 0.8.8 → 0.9.0

package/dist/storage/SourceTable.d.ts

@@ -1,13 +1,12 @@
-import * as replication from '../replication/replication-index.js';
+import { ColumnDescriptor } from './SourceEntity.js';
 export declare class SourceTable {
     readonly id: any;
     readonly connectionTag: string;
-    readonly relationId: number;
+    readonly objectId: number | string;
     readonly schema: string;
     readonly table: string;
-    readonly replicaIdColumns: replication.ReplicationColumn[];
+    readonly replicaIdColumns: ColumnDescriptor[];
     readonly snapshotComplete: boolean;
-    static readonly DEFAULT_SCHEMA = "public";
     static readonly DEFAULT_TAG = "default";
     /**
      * True if the table is used in sync rules for data queries.
@@ -25,7 +24,15 @@ export declare class SourceTable {
      * Defaults to true for tests.
      */
     syncParameters: boolean;
-    constructor(id: any, connectionTag: string, relationId: number, schema: string, table: string, replicaIdColumns: replication.ReplicationColumn[], snapshotComplete: boolean);
+    /**
+     * True if the table is used in sync rules for events.
+     *
+     * This value is resolved externally, and cached here.
+     *
+     * Defaults to true for tests.
+     */
+    syncEvent: boolean;
+    constructor(id: any, connectionTag: string, objectId: number | string, schema: string, table: string, replicaIdColumns: ColumnDescriptor[], snapshotComplete: boolean);
     get hasReplicaIdentity(): boolean;
     /**
      * Usage: db.query({statement: `SELECT $1::regclass`, params: [{type: 'varchar', value: table.qualifiedName}]})

package/dist/storage/SourceTable.js

@@ -1,10 +1,10 @@
-import { DEFAULT_SCHEMA, DEFAULT_TAG } from '@powersync/service-sync-rules';
+import { DEFAULT_TAG } from '@powersync/service-sync-rules';
 import * as util from '../util/util-index.js';
 export class SourceTable {
-    constructor(id, connectionTag, relationId, schema, table, replicaIdColumns, snapshotComplete) {
+    constructor(id, connectionTag, objectId, schema, table, replicaIdColumns, snapshotComplete) {
         this.id = id;
         this.connectionTag = connectionTag;
-        this.relationId = relationId;
+        this.objectId = objectId;
         this.schema = schema;
         this.table = table;
         this.replicaIdColumns = replicaIdColumns;
@@ -25,6 +25,14 @@ export class SourceTable {
          * Defaults to true for tests.
          */
         this.syncParameters = true;
+        /**
+         * True if the table is used in sync rules for events.
+         *
+         * This value is resolved externally, and cached here.
+         *
+         * Defaults to true for tests.
+         */
+        this.syncEvent = true;
     }
     get hasReplicaIdentity() {
         return this.replicaIdColumns.length > 0;
@@ -42,9 +50,8 @@ export class SourceTable {
         return `${util.escapeIdentifier(this.schema)}.${util.escapeIdentifier(this.table)}`;
     }
     get syncAny() {
-        return this.syncData || this.syncParameters;
+        return this.syncData || this.syncParameters || this.syncEvent;
     }
 }
-SourceTable.DEFAULT_SCHEMA = DEFAULT_SCHEMA;
 SourceTable.DEFAULT_TAG = DEFAULT_TAG;
 //# sourceMappingURL=SourceTable.js.map

package/dist/storage/SourceTable.js.map

@@ -1 +1 @@
-{"version":3,"file":"SourceTable.js","sourceRoot":"","sources":["../../src/storage/SourceTable.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAC;AAG5E,OAAO,KAAK,IAAI,MAAM,uBAAuB,CAAC;AAE9C,MAAM,OAAO,WAAW;IAsBtB,YACkB,EAAO,EACP,aAAqB,EACrB,UAAkB,EAClB,MAAc,EACd,KAAa,EAEb,gBAAiD,EACjD,gBAAyB;QAPzB,OAAE,GAAF,EAAE,CAAK;QACP,kBAAa,GAAb,aAAa,CAAQ;QACrB,eAAU,GAAV,UAAU,CAAQ;QAClB,WAAM,GAAN,MAAM,CAAQ;QACd,UAAK,GAAL,KAAK,CAAQ;QAEb,qBAAgB,GAAhB,gBAAgB,CAAiC;QACjD,qBAAgB,GAAhB,gBAAgB,CAAS;QA1B3C;;;;;;WAMG;QACI,aAAQ,GAAG,IAAI,CAAC;QAEvB;;;;;;WAMG;QACI,mBAAc,GAAG,IAAI,CAAC;IAW1B,CAAC;IAEJ,IAAI,kBAAkB;QACpB,OAAO,IAAI,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC;IAC1C,CAAC;IAED;;OAEG;IACH,IAAI,aAAa;QACf,OAAO,IAAI,CAAC,iBAAiB,CAAC;IAChC,CAAC;IAED;;OAEG;IACH,IAAI,iBAAiB;QACnB,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;IACtF,CAAC;IAED,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,cAAc,CAAC;IAC9C,CAAC;;AApDe,0BAAc,GAAG,cAAc,AAAjB,CAAkB;AAChC,uBAAW,GAAG,WAAW,AAAd,CAAe"}
+{"version":3,"file":"SourceTable.js","sourceRoot":"","sources":["../../src/storage/SourceTable.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAC;AAC5D,OAAO,KAAK,IAAI,MAAM,uBAAuB,CAAC;AAG9C,MAAM,OAAO,WAAW;IA8BtB,YACkB,EAAO,EACP,aAAqB,EACrB,QAAyB,EACzB,MAAc,EACd,KAAa,EAEb,gBAAoC,EACpC,gBAAyB;QAPzB,OAAE,GAAF,EAAE,CAAK;QACP,kBAAa,GAAb,aAAa,CAAQ;QACrB,aAAQ,GAAR,QAAQ,CAAiB;QACzB,WAAM,GAAN,MAAM,CAAQ;QACd,UAAK,GAAL,KAAK,CAAQ;QAEb,qBAAgB,GAAhB,gBAAgB,CAAoB;QACpC,qBAAgB,GAAhB,gBAAgB,CAAS;QAnC3C;;;;;;WAMG;QACI,aAAQ,GAAG,IAAI,CAAC;QAEvB;;;;;;WAMG;QACI,mBAAc,GAAG,IAAI,CAAC;QAE7B;;;;;;WAMG;QACI,cAAS,GAAG,IAAI,CAAC;IAWrB,CAAC;IAEJ,IAAI,kBAAkB;QACpB,OAAO,IAAI,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC;IAC1C,CAAC;IAED;;OAEG;IACH,IAAI,aAAa;QACf,OAAO,IAAI,CAAC,iBAAiB,CAAC;IAChC,CAAC;IAED;;OAEG;IACH,IAAI,iBAAiB;QACnB,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;IACtF,CAAC;IAED,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,SAAS,CAAC;IAChE,CAAC;;AA5De,uBAAW,GAAG,WAAW,AAAd,CAAe"}

package/dist/storage/StorageEngine.d.ts

@@ -0,0 +1,28 @@
+import { DisposableListener, DisposableObserver } from '@powersync/lib-services-framework';
+import { ResolvedPowerSyncConfig } from '../util/util-index.js';
+import { BucketStorageFactory } from './BucketStorage.js';
+import { ActiveStorage, BucketStorageProvider } from './StorageProvider.js';
+export type StorageEngineOptions = {
+    configuration: ResolvedPowerSyncConfig;
+};
+export interface StorageEngineListener extends DisposableListener {
+    storageActivated: (storage: BucketStorageFactory) => void;
+}
+export declare class StorageEngine extends DisposableObserver<StorageEngineListener> {
+    private options;
+    private storageProviders;
+    private currentActiveStorage;
+    constructor(options: StorageEngineOptions);
+    get activeBucketStorage(): BucketStorageFactory;
+    get activeStorage(): ActiveStorage;
+    /**
+     * Register a provider which generates a {@link BucketStorageFactory}
+     * given the matching config specified in the loaded {@link ResolvedPowerSyncConfig}
+     */
+    registerProvider(provider: BucketStorageProvider): void;
+    start(): Promise<void>;
+    /**
+     *  Shutdown the storage engine, safely shutting down any activated storage providers.
+     */
+    shutDown(): Promise<void>;
+}

package/dist/storage/StorageEngine.js

@@ -0,0 +1,45 @@
+import { DisposableObserver, logger } from '@powersync/lib-services-framework';
+export class StorageEngine extends DisposableObserver {
+    constructor(options) {
+        super();
+        this.options = options;
+        // TODO: This will need to revisited when we actually support multiple storage providers.
+        this.storageProviders = new Map();
+        this.currentActiveStorage = null;
+    }
+    get activeBucketStorage() {
+        return this.activeStorage.storage;
+    }
+    get activeStorage() {
+        if (!this.currentActiveStorage) {
+            throw new Error(`No storage provider has been initialized yet.`);
+        }
+        return this.currentActiveStorage;
+    }
+    /**
+     * Register a provider which generates a {@link BucketStorageFactory}
+     * given the matching config specified in the loaded {@link ResolvedPowerSyncConfig}
+     */
+    registerProvider(provider) {
+        this.storageProviders.set(provider.type, provider);
+    }
+    async start() {
+        logger.info('Starting Storage Engine...');
+        const { configuration } = this.options;
+        this.currentActiveStorage = await this.storageProviders.get(configuration.storage.type).getStorage({
+            resolvedConfig: configuration
+        });
+        this.iterateListeners((cb) => cb.storageActivated?.(this.activeBucketStorage));
+        logger.info(`Successfully activated storage: ${configuration.storage.type}.`);
+        logger.info('Successfully started Storage Engine.');
+    }
+    /**
+     *  Shutdown the storage engine, safely shutting down any activated storage providers.
+     */
+    async shutDown() {
+        logger.info('Shutting down Storage Engine...');
+        await this.currentActiveStorage?.shutDown();
+        logger.info('Successfully shut down Storage Engine.');
+    }
+}
+//# sourceMappingURL=StorageEngine.js.map

package/dist/storage/StorageEngine.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"StorageEngine.js","sourceRoot":"","sources":["../../src/storage/StorageEngine.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsB,kBAAkB,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAanG,MAAM,OAAO,aAAc,SAAQ,kBAAyC;IAK1E,YAAoB,OAA6B;QAC/C,KAAK,EAAE,CAAC;QADU,YAAO,GAAP,OAAO,CAAsB;QAJjD,yFAAyF;QACjF,qBAAgB,GAAuC,IAAI,GAAG,EAAE,CAAC;QACjE,yBAAoB,GAAyB,IAAI,CAAC;IAI1D,CAAC;IAED,IAAI,mBAAmB;QACrB,OAAO,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC;IACpC,CAAC;IAED,IAAI,aAAa;QACf,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAC/B,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;QACnE,CAAC;QAED,OAAO,IAAI,CAAC,oBAAoB,CAAC;IACnC,CAAC;IAED;;;OAGG;IACH,gBAAgB,CAAC,QAA+B;QAC9C,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IACrD,CAAC;IAEM,KAAK,CAAC,KAAK;QAChB,MAAM,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC;QAC1C,MAAM,EAAE,aAAa,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC;QACvC,IAAI,CAAC,oBAAoB,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,aAAa,CAAC,OAAO,CAAC,IAAI,CAAE,CAAC,UAAU,CAAC;YAClG,cAAc,EAAE,aAAa;SAC9B,CAAC,CAAC;QACH,IAAI,CAAC,gBAAgB,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,gBAAgB,EAAE,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC,CAAC;QAC/E,MAAM,CAAC,IAAI,CAAC,mCAAmC,aAAa,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC,CAAC;QAC9E,MAAM,CAAC,IAAI,CAAC,sCAAsC,CAAC,CAAC;IACtD,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,QAAQ;QACnB,MAAM,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;QAC/C,MAAM,IAAI,CAAC,oBAAoB,EAAE,QAAQ,EAAE,CAAC;QAC5C,MAAM,CAAC,IAAI,CAAC,wCAAwC,CAAC,CAAC;IACxD,CAAC;CACF"}

package/dist/storage/StorageProvider.d.ts

@@ -0,0 +1,21 @@
+import * as util from '../util/util-index.js';
+import { BucketStorageFactory } from './BucketStorage.js';
+export interface ActiveStorage {
+    storage: BucketStorageFactory;
+    shutDown(): Promise<void>;
+    /**
+     *  Tear down / drop the storage permanently
+     */
+    tearDown(): Promise<boolean>;
+}
+export interface GetStorageOptions {
+    resolvedConfig: util.ResolvedPowerSyncConfig;
+}
+export interface BucketStorageProvider {
+    /**
+     *  The storage type that this provider provides.
+     *  The type should match the `type` field in the config.
+     */
+    type: string;
+    getStorage(options: GetStorageOptions): Promise<ActiveStorage>;
+}

package/dist/storage/StorageProvider.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=StorageProvider.js.map

package/dist/storage/StorageProvider.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"StorageProvider.js","sourceRoot":"","sources":["../../src/storage/StorageProvider.ts"],"names":[],"mappings":""}

package/dist/storage/WriteCheckpointAPI.d.ts

@@ -0,0 +1,74 @@
+export declare enum WriteCheckpointMode {
+    /**
+     * Raw mappings of `user_id` to `write_checkpoint`s should
+     * be supplied for each set of sync rules.
+     */
+    CUSTOM = "custom",
+    /**
+     * Write checkpoints are stored as a mapping of `user_id` plus
+     * replication HEAD (lsn in Postgres) to an automatically generated
+     * incrementing `write_checkpoint` (stored as`client_id`).
+     */
+    MANAGED = "managed"
+}
+export interface BaseWriteCheckpointIdentifier {
+    /**
+     * Identifier for User's account.
+     */
+    user_id: string;
+}
+export interface CustomWriteCheckpointFilters extends BaseWriteCheckpointIdentifier {
+    /**
+     * Sync rules which were active when this checkpoint was created.
+     */
+    sync_rules_id: number;
+}
+export interface BatchedCustomWriteCheckpointOptions extends BaseWriteCheckpointIdentifier {
+    /**
+     * A supplied incrementing Write Checkpoint number
+     */
+    checkpoint: bigint;
+}
+export interface CustomWriteCheckpointOptions extends BatchedCustomWriteCheckpointOptions {
+    /**
+     * Sync rules which were active when this checkpoint was created.
+     */
+    sync_rules_id: number;
+}
+/**
+ * Managed Write Checkpoints are a mapping of User ID to replication HEAD
+ */
+export interface ManagedWriteCheckpointFilters extends BaseWriteCheckpointIdentifier {
+    /**
+     * Replication HEAD(s) at the creation of the checkpoint.
+     */
+    heads: Record<string, string>;
+}
+export type ManagedWriteCheckpointOptions = ManagedWriteCheckpointFilters;
+export type SyncStorageLastWriteCheckpointFilters = BaseWriteCheckpointIdentifier | ManagedWriteCheckpointFilters;
+export type LastWriteCheckpointFilters = CustomWriteCheckpointFilters | ManagedWriteCheckpointFilters;
+export interface BaseWriteCheckpointAPI {
+    readonly writeCheckpointMode: WriteCheckpointMode;
+    setWriteCheckpointMode(mode: WriteCheckpointMode): void;
+    createManagedWriteCheckpoint(checkpoint: ManagedWriteCheckpointOptions): Promise<bigint>;
+}
+/**
+ * Write Checkpoint API to be used in conjunction with a {@link SyncRulesBucketStorage}.
+ * This storage corresponds with a set of sync rules. These APIs don't require specifying a
+ * sync rules id.
+ */
+export interface SyncStorageWriteCheckpointAPI extends BaseWriteCheckpointAPI {
+    batchCreateCustomWriteCheckpoints(checkpoints: BatchedCustomWriteCheckpointOptions[]): Promise<void>;
+    createCustomWriteCheckpoint(checkpoint: BatchedCustomWriteCheckpointOptions): Promise<bigint>;
+    lastWriteCheckpoint(filters: SyncStorageLastWriteCheckpointFilters): Promise<bigint | null>;
+}
+/**
+ * Write Checkpoint API which is interfaced directly with the storage layer. This requires
+ * sync rules identifiers for custom write checkpoints.
+ */
+export interface WriteCheckpointAPI extends BaseWriteCheckpointAPI {
+    batchCreateCustomWriteCheckpoints(checkpoints: CustomWriteCheckpointOptions[]): Promise<void>;
+    createCustomWriteCheckpoint(checkpoint: CustomWriteCheckpointOptions): Promise<bigint>;
+    lastWriteCheckpoint(filters: LastWriteCheckpointFilters): Promise<bigint | null>;
+}
+export declare const DEFAULT_WRITE_CHECKPOINT_MODE = WriteCheckpointMode.MANAGED;

package/dist/storage/WriteCheckpointAPI.js

@@ -0,0 +1,16 @@
+export var WriteCheckpointMode;
+(function (WriteCheckpointMode) {
+    /**
+     * Raw mappings of `user_id` to `write_checkpoint`s should
+     * be supplied for each set of sync rules.
+     */
+    WriteCheckpointMode["CUSTOM"] = "custom";
+    /**
+     * Write checkpoints are stored as a mapping of `user_id` plus
+     * replication HEAD (lsn in Postgres) to an automatically generated
+     * incrementing `write_checkpoint` (stored as`client_id`).
+     */
+    WriteCheckpointMode["MANAGED"] = "managed";
+})(WriteCheckpointMode || (WriteCheckpointMode = {}));
+export const DEFAULT_WRITE_CHECKPOINT_MODE = WriteCheckpointMode.MANAGED;
+//# sourceMappingURL=WriteCheckpointAPI.js.map

package/dist/storage/WriteCheckpointAPI.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"WriteCheckpointAPI.js","sourceRoot":"","sources":["../../src/storage/WriteCheckpointAPI.ts"],"names":[],"mappings":"AAAA,MAAM,CAAN,IAAY,mBAYX;AAZD,WAAY,mBAAmB;IAC7B;;;OAGG;IACH,wCAAiB,CAAA;IACjB;;;;OAIG;IACH,0CAAmB,CAAA;AACrB,CAAC,EAZW,mBAAmB,KAAnB,mBAAmB,QAY9B;AAwED,MAAM,CAAC,MAAM,6BAA6B,GAAG,mBAAmB,CAAC,OAAO,CAAC"}

package/dist/storage/mongo/MongoBucketBatch.d.ts

@@ -1,17 +1,30 @@
-import { SqlSyncRules } from '@powersync/service-sync-rules';
+import { SqlEventDescriptor, SqlSyncRules } from '@powersync/service-sync-rules';
 import * as mongo from 'mongodb';
-import { BucketStorageBatch, FlushedResult, SaveOptions } from '../BucketStorage.js';
+import { DisposableObserver } from '@powersync/lib-services-framework';
+import { BucketBatchStorageListener, BucketStorageBatch, FlushedResult, SaveOptions } from '../BucketStorage.js';
 import { SourceTable } from '../SourceTable.js';
+import { BatchedCustomWriteCheckpointOptions } from '../WriteCheckpointAPI.js';
 import { PowerSyncMongo } from './db.js';
 import { CurrentBucket } from './models.js';
-export declare class MongoBucketBatch implements BucketStorageBatch {
+export interface MongoBucketBatchOptions {
+    db: PowerSyncMongo;
+    syncRules: SqlSyncRules;
+    groupId: number;
+    slotName: string;
+    lastCheckpointLsn: string | null;
+    noCheckpointBeforeLsn: string;
+    storeCurrentData: boolean;
+}
+export declare class MongoBucketBatch extends DisposableObserver<BucketBatchStorageListener> implements BucketStorageBatch {
     private readonly client;
     readonly db: PowerSyncMongo;
     readonly session: mongo.ClientSession;
     private readonly sync_rules;
     private readonly group_id;
     private readonly slot_name;
+    private readonly storeCurrentData;
     private batch;
+    private write_checkpoint_batch;
     /**
      * Last LSN received associated with a checkpoint.
      *
@@ -26,14 +39,16 @@ export declare class MongoBucketBatch implements BucketStorageBatch {
      * For tests only - not for persistence logic.
      */
     last_flushed_op: bigint | null;
-    constructor(db: PowerSyncMongo, sync_rules: SqlSyncRules, group_id: number, slot_name: string, last_checkpoint_lsn: string | null, no_checkpoint_before_lsn: string | null);
+    constructor(options: MongoBucketBatchOptions);
+    addCustomWriteCheckpoint(checkpoint: BatchedCustomWriteCheckpointOptions): void;
+    get lastCheckpointLsn(): string | null;
     flush(): Promise<FlushedResult | null>;
     private flushInner;
     private replicateBatch;
     private saveOperation;
     private withTransaction;
     private withReplicationTransaction;
-    abort(): Promise<void>;
+    [Symbol.asyncDispose](): Promise<void>;
     commit(lsn: string): Promise<boolean>;
     keepalive(lsn: string): Promise<boolean>;
     save(record: SaveOptions): Promise<FlushedResult | null>;
@@ -44,5 +59,9 @@ export declare class MongoBucketBatch implements BucketStorageBatch {
     truncate(sourceTables: SourceTable[]): Promise<FlushedResult | null>;
     truncateSingle(sourceTable: SourceTable): Promise<bigint>;
     markSnapshotDone(tables: SourceTable[], no_checkpoint_before_lsn: string): Promise<SourceTable[]>;
+    /**
+     * Gets relevant {@link SqlEventDescriptor}s for the given {@link SourceTable}
+     */
+    protected getTableEvents(table: SourceTable): SqlEventDescriptor[];
 }
 export declare function currentBucketKey(b: CurrentBucket): string;