@powersync/common 1.38.1 → 1.40.0

package/lib/client/AbstractPowerSyncDatabase.js

@@ -2,7 +2,6 @@ import { Mutex } from 'async-mutex';
 import { EventIterator } from 'event-iterator';
 import Logger from 'js-logger';
 import { isBatchedUpdateNotification } from '../db/DBAdapter.js';
-import { FULL_SYNC_PRIORITY } from '../db/crud/SyncProgress.js';
 import { SyncStatus } from '../db/crud/SyncStatus.js';
 import { UploadQueueStats } from '../db/crud/UploadQueueStatus.js';
 import { BaseObserver } from '../utils/BaseObserver.js';
@@ -19,6 +18,7 @@ import { DEFAULT_CRUD_UPLOAD_THROTTLE_MS, DEFAULT_RETRY_DELAY_MS } from './sync/
 import { TriggerManagerImpl } from './triggers/TriggerManagerImpl.js';
 import { DEFAULT_WATCH_THROTTLE_MS } from './watched/WatchedQuery.js';
 import { OnChangeQueryProcessor } from './watched/processors/OnChangeQueryProcessor.js';
+import { coreStatusToJs } from './sync/stream/core-instruction.js';
 const POWERSYNC_TABLE_MATCH = /(^ps_data__|^ps_data_local__)/;
 const DEFAULT_DISCONNECT_CLEAR_OPTIONS = {
     clearLocal: true
@@ -59,9 +59,26 @@ export class AbstractPowerSyncDatabase extends BaseObserver {
     bucketStorageAdapter;
     _isReadyPromise;
     connectionManager;
+    subscriptions;
     get syncStreamImplementation() {
         return this.connectionManager.syncStreamImplementation;
     }
+    /**
+     * The connector used to connect to the PowerSync service.
+     *
+     * @returns The connector used to connect to the PowerSync service or null if `connect()` has not been called.
+     */
+    get connector() {
+        return this.connectionManager.connector;
+    }
+    /**
+     * The resolved connection options used to connect to the PowerSync service.
+     *
+     * @returns The resolved connection options used to connect to the PowerSync service or null if `connect()` has not been called.
+     */
+    get connectionOptions() {
+        return this.connectionManager.connectionOptions;
+    }
     _schema;
     _database;
     runExclusiveMutex;
@@ -100,6 +117,15 @@ export class AbstractPowerSyncDatabase extends BaseObserver {
         this.sdkVersion = '';
         this.runExclusiveMutex = new Mutex();
         // Start async init
+        this.subscriptions = {
+            firstStatusMatching: (predicate, abort) => this.waitForStatus(predicate, abort),
+            resolveOfflineSyncStatus: () => this.resolveOfflineSyncStatus(),
+            rustSubscriptionsCommand: async (payload) => {
+                await this.writeTransaction((tx) => {
+                    return tx.execute('select powersync_control(?,?)', ['subscriptions', JSON.stringify(payload)]);
+                });
+            }
+        };
         this.connectionManager = new ConnectionManager({
             createSyncImplementation: async (connector, options) => {
                 await this.waitForReady();
@@ -176,22 +202,33 @@ export class AbstractPowerSyncDatabase extends BaseObserver {
         const statusMatches = priority === undefined
             ? (status) => status.hasSynced
             : (status) => status.statusForPriority(priority).hasSynced;
-        if (statusMatches(this.currentStatus)) {
+        return this.waitForStatus(statusMatches, signal);
+    }
+    /**
+     * Waits for the first sync status for which the `status` callback returns a truthy value.
+     */
+    async waitForStatus(predicate, signal) {
+        if (predicate(this.currentStatus)) {
             return;
         }
         return new Promise((resolve) => {
             const dispose = this.registerListener({
                 statusChanged: (status) => {
-                    if (statusMatches(status)) {
-                        dispose();
-                        resolve();
+                    if (predicate(status)) {
+                        abort();
                     }
                 }
             });
-            signal?.addEventListener('abort', () => {
+            function abort() {
                 dispose();
                 resolve();
-            });
+            }
+            if (signal?.aborted) {
+                abort();
+            }
+            else {
+                signal?.addEventListener('abort', abort);
+            }
         });
     }
     /**
@@ -203,7 +240,7 @@ export class AbstractPowerSyncDatabase extends BaseObserver {
         await this.bucketStorageAdapter.init();
         await this._loadVersion();
         await this.updateSchema(this.options.schema);
-        await this.updateHasSynced();
+        await this.resolveOfflineSyncStatus();
         await this.database.execute('PRAGMA RECURSIVE_TRIGGERS=TRUE');
         this.ready = true;
         this.iterateListeners((cb) => cb.initialized?.());
@@ -230,26 +267,12 @@ export class AbstractPowerSyncDatabase extends BaseObserver {
             throw new Error(`Unsupported powersync extension version. Need >=0.4.5 <1.0.0, got: ${this.sdkVersion}`);
         }
     }
-    async updateHasSynced() {
-        const result = await this.database.getAll('SELECT priority, last_synced_at FROM ps_sync_state ORDER BY priority DESC');
-        let lastCompleteSync;
-        const priorityStatusEntries = [];
-        for (const { priority, last_synced_at } of result) {
-            const parsedDate = new Date(last_synced_at + 'Z');
-            if (priority == FULL_SYNC_PRIORITY) {
-                // This lowest-possible priority represents a complete sync.
-                lastCompleteSync = parsedDate;
-            }
-            else {
-                priorityStatusEntries.push({ priority, hasSynced: true, lastSyncedAt: parsedDate });
-            }
-        }
-        const hasSynced = lastCompleteSync != null;
+    async resolveOfflineSyncStatus() {
+        const result = await this.database.get('SELECT powersync_offline_sync_status() as r');
+        const parsed = JSON.parse(result.r);
         const updatedStatus = new SyncStatus({
             ...this.currentStatus.toJSON(),
-            hasSynced,
-            priorityStatusEntries,
-            lastSyncedAt: lastCompleteSync
+            ...coreStatusToJs(parsed)
         });
         if (!updatedStatus.isEqual(this.currentStatus)) {
             this.currentStatus = updatedStatus;
@@ -346,6 +369,17 @@ export class AbstractPowerSyncDatabase extends BaseObserver {
         this.currentStatus = new SyncStatus({});
         this.iterateListeners((l) => l.statusChanged?.(this.currentStatus));
     }
+    /**
+     * Create a sync stream to query its status or to subscribe to it.
+     *
+     * @param name The name of the stream to subscribe to.
+     * @param params Optional parameters for the stream subscription.
+     * @returns A {@link SyncStream} instance that can be subscribed to.
+     * @experimental Sync streams are currently in alpha.
+     */
+    syncStream(name, params) {
+        return this.connectionManager.stream(this.subscriptions, name, params ?? null);
+    }
     /**
      * Close the database, releasing resources.
      *

package/lib/client/ConnectionManager.d.ts

@@ -1,7 +1,9 @@
 import { ILogger } from 'js-logger';
 import { BaseListener, BaseObserver } from '../utils/BaseObserver.js';
 import { PowerSyncBackendConnector } from './connection/PowerSyncBackendConnector.js';
-import { InternalConnectionOptions, StreamingSyncImplementation } from './sync/stream/AbstractStreamingSyncImplementation.js';
+import { AdditionalConnectionOptions, InternalConnectionOptions, StreamingSyncImplementation, SubscribedStream } from './sync/stream/AbstractStreamingSyncImplementation.js';
+import { SyncStream } from './sync/sync-streams.js';
+import { SyncStatus } from '../db/crud/SyncStatus.js';
 /**
  * @internal
  */
@@ -13,11 +15,24 @@ export interface ConnectionManagerSyncImplementationResult {
      */
     onDispose: () => Promise<void> | void;
 }
+/**
+ * The subset of {@link AbstractStreamingSyncImplementationOptions} managed by the connection manager.
+ *
+ * @internal
+ */
+export interface CreateSyncImplementationOptions extends AdditionalConnectionOptions {
+    subscriptions: SubscribedStream[];
+}
+export interface InternalSubscriptionAdapter {
+    firstStatusMatching(predicate: (status: SyncStatus) => any, abort?: AbortSignal): Promise<void>;
+    resolveOfflineSyncStatus(): Promise<void>;
+    rustSubscriptionsCommand(payload: any): Promise<void>;
+}
 /**
  * @internal
  */
 export interface ConnectionManagerOptions {
-    createSyncImplementation(connector: PowerSyncBackendConnector, options: InternalConnectionOptions): Promise<ConnectionManagerSyncImplementationResult>;
+    createSyncImplementation(connector: PowerSyncBackendConnector, options: CreateSyncImplementationOptions): Promise<ConnectionManagerSyncImplementationResult>;
     logger: ILogger;
 }
 type StoredConnectionOptions = {
@@ -63,7 +78,15 @@ export declare class ConnectionManager extends BaseObserver<ConnectionManagerLis
      * is disposed.
      */
     protected syncDisposer: (() => Promise<void> | void) | null;
+    /**
+     * Subscriptions managed in this connection manager.
+     *
+     * On the web, these local subscriptions are merged across tabs by a shared worker.
+     */
+    private locallyActiveSubscriptions;
     constructor(options: ConnectionManagerOptions);
+    get connector(): PowerSyncBackendConnector | null;
+    get connectionOptions(): InternalConnectionOptions | null;
     get logger(): ILogger;
     close(): Promise<void>;
     connect(connector: PowerSyncBackendConnector, options: InternalConnectionOptions): Promise<void>;
@@ -76,5 +99,14 @@ export declare class ConnectionManager extends BaseObserver<ConnectionManagerLis
     disconnect(): Promise<void>;
     protected disconnectInternal(): Promise<void>;
     protected performDisconnect(): Promise<void>;
+    stream(adapter: InternalSubscriptionAdapter, name: string, parameters: Record<string, any> | null): SyncStream;
+    /**
+     * @internal exposed for testing
+     */
+    get activeStreams(): {
+        name: string;
+        params: Record<string, any> | null;
+    }[];
+    private subscriptionsMayHaveChanged;
 }
 export {};

package/lib/client/ConnectionManager.js

@@ -32,6 +32,12 @@ export class ConnectionManager extends BaseObserver {
      * is disposed.
      */
     syncDisposer;
+    /**
+     * Subscriptions managed in this connection manager.
+     *
+     * On the web, these local subscriptions are merged across tabs by a shared worker.
+     */
+    locallyActiveSubscriptions = new Map();
     constructor(options) {
         super();
         this.options = options;
@@ -42,6 +48,12 @@ export class ConnectionManager extends BaseObserver {
         this.syncStreamImplementation = null;
         this.syncDisposer = null;
     }
+    get connector() {
+        return this.pendingConnectionOptions?.connector ?? null;
+    }
+    get connectionOptions() {
+        return this.pendingConnectionOptions?.options ?? null;
+    }
     get logger() {
         return this.options.logger;
     }
@@ -55,7 +67,7 @@ export class ConnectionManager extends BaseObserver {
         // Update pending options to the latest values
         this.pendingConnectionOptions = {
             connector,
-            options: options ?? {}
+            options
         };
         // Disconnecting here provides aborting in progress connection attempts.
         // The connectInternal method will clear pending options once it starts connecting (with the options).
@@ -114,7 +126,10 @@ export class ConnectionManager extends BaseObserver {
                 const { connector, options } = this.pendingConnectionOptions;
                 appliedOptions = options;
                 this.pendingConnectionOptions = null;
-                const { sync, onDispose } = await this.options.createSyncImplementation(connector, options);
+                const { sync, onDispose } = await this.options.createSyncImplementation(connector, {
+                    subscriptions: this.activeStreams,
+                    ...options
+                });
                 this.iterateListeners((l) => l.syncStreamCreated?.(sync));
                 this.syncStreamImplementation = sync;
                 this.syncDisposer = onDispose;
@@ -171,4 +186,108 @@ export class ConnectionManager extends BaseObserver {
         await sync?.dispose();
         await disposer?.();
     }
+    stream(adapter, name, parameters) {
+        const desc = { name, parameters };
+        const waitForFirstSync = (abort) => {
+            return adapter.firstStatusMatching((s) => s.forStream(desc)?.subscription.hasSynced, abort);
+        };
+        return {
+            ...desc,
+            subscribe: async (options) => {
+                // NOTE: We also run this command if a subscription already exists, because this increases the expiry date
+                // (relevant if the app is closed before connecting again, where the last subscribe call determines the ttl).
+                await adapter.rustSubscriptionsCommand({
+                    subscribe: {
+                        stream: {
+                            name,
+                            params: parameters
+                        },
+                        ttl: options?.ttl,
+                        priority: options?.priority
+                    }
+                });
+                if (!this.syncStreamImplementation) {
+                    // We're not connected. So, update the offline sync status to reflect the new subscription.
+                    // (With an active iteration, the sync client would include it in its state).
+                    await adapter.resolveOfflineSyncStatus();
+                }
+                const key = `${name}|${JSON.stringify(parameters)}`;
+                let subscription = this.locallyActiveSubscriptions.get(key);
+                if (subscription == null) {
+                    const clearSubscription = () => {
+                        this.locallyActiveSubscriptions.delete(key);
+                        this.subscriptionsMayHaveChanged();
+                    };
+                    subscription = new ActiveSubscription(name, parameters, this.logger, waitForFirstSync, clearSubscription);
+                    this.locallyActiveSubscriptions.set(key, subscription);
+                    this.subscriptionsMayHaveChanged();
+                }
+                return new SyncStreamSubscriptionHandle(subscription);
+            },
+            unsubscribeAll: async () => {
+                await adapter.rustSubscriptionsCommand({ unsubscribe: { name, params: parameters } });
+                this.subscriptionsMayHaveChanged();
+            }
+        };
+    }
+    /**
+     * @internal exposed for testing
+     */
+    get activeStreams() {
+        return [...this.locallyActiveSubscriptions.values()].map((a) => ({ name: a.name, params: a.parameters }));
+    }
+    subscriptionsMayHaveChanged() {
+        this.syncStreamImplementation?.updateSubscriptions(this.activeStreams);
+    }
+}
+class ActiveSubscription {
+    name;
+    parameters;
+    logger;
+    waitForFirstSync;
+    clearSubscription;
+    refcount = 0;
+    constructor(name, parameters, logger, waitForFirstSync, clearSubscription) {
+        this.name = name;
+        this.parameters = parameters;
+        this.logger = logger;
+        this.waitForFirstSync = waitForFirstSync;
+        this.clearSubscription = clearSubscription;
+    }
+    decrementRefCount() {
+        this.refcount--;
+        if (this.refcount == 0) {
+            this.clearSubscription();
+        }
+    }
+}
+class SyncStreamSubscriptionHandle {
+    subscription;
+    active = true;
+    constructor(subscription) {
+        this.subscription = subscription;
+        subscription.refcount++;
+        _finalizer?.register(this, subscription);
+    }
+    get name() {
+        return this.subscription.name;
+    }
+    get parameters() {
+        return this.subscription.parameters;
+    }
+    waitForFirstSync(abort) {
+        return this.subscription.waitForFirstSync(abort);
+    }
+    unsubscribe() {
+        if (this.active) {
+            this.active = false;
+            _finalizer?.unregister(this);
+            this.subscription.decrementRefCount();
+        }
+    }
 }
+const _finalizer = 'FinalizationRegistry' in globalThis
+    ? new FinalizationRegistry((sub) => {
+        sub.logger.warn(`A subscription to ${sub.name} with params ${JSON.stringify(sub.parameters)} leaked! Please ensure calling unsubscribe() when you don't need a subscription anymore. For global subscriptions, consider storing them in global fields to avoid this warning.`);
+    })
+    : null;

package/lib/client/sync/bucket/BucketStorageAdapter.d.ts

@@ -10,6 +10,7 @@ export interface Checkpoint {
     last_op_id: OpId;
     buckets: BucketChecksum[];
     write_checkpoint?: string;
+    streams?: any[];
 }
 export interface BucketState {
     bucket: string;
@@ -43,6 +44,12 @@ export interface BucketChecksum {
      * Count of operations - informational only.
      */
     count?: number;
+    /**
+     * The JavaScript client does not use this field, which is why it's defined to be `any`. We rely on the structure of
+     * this interface to pass custom `BucketChecksum`s to the Rust client in unit tests, which so all fields need to be
+     * present.
+     */
+    subscriptions?: any;
 }
 export declare enum PSInternalTable {
     DATA = "ps_data",
@@ -57,7 +64,8 @@ export declare enum PowerSyncControlCommand {
     STOP = "stop",
     START = "start",
     NOTIFY_TOKEN_REFRESHED = "refreshed_token",
-    NOTIFY_CRUD_UPLOAD_COMPLETED = "completed_upload"
+    NOTIFY_CRUD_UPLOAD_COMPLETED = "completed_upload",
+    UPDATE_SUBSCRIPTIONS = "update_subscriptions"
 }
 export interface BucketStorageListener extends BaseListener {
     crudUpdate: () => void;

package/lib/client/sync/bucket/BucketStorageAdapter.js

@@ -14,4 +14,5 @@ export var PowerSyncControlCommand;
     PowerSyncControlCommand["START"] = "start";
     PowerSyncControlCommand["NOTIFY_TOKEN_REFRESHED"] = "refreshed_token";
     PowerSyncControlCommand["NOTIFY_CRUD_UPLOAD_COMPLETED"] = "completed_upload";
+    PowerSyncControlCommand["UPDATE_SUBSCRIPTIONS"] = "update_subscriptions";
 })(PowerSyncControlCommand || (PowerSyncControlCommand = {}));

package/lib/client/sync/stream/AbstractStreamingSyncImplementation.d.ts

@@ -62,8 +62,9 @@ export interface LockOptions<T> {
     type: LockType;
     signal?: AbortSignal;
 }
-export interface AbstractStreamingSyncImplementationOptions extends AdditionalConnectionOptions {
+export interface AbstractStreamingSyncImplementationOptions extends RequiredAdditionalConnectionOptions {
     adapter: BucketStorageAdapter;
+    subscriptions: SubscribedStream[];
     uploadCrud: () => Promise<void>;
     /**
      * An identifier for which PowerSync DB this sync implementation is
@@ -115,6 +116,12 @@ export interface BaseConnectionOptions {
      * These parameters are passed to the sync rules, and will be available under the`user_parameters` object.
      */
     params?: Record<string, StreamingSyncRequestParameterType>;
+    /**
+     * Whether to include streams that have `auto_subscribe: true` in their definition.
+     *
+     * This defaults to `true`.
+     */
+    includeDefaultStreams?: boolean;
     /**
      * The serialized schema - mainly used to forward information about raw tables to the sync client.
      */
@@ -135,7 +142,9 @@ export interface AdditionalConnectionOptions {
     crudUploadThrottleMs?: number;
 }
 /** @internal */
-export type RequiredAdditionalConnectionOptions = Required<AdditionalConnectionOptions>;
+export interface RequiredAdditionalConnectionOptions extends Required<AdditionalConnectionOptions> {
+    subscriptions: SubscribedStream[];
+}
 export interface StreamingSyncImplementation extends BaseObserverInterface<StreamingSyncImplementationListener>, Disposable {
     /**
      * Connects to the sync service
@@ -155,6 +164,7 @@ export interface StreamingSyncImplementation extends BaseObserverInterface<Strea
     waitForReady(): Promise<void>;
     waitForStatus(status: SyncStatusOptions): Promise<void>;
     waitUntilStatusMatches(predicate: (status: SyncStatus) => boolean): Promise<void>;
+    updateSubscriptions(subscriptions: SubscribedStream[]): void;
 }
 export declare const DEFAULT_CRUD_UPLOAD_THROTTLE_MS = 1000;
 export declare const DEFAULT_RETRY_DELAY_MS = 5000;
@@ -164,6 +174,10 @@ export declare const DEFAULT_STREAMING_SYNC_OPTIONS: {
 };
 export type RequiredPowerSyncConnectionOptions = Required<BaseConnectionOptions>;
 export declare const DEFAULT_STREAM_CONNECTION_OPTIONS: RequiredPowerSyncConnectionOptions;
+export type SubscribedStream = {
+    name: string;
+    params: Record<string, any> | null;
+};
 export declare abstract class AbstractStreamingSyncImplementation extends BaseObserver<StreamingSyncImplementationListener> implements StreamingSyncImplementation {
     protected _lastSyncedAt: Date | null;
     protected options: AbstractStreamingSyncImplementationOptions;
@@ -172,8 +186,10 @@ export declare abstract class AbstractStreamingSyncImplementation extends BaseOb
     protected crudUpdateListener?: () => void;
     protected streamingSyncPromise?: Promise<void>;
     protected logger: ILogger;
+    private activeStreams;
     private isUploadingCrud;
     private notifyCompletedUploads?;
+    private handleActiveStreamsChange?;
     syncStatus: SyncStatus;
     triggerCrudUpload: () => void;
     constructor(options: AbstractStreamingSyncImplementationOptions);
@@ -210,11 +226,16 @@ export declare abstract class AbstractStreamingSyncImplementation extends BaseOb
      * @returns Whether the database is now using the new, fixed subkey format.
      */
     private requireKeyFormat;
-    protected streamingSyncIteration(signal: AbortSignal, options?: PowerSyncConnectionOptions): Promise<void>;
+    protected streamingSyncIteration(signal: AbortSignal, options?: PowerSyncConnectionOptions): Promise<RustIterationResult | null>;
     private legacyStreamingSyncIteration;
     private rustSyncIteration;
     private updateSyncStatusForStartingCheckpoint;
     private applyCheckpoint;
     protected updateSyncStatus(options: SyncStatusOptions): void;
     private delayRetry;
+    updateSubscriptions(subscriptions: SubscribedStream[]): void;
+}
+interface RustIterationResult {
+    immediateRestart: boolean;
 }
+export {};