@powersync/common 1.38.1 → 1.40.0

package/lib/client/sync/stream/AbstractStreamingSyncImplementation.js

@@ -1,5 +1,4 @@
 import Logger from 'js-logger';
-import { FULL_SYNC_PRIORITY } from '../../../db/crud/SyncProgress.js';
 import { SyncStatus } from '../../../db/crud/SyncStatus.js';
 import { AbortOperation } from '../../../utils/AbortOperation.js';
 import { BaseObserver } from '../../../utils/BaseObserver.js';
@@ -7,6 +6,7 @@ import { throttleLeadingTrailing } from '../../../utils/async.js';
 import { PowerSyncControlCommand } from '../bucket/BucketStorageAdapter.js';
 import { SyncDataBucket } from '../bucket/SyncDataBucket.js';
 import { FetchStrategy } from './AbstractRemote.js';
+import { coreStatusToJs } from './core-instruction.js';
 import { isStreamingKeepalive, isStreamingSyncCheckpoint, isStreamingSyncCheckpointComplete, isStreamingSyncCheckpointDiff, isStreamingSyncCheckpointPartiallyComplete, isStreamingSyncData } from './streaming-sync-types.js';
 export var LockType;
 (function (LockType) {
@@ -72,7 +72,8 @@ export const DEFAULT_STREAM_CONNECTION_OPTIONS = {
     clientImplementation: DEFAULT_SYNC_CLIENT_IMPLEMENTATION,
     fetchStrategy: FetchStrategy.Buffered,
     params: {},
-    serializedSchema: undefined
+    serializedSchema: undefined,
+    includeDefaultStreams: true
 };
 // The priority we assume when we receive checkpoint lines where no priority is set.
 // This is the default priority used by the sync service, but can be set to an arbitrary
@@ -89,13 +90,16 @@ export class AbstractStreamingSyncImplementation extends BaseObserver {
     crudUpdateListener;
     streamingSyncPromise;
     logger;
+    activeStreams;
     isUploadingCrud = false;
     notifyCompletedUploads;
+    handleActiveStreamsChange;
     syncStatus;
     triggerCrudUpload;
     constructor(options) {
         super();
-        this.options = { ...DEFAULT_STREAMING_SYNC_OPTIONS, ...options };
+        this.options = options;
+        this.activeStreams = options.subscriptions;
         this.logger = options.logger ?? Logger.get('PowerSyncStream');
         this.syncStatus = new SyncStatus({
             connected: false,
@@ -343,11 +347,12 @@ The next upload iteration will be delayed.`);
         while (true) {
             this.updateSyncStatus({ connecting: true });
             let shouldDelayRetry = true;
+            let result = null;
             try {
                 if (signal?.aborted) {
                     break;
                 }
-                await this.streamingSyncIteration(nestedAbortController.signal, options);
+                result = await this.streamingSyncIteration(nestedAbortController.signal, options);
                 // Continue immediately, streamingSyncIteration will wait before completing if necessary.
             }
             catch (ex) {
@@ -380,13 +385,15 @@ The next upload iteration will be delayed.`);
                     nestedAbortController.abort(new AbortOperation('Closing sync stream network requests before retry.'));
                     nestedAbortController = new AbortController();
                 }
-                this.updateSyncStatus({
-                    connected: false,
-                    connecting: true // May be unnecessary
-                });
-                // On error, wait a little before retrying
-                if (shouldDelayRetry) {
-                    await this.delayRetry(nestedAbortController.signal);
+                if (result?.immediateRestart != true) {
+                    this.updateSyncStatus({
+                        connected: false,
+                        connecting: true // May be unnecessary
+                    });
+                    // On error, wait a little before retrying
+                    if (shouldDelayRetry) {
+                        await this.delayRetry(nestedAbortController.signal);
+                    }
                 }
             }
         }
@@ -430,8 +437,8 @@ The next upload iteration will be delayed.`);
             return hasMigrated;
         }
     }
-    async streamingSyncIteration(signal, options) {
-        await this.obtainLock({
+    streamingSyncIteration(signal, options) {
+        return this.obtainLock({
             type: LockType.SYNC,
             signal,
             callback: async () => {
@@ -443,10 +450,11 @@ The next upload iteration will be delayed.`);
                 this.updateSyncStatus({ clientImplementation });
                 if (clientImplementation == SyncClientImplementation.JAVASCRIPT) {
                     await this.legacyStreamingSyncIteration(signal, resolvedOptions);
+                    return null;
                 }
                 else {
                     await this.requireKeyFormat(true);
-                    await this.rustSyncIteration(signal, resolvedOptions);
+                    return await this.rustSyncIteration(signal, resolvedOptions);
                 }
             }
         });
@@ -695,6 +703,7 @@ The next upload iteration will be delayed.`);
         const remote = this.options.remote;
         let receivingLines = null;
         let hadSyncLine = false;
+        let hideDisconnectOnRestart = false;
         if (signal.aborted) {
             throw new AbortOperation('Connection request has been aborted');
         }
@@ -771,6 +780,8 @@ The next upload iteration will be delayed.`);
         }
         async function control(op, payload) {
             const rawResponse = await adapter.control(op, payload ?? null);
+            const logger = syncImplementation.logger;
+            logger.trace('powersync_control', op, payload == null || typeof payload == 'string' ? payload : '<bytes>', rawResponse);
             await handleInstructions(JSON.parse(rawResponse));
         }
         async function handleInstruction(instruction) {
@@ -788,27 +799,7 @@ The next upload iteration will be delayed.`);
                 }
             }
             else if ('UpdateSyncStatus' in instruction) {
-                function coreStatusToJs(status) {
-                    return {
-                        priority: status.priority,
-                        hasSynced: status.has_synced ?? undefined,
-                        lastSyncedAt: status?.last_synced_at != null ? new Date(status.last_synced_at * 1000) : undefined
-                    };
-                }
-                const info = instruction.UpdateSyncStatus.status;
-                const coreCompleteSync = info.priority_status.find((s) => s.priority == FULL_SYNC_PRIORITY);
-                const completeSync = coreCompleteSync != null ? coreStatusToJs(coreCompleteSync) : null;
-                syncImplementation.updateSyncStatus({
-                    connected: info.connected,
-                    connecting: info.connecting,
-                    dataFlow: {
-                        downloading: info.downloading != null,
-                        downloadProgress: info.downloading?.buckets
-                    },
-                    lastSyncedAt: completeSync?.lastSyncedAt,
-                    hasSynced: completeSync?.hasSynced,
-                    priorityStatusEntries: info.priority_status.map(coreStatusToJs)
-                });
+                syncImplementation.updateSyncStatus(coreStatusToJs(instruction.UpdateSyncStatus.status));
             }
             else if ('EstablishSyncStream' in instruction) {
                 if (receivingLines != null) {
@@ -833,6 +824,7 @@ The next upload iteration will be delayed.`);
             }
             else if ('CloseSyncStream' in instruction) {
                 abortController.abort();
+                hideDisconnectOnRestart = instruction.CloseSyncStream.hide_disconnect;
             }
             else if ('FlushFileSystem' in instruction) {
                 // Not necessary on JS platforms.
@@ -851,7 +843,11 @@ The next upload iteration will be delayed.`);
             }
         }
         try {
-            const options = { parameters: resolvedOptions.params };
+            const options = {
+                parameters: resolvedOptions.params,
+                active_streams: this.activeStreams,
+                include_defaults: resolvedOptions.includeDefaultStreams
+            };
             if (resolvedOptions.serializedSchema) {
                 options.schema = resolvedOptions.serializedSchema;
             }
@@ -861,12 +857,21 @@ The next upload iteration will be delayed.`);
                     controlInvocations.enqueueData({ command: PowerSyncControlCommand.NOTIFY_CRUD_UPLOAD_COMPLETED });
                 }
             };
+            this.handleActiveStreamsChange = () => {
+                if (controlInvocations && !controlInvocations?.closed) {
+                    controlInvocations.enqueueData({
+                        command: PowerSyncControlCommand.UPDATE_SUBSCRIPTIONS,
+                        payload: JSON.stringify(this.activeStreams)
+                    });
+                }
+            };
             await receivingLines;
         }
         finally {
-            this.notifyCompletedUploads = undefined;
+            this.notifyCompletedUploads = this.handleActiveStreamsChange = undefined;
             await stop();
         }
+        return { immediateRestart: hideDisconnectOnRestart };
     }
     async updateSyncStatusForStartingCheckpoint(checkpoint) {
         const localProgress = await this.options.adapter.getBucketOperationProgress();
@@ -971,4 +976,8 @@ The next upload iteration will be delayed.`);
             timeoutId = setTimeout(endDelay, retryDelayMs);
         });
     }
+    updateSubscriptions(subscriptions) {
+        this.activeStreams = subscriptions;
+        this.handleActiveStreamsChange?.();
+    }
 }

package/lib/client/sync/stream/core-instruction.d.ts

@@ -1,4 +1,5 @@
 import { StreamingSyncRequest } from './streaming-sync-types.js';
+import * as sync_status from '../../../db/crud/SyncStatus.js';
 /**
  * An internal instruction emitted by the sync client in the core extension in response to the JS
  * SDK passing sync data into the extension.
@@ -12,7 +13,9 @@ export type Instruction = {
 } | {
     FetchCredentials: FetchCredentials;
 } | {
-    CloseSyncStream: any;
+    CloseSyncStream: {
+        hide_disconnect: boolean;
+    };
 } | {
     FlushFileSystem: any;
 } | {
@@ -33,6 +36,21 @@ export interface CoreSyncStatus {
     connecting: boolean;
     priority_status: SyncPriorityStatus[];
     downloading: DownloadProgress | null;
+    streams: CoreStreamSubscription[];
+}
+export interface CoreStreamSubscription {
+    progress: {
+        total: number;
+        downloaded: number;
+    };
+    name: string;
+    parameters: any;
+    priority: number | null;
+    active: boolean;
+    is_default: boolean;
+    has_explicit_subscription: boolean;
+    expires_at: number | null;
+    last_synced_at: number | null;
 }
 export interface SyncPriorityStatus {
     priority: number;
@@ -51,3 +69,4 @@ export interface BucketProgress {
 export interface FetchCredentials {
     did_expire: boolean;
 }
+export declare function coreStatusToJs(status: CoreSyncStatus): sync_status.SyncStatusOptions;

package/lib/client/sync/stream/core-instruction.js

@@ -1 +1,26 @@
-export {};
+import { FULL_SYNC_PRIORITY } from '../../../db/crud/SyncProgress.js';
+function priorityToJs(status) {
+    return {
+        priority: status.priority,
+        hasSynced: status.has_synced ?? undefined,
+        lastSyncedAt: status.last_synced_at != null ? new Date(status.last_synced_at * 1000) : undefined
+    };
+}
+export function coreStatusToJs(status) {
+    const coreCompleteSync = status.priority_status.find((s) => s.priority == FULL_SYNC_PRIORITY);
+    const completeSync = coreCompleteSync != null ? priorityToJs(coreCompleteSync) : null;
+    return {
+        connected: status.connected,
+        connecting: status.connecting,
+        dataFlow: {
+            // We expose downloading as a boolean field, the core extension reports download information as a nullable
+            // download status. When that status is non-null, a download is in progress.
+            downloading: status.downloading != null,
+            downloadProgress: status.downloading?.buckets,
+            internalStreamSubscriptions: status.streams
+        },
+        lastSyncedAt: completeSync?.lastSyncedAt,
+        hasSynced: completeSync?.hasSynced,
+        priorityStatusEntries: status.priority_status.map(priorityToJs)
+    };
+}

package/lib/client/sync/sync-streams.d.ts

@@ -0,0 +1,98 @@
+/**
+ * A description of a sync stream, consisting of its {@link name} and the {@link parameters} used when subscribing.
+ */
+export interface SyncStreamDescription {
+    /**
+     * The name of the stream as it appears in the stream definition for the PowerSync service.
+     */
+    name: string;
+    /**
+     * The parameters used to subscribe to the stream, if any.
+     *
+     * The same stream can be subscribed to multiple times with different parameters.
+     */
+    parameters: Record<string, any> | null;
+}
+/**
+ * Information about a subscribed sync stream.
+ *
+ * This includes the {@link SyncStreamDescription}, along with information about the current sync status.
+ */
+export interface SyncSubscriptionDescription extends SyncStreamDescription {
+    active: boolean;
+    /**
+     * Whether this stream subscription is included by default, regardless of whether the stream has explicitly been
+     * subscribed to or not.
+     *
+     * It's possible for both {@link isDefault} and {@link hasExplicitSubscription} to be true at the same time - this
+     * happens when a default stream was subscribed explicitly.
+     */
+    isDefault: boolean;
+    /**
+     * Whether this stream has been subscribed to explicitly.
+     *
+     * It's possible for both {@link isDefault} and {@link hasExplicitSubscription} to be true at the same time - this
+     * happens when a default stream was subscribed explicitly.
+     */
+    hasExplicitSubscription: boolean;
+    /**
+     * For sync streams that have a time-to-live, the current time at which the stream would expire if not subscribed to
+     * again.
+     */
+    expiresAt: Date | null;
+    /**
+     * Whether this stream subscription has been synced at least once.
+     */
+    hasSynced: boolean;
+    /**
+     * If {@link hasSynced} is true, the last time data from this stream has been synced.
+     */
+    lastSyncedAt: Date | null;
+}
+export interface SyncStreamSubscribeOptions {
+    /**
+     * A "time to live" for this stream subscription, in seconds.
+     *
+     * The TTL control when a stream gets evicted after not having an active {@link SyncStreamSubscription} object
+     * attached to it.
+     */
+    ttl?: number;
+    /**
+     * A priority to assign to this subscription. This overrides the default priority that may have been set on streams.
+     *
+     * For details on priorities, see [priotized sync](https://docs.powersync.com/usage/use-case-examples/prioritized-sync).
+     */
+    priority?: 0 | 1 | 2 | 3;
+}
+/**
+ * A handle to a {@link SyncStreamDescription} that allows subscribing to the stream.
+ *
+ * To obtain an instance of {@link SyncStream}, call {@link AbstractPowerSyncDatabase.syncStream}.
+ */
+export interface SyncStream extends SyncStreamDescription {
+    /**
+     * Adds a subscription to this stream, requesting it to be included when connecting to the sync service.
+     *
+     * You should keep a reference to the returned {@link SyncStreamSubscription} object along as you need data for that
+     * stream. As soon as {@link SyncStreamSubscription.unsubscribe} is called for all subscriptions on this stream
+     * (including subscriptions created on other tabs), the {@link SyncStreamSubscribeOptions.ttl} starts ticking and will
+     * eventually evict the stream (unless {@link subscribe} is called again).
+     */
+    subscribe(options?: SyncStreamSubscribeOptions): Promise<SyncStreamSubscription>;
+    /**
+     * Clears all subscriptions attached to this stream and resets the TTL for the stream.
+     *
+     * This is a potentially dangerous operations, as it interferes with other stream subscriptions.
+     */
+    unsubscribeAll(): Promise<void>;
+}
+export interface SyncStreamSubscription extends SyncStreamDescription {
+    /**
+     * A promise that resolves once data from in this sync stream has been synced and applied.
+     */
+    waitForFirstSync(abort?: AbortSignal): Promise<void>;
+    /**
+     * Removes this stream subscription.
+     */
+    unsubscribe(): void;
+}

package/lib/client/sync/sync-streams.js

@@ -0,0 +1 @@
+export {};

package/lib/db/crud/SyncStatus.d.ts

@@ -1,5 +1,7 @@
+import { CoreStreamSubscription } from '../../client/sync/stream/core-instruction.js';
 import { SyncClientImplementation } from '../../client/sync/stream/AbstractStreamingSyncImplementation.js';
-import { InternalProgressInformation, SyncProgress } from './SyncProgress.js';
+import { InternalProgressInformation, ProgressWithOperations, SyncProgress } from './SyncProgress.js';
+import { SyncStreamDescription, SyncSubscriptionDescription } from '../../client/sync/sync-streams.js';
 export type SyncDataFlowStatus = Partial<{
     downloading: boolean;
     uploading: boolean;
@@ -20,6 +22,7 @@ export type SyncDataFlowStatus = Partial<{
      * Please use the {@link SyncStatus#downloadProgress} property to track sync progress.
      */
     downloadProgress: InternalProgressInformation | null;
+    internalStreamSubscriptions: CoreStreamSubscription[] | null;
 }>;
 export interface SyncPriorityStatus {
     priority: number;
@@ -99,7 +102,23 @@ export declare class SyncStatus {
          * Please use the {@link SyncStatus#downloadProgress} property to track sync progress.
          */
         downloadProgress: InternalProgressInformation | null;
+        internalStreamSubscriptions: CoreStreamSubscription[] | null;
     }>;
+    /**
+     * All sync streams currently being tracked in teh database.
+     *
+     * This returns null when the database is currently being opened and we don't have reliable information about all
+     * included streams yet.
+     *
+     * @experimental Sync streams are currently in alpha.
+     */
+    get syncStreams(): SyncStreamStatus[] | undefined;
+    /**
+     * If the `stream` appears in {@link syncStreams}, returns the current status for that stream.
+     *
+     * @experimental Sync streams are currently in alpha.
+     */
+    forStream(stream: SyncStreamDescription): SyncStreamStatus | undefined;
     /**
      * Provides sync status information for all bucket priorities, sorted by priority (highest first).
      *
@@ -157,3 +176,11 @@ export declare class SyncStatus {
     toJSON(): SyncStatusOptions;
     private static comparePriorities;
 }
+/**
+ * Information about a sync stream subscription.
+ */
+export interface SyncStreamStatus {
+    progress: ProgressWithOperations | null;
+    subscription: SyncSubscriptionDescription;
+    priority: number | null;
+}

package/lib/db/crud/SyncStatus.js

@@ -68,6 +68,27 @@ export class SyncStatus {
             uploading: false
         });
     }
+    /**
+     * All sync streams currently being tracked in teh database.
+     *
+     * This returns null when the database is currently being opened and we don't have reliable information about all
+     * included streams yet.
+     *
+     * @experimental Sync streams are currently in alpha.
+     */
+    get syncStreams() {
+        return this.options.dataFlow?.internalStreamSubscriptions?.map((core) => new SyncStreamStatusView(this, core));
+    }
+    /**
+     * If the `stream` appears in {@link syncStreams}, returns the current status for that stream.
+     *
+     * @experimental Sync streams are currently in alpha.
+     */
+    forStream(stream) {
+        const asJson = JSON.stringify(stream.parameters);
+        const raw = this.options.dataFlow?.internalStreamSubscriptions?.find((r) => r.name == stream.name && asJson == JSON.stringify(r.parameters));
+        return raw && new SyncStreamStatusView(this, raw);
+    }
     /**
      * Provides sync status information for all bucket priorities, sorted by priority (highest first).
      *
@@ -177,3 +198,34 @@ export class SyncStatus {
         return b.priority - a.priority; // Reverse because higher priorities have lower numbers
     }
 }
+class SyncStreamStatusView {
+    status;
+    core;
+    subscription;
+    constructor(status, core) {
+        this.status = status;
+        this.core = core;
+        this.subscription = {
+            name: core.name,
+            parameters: core.parameters,
+            active: core.active,
+            isDefault: core.is_default,
+            hasExplicitSubscription: core.has_explicit_subscription,
+            expiresAt: core.expires_at != null ? new Date(core.expires_at * 1000) : null,
+            hasSynced: core.last_synced_at != null,
+            lastSyncedAt: core.last_synced_at != null ? new Date(core.last_synced_at * 1000) : null
+        };
+    }
+    get progress() {
+        if (this.status.dataFlowStatus.downloadProgress == null) {
+            // Don't make download progress public if we're not currently downloading.
+            return null;
+        }
+        const { total, downloaded } = this.core.progress;
+        const progress = total == 0 ? 0.0 : downloaded / total;
+        return { totalOperations: total, downloadedOperations: downloaded, downloadedFraction: progress };
+    }
+    get priority() {
+        return this.core.priority;
+    }
+}

package/lib/index.d.ts

@@ -18,6 +18,7 @@ export * from './client/sync/bucket/SyncDataBucket.js';
 export * from './client/sync/stream/AbstractRemote.js';
 export * from './client/sync/stream/AbstractStreamingSyncImplementation.js';
 export * from './client/sync/stream/streaming-sync-types.js';
+export * from './client/sync/sync-streams.js';
 export * from './client/ConnectionManager.js';
 export { ProgressWithOperations, SyncProgress } from './db/crud/SyncProgress.js';
 export * from './db/crud/SyncStatus.js';

package/lib/index.js

@@ -18,6 +18,7 @@ export * from './client/sync/bucket/SyncDataBucket.js';
 export * from './client/sync/stream/AbstractRemote.js';
 export * from './client/sync/stream/AbstractStreamingSyncImplementation.js';
 export * from './client/sync/stream/streaming-sync-types.js';
+export * from './client/sync/sync-streams.js';
 export * from './client/ConnectionManager.js';
 export { SyncProgress } from './db/crud/SyncProgress.js';
 export * from './db/crud/SyncStatus.js';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@powersync/common",
-  "version": "1.38.1",
+  "version": "1.40.0",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"