@powersync/common 0.0.0-dev-20240506092851

package/lib/client/sync/stream/AbstractStreamingSyncImplementation.js

@@ -0,0 +1,477 @@
+import throttle from 'lodash/throttle';
+import Logger from 'js-logger';
+import { isStreamingKeepalive, isStreamingSyncCheckpoint, isStreamingSyncCheckpointComplete, isStreamingSyncCheckpointDiff, isStreamingSyncData } from './streaming-sync-types';
+import ndjsonStream from 'can-ndjson-stream';
+import { SyncStatus } from '../../../db/crud/SyncStatus';
+import { SyncDataBucket } from '../bucket/SyncDataBucket';
+import { BaseObserver } from '../../../utils/BaseObserver';
+import { AbortOperation } from '../../../utils/AbortOperation';
+export var LockType;
+(function (LockType) {
+    LockType["CRUD"] = "crud";
+    LockType["SYNC"] = "sync";
+})(LockType || (LockType = {}));
+export const DEFAULT_CRUD_UPLOAD_THROTTLE_MS = 1000;
+export const DEFAULT_STREAMING_SYNC_OPTIONS = {
+    retryDelayMs: 5000,
+    logger: Logger.get('PowerSyncStream'),
+    crudUploadThrottleMs: DEFAULT_CRUD_UPLOAD_THROTTLE_MS
+};
+export class AbstractStreamingSyncImplementation extends BaseObserver {
+    _lastSyncedAt;
+    options;
+    abortController;
+    crudUpdateListener;
+    streamingSyncPromise;
+    syncStatus;
+    triggerCrudUpload;
+    constructor(options) {
+        super();
+        this.options = { ...DEFAULT_STREAMING_SYNC_OPTIONS, ...options };
+        this.syncStatus = new SyncStatus({
+            connected: false,
+            lastSyncedAt: undefined,
+            dataFlow: {
+                uploading: false,
+                downloading: false
+            }
+        });
+        this.abortController = null;
+        this.triggerCrudUpload = throttle(() => {
+            if (!this.syncStatus.connected || this.syncStatus.dataFlowStatus.uploading) {
+                return;
+            }
+            this._uploadAllCrud();
+        }, this.options.crudUploadThrottleMs, { trailing: true });
+    }
+    async waitForReady() { }
+    waitForStatus(status) {
+        return new Promise((resolve) => {
+            const l = this.registerListener({
+                statusChanged: (updatedStatus) => {
+                    /**
+                     * Match only the partial status options provided in the
+                     * matching status
+                     */
+                    const matchPartialObject = (compA, compB) => {
+                        return Object.entries(compA).every(([key, value]) => {
+                            const comparisonBValue = compB[key];
+                            if (typeof value == 'object' && typeof comparisonBValue == 'object') {
+                                return matchPartialObject(value, comparisonBValue);
+                            }
+                            return value == comparisonBValue;
+                        });
+                    };
+                    if (matchPartialObject(status, updatedStatus.toJSON())) {
+                        resolve();
+                        l?.();
+                    }
+                }
+            });
+        });
+    }
+    get lastSyncedAt() {
+        const lastSynced = this.syncStatus.lastSyncedAt;
+        return lastSynced && new Date(lastSynced);
+    }
+    get isConnected() {
+        return this.syncStatus.connected;
+    }
+    get logger() {
+        return this.options.logger;
+    }
+    async dispose() {
+        this.crudUpdateListener?.();
+        this.crudUpdateListener = undefined;
+    }
+    async hasCompletedSync() {
+        return this.options.adapter.hasCompletedSync();
+    }
+    async getWriteCheckpoint() {
+        const response = await this.options.remote.get('/write-checkpoint2.json');
+        return response['data']['write_checkpoint'];
+    }
+    async _uploadAllCrud() {
+        return this.obtainLock({
+            type: LockType.CRUD,
+            callback: async () => {
+                this.updateSyncStatus({
+                    dataFlow: {
+                        uploading: true
+                    }
+                });
+                while (true) {
+                    try {
+                        const done = await this.uploadCrudBatch();
+                        if (done) {
+                            break;
+                        }
+                    }
+                    catch (ex) {
+                        this.updateSyncStatus({
+                            connected: false,
+                            dataFlow: {
+                                uploading: false
+                            }
+                        });
+                        await this.delayRetry();
+                        break;
+                    }
+                    finally {
+                        this.updateSyncStatus({
+                            dataFlow: {
+                                uploading: false
+                            }
+                        });
+                    }
+                }
+            }
+        });
+    }
+    async uploadCrudBatch() {
+        const hasCrud = await this.options.adapter.hasCrud();
+        if (hasCrud) {
+            await this.options.uploadCrud();
+            return false;
+        }
+        else {
+            await this.options.adapter.updateLocalTarget(() => this.getWriteCheckpoint());
+            return true;
+        }
+    }
+    async connect() {
+        if (this.abortController) {
+            await this.disconnect();
+        }
+        this.abortController = new AbortController();
+        this.streamingSyncPromise = this.streamingSync(this.abortController.signal);
+        // Return a promise that resolves when the connection status is updated
+        return new Promise((resolve) => {
+            const l = this.registerListener({
+                statusUpdated: (update) => {
+                    // This is triggered as soon as a connection is read from
+                    if (typeof update.connected == 'undefined') {
+                        // only concern with connection updates
+                        return;
+                    }
+                    if (update.connected == false) {
+                        /**
+                         * This function does not reject if initial connect attempt failed
+                         */
+                        this.logger.warn('Initial connect attempt did not successfully connect to server');
+                    }
+                    resolve();
+                    l();
+                }
+            });
+        });
+    }
+    async disconnect() {
+        if (!this.abortController) {
+            return;
+        }
+        // This might be called multiple times
+        if (!this.abortController.signal.aborted) {
+            this.abortController.abort(new AbortOperation('Disconnect has been requested'));
+        }
+        // Await any pending operations before completing the disconnect operation
+        try {
+            await this.streamingSyncPromise;
+        }
+        catch (ex) {
+            // The operation might have failed, all we care about is if it has completed
+            this.logger.warn(ex);
+        }
+        this.streamingSyncPromise = undefined;
+        this.abortController = null;
+        this.updateSyncStatus({ connected: false });
+    }
+    /**
+     * @deprecated use [connect instead]
+     */
+    async streamingSync(signal) {
+        if (!signal) {
+            this.abortController = new AbortController();
+            signal = this.abortController.signal;
+        }
+        /**
+         * Listen for CRUD updates and trigger upstream uploads
+         */
+        this.crudUpdateListener = this.options.adapter.registerListener({
+            crudUpdate: () => this.triggerCrudUpload()
+        });
+        /**
+         * Create a new abort controller which aborts items downstream.
+         * This is needed to close any previous connections on exception.
+         */
+        let nestedAbortController = new AbortController();
+        signal.addEventListener('abort', () => {
+            /**
+             * A request for disconnect was received upstream. Relay the request
+             * to the nested abort controller.
+             */
+            nestedAbortController.abort(signal?.reason ?? new AbortOperation('Received command to disconnect from upstream'));
+            this.crudUpdateListener?.();
+            this.crudUpdateListener = undefined;
+            this.updateSyncStatus({
+                connected: false,
+                dataFlow: {
+                    downloading: false
+                }
+            });
+        });
+        /**
+         * This loops runs until [retry] is false or the abort signal is set to aborted.
+         * Aborting the nestedAbortController will:
+         *  - Abort any pending fetch requests
+         *  - Close any sync stream ReadableStreams (which will also close any established network requests)
+         */
+        while (true) {
+            try {
+                if (signal?.aborted) {
+                    break;
+                }
+                const { retry } = await this.streamingSyncIteration(nestedAbortController.signal);
+                if (!retry) {
+                    /**
+                     * A sync error ocurred that we cannot recover from here.
+                     * This loop must terminate.
+                     * The nestedAbortController will close any open network requests and streams below.
+                     */
+                    break;
+                }
+                // Continue immediately
+            }
+            catch (ex) {
+                /**
+                 * Either:
+                 *  - A network request failed with a failed connection or not OKAY response code.
+                 *  - There was a sync processing error.
+                 * This loop will retry.
+                 * The nested abort controller will cleanup any open network requests and streams.
+                 * The WebRemote should only abort pending fetch requests or close active Readable streams.
+                 */
+                if (ex instanceof AbortOperation) {
+                    this.logger.warn(ex);
+                }
+                else {
+                    this.logger.error(ex);
+                }
+                await this.delayRetry();
+            }
+            finally {
+                if (!signal.aborted) {
+                    nestedAbortController.abort(new AbortOperation('Closing sync stream network requests before retry.'));
+                    nestedAbortController = new AbortController();
+                }
+                this.updateSyncStatus({
+                    connected: false
+                });
+                // On error, wait a little before retrying
+            }
+        }
+        // Mark as disconnected if here
+        this.updateSyncStatus({ connected: false });
+    }
+    async streamingSyncIteration(signal, progress) {
+        return await this.obtainLock({
+            type: LockType.SYNC,
+            signal,
+            callback: async () => {
+                this.logger.debug('Streaming sync iteration started');
+                this.options.adapter.startSession();
+                const bucketEntries = await this.options.adapter.getBucketStates();
+                const initialBuckets = new Map();
+                bucketEntries.forEach((entry) => {
+                    initialBuckets.set(entry.bucket, entry.op_id);
+                });
+                const req = Array.from(initialBuckets.entries()).map(([bucket, after]) => ({
+                    name: bucket,
+                    after: after
+                }));
+                // These are compared by reference
+                let targetCheckpoint = null;
+                let validatedCheckpoint = null;
+                let appliedCheckpoint = null;
+                let bucketSet = new Set(initialBuckets.keys());
+                for await (const line of this.streamingSyncRequest({
+                    buckets: req,
+                    include_checksum: true,
+                    raw_data: true
+                }, signal)) {
+                    if (isStreamingSyncCheckpoint(line)) {
+                        targetCheckpoint = line.checkpoint;
+                        const bucketsToDelete = new Set(bucketSet);
+                        const newBuckets = new Set();
+                        for (const checksum of line.checkpoint.buckets) {
+                            newBuckets.add(checksum.bucket);
+                            bucketsToDelete.delete(checksum.bucket);
+                        }
+                        if (bucketsToDelete.size > 0) {
+                            this.logger.debug('Removing buckets', [...bucketsToDelete]);
+                        }
+                        bucketSet = newBuckets;
+                        await this.options.adapter.removeBuckets([...bucketsToDelete]);
+                        await this.options.adapter.setTargetCheckpoint(targetCheckpoint);
+                    }
+                    else if (isStreamingSyncCheckpointComplete(line)) {
+                        this.logger.debug('Checkpoint complete', targetCheckpoint);
+                        const result = await this.options.adapter.syncLocalDatabase(targetCheckpoint);
+                        if (!result.checkpointValid) {
+                            // This means checksums failed. Start again with a new checkpoint.
+                            // TODO: better back-off
+                            await new Promise((resolve) => setTimeout(resolve, 50));
+                            return { retry: true };
+                        }
+                        else if (!result.ready) {
+                            // Checksums valid, but need more data for a consistent checkpoint.
+                            // Continue waiting.
+                            // landing here the whole time
+                        }
+                        else {
+                            appliedCheckpoint = targetCheckpoint;
+                            this.logger.debug('validated checkpoint', appliedCheckpoint);
+                            this.updateSyncStatus({
+                                connected: true,
+                                lastSyncedAt: new Date(),
+                                dataFlow: {
+                                    downloading: false
+                                }
+                            });
+                        }
+                        validatedCheckpoint = targetCheckpoint;
+                    }
+                    else if (isStreamingSyncCheckpointDiff(line)) {
+                        // TODO: It may be faster to just keep track of the diff, instead of the entire checkpoint
+                        if (targetCheckpoint == null) {
+                            throw new Error('Checkpoint diff without previous checkpoint');
+                        }
+                        const diff = line.checkpoint_diff;
+                        const newBuckets = new Map();
+                        for (const checksum of targetCheckpoint.buckets) {
+                            newBuckets.set(checksum.bucket, checksum);
+                        }
+                        for (const checksum of diff.updated_buckets) {
+                            newBuckets.set(checksum.bucket, checksum);
+                        }
+                        for (const bucket of diff.removed_buckets) {
+                            newBuckets.delete(bucket);
+                        }
+                        const newCheckpoint = {
+                            last_op_id: diff.last_op_id,
+                            buckets: [...newBuckets.values()],
+                            write_checkpoint: diff.write_checkpoint
+                        };
+                        targetCheckpoint = newCheckpoint;
+                        bucketSet = new Set(newBuckets.keys());
+                        const bucketsToDelete = diff.removed_buckets;
+                        if (bucketsToDelete.length > 0) {
+                            this.logger.debug('Remove buckets', bucketsToDelete);
+                        }
+                        await this.options.adapter.removeBuckets(bucketsToDelete);
+                        await this.options.adapter.setTargetCheckpoint(targetCheckpoint);
+                    }
+                    else if (isStreamingSyncData(line)) {
+                        const { data } = line;
+                        this.updateSyncStatus({
+                            dataFlow: {
+                                downloading: true
+                            }
+                        });
+                        await this.options.adapter.saveSyncData({ buckets: [SyncDataBucket.fromRow(data)] });
+                    }
+                    else if (isStreamingKeepalive(line)) {
+                        const remaining_seconds = line.token_expires_in;
+                        if (remaining_seconds == 0) {
+                            // Connection would be closed automatically right after this
+                            this.logger.debug('Token expiring; reconnect');
+                            return { retry: true };
+                        }
+                        this.triggerCrudUpload();
+                    }
+                    else {
+                        this.logger.debug('Sync complete');
+                        if (targetCheckpoint === appliedCheckpoint) {
+                            this.updateSyncStatus({
+                                connected: true,
+                                lastSyncedAt: new Date()
+                            });
+                        }
+                        else if (validatedCheckpoint === targetCheckpoint) {
+                            const result = await this.options.adapter.syncLocalDatabase(targetCheckpoint);
+                            if (!result.checkpointValid) {
+                                // This means checksums failed. Start again with a new checkpoint.
+                                // TODO: better back-off
+                                await new Promise((resolve) => setTimeout(resolve, 50));
+                                return { retry: false };
+                            }
+                            else if (!result.ready) {
+                                // Checksums valid, but need more data for a consistent checkpoint.
+                                // Continue waiting.
+                            }
+                            else {
+                                appliedCheckpoint = targetCheckpoint;
+                                this.updateSyncStatus({
+                                    connected: true,
+                                    lastSyncedAt: new Date(),
+                                    dataFlow: {
+                                        downloading: false
+                                    }
+                                });
+                            }
+                        }
+                    }
+                    progress?.();
+                }
+                this.logger.debug('Stream input empty');
+                // Connection closed. Likely due to auth issue.
+                return { retry: true };
+            }
+        });
+    }
+    async *streamingSyncRequest(req, signal) {
+        const body = await this.options.remote.postStreaming('/sync/stream', req, {}, signal);
+        // A connection is active
+        // There is a connection now
+        Promise.resolve().then(() => this.triggerCrudUpload());
+        this.updateSyncStatus({
+            connected: true
+        });
+        const stream = ndjsonStream(body);
+        const reader = stream.getReader();
+        try {
+            while (true) {
+                // Read from the stream
+                const { done, value } = await reader.read();
+                // Exit if we're done
+                if (done)
+                    return;
+                // Else yield the chunk
+                yield value;
+            }
+        }
+        finally {
+            reader.releaseLock();
+        }
+    }
+    updateSyncStatus(options) {
+        const updatedStatus = new SyncStatus({
+            connected: options.connected ?? this.syncStatus.connected,
+            lastSyncedAt: options.lastSyncedAt ?? this.syncStatus.lastSyncedAt,
+            dataFlow: {
+                ...this.syncStatus.dataFlowStatus,
+                ...options.dataFlow
+            }
+        });
+        if (!this.syncStatus.isEqual(updatedStatus)) {
+            this.syncStatus = updatedStatus;
+            // Only trigger this is there was a change
+            this.iterateListeners((cb) => cb.statusChanged?.(updatedStatus));
+        }
+        // trigger this for all updates
+        this.iterateListeners((cb) => cb.statusUpdated?.(options));
+    }
+    async delayRetry() {
+        return new Promise((resolve) => setTimeout(resolve, this.options.retryDelayMs));
+    }
+}

package/lib/client/sync/stream/streaming-sync-types.d.ts

@@ -0,0 +1,116 @@
+import { BucketChecksum, Checkpoint } from '../bucket/BucketStorageAdapter';
+import { CrudEntry, OpId } from '../bucket/CrudEntry';
+import { SyncDataBucketJSON } from '../bucket/SyncDataBucket';
+/**
+ * For sync2.json
+ */
+export interface ContinueCheckpointRequest {
+    /**
+     * Existing bucket states. Only these buckets are synchronized.
+     */
+    buckets: BucketRequest[];
+    checkpoint_token: string;
+    limit?: number;
+}
+export interface SyncNewCheckpointRequest {
+    /**
+     * Existing bucket states. Used if include_data is specified.
+     */
+    buckets?: BucketRequest[];
+    request_checkpoint: {
+        /**
+         * Whether or not to include an initial data request.
+         */
+        include_data: boolean;
+        /**
+         * Whether or not to compute a checksum.
+         */
+        include_checksum: boolean;
+    };
+    limit?: number;
+}
+export type SyncRequest = ContinueCheckpointRequest | SyncNewCheckpointRequest;
+export interface SyncResponse {
+    /**
+     * Data for the buckets returned. May not have an an entry for each bucket in the request.
+     */
+    data?: SyncDataBucketJSON[];
+    /**
+     * True if the response limit has been reached, and another request must be made.
+     */
+    has_more: boolean;
+    checkpoint_token?: string;
+    checkpoint?: Checkpoint;
+}
+export interface StreamingSyncRequest {
+    /**
+     * Existing bucket states.
+     */
+    buckets?: BucketRequest[];
+    /**
+     * If specified, limit the response to only include these buckets.
+     */
+    only?: string[];
+    /**
+     * Whether or not to compute a checksum for each checkpoint
+     */
+    include_checksum: boolean;
+    /**
+     * Changes the response to stringified data in each OplogEntry
+     */
+    raw_data: boolean;
+}
+export interface StreamingSyncCheckpoint {
+    checkpoint: Checkpoint;
+}
+export interface StreamingSyncCheckpointDiff {
+    checkpoint_diff: {
+        last_op_id: OpId;
+        updated_buckets: BucketChecksum[];
+        removed_buckets: string[];
+        write_checkpoint: string;
+    };
+}
+export interface StreamingSyncDataJSON {
+    data: SyncDataBucketJSON;
+}
+export interface StreamingSyncCheckpointComplete {
+    checkpoint_complete: {
+        last_op_id: OpId;
+    };
+}
+export interface StreamingSyncKeepalive {
+    /** If specified, token expires in this many seconds. */
+    token_expires_in: number;
+}
+export type StreamingSyncLine = StreamingSyncDataJSON | StreamingSyncCheckpoint | StreamingSyncCheckpointDiff | StreamingSyncCheckpointComplete | StreamingSyncKeepalive;
+export interface BucketRequest {
+    name: string;
+    /**
+     * Base-10 number. Sync all data from this bucket with op_id > after.
+     */
+    after: OpId;
+}
+export declare function isStreamingSyncData(line: StreamingSyncLine): line is StreamingSyncDataJSON;
+export declare function isStreamingKeepalive(line: StreamingSyncLine): line is StreamingSyncKeepalive;
+export declare function isStreamingSyncCheckpoint(line: StreamingSyncLine): line is StreamingSyncCheckpoint;
+export declare function isStreamingSyncCheckpointComplete(line: StreamingSyncLine): line is StreamingSyncCheckpointComplete;
+export declare function isStreamingSyncCheckpointDiff(line: StreamingSyncLine): line is StreamingSyncCheckpointDiff;
+export declare function isContinueCheckpointRequest(request: SyncRequest): request is ContinueCheckpointRequest;
+export declare function isSyncNewCheckpointRequest(request: SyncRequest): request is SyncNewCheckpointRequest;
+/**
+ * For crud.json
+ */
+export interface CrudRequest {
+    data: CrudEntry[];
+}
+export interface CrudResponse {
+    /**
+     * A sync response with a checkpoint >= this checkpoint would contain all the changes in this request.
+     *
+     * Any earlier checkpoint may or may not contain these changes.
+     *
+     * May be empty when the request contains no ops.
+     */
+    checkpoint?: OpId;
+}

package/lib/client/sync/stream/streaming-sync-types.js

@@ -0,0 +1,22 @@
+export function isStreamingSyncData(line) {
+    return line.data != null;
+}
+export function isStreamingKeepalive(line) {
+    return line.token_expires_in != null;
+}
+export function isStreamingSyncCheckpoint(line) {
+    return line.checkpoint != null;
+}
+export function isStreamingSyncCheckpointComplete(line) {
+    return line.checkpoint_complete != null;
+}
+export function isStreamingSyncCheckpointDiff(line) {
+    return line.checkpoint_diff != null;
+}
+export function isContinueCheckpointRequest(request) {
+    return (Array.isArray(request.buckets) &&
+        typeof request.checkpoint_token == 'string');
+}
+export function isSyncNewCheckpointRequest(request) {
+    return typeof request.request_checkpoint == 'object';
+}

package/lib/db/Column.d.ts

@@ -0,0 +1,19 @@
+export declare enum ColumnType {
+    TEXT = "TEXT",
+    INTEGER = "INTEGER",
+    REAL = "REAL"
+}
+export interface ColumnOptions {
+    name: string;
+    type?: ColumnType;
+}
+export declare class Column {
+    protected options: ColumnOptions;
+    constructor(options: ColumnOptions);
+    get name(): string;
+    get type(): ColumnType | undefined;
+    toJSON(): {
+        name: string;
+        type: ColumnType | undefined;
+    };
+}

package/lib/db/Column.js

@@ -0,0 +1,25 @@
+// https://www.sqlite.org/lang_expr.html#castexpr
+export var ColumnType;
+(function (ColumnType) {
+    ColumnType["TEXT"] = "TEXT";
+    ColumnType["INTEGER"] = "INTEGER";
+    ColumnType["REAL"] = "REAL";
+})(ColumnType || (ColumnType = {}));
+export class Column {
+    options;
+    constructor(options) {
+        this.options = options;
+    }
+    get name() {
+        return this.options.name;
+    }
+    get type() {
+        return this.options.type;
+    }
+    toJSON() {
+        return {
+            name: this.name,
+            type: this.type
+        };
+    }
+}