@powersync/common 0.0.0-dev-20240920103931 → 0.0.0-dev-20250207081035

package/lib/client/AbstractPowerSyncDatabase.d.ts

@@ -10,25 +10,18 @@ import { PowerSyncBackendConnector } from './connection/PowerSyncBackendConnecto
 import { BucketStorageAdapter } from './sync/bucket/BucketStorageAdapter.js';
 import { CrudBatch } from './sync/bucket/CrudBatch.js';
 import { CrudTransaction } from './sync/bucket/CrudTransaction.js';
-import { PowerSyncConnectionOptions, StreamingSyncImplementation, StreamingSyncImplementationListener } from './sync/stream/AbstractStreamingSyncImplementation.js';
+import { type AdditionalConnectionOptions, type PowerSyncConnectionOptions, StreamingSyncImplementation, StreamingSyncImplementationListener, type RequiredAdditionalConnectionOptions } from './sync/stream/AbstractStreamingSyncImplementation.js';
 export interface DisconnectAndClearOptions {
     /** When set to false, data in local-only tables is preserved. */
     clearLocal?: boolean;
 }
-export interface BasePowerSyncDatabaseOptions {
+export interface BasePowerSyncDatabaseOptions extends AdditionalConnectionOptions {
     /** Schema used for the local database. */
     schema: Schema;
     /**
-     * Delay for retrying sync streaming operations
-     * from the PowerSync backend after an error occurs.
+     * @deprecated Use {@link retryDelayMs} instead as this will be removed in future releases.
      */
     retryDelay?: number;
-    /**
-     * Backend Connector CRUD operations are throttled
-     * to occur at most every `crudUploadThrottleMs`
-     * milliseconds.
-     */
-    crudUploadThrottleMs?: number;
     logger?: ILogger;
 }
 export interface PowerSyncDatabaseOptions extends BasePowerSyncDatabaseOptions {
@@ -56,6 +49,8 @@ export interface SQLWatchOptions {
     /** The minimum interval between queries. */
     throttleMs?: number;
     /**
+     * @deprecated All tables specified in {@link tables} will be watched, including PowerSync tables with prefixes.
+     *
      * Allows for watching any SQL table
      * by not removing PowerSync table name prefixes
      */
@@ -74,6 +69,7 @@ export interface WatchOnChangeHandler {
 }
 export interface PowerSyncDBListener extends StreamingSyncImplementationListener {
     initialized: () => void;
+    schemaChanged: (schema: Schema) => void;
 }
 export interface PowerSyncCloseOptions {
     /**
@@ -86,10 +82,11 @@ export interface PowerSyncCloseOptions {
 export declare const DEFAULT_POWERSYNC_CLOSE_OPTIONS: PowerSyncCloseOptions;
 export declare const DEFAULT_WATCH_THROTTLE_MS = 30;
 export declare const DEFAULT_POWERSYNC_DB_OPTIONS: {
-    retryDelay: number;
+    retryDelayMs: number;
     logger: Logger.ILogger;
     crudUploadThrottleMs: number;
 };
+export declare const DEFAULT_CRUD_BATCH_LIMIT = 100;
 /**
  * Requesting nested or recursive locks can block the application in some circumstances.
  * This default lock timeout will act as a failsafe to throw an error if a lock cannot
@@ -144,11 +141,12 @@ export declare abstract class AbstractPowerSyncDatabase extends BaseObserver<Pow
      * Whether a connection to the PowerSync service is currently open.
      */
     get connected(): boolean;
+    get connecting(): boolean;
     /**
      * Opens the DBAdapter given open options using a default open factory
      */
     protected abstract openDBAdapter(options: PowerSyncDatabaseOptionsWithSettings): DBAdapter;
-    protected abstract generateSyncStreamImplementation(connector: PowerSyncBackendConnector): StreamingSyncImplementation;
+    protected abstract generateSyncStreamImplementation(connector: PowerSyncBackendConnector, options: RequiredAdditionalConnectionOptions): StreamingSyncImplementation;
     protected abstract generateBucketStorageAdapter(): BucketStorageAdapter;
     /**
      * @returns A promise which will resolve once initialization is completed.
@@ -181,6 +179,7 @@ export declare abstract class AbstractPowerSyncDatabase extends BaseObserver<Pow
      * While initializing is automatic, this helps to catch and report initialization errors.
      */
     init(): Promise<void>;
+    resolvedConnectionOptions(options?: PowerSyncConnectionOptions): RequiredAdditionalConnectionOptions;
     /**
      * Connects to stream of events from the PowerSync instance.
      */
@@ -230,7 +229,7 @@ export declare abstract class AbstractPowerSyncDatabase extends BaseObserver<Pow
      * data by transaction. One batch may contain data from multiple transactions,
      * and a single transaction may be split over multiple batches.
      */
-    getCrudBatch(limit: number): Promise<CrudBatch | null>;
+    getCrudBatch(limit?: number): Promise<CrudBatch | null>;
     /**
      * Get the next recorded transaction to upload.
      *

package/lib/client/AbstractPowerSyncDatabase.js

@@ -13,7 +13,8 @@ import { PSInternalTable } from './sync/bucket/BucketStorageAdapter.js';
 import { CrudBatch } from './sync/bucket/CrudBatch.js';
 import { CrudEntry } from './sync/bucket/CrudEntry.js';
 import { CrudTransaction } from './sync/bucket/CrudTransaction.js';
-import { DEFAULT_CRUD_UPLOAD_THROTTLE_MS } from './sync/stream/AbstractStreamingSyncImplementation.js';
+import { DEFAULT_CRUD_UPLOAD_THROTTLE_MS, DEFAULT_RETRY_DELAY_MS } from './sync/stream/AbstractStreamingSyncImplementation.js';
+import { runOnSchemaChange } from './runOnSchemaChange.js';
 const POWERSYNC_TABLE_MATCH = /(^ps_data__|^ps_data_local__)/;
 const DEFAULT_DISCONNECT_CLEAR_OPTIONS = {
     clearLocal: true
@@ -23,10 +24,11 @@ export const DEFAULT_POWERSYNC_CLOSE_OPTIONS = {
 };
 export const DEFAULT_WATCH_THROTTLE_MS = 30;
 export const DEFAULT_POWERSYNC_DB_OPTIONS = {
-    retryDelay: 5000,
+    retryDelayMs: 5000,
     logger: Logger.get('PowerSyncDatabase'),
     crudUploadThrottleMs: DEFAULT_CRUD_UPLOAD_THROTTLE_MS
 };
+export const DEFAULT_CRUD_BATCH_LIMIT = 100;
 /**
  * Requesting nested or recursive locks can block the application in some circumstances.
  * This default lock timeout will act as a failsafe to throw an error if a lock cannot
@@ -112,6 +114,9 @@ export class AbstractPowerSyncDatabase extends BaseObserver {
     get connected() {
         return this.currentStatus?.connected || false;
     }
+    get connecting() {
+        return this.currentStatus?.connecting || false;
+    }
     /**
      * @returns A promise which will resolve once initialization is completed.
      */
@@ -172,11 +177,11 @@ export class AbstractPowerSyncDatabase extends BaseObserver {
                 .map((n) => parseInt(n));
         }
         catch (e) {
-            throw new Error(`Unsupported powersync extension version. Need ^0.2.0, got: ${this.sdkVersion}. Details: ${e.message}`);
+            throw new Error(`Unsupported powersync extension version. Need >=0.2.0 <1.0.0, got: ${this.sdkVersion}. Details: ${e.message}`);
         }
-        // Validate ^0.2.0
-        if (versionInts[0] != 0 || versionInts[1] != 2 || versionInts[2] < 0) {
-            throw new Error(`Unsupported powersync extension version. Need ^0.2.0, got: ${this.sdkVersion}`);
+        // Validate >=0.2.0 <1.0.0
+        if (versionInts[0] != 0 || versionInts[1] < 2 || versionInts[2] < 0) {
+            throw new Error(`Unsupported powersync extension version. Need >=0.2.0 <1.0.0, got: ${this.sdkVersion}`);
         }
     }
     async updateHasSynced() {
@@ -210,6 +215,8 @@ export class AbstractPowerSyncDatabase extends BaseObserver {
         }
         this._schema = schema;
         await this.database.execute('SELECT powersync_replace_schema(?)', [JSON.stringify(this.schema.toJSON())]);
+        await this.database.refreshSchema();
+        this.iterateListeners(async (cb) => cb.schemaChanged?.(schema));
     }
     /**
      * Wait for initialization to complete.
@@ -218,6 +225,13 @@ export class AbstractPowerSyncDatabase extends BaseObserver {
     async init() {
         return this.waitForReady();
     }
+    // Use the options passed in during connect, or fallback to the options set during database creation or fallback to the default options
+    resolvedConnectionOptions(options) {
+        return {
+            retryDelayMs: options?.retryDelayMs ?? this.options.retryDelayMs ?? this.options.retryDelay ?? DEFAULT_RETRY_DELAY_MS,
+            crudUploadThrottleMs: options?.crudUploadThrottleMs ?? this.options.crudUploadThrottleMs ?? DEFAULT_CRUD_UPLOAD_THROTTLE_MS
+        };
+    }
     /**
      * Connects to stream of events from the PowerSync instance.
      */
@@ -228,7 +242,11 @@ export class AbstractPowerSyncDatabase extends BaseObserver {
         if (this.closed) {
             throw new Error('Cannot connect using a closed client');
         }
-        this.syncStreamImplementation = this.generateSyncStreamImplementation(connector);
+        const { retryDelayMs, crudUploadThrottleMs } = this.resolvedConnectionOptions(options);
+        this.syncStreamImplementation = this.generateSyncStreamImplementation(connector, {
+            retryDelayMs,
+            crudUploadThrottleMs,
+        });
         this.syncStatusListenerDisposer = this.syncStreamImplementation.registerListener({
             statusChanged: (status) => {
                 this.currentStatus = new SyncStatus({
@@ -326,7 +344,7 @@ export class AbstractPowerSyncDatabase extends BaseObserver {
      * data by transaction. One batch may contain data from multiple transactions,
      * and a single transaction may be split over multiple batches.
      */
-    async getCrudBatch(limit) {
+    async getCrudBatch(limit = DEFAULT_CRUD_BATCH_LIMIT) {
         const result = await this.getAll(`SELECT id, tx_id, data FROM ${PSInternalTable.CRUD} ORDER BY id ASC LIMIT ?`, [limit + 1]);
         const all = result.map((row) => CrudEntry.fromRow(row)) ?? [];
         let haveMore = false;
@@ -386,13 +404,13 @@ export class AbstractPowerSyncDatabase extends BaseObserver {
             if (writeCheckpoint) {
                 const check = await tx.execute(`SELECT 1 FROM ${PSInternalTable.CRUD} LIMIT 1`);
                 if (!check.rows?.length) {
-                    await tx.execute(`UPDATE ${PSInternalTable.BUCKETS} SET target_op = ? WHERE name='$local'`, [
+                    await tx.execute(`UPDATE ${PSInternalTable.BUCKETS} SET target_op = CAST(? as INTEGER) WHERE name='$local'`, [
                         writeCheckpoint
                     ]);
                 }
             }
             else {
-                await tx.execute(`UPDATE ${PSInternalTable.BUCKETS} SET target_op = ? WHERE name='$local'`, [
+                await tx.execute(`UPDATE ${PSInternalTable.BUCKETS} SET target_op = CAST(? as INTEGER) WHERE name='$local'`, [
                     this.bucketStorageAdapter.getMaxOpId()
                 ]);
             }
@@ -502,7 +520,7 @@ export class AbstractPowerSyncDatabase extends BaseObserver {
         if (!onResult) {
             throw new Error('onResult is required');
         }
-        (async () => {
+        const watchQuery = async (abortSignal) => {
             try {
                 const resolvedTables = await this.resolveTables(sql, parameters, options);
                 // Fetch initial data
@@ -521,13 +539,16 @@ export class AbstractPowerSyncDatabase extends BaseObserver {
                     onError
                 }, {
                     ...(options ?? {}),
-                    tables: resolvedTables
+                    tables: resolvedTables,
+                    // Override the abort signal since we intercept it
+                    signal: abortSignal
                 });
             }
             catch (error) {
                 onError?.(error);
             }
-        })();
+        };
+        runOnSchemaChange(watchQuery, this, options);
     }
     /**
      * Execute a read query every time the source tables are modified.
@@ -536,17 +557,18 @@ export class AbstractPowerSyncDatabase extends BaseObserver {
      */
     watchWithAsyncGenerator(sql, parameters, options) {
         return new EventIterator((eventOptions) => {
-            (async () => {
-                const resolvedTables = await this.resolveTables(sql, parameters, options);
-                // Fetch initial data
-                eventOptions.push(await this.executeReadOnly(sql, parameters));
-                for await (const event of this.onChangeWithAsyncGenerator({
-                    ...(options ?? {}),
-                    tables: resolvedTables
-                })) {
-                    eventOptions.push(await this.executeReadOnly(sql, parameters));
+            const handler = {
+                onResult: (result) => {
+                    eventOptions.push(result);
+                },
+                onError: (error) => {
+                    eventOptions.fail(error);
                 }
-            })();
+            };
+            this.watchWithCallback(sql, parameters, handler, options);
+            options?.signal?.addEventListener('abort', () => {
+                eventOptions.stop();
+            });
         });
     }
     async resolveTables(sql, parameters, options) {
@@ -588,7 +610,7 @@ export class AbstractPowerSyncDatabase extends BaseObserver {
             throw new Error('onChange is required');
         }
         const resolvedOptions = options ?? {};
-        const watchedTables = new Set(resolvedOptions.tables ?? []);
+        const watchedTables = new Set((resolvedOptions?.tables ?? []).flatMap((table) => [table, `ps_data__${table}`, `ps_data_local__${table}`]));
         const changedTables = new Set();
         const throttleMs = resolvedOptions.throttleMs ?? DEFAULT_WATCH_THROTTLE_MS;
         const executor = new ControlledExecutor(async (e) => {
@@ -602,8 +624,7 @@ export class AbstractPowerSyncDatabase extends BaseObserver {
         const dispose = this.database.registerListener({
             tablesUpdated: async (update) => {
                 try {
-                    const { rawTableNames } = resolvedOptions;
-                    this.processTableUpdates(update, rawTableNames, changedTables);
+                    this.processTableUpdates(update, changedTables);
                     flushTableUpdates();
                 }
                 catch (error) {
@@ -652,19 +673,11 @@ export class AbstractPowerSyncDatabase extends BaseObserver {
         }
         changedTables.clear();
     }
-    processTableUpdates(updateNotification, rawTableNames, changedTables) {
+    processTableUpdates(updateNotification, changedTables) {
         const tables = isBatchedUpdateNotification(updateNotification)
             ? updateNotification.tables
             : [updateNotification.table];
-        const filteredTables = rawTableNames ? tables : tables.filter((t) => !!t.match(POWERSYNC_TABLE_MATCH));
-        if (!filteredTables.length) {
-            return;
-        }
-        // Remove any PowerSync table prefixes if necessary
-        const mappedTableNames = rawTableNames
-            ? filteredTables
-            : filteredTables.map((t) => t.replace(POWERSYNC_TABLE_MATCH, ''));
-        for (const table of mappedTableNames) {
+        for (const table of tables) {
             changedTables.add(table);
         }
     }

package/lib/client/compilableQueryWatch.d.ts

@@ -0,0 +1,7 @@
+import { CompilableQuery } from './../types/types.js';
+import { AbstractPowerSyncDatabase, SQLWatchOptions } from './AbstractPowerSyncDatabase.js';
+export interface CompilableQueryWatchHandler<T> {
+    onResult: (results: T[]) => void;
+    onError?: (error: Error) => void;
+}
+export declare function compilableQueryWatch<T>(db: AbstractPowerSyncDatabase, query: CompilableQuery<T>, handler: CompilableQueryWatchHandler<T>, options?: SQLWatchOptions): void;

package/lib/client/compilableQueryWatch.js

@@ -0,0 +1,37 @@
+import { runOnSchemaChange } from './runOnSchemaChange.js';
+export function compilableQueryWatch(db, query, handler, options) {
+    const { onResult, onError = (e) => { } } = handler ?? {};
+    if (!onResult) {
+        throw new Error('onResult is required');
+    }
+    const watchQuery = async (abortSignal) => {
+        try {
+            const toSql = query.compile();
+            const resolvedTables = await db.resolveTables(toSql.sql, toSql.parameters, options);
+            // Fetch initial data
+            const result = await query.execute();
+            onResult(result);
+            db.onChangeWithCallback({
+                onChange: async () => {
+                    try {
+                        const result = await query.execute();
+                        onResult(result);
+                    }
+                    catch (error) {
+                        onError(error);
+                    }
+                },
+                onError
+            }, {
+                ...(options ?? {}),
+                tables: resolvedTables,
+                // Override the abort signal since we intercept it
+                signal: abortSignal
+            });
+        }
+        catch (error) {
+            onError(error);
+        }
+    };
+    runOnSchemaChange(watchQuery, db, options);
+}

package/lib/client/runOnSchemaChange.d.ts

@@ -0,0 +1,2 @@
+import { AbstractPowerSyncDatabase, SQLWatchOptions } from './AbstractPowerSyncDatabase.js';
+export declare function runOnSchemaChange(callback: (signal: AbortSignal) => void, db: AbstractPowerSyncDatabase, options?: SQLWatchOptions): void;

package/lib/client/runOnSchemaChange.js

@@ -0,0 +1,23 @@
+export function runOnSchemaChange(callback, db, options) {
+    const triggerWatchedQuery = () => {
+        const abortController = new AbortController();
+        let disposeSchemaListener = null;
+        const stopWatching = () => {
+            abortController.abort('Abort triggered');
+            disposeSchemaListener?.();
+            disposeSchemaListener = null;
+            // Stop listening to upstream abort for this watch
+            options?.signal?.removeEventListener('abort', stopWatching);
+        };
+        options?.signal?.addEventListener('abort', stopWatching);
+        disposeSchemaListener = db.registerListener({
+            schemaChanged: async () => {
+                stopWatching();
+                // Re trigger the watched query (recursively), setTimeout ensures that we don't modify the list of listeners while iterating through them
+                setTimeout(() => triggerWatchedQuery(), 0);
+            }
+        });
+        callback(abortController.signal);
+    };
+    triggerWatchedQuery();
+}

package/lib/client/sync/bucket/SqliteBucketStorage.js

@@ -63,7 +63,7 @@ export class SqliteBucketStorage extends BaseObserver {
      */
     startSession() { }
     async getBucketStates() {
-        const result = await this.db.getAll('SELECT name as bucket, cast(last_op as TEXT) as op_id FROM ps_buckets WHERE pending_delete = 0');
+        const result = await this.db.getAll("SELECT name as bucket, cast(last_op as TEXT) as op_id FROM ps_buckets WHERE pending_delete = 0 AND name != '$local'");
         return result;
     }
     async saveSyncData(batch) {
@@ -204,9 +204,7 @@ export class SqliteBucketStorage extends BaseObserver {
         this.compactCounter = 0;
     }
     async updateLocalTarget(cb) {
-        const rs1 = await this.db.getAll("SELECT target_op FROM ps_buckets WHERE name = '$local' AND target_op = ?", [
-            MAX_OP_ID
-        ]);
+        const rs1 = await this.db.getAll("SELECT target_op FROM ps_buckets WHERE name = '$local' AND target_op = CAST(? as INTEGER)", [MAX_OP_ID]);
         if (!rs1.length) {
             // Nothing to update
             return false;

package/lib/client/sync/stream/AbstractRemote.js

@@ -6,14 +6,15 @@ import { WebsocketClientTransport } from 'rsocket-websocket-client';
 import { AbortOperation } from '../../../utils/AbortOperation.js';
 import { DataStream } from '../../../utils/DataStream.js';
 import { version as POWERSYNC_JS_VERSION } from '../../../../package.json';
+const POWERSYNC_TRAILING_SLASH_MATCH = /\/+$/;
 // Refresh at least 30 sec before it expires
 const REFRESH_CREDENTIALS_SAFETY_PERIOD_MS = 30_000;
-const SYNC_QUEUE_REQUEST_N = 10;
+const SYNC_QUEUE_REQUEST_N = 1;
 const SYNC_QUEUE_REQUEST_LOW_WATER = 5;
 // Keep alive message is sent every period
-const KEEP_ALIVE_MS = 20_000;
+const KEEP_ALIVE_MS = 60_000;
 // The ACK must be received in this period
-const KEEP_ALIVE_LIFETIME_MS = 30_000;
+const KEEP_ALIVE_LIFETIME_MS = 90_000;
 export const DEFAULT_REMOTE_LOGGER = Logger.get('PowerSyncRemote');
 /**
  * Class wrapper for providing a fetch implementation.
@@ -61,6 +62,9 @@ export class AbstractRemote {
             return this.credentials;
         }
         this.credentials = await this.connector.fetchCredentials();
+        if (this.credentials?.endpoint.match(POWERSYNC_TRAILING_SLASH_MATCH)) {
+            throw new Error(`A trailing forward slash "/" was found in the fetchCredentials endpoint: "${this.credentials.endpoint}". Remove the trailing forward slash "/" to fix this error.`);
+        }
         return this.credentials;
     }
     getUserAgent() {

package/lib/client/sync/stream/AbstractStreamingSyncImplementation.d.ts

@@ -20,10 +20,9 @@ export interface LockOptions<T> {
     type: LockType;
     signal?: AbortSignal;
 }
-export interface AbstractStreamingSyncImplementationOptions {
+export interface AbstractStreamingSyncImplementationOptions extends AdditionalConnectionOptions {
     adapter: BucketStorageAdapter;
     uploadCrud: () => Promise<void>;
-    crudUploadThrottleMs?: number;
     /**
      * An identifier for which PowerSync DB this sync implementation is
      * linked to. Most commonly DB name, but not restricted to DB name.
@@ -31,7 +30,6 @@ export interface AbstractStreamingSyncImplementationOptions {
     identifier?: string;
     logger?: ILogger;
     remote: AbstractRemote;
-    retryDelayMs?: number;
 }
 export interface StreamingSyncImplementationListener extends BaseListener {
     /**
@@ -48,7 +46,10 @@ export interface StreamingSyncImplementationListener extends BaseListener {
  * Configurable options to be used when connecting to the PowerSync
  * backend instance.
  */
-export interface PowerSyncConnectionOptions {
+export interface PowerSyncConnectionOptions extends BaseConnectionOptions, AdditionalConnectionOptions {
+}
+/** @internal */
+export interface BaseConnectionOptions {
     /**
      * The connection method to use when streaming updates from
      * the PowerSync backend instance.
@@ -60,6 +61,22 @@ export interface PowerSyncConnectionOptions {
      */
     params?: Record<string, StreamingSyncRequestParameterType>;
 }
+/** @internal */
+export interface AdditionalConnectionOptions {
+    /**
+     * Delay for retrying sync streaming operations
+     * from the PowerSync backend after an error occurs.
+     */
+    retryDelayMs?: number;
+    /**
+     * Backend Connector CRUD operations are throttled
+     * to occur at most every `crudUploadThrottleMs`
+     * milliseconds.
+     */
+    crudUploadThrottleMs?: number;
+}
+/** @internal */
+export type RequiredAdditionalConnectionOptions = Required<AdditionalConnectionOptions>;
 export interface StreamingSyncImplementation extends BaseObserver<StreamingSyncImplementationListener>, Disposable {
     /**
      * Connects to the sync service
@@ -80,12 +97,14 @@ export interface StreamingSyncImplementation extends BaseObserver<StreamingSyncI
     waitForStatus(status: SyncStatusOptions): Promise<void>;
 }
 export declare const DEFAULT_CRUD_UPLOAD_THROTTLE_MS = 1000;
+export declare const DEFAULT_RETRY_DELAY_MS = 5000;
 export declare const DEFAULT_STREAMING_SYNC_OPTIONS: {
     retryDelayMs: number;
     logger: Logger.ILogger;
     crudUploadThrottleMs: number;
 };
-export declare const DEFAULT_STREAM_CONNECTION_OPTIONS: Required<PowerSyncConnectionOptions>;
+export type RequiredPowerSyncConnectionOptions = Required<BaseConnectionOptions>;
+export declare const DEFAULT_STREAM_CONNECTION_OPTIONS: RequiredPowerSyncConnectionOptions;
 export declare abstract class AbstractStreamingSyncImplementation extends BaseObserver<StreamingSyncImplementationListener> implements StreamingSyncImplementation {
     protected _lastSyncedAt: Date | null;
     protected options: AbstractStreamingSyncImplementationOptions;

package/lib/client/sync/stream/AbstractStreamingSyncImplementation.js

@@ -2,9 +2,9 @@ import Logger from 'js-logger';
 import { SyncStatus } from '../../../db/crud/SyncStatus.js';
 import { AbortOperation } from '../../../utils/AbortOperation.js';
 import { BaseObserver } from '../../../utils/BaseObserver.js';
+import { throttleLeadingTrailing } from '../../../utils/throttle.js';
 import { SyncDataBucket } from '../bucket/SyncDataBucket.js';
 import { isStreamingKeepalive, isStreamingSyncCheckpoint, isStreamingSyncCheckpointComplete, isStreamingSyncCheckpointDiff, isStreamingSyncData } from './streaming-sync-types.js';
-import { throttleLeadingTrailing } from '../../../utils/throttle.js';
 export var LockType;
 (function (LockType) {
     LockType["CRUD"] = "crud";
@@ -16,8 +16,9 @@ export var SyncStreamConnectionMethod;
     SyncStreamConnectionMethod["WEB_SOCKET"] = "web-socket";
 })(SyncStreamConnectionMethod || (SyncStreamConnectionMethod = {}));
 export const DEFAULT_CRUD_UPLOAD_THROTTLE_MS = 1000;
+export const DEFAULT_RETRY_DELAY_MS = 5000;
 export const DEFAULT_STREAMING_SYNC_OPTIONS = {
-    retryDelayMs: 5000,
+    retryDelayMs: DEFAULT_RETRY_DELAY_MS,
     logger: Logger.get('PowerSyncStream'),
     crudUploadThrottleMs: DEFAULT_CRUD_UPLOAD_THROTTLE_MS
 };
@@ -38,6 +39,7 @@ export class AbstractStreamingSyncImplementation extends BaseObserver {
         this.options = { ...DEFAULT_STREAMING_SYNC_OPTIONS, ...options };
         this.syncStatus = new SyncStatus({
             connected: false,
+            connecting: false,
             lastSyncedAt: undefined,
             dataFlow: {
                 uploading: false,
@@ -121,7 +123,7 @@ export class AbstractStreamingSyncImplementation extends BaseObserver {
                          */
                         const nextCrudItem = await this.options.adapter.nextCrudItem();
                         if (nextCrudItem) {
-                            if (nextCrudItem.id == checkedCrudItem?.id) {
+                            if (nextCrudItem.clientId == checkedCrudItem?.clientId) {
                                 // This will force a higher log level than exceptions which are caught here.
                                 this.logger.warn(`Potentially previously uploaded CRUD entries are still present in the upload queue.
 Make sure to handle uploads and complete CRUD transactions or batches by calling and awaiting their [.complete()] method.
@@ -207,7 +209,7 @@ The next upload iteration will be delayed.`);
         }
         this.streamingSyncPromise = undefined;
         this.abortController = null;
-        this.updateSyncStatus({ connected: false });
+        this.updateSyncStatus({ connected: false, connecting: false });
     }
     /**
      * @deprecated use [connect instead]
@@ -238,6 +240,7 @@ The next upload iteration will be delayed.`);
             this.crudUpdateListener = undefined;
             this.updateSyncStatus({
                 connected: false,
+                connecting: false,
                 dataFlow: {
                     downloading: false
                 }
@@ -250,6 +253,7 @@ The next upload iteration will be delayed.`);
          *  - Close any sync stream ReadableStreams (which will also close any established network requests)
          */
         while (true) {
+            this.updateSyncStatus({ connecting: true });
             try {
                 if (signal?.aborted) {
                     break;
@@ -280,6 +284,7 @@ The next upload iteration will be delayed.`);
                 else {
                     this.logger.error(ex);
                 }
+                // On error, wait a little before retrying
                 await this.delayRetry();
             }
             finally {
@@ -288,13 +293,13 @@ The next upload iteration will be delayed.`);
                     nestedAbortController = new AbortController();
                 }
                 this.updateSyncStatus({
-                    connected: false
+                    connected: false,
+                    connecting: true // May be unnecessary
                 });
-                // On error, wait a little before retrying
             }
         }
         // Mark as disconnected if here
-        this.updateSyncStatus({ connected: false });
+        this.updateSyncStatus({ connected: false, connecting: false });
     }
     async streamingSyncIteration(signal, options) {
         return await this.obtainLock({
@@ -489,6 +494,7 @@ The next upload iteration will be delayed.`);
     updateSyncStatus(options) {
         const updatedStatus = new SyncStatus({
             connected: options.connected ?? this.syncStatus.connected,
+            connecting: !options.connected && (options.connecting ?? this.syncStatus.connecting),
             lastSyncedAt: options.lastSyncedAt ?? this.syncStatus.lastSyncedAt,
             dataFlow: {
                 ...this.syncStatus.dataFlowStatus,

package/lib/db/DBAdapter.d.ts

@@ -90,6 +90,10 @@ export interface DBAdapter extends BaseObserverInterface<DBAdapterListener>, DBG
     readTransaction: <T>(fn: (tx: Transaction) => Promise<T>, options?: DBLockOptions) => Promise<T>;
     writeLock: <T>(fn: (tx: LockContext) => Promise<T>, options?: DBLockOptions) => Promise<T>;
     writeTransaction: <T>(fn: (tx: Transaction) => Promise<T>, options?: DBLockOptions) => Promise<T>;
+    /**
+     * This method refreshes the schema information across all connections. This is for advanced use cases, and should generally not be needed.
+     */
+    refreshSchema: () => Promise<void>;
 }
 export declare function isBatchedUpdateNotification(update: BatchedUpdateNotification | UpdateNotification): update is BatchedUpdateNotification;
 export declare function extractTableUpdates(update: BatchedUpdateNotification | UpdateNotification): string[];

package/lib/db/crud/SyncStatus.d.ts

@@ -4,6 +4,7 @@ export type SyncDataFlowStatus = Partial<{
 }>;
 export type SyncStatusOptions = {
     connected?: boolean;
+    connecting?: boolean;
     dataFlow?: SyncDataFlowStatus;
     lastSyncedAt?: Date;
     hasSynced?: boolean;
@@ -15,6 +16,7 @@ export declare class SyncStatus {
      * true if currently connected.
      */
     get connected(): boolean;
+    get connecting(): boolean;
     /**
      *  Time that a last sync has fully completed, if any.
      *  Currently this is reset to null after a restart.

package/lib/db/crud/SyncStatus.js

@@ -9,6 +9,9 @@ export class SyncStatus {
     get connected() {
         return this.options.connected ?? false;
     }
+    get connecting() {
+        return this.options.connecting ?? false;
+    }
     /**
      *  Time that a last sync has fully completed, if any.
      *  Currently this is reset to null after a restart.
@@ -44,11 +47,12 @@ export class SyncStatus {
     }
     getMessage() {
         const dataFlow = this.dataFlowStatus;
-        return `SyncStatus<connected: ${this.connected} lastSyncedAt: ${this.lastSyncedAt} hasSynced: ${this.hasSynced}. Downloading: ${dataFlow.downloading}. Uploading: ${dataFlow.uploading}`;
+        return `SyncStatus<connected: ${this.connected} connecting: ${this.connecting} lastSyncedAt: ${this.lastSyncedAt} hasSynced: ${this.hasSynced}. Downloading: ${dataFlow.downloading}. Uploading: ${dataFlow.uploading}`;
     }
     toJSON() {
         return {
             connected: this.connected,
+            connecting: this.connecting,
             dataFlow: this.dataFlowStatus,
             lastSyncedAt: this.lastSyncedAt,
             hasSynced: this.hasSynced

package/lib/db/schema/Column.d.ts

@@ -12,7 +12,7 @@ export type BaseColumnType<T extends number | string | null> = {
 };
 export type ColumnsType = Record<string, BaseColumnType<any>>;
 export type ExtractColumnValueType<T extends BaseColumnType<any>> = T extends BaseColumnType<infer R> ? R : unknown;
-export declare const MAX_AMOUNT_OF_COLUMNS = 63;
+export declare const MAX_AMOUNT_OF_COLUMNS = 1999;
 export declare const column: {
     text: BaseColumnType<string | null>;
     integer: BaseColumnType<number | null>;

package/lib/db/schema/Column.js

@@ -14,9 +14,9 @@ const integer = {
 const real = {
     type: ColumnType.REAL
 };
-// There is maximum of 127 arguments for any function in SQLite. Currently we use json_object which uses 1 arg per key (column name)
-// and one per value, which limits it to 63 arguments.
-export const MAX_AMOUNT_OF_COLUMNS = 63;
+// powersync-sqlite-core limits the number of column per table to 1999, due to internal SQLite limits.
+// In earlier versions this was limited to 63.
+export const MAX_AMOUNT_OF_COLUMNS = 1999;
 export const column = {
     text,
     integer,