@powersync/common 1.32.0 → 1.33.1

package/lib/client/sync/stream/core-instruction.d.ts

@@ -0,0 +1,53 @@
+import { StreamingSyncRequest } from './streaming-sync-types.js';
+/**
+ * An internal instruction emitted by the sync client in the core extension in response to the JS
+ * SDK passing sync data into the extension.
+ */
+export type Instruction = {
+    LogLine: LogLine;
+} | {
+    UpdateSyncStatus: UpdateSyncStatus;
+} | {
+    EstablishSyncStream: EstablishSyncStream;
+} | {
+    FetchCredentials: FetchCredentials;
+} | {
+    CloseSyncStream: any;
+} | {
+    FlushFileSystem: any;
+} | {
+    DidCompleteSync: any;
+};
+export interface LogLine {
+    severity: 'DEBUG' | 'INFO' | 'WARNING';
+    line: string;
+}
+export interface EstablishSyncStream {
+    request: StreamingSyncRequest;
+}
+export interface UpdateSyncStatus {
+    status: CoreSyncStatus;
+}
+export interface CoreSyncStatus {
+    connected: boolean;
+    connecting: boolean;
+    priority_status: SyncPriorityStatus[];
+    downloading: DownloadProgress | null;
+}
+export interface SyncPriorityStatus {
+    priority: number;
+    last_synced_at: number | number;
+    has_synced: boolean | null;
+}
+export interface DownloadProgress {
+    buckets: Record<string, BucketProgress>;
+}
+export interface BucketProgress {
+    priority: number;
+    at_last: number;
+    since_last: number;
+    target_count: number;
+}
+export interface FetchCredentials {
+    did_expire: boolean;
+}

package/lib/client/sync/stream/core-instruction.js

@@ -0,0 +1 @@
+export {};

package/lib/db/crud/SyncProgress.d.ts

@@ -1,10 +1,6 @@
+import { BucketProgress } from 'src/client/sync/stream/core-instruction.js';
 /** @internal */
-export type InternalProgressInformation = Record<string, {
-    priority: number;
-    atLast: number;
-    sinceLast: number;
-    targetCount: number;
-}>;
+export type InternalProgressInformation = Record<string, BucketProgress>;
 /**
  * @internal The priority used by the core extension to indicate that a full sync was completed.
  */

package/lib/db/crud/SyncProgress.js

@@ -46,8 +46,8 @@ export class SyncProgress {
         for (const progress of Object.values(this.internal)) {
             // Include higher-priority buckets, which are represented by lower numbers.
             if (progress.priority <= priority) {
-                downloaded += progress.sinceLast;
-                total += progress.targetCount - progress.atLast;
+                downloaded += progress.since_last;
+                total += progress.target_count - progress.at_last;
             }
         }
         let progress = total == 0 ? 0.0 : downloaded / total;

package/lib/utils/DataStream.d.ts

@@ -1,6 +1,7 @@
 import { ILogger } from 'js-logger';
 import { BaseListener, BaseObserver } from './BaseObserver.js';
-export type DataStreamOptions = {
+export type DataStreamOptions<ParsedData, SourceData> = {
+    mapLine?: (line: SourceData) => ParsedData;
     /**
      * Close the stream if any consumer throws an error
      */
@@ -28,13 +29,15 @@ export declare const DEFAULT_PRESSURE_LIMITS: {
  * native JS streams or async iterators.
  * This is handy for environments such as React Native which need polyfills for the above.
  */
-export declare class DataStream<Data extends any = any> extends BaseObserver<DataStreamListener<Data>> {
-    protected options?: DataStreamOptions | undefined;
-    dataQueue: Data[];
+export declare class DataStream<ParsedData, SourceData = any> extends BaseObserver<DataStreamListener<ParsedData>> {
+    protected options?: DataStreamOptions<ParsedData, SourceData> | undefined;
+    dataQueue: SourceData[];
     protected isClosed: boolean;
     protected processingPromise: Promise<void> | null;
+    protected notifyDataAdded: (() => void) | null;
     protected logger: ILogger;
-    constructor(options?: DataStreamOptions | undefined);
+    protected mapLine: (line: SourceData) => ParsedData;
+    constructor(options?: DataStreamOptions<ParsedData, SourceData> | undefined);
     get highWatermark(): number;
     get lowWatermark(): number;
     get closed(): boolean;
@@ -42,22 +45,18 @@ export declare class DataStream<Data extends any = any> extends BaseObserver<Dat
     /**
      * Enqueues data for the consumers to read
      */
-    enqueueData(data: Data): void;
+    enqueueData(data: SourceData): void;
     /**
      * Reads data once from the data stream
      * @returns a Data payload or Null if the stream closed.
      */
-    read(): Promise<Data | null>;
+    read(): Promise<ParsedData | null>;
     /**
      * Executes a callback for each data item in the stream
      */
-    forEach(callback: DataStreamCallback<Data>): () => void;
-    protected processQueue(): Promise<void>;
-    /**
-     * Creates a new data stream which is a map of the original
-     */
-    map<ReturnData>(callback: (data: Data) => ReturnData): DataStream<ReturnData>;
+    forEach(callback: DataStreamCallback<ParsedData>): () => void;
+    protected processQueue(): Promise<void> | undefined;
     protected hasDataReader(): boolean;
     protected _processQueue(): Promise<void>;
-    protected iterateAsyncErrored(cb: (l: BaseListener) => Promise<void>): Promise<void>;
+    protected iterateAsyncErrored(cb: (l: Partial<DataStreamListener<ParsedData>>) => Promise<void>): Promise<void>;
 }

package/lib/utils/DataStream.js

@@ -14,13 +14,16 @@ export class DataStream extends BaseObserver {
     dataQueue;
     isClosed;
     processingPromise;
+    notifyDataAdded;
     logger;
+    mapLine;
     constructor(options) {
         super();
         this.options = options;
         this.processingPromise = null;
         this.isClosed = false;
         this.dataQueue = [];
+        this.mapLine = options?.mapLine ?? ((line) => line);
         this.logger = options?.logger ?? Logger.get('DataStream');
         if (options?.closeOnError) {
             const l = this.registerListener({
@@ -56,6 +59,7 @@ export class DataStream extends BaseObserver {
             throw new Error('Cannot enqueue data into closed stream.');
         }
         this.dataQueue.push(data);
+        this.notifyDataAdded?.();
         this.processQueue();
     }
     /**
@@ -96,10 +100,20 @@ export class DataStream extends BaseObserver {
             data: callback
         });
     }
-    async processQueue() {
+    processQueue() {
         if (this.processingPromise) {
             return;
         }
+        const promise = (this.processingPromise = this._processQueue());
+        promise.finally(() => {
+            return (this.processingPromise = null);
+        });
+        return promise;
+    }
+    hasDataReader() {
+        return Array.from(this.listeners.values()).some((l) => !!l.data);
+    }
+    async _processQueue() {
         /**
          * Allow listeners to mutate the queue before processing.
          * This allows for operations such as dropping or compressing data
@@ -108,47 +122,31 @@ export class DataStream extends BaseObserver {
         if (this.dataQueue.length >= this.highWatermark) {
             await this.iterateAsyncErrored(async (l) => l.highWater?.());
         }
-        return (this.processingPromise = this._processQueue());
-    }
-    /**
-     * Creates a new data stream which is a map of the original
-     */
-    map(callback) {
-        const stream = new DataStream(this.options);
-        const l = this.registerListener({
-            data: async (data) => {
-                stream.enqueueData(callback(data));
-            },
-            closed: () => {
-                stream.close();
-                l?.();
-            }
-        });
-        return stream;
-    }
-    hasDataReader() {
-        return Array.from(this.listeners.values()).some((l) => !!l.data);
-    }
-    async _processQueue() {
         if (this.isClosed || !this.hasDataReader()) {
-            Promise.resolve().then(() => (this.processingPromise = null));
             return;
         }
         if (this.dataQueue.length) {
             const data = this.dataQueue.shift();
-            await this.iterateAsyncErrored(async (l) => l.data?.(data));
+            const mapped = this.mapLine(data);
+            await this.iterateAsyncErrored(async (l) => l.data?.(mapped));
         }
         if (this.dataQueue.length <= this.lowWatermark) {
-            await this.iterateAsyncErrored(async (l) => l.lowWater?.());
+            const dataAdded = new Promise((resolve) => {
+                this.notifyDataAdded = resolve;
+            });
+            await Promise.race([this.iterateAsyncErrored(async (l) => l.lowWater?.()), dataAdded]);
+            this.notifyDataAdded = null;
         }
-        this.processingPromise = null;
-        if (this.dataQueue.length) {
+        if (this.dataQueue.length > 0) {
             // Next tick
             setTimeout(() => this.processQueue());
         }
     }
     async iterateAsyncErrored(cb) {
-        for (let i of Array.from(this.listeners.values())) {
+        // Important: We need to copy the listeners, as calling a listener could result in adding another
+        // listener, resulting in infinite loops.
+        const listeners = Array.from(this.listeners.values());
+        for (let i of listeners) {
             try {
                 await cb(i);
             }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@powersync/common",
-  "version": "1.32.0",
+  "version": "1.33.1",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -13,8 +13,9 @@
   "exports": {
     ".": {
       "import": "./dist/bundle.mjs",
-      "default": "./dist/bundle.mjs",
-      "types": "./lib/index.d.ts"
+      "require": "./dist/bundle.cjs",
+      "types": "./lib/index.d.ts",
+      "default": "./dist/bundle.mjs"
     }
   },
   "author": "JOURNEYAPPS",
@@ -45,7 +46,6 @@
     "async-mutex": "^0.4.0",
     "bson": "^6.6.0",
     "buffer": "^6.0.3",
-    "can-ndjson-stream": "^1.0.2",
     "cross-fetch": "^4.0.0",
     "event-iterator": "^2.0.0",
     "rollup": "4.14.3",