@powersync/common 0.0.0-dev-20250609122429 → 0.0.0-dev-20250625140957

package/lib/client/sync/bucket/BucketStorageAdapter.d.ts

@@ -51,17 +51,27 @@ export declare enum PSInternalTable {
     OPLOG = "ps_oplog",
     UNTYPED = "ps_untyped"
 }
+export declare enum PowerSyncControlCommand {
+    PROCESS_TEXT_LINE = "line_text",
+    PROCESS_BSON_LINE = "line_binary",
+    STOP = "stop",
+    START = "start",
+    NOTIFY_TOKEN_REFRESHED = "refreshed_token",
+    NOTIFY_CRUD_UPLOAD_COMPLETED = "completed_upload"
+}
 export interface BucketStorageListener extends BaseListener {
     crudUpdate: () => void;
 }
 export interface BucketStorageAdapter extends BaseObserver<BucketStorageListener>, Disposable {
     init(): Promise<void>;
-    saveSyncData(batch: SyncDataBatch): Promise<void>;
+    saveSyncData(batch: SyncDataBatch, fixedKeyFormat?: boolean): Promise<void>;
     removeBuckets(buckets: string[]): Promise<void>;
     setTargetCheckpoint(checkpoint: Checkpoint): Promise<void>;
     startSession(): void;
     getBucketStates(): Promise<BucketState[]>;
     getBucketOperationProgress(): Promise<BucketOperationProgress>;
+    hasMigratedSubkeys(): Promise<boolean>;
+    migrateToFixedSubkeys(): Promise<void>;
     syncLocalDatabase(checkpoint: Checkpoint, priority?: number): Promise<{
         checkpointValid: boolean;
         ready: boolean;
@@ -72,17 +82,13 @@ export interface BucketStorageAdapter extends BaseObserver<BucketStorageListener
     getCrudBatch(limit?: number): Promise<CrudBatch | null>;
     hasCompletedSync(): Promise<boolean>;
     updateLocalTarget(cb: () => Promise<string>): Promise<boolean>;
-    /**
-     * Exposed for tests only.
-     */
-    autoCompact(): Promise<void>;
-    /**
-     * Exposed for tests only.
-     */
-    forceCompact(): Promise<void>;
     getMaxOpId(): string;
     /**
      * Get an unique client id.
      */
     getClientId(): Promise<string>;
+    /**
+     * Invokes the `powersync_control` function for the sync client.
+     */
+    control(op: PowerSyncControlCommand, payload: string | ArrayBuffer | null): Promise<string>;
 }

package/lib/client/sync/bucket/BucketStorageAdapter.js

@@ -6,3 +6,12 @@ export var PSInternalTable;
     PSInternalTable["OPLOG"] = "ps_oplog";
     PSInternalTable["UNTYPED"] = "ps_untyped";
 })(PSInternalTable || (PSInternalTable = {}));
+export var PowerSyncControlCommand;
+(function (PowerSyncControlCommand) {
+    PowerSyncControlCommand["PROCESS_TEXT_LINE"] = "line_text";
+    PowerSyncControlCommand["PROCESS_BSON_LINE"] = "line_binary";
+    PowerSyncControlCommand["STOP"] = "stop";
+    PowerSyncControlCommand["START"] = "start";
+    PowerSyncControlCommand["NOTIFY_TOKEN_REFRESHED"] = "refreshed_token";
+    PowerSyncControlCommand["NOTIFY_CRUD_UPLOAD_COMPLETED"] = "completed_upload";
+})(PowerSyncControlCommand || (PowerSyncControlCommand = {}));

package/lib/client/sync/bucket/CrudEntry.d.ts

@@ -30,6 +30,8 @@ type CrudEntryOutputJSON = {
     id: string;
     tx_id?: number;
     data?: Record<string, any>;
+    old?: Record<string, any>;
+    metadata?: string;
 };
 /**
  * A single client-side change.

package/lib/client/sync/bucket/CrudEntry.js

@@ -74,7 +74,9 @@ export class CrudEntry {
             type: this.table,
             id: this.id,
             tx_id: this.transactionId,
-            data: this.opData
+            data: this.opData,
+            old: this.previousValues,
+            metadata: this.metadata
         };
     }
     equals(entry) {
@@ -93,6 +95,15 @@ export class CrudEntry {
      * Generates an array for use in deep comparison operations
      */
     toComparisonArray() {
-        return [this.transactionId, this.clientId, this.op, this.table, this.id, this.opData];
+        return [
+            this.transactionId,
+            this.clientId,
+            this.op,
+            this.table,
+            this.id,
+            this.opData,
+            this.previousValues,
+            this.metadata
+        ];
     }
 }

package/lib/client/sync/bucket/OplogEntry.d.ts

@@ -7,17 +7,17 @@ export interface OplogEntryJSON {
     object_type?: string;
     op_id: string;
     op: OpTypeJSON;
-    subkey?: string | object;
+    subkey?: string;
 }
 export declare class OplogEntry {
     op_id: OpId;
     op: OpType;
     checksum: number;
-    subkey: string;
+    subkey?: string | undefined;
     object_type?: string | undefined;
     object_id?: string | undefined;
     data?: string | undefined;
     static fromRow(row: OplogEntryJSON): OplogEntry;
-    constructor(op_id: OpId, op: OpType, checksum: number, subkey: string, object_type?: string | undefined, object_id?: string | undefined, data?: string | undefined);
-    toJSON(): OplogEntryJSON;
+    constructor(op_id: OpId, op: OpType, checksum: number, subkey?: string | undefined, object_type?: string | undefined, object_id?: string | undefined, data?: string | undefined);
+    toJSON(fixedKeyEncoding?: boolean): OplogEntryJSON;
 }

package/lib/client/sync/bucket/OplogEntry.js

@@ -8,7 +8,7 @@ export class OplogEntry {
     object_id;
     data;
     static fromRow(row) {
-        return new OplogEntry(row.op_id, OpType.fromJSON(row.op), row.checksum, typeof row.subkey == 'string' ? row.subkey : JSON.stringify(row.subkey), row.object_type, row.object_id, row.data);
+        return new OplogEntry(row.op_id, OpType.fromJSON(row.op), row.checksum, row.subkey, row.object_type, row.object_id, row.data);
     }
     constructor(op_id, op, checksum, subkey, object_type, object_id, data) {
         this.op_id = op_id;
@@ -19,7 +19,7 @@ export class OplogEntry {
         this.object_id = object_id;
         this.data = data;
     }
-    toJSON() {
+    toJSON(fixedKeyEncoding = false) {
         return {
             op_id: this.op_id,
             op: this.op.toJSON(),
@@ -27,7 +27,9 @@ export class OplogEntry {
             object_id: this.object_id,
             checksum: this.checksum,
             data: this.data,
-            subkey: JSON.stringify(this.subkey)
+            // Older versions of the JS SDK used to always JSON.stringify here. That has always been wrong,
+            // but we need to migrate gradually to not break existing databases.
+            subkey: fixedKeyEncoding ? this.subkey : JSON.stringify(this.subkey)
         };
     }
 }

package/lib/client/sync/bucket/SqliteBucketStorage.d.ts

@@ -2,7 +2,7 @@ import { Mutex } from 'async-mutex';
 import { ILogger } from 'js-logger';
 import { DBAdapter, Transaction } from '../../../db/DBAdapter.js';
 import { BaseObserver } from '../../../utils/BaseObserver.js';
-import { BucketOperationProgress, BucketState, BucketStorageAdapter, BucketStorageListener, Checkpoint, SyncLocalDatabaseResult } from './BucketStorageAdapter.js';
+import { BucketOperationProgress, BucketState, BucketStorageAdapter, BucketStorageListener, Checkpoint, PowerSyncControlCommand, SyncLocalDatabaseResult } from './BucketStorageAdapter.js';
 import { CrudBatch } from './CrudBatch.js';
 import { CrudEntry } from './CrudEntry.js';
 import { SyncDataBatch } from './SyncDataBatch.js';
@@ -11,14 +11,9 @@ export declare class SqliteBucketStorage extends BaseObserver<BucketStorageListe
     private mutex;
     private logger;
     tableNames: Set<string>;
-    private pendingBucketDeletes;
     private _hasCompletedSync;
     private updateListener;
     private _clientId?;
-    /**
-     * Count up, and do a compact on startup.
-     */
-    private compactCounter;
     constructor(db: DBAdapter, mutex: Mutex, logger?: ILogger);
     init(): Promise<void>;
     dispose(): Promise<void>;
@@ -31,7 +26,7 @@ export declare class SqliteBucketStorage extends BaseObserver<BucketStorageListe
     startSession(): void;
     getBucketStates(): Promise<BucketState[]>;
     getBucketOperationProgress(): Promise<BucketOperationProgress>;
-    saveSyncData(batch: SyncDataBatch): Promise<void>;
+    saveSyncData(batch: SyncDataBatch, fixedKeyFormat?: boolean): Promise<void>;
     removeBuckets(buckets: string[]): Promise<void>;
     /**
      * Mark a bucket for deletion.
@@ -46,13 +41,6 @@ export declare class SqliteBucketStorage extends BaseObserver<BucketStorageListe
      */
     private updateObjectsFromBuckets;
     validateChecksums(checkpoint: Checkpoint, priority: number | undefined): Promise<SyncLocalDatabaseResult>;
-    /**
-     * Force a compact, for tests.
-     */
-    forceCompact(): Promise<void>;
-    autoCompact(): Promise<void>;
-    private deletePendingBuckets;
-    private clearRemoveOps;
     updateLocalTarget(cb: () => Promise<string>): Promise<boolean>;
     nextCrudItem(): Promise<CrudEntry | undefined>;
     hasCrud(): Promise<boolean>;
@@ -68,4 +56,8 @@ export declare class SqliteBucketStorage extends BaseObserver<BucketStorageListe
      * Set a target checkpoint.
      */
     setTargetCheckpoint(checkpoint: Checkpoint): Promise<void>;
+    control(op: PowerSyncControlCommand, payload: string | ArrayBuffer | null): Promise<string>;
+    hasMigratedSubkeys(): Promise<boolean>;
+    migrateToFixedSubkeys(): Promise<void>;
+    static _subkeyMigrationKey: string;
 }

package/lib/client/sync/bucket/SqliteBucketStorage.js

@@ -4,27 +4,20 @@ import { BaseObserver } from '../../../utils/BaseObserver.js';
 import { MAX_OP_ID } from '../../constants.js';
 import { PSInternalTable } from './BucketStorageAdapter.js';
 import { CrudEntry } from './CrudEntry.js';
-const COMPACT_OPERATION_INTERVAL = 1_000;
 export class SqliteBucketStorage extends BaseObserver {
     db;
     mutex;
     logger;
     tableNames;
-    pendingBucketDeletes;
     _hasCompletedSync;
     updateListener;
     _clientId;
-    /**
-     * Count up, and do a compact on startup.
-     */
-    compactCounter = COMPACT_OPERATION_INTERVAL;
     constructor(db, mutex, logger = Logger.get('SqliteBucketStorage')) {
         super();
         this.db = db;
         this.mutex = mutex;
         this.logger = logger;
         this._hasCompletedSync = false;
-        this.pendingBucketDeletes = true;
         this.tableNames = new Set();
         this.updateListener = db.registerListener({
             tablesUpdated: (update) => {
@@ -70,18 +63,15 @@ export class SqliteBucketStorage extends BaseObserver {
         const rows = await this.db.getAll('SELECT name, count_at_last, count_since_last FROM ps_buckets');
         return Object.fromEntries(rows.map((r) => [r.name, { atLast: r.count_at_last, sinceLast: r.count_since_last }]));
     }
-    async saveSyncData(batch) {
+    async saveSyncData(batch, fixedKeyFormat = false) {
         await this.writeTransaction(async (tx) => {
-            let count = 0;
             for (const b of batch.buckets) {
                 const result = await tx.execute('INSERT INTO powersync_operations(op, data) VALUES(?, ?)', [
                     'save',
-                    JSON.stringify({ buckets: [b.toJSON()] })
+                    JSON.stringify({ buckets: [b.toJSON(fixedKeyFormat)] })
                 ]);
                 this.logger.debug('saveSyncData', JSON.stringify(result));
-                count += b.data.length;
             }
-            this.compactCounter += count;
         });
     }
     async removeBuckets(buckets) {
@@ -97,7 +87,6 @@ export class SqliteBucketStorage extends BaseObserver {
             await tx.execute('INSERT INTO powersync_operations(op, data) VALUES(?, ?)', ['delete_bucket', bucket]);
         });
         this.logger.debug('done deleting bucket');
-        this.pendingBucketDeletes = true;
     }
     async hasCompletedSync() {
         if (this._hasCompletedSync) {
@@ -138,7 +127,6 @@ export class SqliteBucketStorage extends BaseObserver {
             this.logger.debug('Not at a consistent checkpoint - cannot update local db');
             return { ready: false, checkpointValid: true };
         }
-        await this.forceCompact();
         return {
             ready: true,
             checkpointValid: true
@@ -209,36 +197,6 @@ export class SqliteBucketStorage extends BaseObserver {
             };
         }
     }
-    /**
-     * Force a compact, for tests.
-     */
-    async forceCompact() {
-        this.compactCounter = COMPACT_OPERATION_INTERVAL;
-        this.pendingBucketDeletes = true;
-        await this.autoCompact();
-    }
-    async autoCompact() {
-        await this.deletePendingBuckets();
-        await this.clearRemoveOps();
-    }
-    async deletePendingBuckets() {
-        if (this.pendingBucketDeletes !== false) {
-            await this.writeTransaction(async (tx) => {
-                await tx.execute('INSERT INTO powersync_operations(op, data) VALUES (?, ?)', ['delete_pending_buckets', '']);
-            });
-            // Executed once after start-up, and again when there are pending deletes.
-            this.pendingBucketDeletes = false;
-        }
-    }
-    async clearRemoveOps() {
-        if (this.compactCounter < COMPACT_OPERATION_INTERVAL) {
-            return;
-        }
-        await this.writeTransaction(async (tx) => {
-            await tx.execute('INSERT INTO powersync_operations(op, data) VALUES (?, ?)', ['clear_remove_ops', '']);
-        });
-        this.compactCounter = 0;
-    }
     async updateLocalTarget(cb) {
         const rs1 = await this.db.getAll("SELECT target_op FROM ps_buckets WHERE name = '$local' AND target_op = CAST(? as INTEGER)", [MAX_OP_ID]);
         if (!rs1.length) {
@@ -339,6 +297,28 @@ export class SqliteBucketStorage extends BaseObserver {
     async setTargetCheckpoint(checkpoint) {
         // No-op for now
     }
+    async control(op, payload) {
+        return await this.writeTransaction(async (tx) => {
+            const [[raw]] = await tx.executeRaw('SELECT powersync_control(?, ?)', [op, payload]);
+            return raw;
+        });
+    }
+    async hasMigratedSubkeys() {
+        const { r } = await this.db.get('SELECT EXISTS(SELECT * FROM ps_kv WHERE key = ?) as r', [
+            SqliteBucketStorage._subkeyMigrationKey
+        ]);
+        return r != 0;
+    }
+    async migrateToFixedSubkeys() {
+        await this.writeTransaction(async (tx) => {
+            await tx.execute('UPDATE ps_oplog SET key = powersync_remove_duplicate_key_encoding(key);');
+            await tx.execute('INSERT OR REPLACE INTO ps_kv (key, value) VALUES (?, ?);', [
+                SqliteBucketStorage._subkeyMigrationKey,
+                '1'
+            ]);
+        });
+    }
+    static _subkeyMigrationKey = 'powersync_js_migrated_subkeys';
 }
 function hasMatchingPriority(priority, bucket) {
     return bucket.priority != null && bucket.priority <= priority;

package/lib/client/sync/bucket/SyncDataBucket.d.ts

@@ -36,5 +36,5 @@ export declare class SyncDataBucket {
      * Use this for the next request.
      */
     next_after?: OpId | undefined);
-    toJSON(): SyncDataBucketJSON;
+    toJSON(fixedKeyEncoding?: boolean): SyncDataBucketJSON;
 }

package/lib/client/sync/bucket/SyncDataBucket.js

@@ -27,13 +27,13 @@ export class SyncDataBucket {
         this.after = after;
         this.next_after = next_after;
     }
-    toJSON() {
+    toJSON(fixedKeyEncoding = false) {
         return {
             bucket: this.bucket,
             has_more: this.has_more,
             after: this.after,
             next_after: this.next_after,
-            data: this.data.map((entry) => entry.toJSON())
+            data: this.data.map((entry) => entry.toJSON(fixedKeyEncoding))
         };
     }
 }

package/lib/client/sync/stream/AbstractRemote.d.ts

@@ -1,4 +1,5 @@
 import type { BSON } from 'bson';
+import { Buffer } from 'buffer';
 import { type fetch } from 'cross-fetch';
 import Logger, { ILogger } from 'js-logger';
 import { DataStream } from '../../../utils/DataStream.js';
@@ -115,18 +116,30 @@ export declare abstract class AbstractRemote {
     }>;
     post(path: string, data: any, headers?: Record<string, string>): Promise<any>;
     get(path: string, headers?: Record<string, string>): Promise<any>;
-    postStreaming(path: string, data: any, headers?: Record<string, string>, signal?: AbortSignal): Promise<any>;
     /**
      * Provides a BSON implementation. The import nature of this varies depending on the platform
      */
     abstract getBSON(): Promise<BSONImplementation>;
     protected createSocket(url: string): WebSocket;
     /**
-     * Connects to the sync/stream websocket endpoint
+     * Connects to the sync/stream websocket endpoint and delivers sync lines by decoding the BSON events
+     * sent by the server.
      */
     socketStream(options: SocketSyncStreamOptions): Promise<DataStream<StreamingSyncLine>>;
     /**
-     * Connects to the sync/stream http endpoint
+     * Returns a data stream of sync line data.
+     *
+     * @param map Maps received payload frames to the typed event value.
+     * @param bson A BSON encoder and decoder. When set, the data stream will be requested with a BSON payload
+     * (required for compatibility with older sync services).
+     */
+    socketStreamRaw<T>(options: SocketSyncStreamOptions, map: (buffer: Buffer) => T, bson?: typeof BSON): Promise<DataStream>;
+    /**
+     * Connects to the sync/stream http endpoint, parsing lines as JSON.
      */
     postStream(options: SyncStreamOptions): Promise<DataStream<StreamingSyncLine>>;
+    /**
+     * Connects to the sync/stream http endpoint, mapping and emitting each received string line.
+     */
+    postStreamRaw<T>(options: SyncStreamOptions, mapLine: (line: string) => T): Promise<DataStream<T>>;
 }

package/lib/client/sync/stream/AbstractRemote.js

@@ -1,5 +1,4 @@
 import { Buffer } from 'buffer';
-import ndjsonStream from 'can-ndjson-stream';
 import Logger from 'js-logger';
 import { RSocketConnector } from 'rsocket-core';
 import PACKAGE from '../../../../package.json' with { type: 'json' };
@@ -172,41 +171,39 @@ export class AbstractRemote {
         }
         return res.json();
     }
-    async postStreaming(path, data, headers = {}, signal) {
-        const request = await this.buildRequest(path);
-        const res = await this.fetch(request.url, {
-            method: 'POST',
-            headers: { ...headers, ...request.headers },
-            body: JSON.stringify(data),
-            signal,
-            cache: 'no-store'
-        }).catch((ex) => {
-            this.logger.error(`Caught ex when POST streaming to ${path}`, ex);
-            throw ex;
-        });
-        if (res.status === 401) {
-            this.invalidateCredentials();
-        }
-        if (!res.ok) {
-            const text = await res.text();
-            this.logger.error(`Could not POST streaming to ${path} - ${res.status} - ${res.statusText}: ${text}`);
-            const error = new Error(`HTTP ${res.statusText}: ${text}`);
-            error.status = res.status;
-            throw error;
-        }
-        return res;
-    }
     createSocket(url) {
         return new WebSocket(url);
     }
     /**
-     * Connects to the sync/stream websocket endpoint
+     * Connects to the sync/stream websocket endpoint and delivers sync lines by decoding the BSON events
+     * sent by the server.
      */
     async socketStream(options) {
+        const bson = await this.getBSON();
+        return await this.socketStreamRaw(options, (data) => bson.deserialize(data), bson);
+    }
+    /**
+     * Returns a data stream of sync line data.
+     *
+     * @param map Maps received payload frames to the typed event value.
+     * @param bson A BSON encoder and decoder. When set, the data stream will be requested with a BSON payload
+     * (required for compatibility with older sync services).
+     */
+    async socketStreamRaw(options, map, bson) {
         const { path, fetchStrategy = FetchStrategy.Buffered } = options;
+        const mimeType = bson == null ? 'application/json' : 'application/bson';
+        function toBuffer(js) {
+            let contents;
+            if (bson != null) {
+                contents = bson.serialize(js);
+            }
+            else {
+                contents = JSON.stringify(js);
+            }
+            return Buffer.from(contents);
+        }
         const syncQueueRequestSize = fetchStrategy == FetchStrategy.Buffered ? 10 : 1;
         const request = await this.buildRequest(path);
-        const bson = await this.getBSON();
         // Add the user agent in the setup payload - we can't set custom
         // headers with websockets on web. The browser userAgent is however added
         // automatically as a header.
@@ -222,14 +219,14 @@ export class AbstractRemote {
             setup: {
                 keepAlive: KEEP_ALIVE_MS,
                 lifetime: KEEP_ALIVE_LIFETIME_MS,
-                dataMimeType: 'application/bson',
-                metadataMimeType: 'application/bson',
+                dataMimeType: mimeType,
+                metadataMimeType: mimeType,
                 payload: {
                     data: null,
-                    metadata: Buffer.from(bson.serialize({
+                    metadata: toBuffer({
                         token: request.headers.Authorization,
                         user_agent: userAgent
-                    }))
+                    })
                 }
             }
         });
@@ -268,10 +265,10 @@ export class AbstractRemote {
         const socket = await new Promise((resolve, reject) => {
             let connectionEstablished = false;
             const res = rsocket.requestStream({
-                data: Buffer.from(bson.serialize(options.data)),
-                metadata: Buffer.from(bson.serialize({
+                data: toBuffer(options.data),
+                metadata: toBuffer({
                     path
-                }))
+                })
             }, syncQueueRequestSize, // The initial N amount
             {
                 onError: (e) => {
@@ -310,8 +307,7 @@ export class AbstractRemote {
                     if (!data) {
                         return;
                     }
-                    const deserializedData = bson.deserialize(data);
-                    stream.enqueueData(deserializedData);
+                    stream.enqueueData(map(data));
                 },
                 onComplete: () => {
                     stream.close();
@@ -347,9 +343,17 @@ export class AbstractRemote {
         return stream;
     }
     /**
-     * Connects to the sync/stream http endpoint
+     * Connects to the sync/stream http endpoint, parsing lines as JSON.
      */
     async postStream(options) {
+        return await this.postStreamRaw(options, (line) => {
+            return JSON.parse(line);
+        });
+    }
+    /**
+     * Connects to the sync/stream http endpoint, mapping and emitting each received string line.
+     */
+    async postStreamRaw(options, mapLine) {
         const { data, path, headers, abortSignal } = options;
         const request = await this.buildRequest(path);
         /**
@@ -395,11 +399,8 @@ export class AbstractRemote {
             error.status = res.status;
             throw error;
         }
-        /**
-         * The can-ndjson-stream does not handle aborted streams well.
-         * This will intercept the readable stream and close the stream if
-         * aborted.
-         */
+        // Create a new stream splitting the response at line endings while also handling cancellations
+        // by closing the reader.
         const reader = res.body.getReader();
         // This will close the network request and read stream
         const closeReader = async () => {
@@ -414,49 +415,38 @@ export class AbstractRemote {
         abortSignal?.addEventListener('abort', () => {
             closeReader();
         });
-        const outputStream = new ReadableStream({
-            start: (controller) => {
-                const processStream = async () => {
-                    while (!abortSignal?.aborted) {
-                        try {
-                            const { done, value } = await reader.read();
-                            // When no more data needs to be consumed, close the stream
-                            if (done) {
-                                break;
-                            }
-                            // Enqueue the next data chunk into our target stream
-                            controller.enqueue(value);
-                        }
-                        catch (ex) {
-                            this.logger.error('Caught exception when reading sync stream', ex);
-                            break;
-                        }
-                    }
-                    if (!abortSignal?.aborted) {
-                        // Close the downstream readable stream
-                        await closeReader();
-                    }
-                    controller.close();
-                };
-                processStream();
-            }
-        });
-        const jsonS = ndjsonStream(outputStream);
+        const decoder = new TextDecoder();
+        let buffer = '';
         const stream = new DataStream({
             logger: this.logger
         });
-        const r = jsonS.getReader();
         const l = stream.registerListener({
             lowWater: async () => {
                 try {
-                    const { done, value } = await r.read();
-                    // Exit if we're done
-                    if (done) {
-                        stream.close();
-                        l?.();
-                        return;
+                    let didCompleteLine = false;
+                    while (!didCompleteLine) {
+                        const { done, value } = await reader.read();
+                        if (done) {
+                            const remaining = buffer.trim();
+                            if (remaining.length != 0) {
+                                stream.enqueueData(mapLine(remaining));
+                            }
+                            stream.close();
+                            await closeReader();
+                            return;
+                        }
+                        const data = decoder.decode(value, { stream: true });
+                        buffer += data;
+                        const lines = buffer.split('\n');
+                        for (var i = 0; i < lines.length - 1; i++) {
+                            var l = lines[i].trim();
+                            if (l.length > 0) {
+                                stream.enqueueData(mapLine(l));
+                                didCompleteLine = true;
+                            }
+                        }
+                        buffer = lines[lines.length - 1];
                     }
-                    stream.enqueueData(value);
                 }
                 catch (ex) {
                     stream.close();