@push.rocks/smartstream 3.3.0 → 3.4.0

package/ts/smartstream.classes.smartduplex.ts

@@ -56,67 +56,116 @@ export class SmartDuplex<TInput = any, TOutput = any> extends Duplex {
     readableStream: ReadableStream<T>
   ): SmartDuplex<T, T> {
     const smartDuplex = new SmartDuplex<T, T>({
-      /**
-       * this function is called whenever the stream is being read from and at the same time if nothing is enqueued
-       * therefor it is important to always unlock the reader after reading
-       */
-      readFunction: async () => {
-        const reader = readableStream.getReader();
-        const { value, done } = await reader.read();
-        if (value !== undefined) {
-          smartDuplex.push(value);
-        }
-        reader.releaseLock();
-        if (done) {
-          smartDuplex.push(null);
+      objectMode: true,
+    });
+
+    // Acquire reader ONCE
+    const reader = readableStream.getReader();
+    let reading = false;
+
+    // Override _read to pull from the web reader
+    smartDuplex._read = function (_size: number) {
+      if (reading) return;
+      reading = true;
+      reader.read().then(
+        ({ value, done }) => {
+          reading = false;
+          if (done) {
+            smartDuplex.push(null);
+          } else {
+            smartDuplex.push(value);
+          }
+        },
+        (err) => {
+          reading = false;
+          smartDuplex.destroy(err);
         }
-      },
+      );
+    };
+
+    // Cancel reader on destroy
+    smartDuplex.on('close', () => {
+      reader.cancel().catch(() => {});
     });
+
     return smartDuplex;
   }
 
   // INSTANCE
-  private backpressuredArray: plugins.lik.BackpressuredArray<TOutput>; // an array that only takes a defined amount of items
+  private backpressuredArray: plugins.lik.BackpressuredArray<TOutput>;
   public options: ISmartDuplexOptions<TInput, TOutput>;
-  private observableSubscription?: plugins.smartrx.rxjs.Subscription;
+  private _consumerWantsData = false;
+  private _readFunctionRunning = false;
+
   private debugLog(messageArg: string) {
-    // optional debug log
     if (this.options.debug) {
       console.log(messageArg);
     }
   }
 
   constructor(optionsArg?: ISmartDuplexOptions<TInput, TOutput>) {
+    const safeOptions = optionsArg || {} as ISmartDuplexOptions<TInput, TOutput>;
     super(
       Object.assign(
         {
           highWaterMark: 1,
         },
-        optionsArg
+        safeOptions
       )
     );
-    this.options = optionsArg;
+    this.options = safeOptions;
     this.backpressuredArray = new plugins.lik.BackpressuredArray<TOutput>(
       this.options.highWaterMark || 1
     );
   }
 
-  public async _read(size: number): Promise<void> {
+  /**
+   * Synchronously drains items from the backpressuredArray into the readable side.
+   * Stops when push() returns false (consumer is full) or array is empty.
+   */
+  private _drainBackpressuredArray(): void {
+    while (this.backpressuredArray.data.length > 0) {
+      const nextChunk = this.backpressuredArray.shift();
+      if (nextChunk === null) {
+        // EOF signal — push null to end readable side
+        this.push(null);
+        this._consumerWantsData = false;
+        return;
+      }
+      const canPushMore = this.push(nextChunk);
+      if (!canPushMore) {
+        this._consumerWantsData = false;
+        return;
+      }
+    }
+  }
+
+  // _read must NOT be async — Node.js ignores the return value
+  public _read(size: number): void {
     this.debugLog(`${this.options.name}: read was called`);
-    if (this.options.readFunction) {
-      await this.options.readFunction();
+    this._consumerWantsData = true;
+
+    // Drain any buffered items first
+    if (this.backpressuredArray.data.length > 0) {
+      this._drainBackpressuredArray();
     }
-    await this.backpressuredArray.waitForItems();
-    this.debugLog(`${this.options.name}: successfully waited for items.`);
-    let canPushMore = true;
-    while (this.backpressuredArray.data.length > 0 && canPushMore) {
-      const nextChunk = this.backpressuredArray.shift();
-      canPushMore = this.push(nextChunk);
+
+    // If readFunction exists and is not already running, start it
+    if (this.options.readFunction && !this._readFunctionRunning) {
+      this._readFunctionRunning = true;
+      this.options.readFunction().then(
+        () => { this._readFunctionRunning = false; },
+        (err) => { this._readFunctionRunning = false; this.destroy(err); }
+      );
     }
   }
 
   public async backpressuredPush(pushArg: TOutput) {
     const canPushMore = this.backpressuredArray.push(pushArg);
+    // Try to drain if the consumer wants data
+    if (this._consumerWantsData) {
+      this._drainBackpressuredArray();
+    }
     if (!canPushMore) {
       this.debugLog(`${this.options.name}: cannot push more`);
       await this.backpressuredArray.waitForSpace();
@@ -126,83 +175,151 @@ export class SmartDuplex<TInput = any, TOutput = any> extends Duplex {
   }
 
   private asyncWritePromiseObjectmap = new plugins.lik.ObjectMap<Promise<any>>();
-  // Ensure the _write method types the chunk as TInput and encodes TOutput
-  public async _write(chunk: TInput, encoding: string, callback: (error?: Error | null) => void) {
+
+  // _write must NOT be async — Node.js ignores the return value
+  public _write(chunk: TInput, encoding: string, callback: (error?: Error | null) => void) {
     if (!this.options.writeFunction) {
       return callback(new Error('No stream function provided'));
     }
 
+    let callbackCalled = false;
+    const safeCallback = (err?: Error | null) => {
+      if (!callbackCalled) {
+        callbackCalled = true;
+        callback(err);
+      }
+    };
+
     let isTruncated = false;
     const tools: IStreamTools = {
       truncate: () => {
-        this.push(null);
         isTruncated = true;
-        callback();
+        safeCallback();
+        this.push(null);
       },
       push: async (pushArg: TOutput) => {
         return await this.backpressuredPush(pushArg);
       },
     };
 
-    try {
-      const writeDeferred = plugins.smartpromise.defer();
-      this.asyncWritePromiseObjectmap.add(writeDeferred.promise);
-      const modifiedChunk = await this.options.writeFunction(chunk, tools);
-      if (isTruncated) {
-        return;
-      }
-      if (modifiedChunk) {
-        await tools.push(modifiedChunk);
-      }
-      callback();
-      writeDeferred.resolve();
-      writeDeferred.promise.then(() => {
+    const writeDeferred = plugins.smartpromise.defer();
+    this.asyncWritePromiseObjectmap.add(writeDeferred.promise);
+
+    this.options.writeFunction(chunk, tools).then(
+      (modifiedChunk) => {
+        if (isTruncated) {
+          writeDeferred.resolve();
+          this.asyncWritePromiseObjectmap.remove(writeDeferred.promise);
+          return;
+        }
+        const finish = () => {
+          safeCallback();
+          writeDeferred.resolve();
+          this.asyncWritePromiseObjectmap.remove(writeDeferred.promise);
+        };
+        if (modifiedChunk !== undefined && modifiedChunk !== null) {
+          this.backpressuredPush(modifiedChunk).then(finish, (err) => {
+            safeCallback(err);
+            writeDeferred.resolve();
+            this.asyncWritePromiseObjectmap.remove(writeDeferred.promise);
+          });
+        } else {
+          finish();
+        }
+      },
+      (err) => {
+        safeCallback(err);
+        writeDeferred.resolve();
         this.asyncWritePromiseObjectmap.remove(writeDeferred.promise);
-      });
-    } catch (err) {
-      callback(err);
-    }
+      }
+    );
   }
 
-  public async _final(callback: (error?: Error | null) => void) {
-    await Promise.all(this.asyncWritePromiseObjectmap.getArray());
-    if (this.options.finalFunction) {
-      const tools: IStreamTools = {
-        truncate: () => callback(),
-        push: async (pipeObject) => {
-          return this.backpressuredArray.push(pipeObject);
-        },
-      };
+  // _final must NOT be async — Node.js ignores the return value
+  public _final(callback: (error?: Error | null) => void) {
+    let callbackCalled = false;
+    const safeCallback = (err?: Error | null) => {
+      if (!callbackCalled) {
+        callbackCalled = true;
+        callback(err);
+      }
+    };
 
-      try {
-        const finalChunk = await this.options.finalFunction(tools);
-        if (finalChunk) {
-          this.backpressuredArray.push(finalChunk);
-        }
-      } catch (err) {
+    Promise.all(this.asyncWritePromiseObjectmap.getArray()).then(() => {
+      if (this.options.finalFunction) {
+        const tools: IStreamTools = {
+          truncate: () => safeCallback(),
+          push: async (pipeObject) => {
+            return await this.backpressuredPush(pipeObject);
+          },
+        };
+
+        this.options.finalFunction(tools).then(
+          (finalChunk) => {
+            const pushNull = () => {
+              this.backpressuredArray.push(null);
+              if (this._consumerWantsData) {
+                this._drainBackpressuredArray();
+              }
+              safeCallback();
+            };
+
+            if (finalChunk !== undefined && finalChunk !== null) {
+              this.backpressuredPush(finalChunk).then(pushNull, (err) => {
+                safeCallback(err);
+              });
+            } else {
+              pushNull();
+            }
+          },
+          (err) => {
+            this.backpressuredArray.push(null);
+            if (this._consumerWantsData) {
+              this._drainBackpressuredArray();
+            }
+            safeCallback(err);
+          }
+        );
+      } else {
         this.backpressuredArray.push(null);
-        callback(err);
-        return;
+        if (this._consumerWantsData) {
+          this._drainBackpressuredArray();
+        }
+        safeCallback();
       }
-    }
-    this.backpressuredArray.push(null);
-    callback();
+    }, (err) => {
+      safeCallback(err);
+    });
   }
 
   public async getWebStreams(): Promise<{ readable: ReadableStream; writable: WritableStream }> {
     const duplex = this;
+    let readableClosed = false;
+
     const readable = new ReadableStream({
       start(controller) {
-        duplex.on('readable', () => {
+        const onReadable = () => {
           let chunk;
           while (null !== (chunk = duplex.read())) {
             controller.enqueue(chunk);
           }
-        });
+        };
 
-        duplex.on('end', () => {
-          controller.close();
-        });
+        const onEnd = () => {
+          if (!readableClosed) {
+            readableClosed = true;
+            controller.close();
+          }
+          cleanup();
+        };
+
+        const cleanup = () => {
+          duplex.removeListener('readable', onReadable);
+          duplex.removeListener('end', onEnd);
+        };
+
+        duplex.on('readable', onReadable);
+        duplex.on('end', onEnd);
       },
       cancel(reason) {
         duplex.destroy(new Error(reason));
@@ -212,22 +329,38 @@ export class SmartDuplex<TInput = any, TOutput = any> extends Duplex {
     const writable = new WritableStream({
       write(chunk) {
         return new Promise<void>((resolve, reject) => {
+          let resolved = false;
+          const onDrain = () => {
+            if (!resolved) {
+              resolved = true;
+              resolve();
+            }
+          };
+
           const isBackpressured = !duplex.write(chunk, (error) => {
             if (error) {
-              reject(error);
-            } else {
+              if (!resolved) {
+                resolved = true;
+                duplex.removeListener('drain', onDrain);
+                reject(error);
+              }
+            } else if (!isBackpressured && !resolved) {
+              resolved = true;
               resolve();
             }
           });
 
           if (isBackpressured) {
-            duplex.once('drain', resolve);
+            duplex.once('drain', onDrain);
           }
         });
       },
       close() {
         return new Promise<void>((resolve, reject) => {
-          duplex.end(resolve);
+          duplex.end((err: Error | null) => {
+            if (err) reject(err);
+            else resolve();
+          });
         });
       },
       abort(reason) {

package/ts/smartstream.classes.streamintake.ts

@@ -6,8 +6,11 @@ export class StreamIntake<T> extends plugins.stream.Readable {
     const intakeStream = new StreamIntake<U>(options);
 
     if (inputStream instanceof plugins.stream.Readable) {
-      inputStream.on('data', (chunk: U) => {
-        intakeStream.pushData(chunk);
+      inputStream.on('readable', () => {
+        let chunk: U;
+        while (null !== (chunk = inputStream.read() as U)) {
+          intakeStream.pushData(chunk);
+        }
       });
 
       inputStream.on('end', () => {

package/ts/smartstream.classes.streamwrapper.ts

@@ -1,8 +1,5 @@
 import * as plugins from './smartstream.plugins.js';
 
-// interfaces
-import { Transform } from 'stream';
-
 export interface IErrorFunction {
   (err: Error): any;
 }
@@ -82,15 +79,17 @@ export class StreamWrapper {
 
     this.streamStartedDeferred.resolve();
 
-    finalStream.on('end', () => {
-      done.resolve();
-    });
-    finalStream.on('close', () => {
-      done.resolve();
-    });
-    finalStream.on('finish', () => {
-      done.resolve();
-    });
+    let resolved = false;
+    const safeResolve = () => {
+      if (!resolved) {
+        resolved = true;
+        done.resolve();
+      }
+    };
+
+    finalStream.on('end', safeResolve);
+    finalStream.on('close', safeResolve);
+    finalStream.on('finish', safeResolve);
     return done.promise;
   }
 }

package/ts/smartstream.nodewebhelpers.ts

@@ -1,7 +1,7 @@
 import * as plugins from './smartstream.plugins.js';
 
 /**
- * Creates a Web ReadableStream from a file.
+ * Creates a Web ReadableStream from a file using pull-based backpressure.
  *
  * @param filePath - The path to the file to be read
  * @returns A Web ReadableStream that reads the file in chunks
@@ -11,23 +11,53 @@ export function createWebReadableStreamFromFile(filePath: string): ReadableStrea
 
   return new ReadableStream({
     start(controller) {
-      // When data is available, enqueue it into the Web ReadableStream
-      fileStream.on('data', (chunk) => {
-        controller.enqueue(chunk as Uint8Array);
+      fileStream.on('error', (err) => {
+        controller.error(err);
       });
 
-      // When the file stream ends, close the Web ReadableStream
       fileStream.on('end', () => {
         controller.close();
       });
 
-      // If there's an error, error the Web ReadableStream
-      fileStream.on('error', (err) => {
-        controller.error(err);
+      // Pause immediately — pull() will drive reads
+      fileStream.pause();
+    },
+    pull(controller) {
+      return new Promise<void>((resolve, reject) => {
+        const chunk = fileStream.read();
+        if (chunk !== null) {
+          controller.enqueue(chunk as Uint8Array);
+          resolve();
+          return;
+        }
+        // No data available yet — wait for 'readable' or 'end'
+        const onReadable = () => {
+          cleanup();
+          const data = fileStream.read();
+          if (data !== null) {
+            controller.enqueue(data as Uint8Array);
+          }
+          resolve();
+        };
+        const onEnd = () => {
+          cleanup();
+          resolve();
+        };
+        const onError = (err: Error) => {
+          cleanup();
+          reject(err);
+        };
+        const cleanup = () => {
+          fileStream.removeListener('readable', onReadable);
+          fileStream.removeListener('end', onEnd);
+          fileStream.removeListener('error', onError);
+        };
+        fileStream.once('readable', onReadable);
+        fileStream.once('end', onEnd);
+        fileStream.once('error', onError);
       });
     },
     cancel() {
-      // If the Web ReadableStream is canceled, destroy the file stream
       fileStream.destroy();
     }
   });
@@ -43,23 +73,25 @@ export function convertWebReadableToNodeReadable(webStream: ReadableStream<Uint8
   const reader = webStream.getReader();
 
   return new plugins.stream.Readable({
-    async read() {
-      try {
-        const { value, done } = await reader.read();
-        if (done) {
-          this.push(null); // Signal end of stream
-        } else {
-          this.push(Buffer.from(value)); // Convert Uint8Array to Buffer for Node.js Readable
+    read() {
+      reader.read().then(
+        ({ value, done }) => {
+          if (done) {
+            this.push(null);
+          } else {
+            this.push(Buffer.from(value));
+          }
+        },
+        (err) => {
+          this.destroy(err);
         }
-      } catch (err) {
-        this.destroy(err); // Handle errors by destroying the stream
-      }
+      );
     }
   });
 }
 
 /**
- * Converts a Node.js Readable stream to a Web ReadableStream.
+ * Converts a Node.js Readable stream to a Web ReadableStream using pull-based backpressure.
  *
  * @param nodeStream - The Node.js Readable stream to convert
  * @returns A Web ReadableStream that reads data from the Node.js Readable stream
@@ -67,16 +99,50 @@ export function convertWebReadableToNodeReadable(webStream: ReadableStream<Uint8
 export function convertNodeReadableToWebReadable(nodeStream: plugins.stream.Readable): ReadableStream<Uint8Array> {
   return new ReadableStream({
     start(controller) {
-      nodeStream.on('data', (chunk) => {
-        controller.enqueue(new Uint8Array(chunk));
+      nodeStream.on('error', (err) => {
+        controller.error(err);
       });
 
       nodeStream.on('end', () => {
         controller.close();
       });
 
-      nodeStream.on('error', (err) => {
-        controller.error(err);
+      // Pause immediately — pull() will drive reads
+      nodeStream.pause();
+    },
+    pull(controller) {
+      return new Promise<void>((resolve, reject) => {
+        const chunk = nodeStream.read();
+        if (chunk !== null) {
+          controller.enqueue(new Uint8Array(chunk));
+          resolve();
+          return;
+        }
+        // No data available yet — wait for 'readable' or 'end'
+        const onReadable = () => {
+          cleanup();
+          const data = nodeStream.read();
+          if (data !== null) {
+            controller.enqueue(new Uint8Array(data));
+          }
+          resolve();
+        };
+        const onEnd = () => {
+          cleanup();
+          resolve();
+        };
+        const onError = (err: Error) => {
+          cleanup();
+          reject(err);
+        };
+        const cleanup = () => {
+          nodeStream.removeListener('readable', onReadable);
+          nodeStream.removeListener('end', onEnd);
+          nodeStream.removeListener('error', onError);
+        };
+        nodeStream.once('readable', onReadable);
+        nodeStream.once('end', onEnd);
+        nodeStream.once('error', onError);
       });
     },
     cancel() {
@@ -95,19 +161,23 @@ export function convertWebWritableToNodeWritable(webWritable: WritableStream<Uin
   const writer = webWritable.getWriter();
 
   return new plugins.stream.Writable({
-    async write(chunk, encoding, callback) {
-      try {
-        await writer.write(new Uint8Array(chunk));
-        callback();
-      } catch (err) {
-        callback(err);
-      }
+    write(chunk, encoding, callback) {
+      writer.write(new Uint8Array(chunk)).then(
+        () => callback(),
+        (err) => callback(err)
+      );
     },
     final(callback) {
       writer.close().then(() => callback()).catch(callback);
     },
     destroy(err, callback) {
-      writer.abort(err).then(() => callback(err)).catch(callback);
+      if (err) {
+        writer.abort(err).then(() => callback(err)).catch(() => callback(err));
+      } else {
+        // Clean destroy — just release the lock
+        writer.releaseLock();
+        callback(null);
+      }
     }
   });
 }
@@ -133,7 +203,7 @@ export function convertNodeWritableToWebWritable(nodeWritable: plugins.stream.Wr
     },
     close() {
       return new Promise((resolve, reject) => {
-        nodeWritable.end((err) => {
+        nodeWritable.end((err: Error | null) => {
           if (err) {
             reject(err);
           } else {
@@ -143,9 +213,7 @@ export function convertNodeWritableToWebWritable(nodeWritable: plugins.stream.Wr
       });
     },
     abort(reason) {
-      return new Promise((resolve, reject) => {
-        nodeWritable.destroy(reason);
-      });
+      nodeWritable.destroy(reason instanceof Error ? reason : new Error(String(reason)));
     }
   });
-}
+}

package/ts_web/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@push.rocks/smartstream',
-  version: '3.3.0',
+  version: '3.4.0',
   description: 'A library to simplify the creation and manipulation of Node.js streams, providing utilities for handling transform, duplex, and readable/writable streams effectively in TypeScript.'
 }