@promistream/from-node-stream 0.1.4 → 0.1.6

package/package.json

@@ -1,36 +1,38 @@
 {
   "name": "@promistream/from-node-stream",
-  "version": "0.1.4",
+  "version": "0.1.6",
   "main": "src/index.js",
-  "repository": "http://git.cryto.net/promistream/from-node-stream.git",
+  "repository": "https://codeberg.org/promistream/from-node-stream.git",
   "author": "Sven Slootweg <admin@cryto.net>",
   "license": "WTFPL OR CC0-1.0",
   "dependencies": {
     "@joepie91/consumable": "^1.0.1",
     "@joepie91/unreachable": "^1.0.0",
     "@joepie91/warning": "^1.1.0",
-    "@promistream/end-of-stream": "^0.1.2",
+    "@promistream/end-of-stream": "^0.1.3",
     "@promistream/is-aborted": "^0.1.1",
     "@promistream/is-end-of-stream": "^0.1.1",
     "@promistream/propagate-abort": "^0.1.7",
-    "@promistream/propagate-peek": "^0.1.1",
-    "@promistream/simple-sink": "^0.3.2",
-    "@promistream/simple-source": "^0.1.5",
+    "@promistream/propagate-peek": "^0.1.2",
+    "@promistream/simple-sink": "^0.3.5",
+    "@promistream/simple-source": "^0.1.10",
     "debug-instanced": "^1.0.0",
     "promisify-event": "^1.0.0",
-    "push-buffer": "^1.5.5",
+    "push-buffer": "^1.5.7",
     "single-concurrent": "^1.0.0",
     "split-filter": "^1.1.3"
   },
   "devDependencies": {
     "@promistream/buffered-map": "^0.1.1",
     "@promistream/collect": "^0.1.2",
-    "@promistream/debug": "^0.1.0",
-    "@promistream/from-iterable": "^0.1.0",
-    "@promistream/map": "^0.1.5",
-    "@promistream/pipe": "^0.1.6",
+    "@promistream/debug": "^0.1.2",
+    "@promistream/from-iterable": "^0.1.1",
+    "@promistream/map": "^0.1.10",
+    "@promistream/parallelize": "^0.1.8",
+    "@promistream/pipe": "^0.1.8",
     "@promistream/range-numbers": "^0.1.2",
     "bluebird": "^3.7.2",
+    "p-defer": "^4.0.1",
     "through2": "^4.0.2"
   }
 }

package/src/index.js

@@ -9,6 +9,7 @@ const isEndOfStream = require("@promistream/is-end-of-stream");
 const isAborted = require("@promistream/is-aborted");
 const makeDebug = require("debug-instanced")("promistream:from-node-stream");
 
+const { once } = require("node:events");
 const pushBuffer = require("push-buffer");
 const promisifyEvent = require("promisify-event");
 const singleConcurrent = require("single-concurrent");
@@ -75,6 +76,7 @@ function wireReadable(debug, stream) {
 			// We have run out of values
 			if (isPaused && buffer.countLane().requests > 0) {
 				debug("Resuming underlying Readable stream, as our buffer ran out");
+				isPaused = false;
 				stream.resume();
 			}
 
@@ -125,6 +127,14 @@ function wireWritable(debug, stream) {
 		errorEncountered = ensureValidError(error);
 	});
 
+	async function maybeWaitForDrain() {
+		if (stream.writableNeedDrain) {
+			/* Wait for the write buffer to be fully processed */
+			// FIXME: Verify that this cannot cause a never-ending wait; if we wait to close the stream until it has drained, then during that time writes to it can continue to happen indefinitely, never letting it drain? Or is this prevented by the backpressure mechanisms we already have (re: .pause and .resume)?
+			await once(stream, "drain");
+		}
+	}
+
 	// We're not wiring into the `finish` event here because we detect the ended stream on the next write attempt instead (inside of `tryWrite`).
 
 	return {
@@ -132,12 +142,14 @@ function wireWritable(debug, stream) {
 			// TODO: Do we need to detach any event handlers prior to destroying streams? ref. https://www.npmjs.com/package/yauzl:
 			//   "You must unpipe() the readStream from any destination before calling readStream.destroy()."
 			// TODO: In a previous version, we skipped the .destroy step if the stream didn't have a .destroy method. What was this for? Streams v1?
+			await maybeWaitForDrain();
 			if (!isStdioStream(stream) && !isWritableStreamEnded(stream)) {
 				debug("Calling .destroy on Writable stream");
 				stream.destroy(error);
 			}
 		},
 		end: async function () {
+			await maybeWaitForDrain();
 			if (!isStdioStream(stream) && !isWritableStreamEnded(stream)) {
 				debug("Calling .end on Writable stream");
 				stream.end();
@@ -181,10 +193,11 @@ function convertFromWritable(stream) {
 
 	return simpleSink({
 		onAbort: async (error) => {
-			if (error === true) {
-				return end();
-			} else {
+			// TODO: this is mostly-duplicated logic with that in the transform stream implementation
+			if (error instanceof Error) {
 				return destroy(error);
+			} else {
+				return end();
 			}
 		},
 		onEnd: async () => {
@@ -213,10 +226,9 @@ function convertFromTransform(stream) {
 				await tryWrite(await lastSource.read());
 			}
 		} catch (error) {
-			if (endMarker == null && (isEndOfStream(error) || isAborted(error))) {
+			if (endMarker == null && (isEndOfStream(error) || isAborted(error))) { // FIXME: isExhausted?
 				endMarker = error;
 
-				// TODO: Is this necessary? Is this not already handled by wiring for the readable/writable interface?
 				if (isAborted(error)) {
 					if (error.cause instanceof Error) {
 						destroy(error);
@@ -228,8 +240,6 @@ function convertFromTransform(stream) {
 					end();
 				}
 			}
-
-			buffer.pushError(error);
 		}
 	});
 

package/test.js

@@ -5,11 +5,14 @@ const assert = require("node:assert");
 const fs = require("node:fs");
 const path = require("node:path");
 const stream = require("node:stream");
+const net = require("node:net");
+const pDefer = require("p-defer").default;
 
 const pipe = require("@promistream/pipe");
 const fromIterable = require("@promistream/from-iterable");
 const fromNodeStream = require(".");
 const collect = require("@promistream/collect");
+const parallelize = require("@promistream/parallelize");
 const debug = require("@promistream/debug");
 
 test("readable stream", async () => {
@@ -55,9 +58,12 @@ test("transform stream", async () => {
 		fromNodeStream(new stream.Transform({
 			objectMode: true,
 			transform(chunk, _encoding, callback) {
-				this.push(chunk);
-				this.push(chunk * 2);
-				callback();
+				/* Intentionally delay things, to validate that processing in the Node stream fully completes before the pipeline gets torn down */
+				setTimeout(() => {
+					this.push(chunk);
+					this.push(chunk * 2);
+					callback();
+				}, 10);
 			}
 		})),
 		collect()
@@ -65,3 +71,89 @@ test("transform stream", async () => {
 
 	assert.deepStrictEqual(result, [ 0, 0, 1, 2, 2, 4, 3, 6, 4, 8, 5, 10, 6, 12, 7, 14, 8, 16, 9, 18, 10, 20 ]);
 });
+
+test("transform stream with abort", async () => {
+	assert.rejects(async () => {
+		let pipeline = pipe([
+			fromIterable([ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 ]),
+			fromNodeStream(new stream.Transform({
+				objectMode: true,
+				transform(chunk, _encoding, callback) {
+					setTimeout(() => {
+						this.push(chunk);
+						this.push(chunk * 2);
+						callback();
+					}, 10);
+				}
+			})),
+			collect()
+		]);
+
+		pipeline.abort(new Error("Some arbitrary error"));
+		let result = await pipeline.read();
+	}, { message: "Some arbitrary error" });
+});
+
+test("transform stream, parallel", async () => {
+	let result = await pipe([
+		fromIterable([ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 ]),
+		fromNodeStream(new stream.Transform({
+			objectMode: true,
+			transform(chunk, _encoding, callback) {
+				/* Intentionally delay things, to validate that processing in the Node stream fully completes before the pipeline gets torn down */
+				setTimeout(() => {
+					this.push(chunk);
+					this.push(chunk * 2);
+					callback();
+				}, 10);
+			}
+		})),
+		parallelize(8),
+		collect()
+	]).read();
+
+	assert.deepStrictEqual(result, [ 0, 0, 1, 2, 2, 4, 3, 6, 4, 8, 5, 10, 6, 12, 7, 14, 8, 16, 9, 18, 10, 20 ]);
+});
+
+/* FIXME: Do network socket test with both happy abort and error abort */
+test("network socket", async () => {
+	let { resolve, promise } = pDefer();
+
+	let server = net.createServer((socket) => {
+		// console.log("connected", socket);
+		let serverRead = fromNodeStream.fromReadable(socket);
+		let serverWrite = fromNodeStream.fromWritable(socket);
+		resolve({ serverRead, serverWrite });
+	});
+	server.listen(14292);
+
+	try {
+		let client = net.connect({ host: "localhost", port: 14292 });
+		let clientRead = fromNodeStream.fromReadable(client);
+		let clientWrite = fromNodeStream.fromWritable(client);
+		let { serverRead, serverWrite } = await promise;
+		
+		let serverPipe = pipe([ serverRead, collect() ]);
+		let serverWritePipe = pipe([ fromIterable([ Buffer.from("c"), Buffer.from("d") ]), serverWrite ]);
+		let clientPipe = pipe([ clientRead, collect() ]);
+		let clientWritePipe = pipe([ fromIterable([ Buffer.from("a"), Buffer.from("b") ]), clientWrite ]);
+
+		let clientWritePromise = clientWritePipe.read();
+		await new Promise((resolve) => setTimeout(resolve, 1)); /* give it a couple ticks to actually read from the iterable source before aborting */
+		clientWritePipe.abort(true);
+
+		let [ serverResult, serverWriteResult, clientResult, clientWriteResult ] = await Promise.all([
+			serverPipe.read(),
+			serverWritePipe.read(),
+			clientPipe.read(),
+			clientWritePromise,
+		]);
+
+		assert.deepStrictEqual(Buffer.concat(serverResult), Buffer.from("ab"));
+		assert.deepStrictEqual(Buffer.concat(clientResult), Buffer.from("cd"));
+		assert.deepStrictEqual(serverWriteResult, undefined);
+		assert.deepStrictEqual(clientWriteResult, undefined);
+	} finally {
+		server.close();
+	}
+});