@promistream/from-node-stream 0.1.8 → 0.1.10

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@promistream/from-node-stream",
-  "version": "0.1.8",
+  "version": "0.1.10",
   "main": "src/index.js",
   "repository": {
     "type": "git",
@@ -33,6 +33,7 @@
     "@promistream/parallelize": "^0.1.10",
     "@promistream/pipe": "^0.1.8",
     "@promistream/range-numbers": "^0.1.2",
+    "@promistream/spy": "^0.1.0",
     "@promistream/test-abort": "^0.1.0",
     "bluebird": "^3.7.2",
     "p-defer": "^4.0.1",

package/src/index.js

@@ -36,6 +36,12 @@ function wireReadable(debug, stream) {
 	let buffer = pushBuffer({ mode: "push" });
 	let isPaused = false;
 	let abortSeen = false;
+	let endError;
+
+	function finishStream(error) {
+		endError = error;
+		buffer.pushError(error);
+	}
 
 	stream.on("data", (value) => {
 		debug("Received value from underlying Readable stream");
@@ -58,13 +64,14 @@ function wireReadable(debug, stream) {
 		if (!abortSeen) {
 			debug(`Underlying Readable stream produced an error: ${error.message}`);
 			abortSeen = true;
-			buffer.pushError(ensureValidError(error));
+			finishStream(ensureValidError(error));
 		}
 	});
 
 	stream.on("end", () => {
+		// TODO: How/where do we propagate Aborted markers, especially on subsequent overreads?
 		debug("Reached end of underlying Readable stream");
-		buffer.pushError(new EndOfStream("The underlying stream has ended"));
+		finishStream(new EndOfStream("The underlying stream has ended"));
 	});
 
 	return {
@@ -72,11 +79,16 @@ function wireReadable(debug, stream) {
 		read: async function () {
 			let promise = buffer.request();
 
-			// We have run out of values
-			if (isPaused && buffer.countLane().requests > 0) {
-				debug("Resuming underlying Readable stream, as our buffer ran out");
-				isPaused = false;
-				stream.resume();
+			let counts = buffer.countLane();
+			if (counts.requests > 0 && counts.values === 0) {
+				// We have run out of values
+				if (endError != null) {
+					buffer.pushError(endError);
+				} else if (isPaused) {
+					debug("Resuming underlying Readable stream, as our buffer ran out");
+					isPaused = false;
+					stream.resume();
+				}
 			}
 
 			return promise;
@@ -123,11 +135,26 @@ function isWritableStreamEnded(stream) {
 }
 
 function wireWritable(debug, stream) {
-	let errorEncountered;
+	let errorEncountered, lastAbort;
+	let aborted = false;
+	
+	function abortPromistream(reason) {
+		/* If an abort/end of the underlying Node stream occurred for a reason that's entirely unrelated to our current pipeline - most notably, this happens if the error originates from the *read* side of a non-transform Duplex stream - then we want to make sure that we tear down the Promistream pipeline on this side too. */
+		if (lastAbort != null && !aborted) {
+			aborted = true;
+			lastAbort(reason);
+		}
+	}
 
 	stream.on("error", (error) => {
 		debug("Logging writable interface error:", error);
 		errorEncountered = ensureValidError(error);
+		abortPromistream(error);
+	});
+
+	stream.on("end", () => {
+		debug("Reached end of underlying Writable stream");
+		abortPromistream(true);
 	});
 
 	async function maybeWaitForDrain() {
@@ -149,6 +176,7 @@ function wireWritable(debug, stream) {
 			await maybeWaitForDrain();
 			if (!isStdioStream(stream) && !isWritableStreamEnded(stream)) {
 				debug("Calling .destroy on Writable stream");
+				aborted = true;
 				stream.destroy(error);
 			}
 		},
@@ -156,10 +184,13 @@ function wireWritable(debug, stream) {
 			await maybeWaitForDrain();
 			if (!isStdioStream(stream) && !isWritableStreamEnded(stream)) {
 				debug("Calling .end on Writable stream");
+				aborted = true;
 				stream.end();
 			}
 		},
 		tryWrite: async function (value, abort) {
+			lastAbort = abort;
+			
 			if (isWritableStreamEnded(stream)) {
 				debug("Could not write because underlying stream has ended");
 				if (errorEncountered != null) {
@@ -227,7 +258,7 @@ function convertFromTransform(stream) {
 		// This is essentially a 'background process' of sorts, with manual error handling wiring.
 		try {
 			while (endMarker == null) {
-				await tryWrite(await lastSource.read());
+				await tryWrite(await lastSource.read(), lastSource.abort);
 			}
 		} catch (error) {
 			if (endMarker == null && (isEndOfStream(error) || isAborted(error))) { // FIXME: isExhausted?

package/test.js

@@ -16,6 +16,7 @@ const parallelize = require("@promistream/parallelize");
 const testAbort = require("@promistream/test-abort");
 const debug = require("@promistream/debug");
 const { once } = require("node:events");
+const spy = require("@promistream/spy");
 
 test("readable stream", async () => {
 	let testFile = path.join(__dirname, "pnpm-lock.yaml"); // Big-ish file, so we're likely to read in multiple chunks
@@ -60,7 +61,6 @@ test("readable stream with happy abort", { timeout: 500 }, async () => {
 	assert.deepStrictEqual(result, []);
 });
 
-
 test("writable stream", async () => {
 	let buffer = Buffer.alloc(400000);
 
@@ -86,6 +86,34 @@ test("writable stream", async () => {
 	assert.strictEqual(endReached, true);
 });
 
+test("writable stream with error abort", async () => {
+	let buffer = Buffer.alloc(400000);
+
+	let writableBuffer = Buffer.alloc(0);
+	let endReached = false;
+	let nodeWritable = new stream.Writable({
+		write(chunk, _encoding, callback) {
+			writableBuffer = Buffer.concat([ writableBuffer, chunk ]);
+			callback();
+		},
+		final(callback) {
+			endReached = true;
+			callback();
+		}
+	});
+
+	await assert.rejects(async () => {
+		await pipe([
+			fromIterable([ buffer, buffer, buffer ]),
+			testAbort(0, new Error("sample error")),
+			fromNodeStream(nodeWritable)
+		]).read();
+	}, { message: "sample error" });
+
+	assert.strictEqual(writableBuffer.length, 0);
+	assert.strictEqual(endReached, false);
+});
+
 test("transform stream", async () => {
 	let result = await pipe([
 		fromIterable([ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 ]),
@@ -107,7 +135,7 @@ test("transform stream", async () => {
 });
 
 test("transform stream with abort", async () => {
-	assert.rejects(async () => {
+	await assert.rejects(async () => {
 		let pipeline = pipe([
 			fromIterable([ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 ]),
 			fromNodeStream(new stream.Transform({
@@ -149,6 +177,48 @@ test("transform stream, parallel", async () => {
 	assert.deepStrictEqual(result, [ 0, 0, 1, 2, 2, 4, 3, 6, 4, 8, 5, 10, 6, 12, 7, 14, 8, 16, 9, 18, 10, 20 ]);
 });
 
+test("transform stream, overread", { timeout: 500 }, async () => {
+	let pipeline = pipe([
+		fromIterable([ 0 ]),
+		fromNodeStream(new stream.Transform({
+			objectMode: true,
+			transform(chunk, _encoding, callback) {
+				setTimeout(() => {
+					this.push(chunk);
+					this.push(chunk * 2);
+					callback();
+				}, 10);
+			}
+		}))
+	]);
+
+	assert.strictEqual(await pipeline.read(), 0);
+	assert.strictEqual(await pipeline.read(), 0);
+	await assert.rejects(() => pipeline.read(), { name: "EndOfStream" });
+	await assert.rejects(() => pipeline.read(), { name: "EndOfStream" }); /* This should repeat the last error, just like a normal source stream would */
+});
+
+test("transform stream, abort overread", { timeout: 500 }, async () => {
+	let pipeline = pipe([
+		fromIterable([ 0 ]),
+		testAbort(0, new Error("sample error")),
+		fromNodeStream(new stream.Transform({
+			objectMode: true,
+			transform(chunk, _encoding, callback) {
+				setTimeout(() => {
+					this.push(chunk);
+					this.push(chunk * 2);
+					callback();
+				}, 10);
+			}
+		}))
+	]);
+
+	/* Since we're reading directly from the pipeline, we will get an Aborted marker in both cases; it is normally the sink stream that is responsible for unpacking and throwing the original cause, but we don't have a sink stream in this case. */
+	await assert.rejects(() => pipeline.read(), { name: "Aborted", message: "Stream was aborted due to error: sample error" });
+	await assert.rejects(() => pipeline.read(), { name: "Aborted", message: "Stream was aborted due to error: sample error" });
+});
+
 /* FIXME: Do network socket test with both happy abort and error abort */
 /* TODO: Flaky test behaviour was observed for the below test:
 
@@ -203,3 +273,50 @@ test("network socket", async () => {
 		server.close();
 	}
 });
+
+async function wait10() {
+	await new Promise((resolve) => setTimeout(resolve, 10));
+}
+
+test("network socket with error abort", { timeout: 500 }, async () => {
+	let { resolve, promise } = pDefer();
+
+	let server = net.createServer((socket) => {
+		let serverRead = fromNodeStream.fromReadable(socket);
+		let serverWrite = fromNodeStream.fromWritable(socket);
+		resolve({ serverRead, serverWrite });
+	});
+	server.listen(14292);
+
+	try {
+		let client = net.connect({ host: "localhost", port: 14292 });
+		let clientRead = fromNodeStream.fromReadable(client);
+		let clientWrite = fromNodeStream.fromWritable(client);
+		let { serverRead, serverWrite } = await promise;
+		
+		let serverPipe = pipe([ serverRead, collect() ]);
+		let serverWritePipe = pipe([
+			fromIterable([ Buffer.from("c"), Buffer.from("d") ]),
+			testAbort(0, new Error("sample error")),
+			serverWrite
+		]);
+
+		let clientPipe = pipe([ clientRead, collect() ]);
+		let clientWritePipe = pipe([
+			fromIterable([ Buffer.from("a"), Buffer.from("b") ]),
+			/* Give the client side some time to notice that the server side closed the connection */
+			spy(async(value) => { await wait10(); return value; }),
+			clientWrite
+		]);
+
+		let results = await Promise.all([
+			assert.rejects(() => serverPipe.read(), { message: "sample error" }),
+			assert.rejects(() => serverWritePipe.read(), { message: "sample error" }),
+			/* The other side of the connection closes normally, like it would for any disconnection */
+			clientPipe.read(),
+			clientWritePipe.read(),
+		]);
+	} finally {
+		server.close();
+	}
+});