@8ms/helpers 3.2.7 → 3.2.9

package/dist/stream/server/json/logic.mjs

@@ -0,0 +1,16 @@
+import { Readable, Transform } from "stream";
+//#region src/stream/server/json/logic.ts
+const toRowReadable = (source) => {
+	return source instanceof Readable ? source : Readable.from(source, { objectMode: true });
+};
+/**
+* Applies the optional per-row callback, dropping rows for which it resolves null/undefined.
+*/
+const rowCallbackTransform = (rowCallback) => new Transform({
+	objectMode: true,
+	transform(row, _encoding, callback) {
+		Promise.resolve(rowCallback(row)).then((result) => callback(null, null === result || void 0 === result ? void 0 : result)).catch((error) => callback(error));
+	}
+});
+//#endregion
+export { rowCallbackTransform, toRowReadable };

package/dist/stream/server/json/read.cjs

@@ -0,0 +1,46 @@
+Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
+const require_runtime = require("../../../_virtual/_rolldown/runtime.cjs");
+const require_stream_server_sort = require("../sort.cjs");
+const require_stream_server_json_logic = require("./logic.cjs");
+let stream = require("stream");
+let zlib = require("zlib");
+zlib = require_runtime.__toESM(zlib);
+let stream_promises = require("stream/promises");
+//#region src/stream/server/json/read.ts
+const unwrapStreamArrayValue = () => new stream.Transform({
+	objectMode: true,
+	transform({ value }, _encoding, callback) {
+		callback(null, value);
+	}
+});
+const collectJsonArray = async (source, rowCallback, sortCallback) => {
+	const { streamArray } = await import("stream-json/streamers/stream-array.js");
+	const rows = [];
+	const steps = [
+		source,
+		streamArray.withParserAsStream(),
+		unwrapStreamArrayValue()
+	];
+	if (rowCallback) steps.push(require_stream_server_json_logic.rowCallbackTransform(rowCallback));
+	if (sortCallback) steps.push(require_stream_server_sort.sort(sortCallback));
+	steps.push(new stream.Transform({
+		objectMode: true,
+		transform(row, _encoding, callback) {
+			rows.push(row);
+			callback();
+		}
+	}));
+	await (0, stream_promises.pipeline)(steps);
+	return rows;
+};
+/** Parses a JSON array source incrementally, row by row, without buffering the whole document first. */
+const readJsonStream = async (props) => {
+	return collectJsonArray(props.source, props.rowCallback, props.sortCallback);
+};
+/** Same as readJsonStream, but gunzips `source` before parsing it as a JSON array. */
+const readJsonGzipStream = async (props) => {
+	return collectJsonArray(props.source.pipe(zlib.default.createGunzip()), props.rowCallback, props.sortCallback);
+};
+//#endregion
+exports.readJsonGzipStream = readJsonGzipStream;
+exports.readJsonStream = readJsonStream;

package/dist/stream/server/json/read.d.cts

@@ -0,0 +1,7 @@
+import { ReadJsonStreamProps } from "./logic.cjs";
+//#region src/stream/server/json/read.d.ts
+/** Parses a JSON array source incrementally, row by row, without buffering the whole document first. */
+export declare const readJsonStream: <T = any>(props: ReadJsonStreamProps<T>) => Promise<T[]>;
+/** Same as readJsonStream, but gunzips `source` before parsing it as a JSON array. */
+export declare const readJsonGzipStream: <T = any>(props: ReadJsonStreamProps<T>) => Promise<T[]>;
+//#endregion

package/dist/stream/server/json/read.d.mts

@@ -0,0 +1,7 @@
+import { ReadJsonStreamProps } from "./logic.mjs";
+//#region src/stream/server/json/read.d.ts
+/** Parses a JSON array source incrementally, row by row, without buffering the whole document first. */
+export declare const readJsonStream: <T = any>(props: ReadJsonStreamProps<T>) => Promise<T[]>;
+/** Same as readJsonStream, but gunzips `source` before parsing it as a JSON array. */
+export declare const readJsonGzipStream: <T = any>(props: ReadJsonStreamProps<T>) => Promise<T[]>;
+//#endregion

package/dist/stream/server/json/read.mjs

@@ -0,0 +1,42 @@
+import { sort } from "../sort.mjs";
+import { rowCallbackTransform } from "./logic.mjs";
+import { Transform } from "stream";
+import zlib from "zlib";
+import { pipeline } from "stream/promises";
+//#region src/stream/server/json/read.ts
+const unwrapStreamArrayValue = () => new Transform({
+	objectMode: true,
+	transform({ value }, _encoding, callback) {
+		callback(null, value);
+	}
+});
+const collectJsonArray = async (source, rowCallback, sortCallback) => {
+	const { streamArray } = await import("stream-json/streamers/stream-array.js");
+	const rows = [];
+	const steps = [
+		source,
+		streamArray.withParserAsStream(),
+		unwrapStreamArrayValue()
+	];
+	if (rowCallback) steps.push(rowCallbackTransform(rowCallback));
+	if (sortCallback) steps.push(sort(sortCallback));
+	steps.push(new Transform({
+		objectMode: true,
+		transform(row, _encoding, callback) {
+			rows.push(row);
+			callback();
+		}
+	}));
+	await pipeline(steps);
+	return rows;
+};
+/** Parses a JSON array source incrementally, row by row, without buffering the whole document first. */
+const readJsonStream = async (props) => {
+	return collectJsonArray(props.source, props.rowCallback, props.sortCallback);
+};
+/** Same as readJsonStream, but gunzips `source` before parsing it as a JSON array. */
+const readJsonGzipStream = async (props) => {
+	return collectJsonArray(props.source.pipe(zlib.createGunzip()), props.rowCallback, props.sortCallback);
+};
+//#endregion
+export { readJsonGzipStream, readJsonStream };

package/dist/stream/server/json/write.cjs

@@ -0,0 +1,69 @@
+Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
+const require_runtime = require("../../../_virtual/_rolldown/runtime.cjs");
+const require_stream_server_sort = require("../sort.cjs");
+const require_stream_server_json_logic = require("./logic.cjs");
+let stream = require("stream");
+let zlib = require("zlib");
+zlib = require_runtime.__toESM(zlib);
+let stream_promises = require("stream/promises");
+//#region src/stream/server/json/write.ts
+/**
+* Serialises a stream of row objects into a single JSON array without ever holding
+* the full collection in memory - one stringified row at a time.
+*/
+var JsonArrayStringifier = class extends stream.Transform {
+	isFirstRow = true;
+	constructor() {
+		super({
+			writableObjectMode: true,
+			readableObjectMode: false
+		});
+	}
+	_transform(row, _encoding, callback) {
+		this.push((this.isFirstRow ? "[" : ",") + JSON.stringify(row));
+		this.isFirstRow = false;
+		callback();
+	}
+	_flush(callback) {
+		this.push(this.isFirstRow ? "[]" : "]");
+		callback();
+	}
+};
+/** Serialises a stream of row objects into newline-delimited JSON, one stringified row at a time. */
+var JsonNewLineStringifier = class extends stream.Transform {
+	constructor() {
+		super({
+			writableObjectMode: true,
+			readableObjectMode: false
+		});
+	}
+	_transform(row, _encoding, callback) {
+		this.push(JSON.stringify(row) + "\n");
+		callback();
+	}
+};
+const jsonNewLineStringifier = () => new JsonNewLineStringifier();
+const buildWriteSteps = ({ source, rowCallback, sortCallback }) => {
+	const steps = [require_stream_server_json_logic.toRowReadable(source)];
+	if (rowCallback) steps.push(require_stream_server_json_logic.rowCallbackTransform(rowCallback));
+	if (sortCallback) steps.push(require_stream_server_sort.sort(sortCallback));
+	steps.push(new JsonArrayStringifier());
+	return steps;
+};
+/** Streams a row source into a single JSON array on `destination`, never buffering the full collection in memory. */
+const writeJsonStream = async (props) => {
+	await (0, stream_promises.pipeline)([...buildWriteSteps(props), props.destination]);
+};
+/** Same as writeJsonStream, but gzip-compresses the JSON array before it reaches `destination`. */
+const writeJsonGzipStream = async (props) => {
+	await (0, stream_promises.pipeline)([
+		...buildWriteSteps(props),
+		zlib.default.createGzip(),
+		props.destination
+	]);
+};
+//#endregion
+exports.JsonNewLineStringifier = JsonNewLineStringifier;
+exports.jsonNewLineStringifier = jsonNewLineStringifier;
+exports.writeJsonGzipStream = writeJsonGzipStream;
+exports.writeJsonStream = writeJsonStream;

package/dist/stream/server/json/write.d.cts

@@ -0,0 +1,14 @@
+import { WriteJsonStreamProps } from "./logic.cjs";
+import { Transform } from "stream";
+//#region src/stream/server/json/write.d.ts
+/** Serialises a stream of row objects into newline-delimited JSON, one stringified row at a time. */
+export declare class JsonNewLineStringifier extends Transform {
+  constructor();
+  _transform(row: any, _encoding: BufferEncoding, callback: (error?: Error | null) => void): void;
+}
+export declare const jsonNewLineStringifier: () => JsonNewLineStringifier;
+/** Streams a row source into a single JSON array on `destination`, never buffering the full collection in memory. */
+export declare const writeJsonStream: <T = any>(props: WriteJsonStreamProps<T>) => Promise<void>;
+/** Same as writeJsonStream, but gzip-compresses the JSON array before it reaches `destination`. */
+export declare const writeJsonGzipStream: <T = any>(props: WriteJsonStreamProps<T>) => Promise<void>;
+//#endregion

package/dist/stream/server/json/write.d.mts

@@ -0,0 +1,14 @@
+import { WriteJsonStreamProps } from "./logic.mjs";
+import { Transform } from "stream";
+//#region src/stream/server/json/write.d.ts
+/** Serialises a stream of row objects into newline-delimited JSON, one stringified row at a time. */
+export declare class JsonNewLineStringifier extends Transform {
+  constructor();
+  _transform(row: any, _encoding: BufferEncoding, callback: (error?: Error | null) => void): void;
+}
+export declare const jsonNewLineStringifier: () => JsonNewLineStringifier;
+/** Streams a row source into a single JSON array on `destination`, never buffering the full collection in memory. */
+export declare const writeJsonStream: <T = any>(props: WriteJsonStreamProps<T>) => Promise<void>;
+/** Same as writeJsonStream, but gzip-compresses the JSON array before it reaches `destination`. */
+export declare const writeJsonGzipStream: <T = any>(props: WriteJsonStreamProps<T>) => Promise<void>;
+//#endregion

package/dist/stream/server/json/write.mjs

@@ -0,0 +1,63 @@
+import { sort } from "../sort.mjs";
+import { rowCallbackTransform, toRowReadable } from "./logic.mjs";
+import { Transform } from "stream";
+import zlib from "zlib";
+import { pipeline } from "stream/promises";
+//#region src/stream/server/json/write.ts
+/**
+* Serialises a stream of row objects into a single JSON array without ever holding
+* the full collection in memory - one stringified row at a time.
+*/
+var JsonArrayStringifier = class extends Transform {
+	isFirstRow = true;
+	constructor() {
+		super({
+			writableObjectMode: true,
+			readableObjectMode: false
+		});
+	}
+	_transform(row, _encoding, callback) {
+		this.push((this.isFirstRow ? "[" : ",") + JSON.stringify(row));
+		this.isFirstRow = false;
+		callback();
+	}
+	_flush(callback) {
+		this.push(this.isFirstRow ? "[]" : "]");
+		callback();
+	}
+};
+/** Serialises a stream of row objects into newline-delimited JSON, one stringified row at a time. */
+var JsonNewLineStringifier = class extends Transform {
+	constructor() {
+		super({
+			writableObjectMode: true,
+			readableObjectMode: false
+		});
+	}
+	_transform(row, _encoding, callback) {
+		this.push(JSON.stringify(row) + "\n");
+		callback();
+	}
+};
+const jsonNewLineStringifier = () => new JsonNewLineStringifier();
+const buildWriteSteps = ({ source, rowCallback, sortCallback }) => {
+	const steps = [toRowReadable(source)];
+	if (rowCallback) steps.push(rowCallbackTransform(rowCallback));
+	if (sortCallback) steps.push(sort(sortCallback));
+	steps.push(new JsonArrayStringifier());
+	return steps;
+};
+/** Streams a row source into a single JSON array on `destination`, never buffering the full collection in memory. */
+const writeJsonStream = async (props) => {
+	await pipeline([...buildWriteSteps(props), props.destination]);
+};
+/** Same as writeJsonStream, but gzip-compresses the JSON array before it reaches `destination`. */
+const writeJsonGzipStream = async (props) => {
+	await pipeline([
+		...buildWriteSteps(props),
+		zlib.createGzip(),
+		props.destination
+	]);
+};
+//#endregion
+export { JsonNewLineStringifier, jsonNewLineStringifier, writeJsonGzipStream, writeJsonStream };

package/dist/stream/server/sort.cjs

@@ -1,41 +1,30 @@
 Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
 let node_stream = require("node:stream");
 //#region src/stream/server/sort.ts
-const batch = 15e3;
+/**
+* Buffers every row that passes through, then emits them all sorted by `comp` once the source ends.
+* Sorting requires seeing the whole collection first, so this is a full in-memory buffer - not a
+* bounded or windowed sort - and should only be used where the full stream fits comfortably in memory.
+*/
 var SortStreamer = class extends node_stream.Transform {
-	a;
-	i;
-	k;
-	currentSize;
-	sortedArray;
+	rows = [];
 	comp;
 	constructor(options, comp) {
 		super(options);
-		this.a = new Array(batch);
-		this.i = 0;
-		this.k = 0;
-		this.currentSize = batch;
-		this.sortedArray = void 0;
 		this.comp = comp;
 	}
-	_transform(data, _, callback) {
-		if (data) {
-			this.a[this.i++] = data;
-			if (this.i === this.currentSize) {
-				this.currentSize *= 2;
-				const tmp = new Array(this.currentSize);
-				for (let j = 0; j < this.i; j++) tmp[j] = this.a[j];
-				this.a = tmp;
-			}
-		}
+	/** Collects each row as it arrives; nothing is emitted downstream yet. */
+	_transform(data, _encoding, callback) {
+		this.rows.push(data);
 		callback();
 	}
+	/** The source has ended - sort everything collected so far, then emit it all in order. */
 	_flush(callback) {
-		this.sortedArray = this.a.slice(0, this.i).sort(this.comp);
-		while (this.k < this.i) this.push(this.sortedArray[this.k++]);
+		this.rows.sort(this.comp).forEach((row) => this.push(row));
 		callback();
 	}
 };
+/** Creates a sorting Transform: buffers every row, sorts by `comp`, and emits the result once the source ends. */
 const sort = (comp, options) => {
 	return new SortStreamer(options ?? { objectMode: true }, comp);
 };

package/dist/stream/server/sort.d.cts

@@ -1,15 +1,19 @@
 import { Transform, TransformCallback, TransformOptions } from "node:stream";
 //#region src/stream/server/sort.d.ts
+/**
+ * Buffers every row that passes through, then emits them all sorted by `comp` once the source ends.
+ * Sorting requires seeing the whole collection first, so this is a full in-memory buffer - not a
+ * bounded or windowed sort - and should only be used where the full stream fits comfortably in memory.
+ */
 declare class SortStreamer<T> extends Transform {
-  private a;
-  private i;
-  private k;
-  private currentSize;
-  private sortedArray;
+  private readonly rows;
   private readonly comp;
   constructor(options: TransformOptions, comp: (a: T, b: T) => number);
-  _transform(data: T, _: BufferEncoding, callback: TransformCallback): void;
+  /** Collects each row as it arrives; nothing is emitted downstream yet. */
+  _transform(data: T, _encoding: BufferEncoding, callback: TransformCallback): void;
+  /** The source has ended - sort everything collected so far, then emit it all in order. */
   _flush(callback: TransformCallback): void;
 }
+/** Creates a sorting Transform: buffers every row, sorts by `comp`, and emits the result once the source ends. */
 export declare const sort: <T>(comp: (a: T, b: T) => number, options?: TransformOptions) => SortStreamer<T>;
 //#endregion

package/dist/stream/server/sort.d.mts

@@ -1,15 +1,19 @@
 import { Transform, TransformCallback, TransformOptions } from "node:stream";
 //#region src/stream/server/sort.d.ts
+/**
+ * Buffers every row that passes through, then emits them all sorted by `comp` once the source ends.
+ * Sorting requires seeing the whole collection first, so this is a full in-memory buffer - not a
+ * bounded or windowed sort - and should only be used where the full stream fits comfortably in memory.
+ */
 declare class SortStreamer<T> extends Transform {
-  private a;
-  private i;
-  private k;
-  private currentSize;
-  private sortedArray;
+  private readonly rows;
   private readonly comp;
   constructor(options: TransformOptions, comp: (a: T, b: T) => number);
-  _transform(data: T, _: BufferEncoding, callback: TransformCallback): void;
+  /** Collects each row as it arrives; nothing is emitted downstream yet. */
+  _transform(data: T, _encoding: BufferEncoding, callback: TransformCallback): void;
+  /** The source has ended - sort everything collected so far, then emit it all in order. */
   _flush(callback: TransformCallback): void;
 }
+/** Creates a sorting Transform: buffers every row, sorts by `comp`, and emits the result once the source ends. */
 export declare const sort: <T>(comp: (a: T, b: T) => number, options?: TransformOptions) => SortStreamer<T>;
 //#endregion

package/dist/stream/server/sort.mjs

@@ -1,40 +1,29 @@
 import { Transform } from "node:stream";
 //#region src/stream/server/sort.ts
-const batch = 15e3;
+/**
+* Buffers every row that passes through, then emits them all sorted by `comp` once the source ends.
+* Sorting requires seeing the whole collection first, so this is a full in-memory buffer - not a
+* bounded or windowed sort - and should only be used where the full stream fits comfortably in memory.
+*/
 var SortStreamer = class extends Transform {
-	a;
-	i;
-	k;
-	currentSize;
-	sortedArray;
+	rows = [];
 	comp;
 	constructor(options, comp) {
 		super(options);
-		this.a = new Array(batch);
-		this.i = 0;
-		this.k = 0;
-		this.currentSize = batch;
-		this.sortedArray = void 0;
 		this.comp = comp;
 	}
-	_transform(data, _, callback) {
-		if (data) {
-			this.a[this.i++] = data;
-			if (this.i === this.currentSize) {
-				this.currentSize *= 2;
-				const tmp = new Array(this.currentSize);
-				for (let j = 0; j < this.i; j++) tmp[j] = this.a[j];
-				this.a = tmp;
-			}
-		}
+	/** Collects each row as it arrives; nothing is emitted downstream yet. */
+	_transform(data, _encoding, callback) {
+		this.rows.push(data);
 		callback();
 	}
+	/** The source has ended - sort everything collected so far, then emit it all in order. */
 	_flush(callback) {
-		this.sortedArray = this.a.slice(0, this.i).sort(this.comp);
-		while (this.k < this.i) this.push(this.sortedArray[this.k++]);
+		this.rows.sort(this.comp).forEach((row) => this.push(row));
 		callback();
 	}
 };
+/** Creates a sorting Transform: buffers every row, sorts by `comp`, and emits the result once the source ends. */
 const sort = (comp, options) => {
 	return new SortStreamer(options ?? { objectMode: true }, comp);
 };

package/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@8ms/helpers",
 	"license": "UNLICENSED",
-	"version": "3.2.7",
+	"version": "3.2.9",
 	"packageManager": "yarn@4.18.0",
 	"repository": {
 		"type": "git",
@@ -77,6 +77,7 @@
 		"pg": "^8.0.0",
 		"react": "^19.0.0",
 		"react-intersection-observer": "^11.0.0",
+		"stream-json": "^3.7.0",
 		"swr": "^2.0.0",
 		"tailwind-merge": "^3.0.0"
 	},
@@ -183,6 +184,9 @@
 		"react": {
 			"optional": true
 		},
+		"stream-json": {
+			"optional": true
+		},
 		"swr": {
 			"optional": true
 		},
@@ -238,6 +242,7 @@
 		"pg": "^8.23.0",
 		"react": "^19.3.0",
 		"react-intersection-observer": "^11.0.1",
+		"stream-json": "^3.7.0",
 		"swr": "^2.5.1",
 		"tailwind-merge": "^3.7.0",
 		"ts-node": "^10.9.2",
@@ -430,6 +435,18 @@
 			"import": "./dist/aws/lambda/server/getHandlerPath.mjs",
 			"require": "./dist/aws/lambda/server/getHandlerPath.cjs"
 		},
+		"./aws/s3/_shared": {
+			"import": "./dist/aws/s3/_shared/index.mjs",
+			"require": "./dist/aws/s3/_shared/index.cjs"
+		},
+		"./aws/s3/_shared/awsS3File": {
+			"import": "./dist/aws/s3/_shared/awsS3File.mjs",
+			"require": "./dist/aws/s3/_shared/awsS3File.cjs"
+		},
+		"./aws/s3/_shared/types": {
+			"import": "./dist/aws/s3/_shared/types.mjs",
+			"require": "./dist/aws/s3/_shared/types.cjs"
+		},
 		"./aws/s3/server": {
 			"import": "./dist/aws/s3/server/index.mjs",
 			"require": "./dist/aws/s3/server/index.cjs"
@@ -438,10 +455,6 @@
 			"import": "./dist/aws/s3/server/AwsS3.mjs",
 			"require": "./dist/aws/s3/server/AwsS3.cjs"
 		},
-		"./aws/s3/server/awsS3File": {
-			"import": "./dist/aws/s3/server/awsS3File.mjs",
-			"require": "./dist/aws/s3/server/awsS3File.cjs"
-		},
 		"./aws/s3/server/payload": {
 			"import": "./dist/aws/s3/server/payload.mjs",
 			"require": "./dist/aws/s3/server/payload.cjs"
@@ -1542,6 +1555,18 @@
 			"import": "./dist/stream/server/index.mjs",
 			"require": "./dist/stream/server/index.cjs"
 		},
+		"./stream/server/json/logic": {
+			"import": "./dist/stream/server/json/logic.mjs",
+			"require": "./dist/stream/server/json/logic.cjs"
+		},
+		"./stream/server/json/read": {
+			"import": "./dist/stream/server/json/read.mjs",
+			"require": "./dist/stream/server/json/read.cjs"
+		},
+		"./stream/server/json/write": {
+			"import": "./dist/stream/server/json/write.mjs",
+			"require": "./dist/stream/server/json/write.cjs"
+		},
 		"./stream/server/sort": {
 			"import": "./dist/stream/server/sort.mjs",
 			"require": "./dist/stream/server/sort.cjs"
@@ -1847,5 +1872,8 @@
 			"require": "./dist/zip/server/index.cjs"
 		},
 		"./package.json": "./package.json"
+	},
+	"inlinedDependencies": {
+		"@google-cloud/paginator": "7.1.0"
 	}
 }