@8ms/helpers 3.2.6 → 3.2.8

package/dist/googleCloud/bigquery/server/GoogleCloudBigquery.mjs

@@ -1,6 +1,22 @@
 import { isObject } from "../../../lodash/_shared/isObject.mjs";
 import { BaseNamespace } from "../../../_class/_shared/BaseNamespace.mjs";
+import { getYmdHisNumber } from "../../../date/_shared/format.mjs";
+import "../../../date/_shared/index.mjs";
+import { fileType } from "../../../file/_shared/fileType.mjs";
+import { fileMimeType } from "../../../file/_shared/fileMimeType.mjs";
+import "../../../file/_shared/index.mjs";
+import { getJsonNewline } from "../../../json/_shared/getJsonNewline.mjs";
+import { toRowReadable } from "../../../stream/server/json/logic.mjs";
+import { jsonNewLineStringifier } from "../../../stream/server/json/write.mjs";
+import "../../../stream/server/index.mjs";
+import { pipeline } from "node:stream/promises";
 //#region src/googleCloud/bigquery/server/GoogleCloudBigquery.ts
+/** Maps this library's generic file-type catalog onto BigQuery's own load-job sourceFormat strings. */
+const bigQuerySourceFormat = {
+	[fileType.CSV]: "CSV",
+	[fileType.JSON_NEWLINE]: "NEWLINE_DELIMITED_JSON",
+	[fileType.PARQUET]: "PARQUET"
+};
 var GoogleCloudBigquery = class extends BaseNamespace {
 	/**
 	* Either a Google Cloud JSON Config or V10 Auth helper.
@@ -64,7 +80,7 @@ var GoogleCloudBigquery = class extends BaseNamespace {
 		});
 		return response;
 	};
-	query = async (props) => {
+	buildQueryOptions = (props) => {
 		let options = { query: "" };
 		if ("string" === typeof props) {
 			options.query = props;
@@ -75,7 +91,19 @@ var GoogleCloudBigquery = class extends BaseNamespace {
 			if (resolvedLocation) options.location = resolvedLocation;
 			if (props?.params) options["params"] = props.params;
 		}
-		return await this.client.query(options);
+		return options;
+	};
+	/**
+	* Make a request to the database and return the object per row.
+	*/
+	query = async (props) => {
+		return await this.client.query(this.buildQueryOptions(props));
+	};
+	/**
+	* Return the raw row stream for a query, without buffering the full result set in memory.
+	*/
+	queryStream = (props) => {
+		return this.client.createQueryStream(this.buildQueryOptions(props));
 	};
 	/**
 	* Create a Table if it doesn't already exist.
@@ -112,14 +140,137 @@ var GoogleCloudBigquery = class extends BaseNamespace {
 		return true;
 	};
 	/**
-	* Load data into this table using Google Cloud Storage.
+	* Load a Google Cloud Storage file into a table. BigQuery load jobs default to
+	* createDisposition "CREATE_IF_NEEDED"; paired with autodetect, this creates the table itself
+	* from the source file's inferred schema if it doesn't already exist, rather than this library
+	* pre-checking existence and silently skipping the load if it isn't there.
 	*/
-	loadData = async ({ file, googleCloudStorageClient, table }) => {
-		if (await this.tableExists(table.datasetId, table.tableId)) await this.client.dataset(table.datasetId, { location: table?.region }).table(table.tableId).load(googleCloudStorageClient.client.bucket(file.bucket).file(file.key), {
-			...file?.metadata ?? {},
-			location: file?.region
+	loadData = async (props) => {
+		const finalTableId = props.table.tableId;
+		await this.createTable(props.table.datasetId, {}, finalTableId, props.table.options);
+		await this.client.dataset(props.table.datasetId, { location: props.table.region }).table(finalTableId).load(props.googleCloudStorage.client.bucket(props.file.bucket).file(props.file.key), {
+			autodetect: true,
+			sourceFormat: bigQuerySourceFormat[props.file.sourceFormat ?? fileType.JSON_NEWLINE],
+			schemaUpdateOptions: ["ALLOW_FIELD_ADDITION"],
+			location: props.table.region
 		});
 	};
+	/**
+	* Write rows to Google Cloud Storage, then load them into the table - creating the table first
+	* if it doesn't already exist. Only JSON_NEWLINE (the default) is actually serialised by this
+	* library (getJsonNewline); for PARQUET, `data` is written as-is and must already be encoded -
+	* this library has no Parquet encoder.
+	*/
+	loadTable = async (props) => {
+		const finalTableId = props.table.tableId ?? props.table.definition.tableId;
+		const fileSourceFormat = props.file.sourceFormat ?? fileType.JSON_NEWLINE;
+		const finalData = fileType.JSON_NEWLINE === fileSourceFormat ? getJsonNewline(props.file.data) : props.file.data;
+		await props.googleCloudStorage.writeFile(props.file.bucket, props.file.key, finalData);
+		await this.loadData({
+			...props,
+			table: {
+				...props.table,
+				tableId: finalTableId
+			}
+		});
+	};
+	/**
+	* Same as loadTable, but streams a row source into Google Cloud Storage instead of buffering the
+	* full collection in memory first. Writes directly against the Storage client's own write stream
+	* (GoogleCloudStorage.writeFile can't stream a Readable source) - creates the table first if it
+	* doesn't already exist. For JSON_NEWLINE (the default) `source` is a stream of row objects,
+	* stringified one at a time; for PARQUET there is no stringify stage - `source` must already be a
+	* byte-level Readable of encoded Parquet content, piped straight through.
+	*/
+	loadStreamTable = async (props) => {
+		const finalTableId = props.table.tableId ?? props.table.definition.tableId;
+		const fileSourceFormat = props.file.sourceFormat ?? fileType.JSON_NEWLINE;
+		const destination = props.googleCloudStorage.client.bucket(props.file.bucket).file(props.file.key).createWriteStream({
+			contentType: fileMimeType[fileSourceFormat],
+			resumable: false
+		});
+		const source = toRowReadable(props.file.data);
+		if (fileType.JSON_NEWLINE === fileSourceFormat) await pipeline([
+			source,
+			jsonNewLineStringifier(),
+			destination
+		]);
+		else await pipeline([source, destination]);
+		await this.loadData({
+			...props,
+			table: {
+				...props.table,
+				tableId: finalTableId
+			}
+		});
+	};
+	/**
+	* Brings an existing table's clustering in line with `definition.options.clustering`, if it differs.
+	* Partitioning is immutable on an existing BigQuery table - changing it requires recreating the
+	* table and backfilling data, which this method deliberately does not attempt.
+	*/
+	syncTableClustering = async (datasetId, tableId, clustering) => {
+		const [metadata] = await this.client.dataset(datasetId).table(tableId).getMetadata();
+		const currentFields = metadata?.clustering?.fields ?? [];
+		const desiredFields = clustering?.fields ?? [];
+		if (currentFields.length === desiredFields.length && currentFields.every((field, index) => field === desiredFields[index])) return;
+		await this.client.dataset(datasetId).table(tableId).setMetadata(desiredFields.length > 0 ? { clustering: { fields: desiredFields } } : { clustering: void 0 });
+	};
+	/**
+	* Upsert rows into a table via a staging table + a single MERGE statement.
+	*
+	* BigQuery has no efficient row-level UPDATE/DELETE - both rewrite the affected partitions and
+	* are subject to daily DML mutation quotas. Loading new rows into a disposable staging table and
+	* running one MERGE is the standard, quota-friendly way to upsert in BigQuery.
+	*/
+	mergeTable = async (props) => {
+		if (0 === props.file.data.length) return;
+		const finalTableId = props.table.tableId ?? props.table.definition.tableId;
+		if (await this.tableExists(props.table.datasetId, finalTableId)) await this.syncTableClustering(props.table.datasetId, finalTableId, props.table.definition.options?.clustering);
+		const stagingTableId = `${finalTableId}_staging_${getYmdHisNumber()}`;
+		const stagingTableRef = `\`${props.table.datasetId}.${stagingTableId}\``;
+		try {
+			const stagingDefinition = {
+				...props.table.definition,
+				tableId: stagingTableId,
+				options: props.table.definition.options ? {
+					...props.table.definition.options,
+					timePartitioning: void 0,
+					clustering: void 0
+				} : void 0
+			};
+			await this.loadTable({
+				googleCloudStorage: props.googleCloudStorage,
+				file: props.file,
+				table: {
+					datasetId: props.table.datasetId,
+					definition: stagingDefinition,
+					tableId: stagingTableId,
+					region: props.table.region
+				}
+			});
+			const mainTableRef = `\`${props.table.datasetId}.${finalTableId}\``;
+			const updateSet = props.table.updateColumns.map((column) => `${column} = S.${column}`).join(", ");
+			const allColumns = props.table.definition.options?.schema.map((field) => field.name) ?? [];
+			const mergeQuery = `
+				MERGE ${mainTableRef} AS T
+				USING ${stagingTableRef} AS S
+				ON T.${props.table.mergeKey} = S.${props.table.mergeKey}
+				WHEN MATCHED THEN
+					UPDATE SET ${updateSet}
+				WHEN NOT MATCHED THEN
+					INSERT (${allColumns.join(", ")})
+					VALUES (${allColumns.map((column) => `S.${column}`).join(", ")})
+			`;
+			await this.query(mergeQuery);
+		} finally {
+			try {
+				await this.query(`DROP TABLE IF EXISTS ${stagingTableRef}`);
+			} catch (error) {
+				console.error(`mergeTable: failed to drop staging table ${stagingTableRef} - it must be dropped manually.`, error);
+			}
+		}
+	};
 };
 //#endregion
 export { GoogleCloudBigquery };

package/dist/googleCloud/bigquery/server/index.d.cts

@@ -17,8 +17,8 @@ import { GoogleCloudBQRow, googleCloudBQRow } from "../_shared/lookup/googleClou
 import { LanguageBQRow, languageBQRow } from "../_shared/lookup/language.cjs";
 import { PartitionType, partitionType } from "../_shared/partitionType.cjs";
 import { BQField, zodToBQSchema } from "../_shared/zodToBQDefinition.cjs";
-import { TableDefinition } from "../_shared/types.cjs";
+import { BigqueryLoadFormat, Dataset, LoadData, LoadStreamTableProps, LoadTableProps, MergeTableProps, QueryInput, Table, TableDefinition } from "../_shared/types.cjs";
 import "../_shared/index.cjs";
-import { GoogleCloudBigquery, QueryInput } from "./GoogleCloudBigquery.cjs";
+import { GoogleCloudBigquery } from "./GoogleCloudBigquery.cjs";
 import { getHandlerPath } from "./getHandlerPath.cjs";
-export { BQBool, BQBytes, BQDate, BQDateTime, BQDatetime, BQField, BQFieldMode, BQFieldModeSchema, BQFieldType, BQFieldTypeSchema, BQFloat, BQInt64, BQNumeric, BQString, BQTime, BQTimestamp, CountryBQRow, GoogleAdsAccountBQRow, GoogleCloudBQRow, GoogleCloudBigquery, LanguageBQRow, PartitionType, QueryInput, TableDefinition, countryBQRow, decodeBQDate, decodeBQDateTime, encodeBQDate, encodeBQDateTime, getHandlerPath, googleAdsAccountBQRow, googleCloudBQRow, languageBQRow, partitionType, zBQBytes, zBQInt, zBQInt64, zBQNumeric, zBQTime, zLuxonDate, zLuxonDateTime, zodToBQSchema };
+export { BQBool, BQBytes, BQDate, BQDateTime, BQDatetime, BQField, BQFieldMode, BQFieldModeSchema, BQFieldType, BQFieldTypeSchema, BQFloat, BQInt64, BQNumeric, BQString, BQTime, BQTimestamp, BigqueryLoadFormat, CountryBQRow, Dataset, GoogleAdsAccountBQRow, GoogleCloudBQRow, GoogleCloudBigquery, LanguageBQRow, LoadData, LoadStreamTableProps, LoadTableProps, MergeTableProps, PartitionType, QueryInput, Table, TableDefinition, countryBQRow, decodeBQDate, decodeBQDateTime, encodeBQDate, encodeBQDateTime, getHandlerPath, googleAdsAccountBQRow, googleCloudBQRow, languageBQRow, partitionType, zBQBytes, zBQInt, zBQInt64, zBQNumeric, zBQTime, zLuxonDate, zLuxonDateTime, zodToBQSchema };

package/dist/googleCloud/bigquery/server/index.d.mts

@@ -17,8 +17,8 @@ import { GoogleCloudBQRow, googleCloudBQRow } from "../_shared/lookup/googleClou
 import { LanguageBQRow, languageBQRow } from "../_shared/lookup/language.mjs";
 import { PartitionType, partitionType } from "../_shared/partitionType.mjs";
 import { BQField, zodToBQSchema } from "../_shared/zodToBQDefinition.mjs";
-import { TableDefinition } from "../_shared/types.mjs";
+import { BigqueryLoadFormat, Dataset, LoadData, LoadStreamTableProps, LoadTableProps, MergeTableProps, QueryInput, Table, TableDefinition } from "../_shared/types.mjs";
 import "../_shared/index.mjs";
-import { GoogleCloudBigquery, QueryInput } from "./GoogleCloudBigquery.mjs";
+import { GoogleCloudBigquery } from "./GoogleCloudBigquery.mjs";
 import { getHandlerPath } from "./getHandlerPath.mjs";
-export { BQBool, BQBytes, BQDate, BQDateTime, BQDatetime, BQField, BQFieldMode, BQFieldModeSchema, BQFieldType, BQFieldTypeSchema, BQFloat, BQInt64, BQNumeric, BQString, BQTime, BQTimestamp, CountryBQRow, GoogleAdsAccountBQRow, GoogleCloudBQRow, GoogleCloudBigquery, LanguageBQRow, PartitionType, QueryInput, TableDefinition, countryBQRow, decodeBQDate, decodeBQDateTime, encodeBQDate, encodeBQDateTime, getHandlerPath, googleAdsAccountBQRow, googleCloudBQRow, languageBQRow, partitionType, zBQBytes, zBQInt, zBQInt64, zBQNumeric, zBQTime, zLuxonDate, zLuxonDateTime, zodToBQSchema };
+export { BQBool, BQBytes, BQDate, BQDateTime, BQDatetime, BQField, BQFieldMode, BQFieldModeSchema, BQFieldType, BQFieldTypeSchema, BQFloat, BQInt64, BQNumeric, BQString, BQTime, BQTimestamp, BigqueryLoadFormat, CountryBQRow, Dataset, GoogleAdsAccountBQRow, GoogleCloudBQRow, GoogleCloudBigquery, LanguageBQRow, LoadData, LoadStreamTableProps, LoadTableProps, MergeTableProps, PartitionType, QueryInput, Table, TableDefinition, countryBQRow, decodeBQDate, decodeBQDateTime, encodeBQDate, encodeBQDateTime, getHandlerPath, googleAdsAccountBQRow, googleCloudBQRow, languageBQRow, partitionType, zBQBytes, zBQInt, zBQInt64, zBQNumeric, zBQTime, zLuxonDate, zLuxonDateTime, zodToBQSchema };

package/dist/googleCloud/drive/server/GoogleCloudDrive.cjs

@@ -1,8 +1,12 @@
 Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
 const require__class__shared_BaseNamespace = require("../../../_class/_shared/BaseNamespace.cjs");
+const require_stream_server_json_read = require("../../../stream/server/json/read.cjs");
+const require_stream_server_json_write = require("../../../stream/server/json/write.cjs");
+require("../../../stream/server/index.cjs");
 const require_util_server_getGzipDecompressed = require("../../../util/server/getGzipDecompressed.cjs");
 const require_util_server_getGzipCompressed = require("../../../util/server/getGzipCompressed.cjs");
 require("../../../util/server/index.cjs");
+let stream = require("stream");
 //#region src/googleCloud/drive/server/GoogleCloudDrive.ts
 var GoogleCloudDrive = class extends require__class__shared_BaseNamespace.BaseNamespace {
 	/**
@@ -172,6 +176,57 @@ var GoogleCloudDrive = class extends require__class__shared_BaseNamespace.BaseNa
 		return await this.writeFile(parentFolderId, fileName, finalData, driveOptions);
 	};
 	/**
+	* Write a row source (stream, array, or async iterable) as a JSON array to Drive,
+	* without buffering the full payload in memory. `media.body` being a stream makes
+	* @googleapis/drive perform a resumable upload automatically.
+	*/
+	writeJsonStream = async (parentFolderId, fileName, source, options = {}) => {
+		const googleApisDrive = await import("@googleapis/drive");
+		const destination = new stream.PassThrough();
+		const createPromise = googleApisDrive.drive("v3").files.create({
+			auth: this.client,
+			fields: "id, name",
+			media: {
+				body: destination,
+				mimeType: options.driveOptions?.mimeType ?? "application/json"
+			},
+			requestBody: {
+				name: fileName,
+				parents: [parentFolderId]
+			}
+		});
+		const [, createResponse] = await Promise.all([require_stream_server_json_write.writeJsonStream({
+			source,
+			destination,
+			rowCallback: options.rowCallback,
+			sortCallback: options.sortCallback
+		}), createPromise]);
+		return createResponse.data;
+	};
+	writeJsonGzipStream = async (parentFolderId, fileName, source, options = {}) => {
+		const googleApisDrive = await import("@googleapis/drive");
+		const destination = new stream.PassThrough();
+		const createPromise = googleApisDrive.drive("v3").files.create({
+			auth: this.client,
+			fields: "id, name",
+			media: {
+				body: destination,
+				mimeType: options.driveOptions?.mimeType ?? "application/gzip"
+			},
+			requestBody: {
+				name: fileName,
+				parents: [parentFolderId]
+			}
+		});
+		const [, createResponse] = await Promise.all([require_stream_server_json_write.writeJsonGzipStream({
+			source,
+			destination,
+			rowCallback: options.rowCallback,
+			sortCallback: options.sortCallback
+		}), createPromise]);
+		return createResponse.data;
+	};
+	/**
 	* Read a file's contents from Google Drive.
 	*/
 	readFile = async (fileId, options = {}) => {
@@ -182,6 +237,36 @@ var GoogleCloudDrive = class extends require__class__shared_BaseNamespace.BaseNa
 			...options
 		}, { responseType: "text" })).data };
 	};
+	/**
+	* Return a file's contents as a stream, without buffering it into memory.
+	*/
+	readStream = async (fileId, options = {}) => {
+		return (await (await import("@googleapis/drive")).drive("v3").files.get({
+			auth: this.client,
+			fileId,
+			alt: "media",
+			...options
+		}, { responseType: "stream" })).data;
+	};
+	/**
+	* Read a JSON array file back, parsing it incrementally without buffering the full document.
+	*/
+	readJsonStream = async (fileId, options = {}) => {
+		const source = await this.readStream(fileId, options.driveOptions);
+		return require_stream_server_json_read.readJsonStream({
+			source,
+			rowCallback: options.rowCallback,
+			sortCallback: options.sortCallback
+		});
+	};
+	readJsonGzipStream = async (fileId, options = {}) => {
+		const source = await this.readStream(fileId, options.driveOptions);
+		return require_stream_server_json_read.readJsonGzipStream({
+			source,
+			rowCallback: options.rowCallback,
+			sortCallback: options.sortCallback
+		});
+	};
 	readJson = async (fileId) => {
 		const apiResponse = await this.readFile(fileId);
 		apiResponse.body = JSON.parse(apiResponse.body) || null;

package/dist/googleCloud/drive/server/GoogleCloudDrive.d.cts

@@ -1,4 +1,6 @@
 import { BaseNamespace } from "../../../_class/_shared/BaseNamespace.cjs";
+import { RowCallback, RowSource, SortCallback } from "../../../stream/server/json/logic.cjs";
+import { Readable } from "stream";
 //#region src/googleCloud/drive/server/GoogleCloudDrive.d.ts
 type Options = {
   [key: string]: any;
@@ -50,12 +52,44 @@ export declare class GoogleCloudDrive extends BaseNamespace {
   writeFile: (parentFolderId: string, fileName: string, data: any, options?: Options) => Promise<import("@googleapis/drive").drive_v3.Schema$File>;
   writeJson: (parentFolderId: string, fileName: string, data: any, options?: Options) => Promise<import("@googleapis/drive").drive_v3.Schema$File>;
   writeJsonGzip: (parentFolderId: string, fileName: string, data: any, gzipOptions?: object, driveOptions?: Options) => Promise<import("@googleapis/drive").drive_v3.Schema$File>;
+  /**
+   * Write a row source (stream, array, or async iterable) as a JSON array to Drive,
+   * without buffering the full payload in memory. `media.body` being a stream makes
+   * @googleapis/drive perform a resumable upload automatically.
+   */
+  writeJsonStream: <T = any>(parentFolderId: string, fileName: string, source: RowSource<T>, options?: {
+    rowCallback?: RowCallback<T>;
+    sortCallback?: SortCallback<T>;
+    driveOptions?: Options;
+  }) => Promise<import("@googleapis/drive").drive_v3.Schema$File>;
+  writeJsonGzipStream: <T = any>(parentFolderId: string, fileName: string, source: RowSource<T>, options?: {
+    rowCallback?: RowCallback<T>;
+    sortCallback?: SortCallback<T>;
+    driveOptions?: Options;
+  }) => Promise<import("@googleapis/drive").drive_v3.Schema$File>;
   /**
    * Read a file's contents from Google Drive.
    */
   readFile: (fileId: string, options?: Options) => Promise<{
     body: string;
   }>;
+  /**
+   * Return a file's contents as a stream, without buffering it into memory.
+   */
+  readStream: (fileId: string, options?: Options) => Promise<Readable>;
+  /**
+   * Read a JSON array file back, parsing it incrementally without buffering the full document.
+   */
+  readJsonStream: <T = any>(fileId: string, options?: {
+    rowCallback?: RowCallback<T>;
+    sortCallback?: SortCallback<T>;
+    driveOptions?: Options;
+  }) => Promise<T[]>;
+  readJsonGzipStream: <T = any>(fileId: string, options?: {
+    rowCallback?: RowCallback<T>;
+    sortCallback?: SortCallback<T>;
+    driveOptions?: Options;
+  }) => Promise<T[]>;
   readJson: (fileId: string) => Promise<{
     body: string;
   }>;

package/dist/googleCloud/drive/server/GoogleCloudDrive.d.mts

@@ -1,4 +1,6 @@
 import { BaseNamespace } from "../../../_class/_shared/BaseNamespace.mjs";
+import { RowCallback, RowSource, SortCallback } from "../../../stream/server/json/logic.mjs";
+import { Readable } from "stream";
 //#region src/googleCloud/drive/server/GoogleCloudDrive.d.ts
 type Options = {
   [key: string]: any;
@@ -50,12 +52,44 @@ export declare class GoogleCloudDrive extends BaseNamespace {
   writeFile: (parentFolderId: string, fileName: string, data: any, options?: Options) => Promise<import("@googleapis/drive").drive_v3.Schema$File>;
   writeJson: (parentFolderId: string, fileName: string, data: any, options?: Options) => Promise<import("@googleapis/drive").drive_v3.Schema$File>;
   writeJsonGzip: (parentFolderId: string, fileName: string, data: any, gzipOptions?: object, driveOptions?: Options) => Promise<import("@googleapis/drive").drive_v3.Schema$File>;
+  /**
+   * Write a row source (stream, array, or async iterable) as a JSON array to Drive,
+   * without buffering the full payload in memory. `media.body` being a stream makes
+   * @googleapis/drive perform a resumable upload automatically.
+   */
+  writeJsonStream: <T = any>(parentFolderId: string, fileName: string, source: RowSource<T>, options?: {
+    rowCallback?: RowCallback<T>;
+    sortCallback?: SortCallback<T>;
+    driveOptions?: Options;
+  }) => Promise<import("@googleapis/drive").drive_v3.Schema$File>;
+  writeJsonGzipStream: <T = any>(parentFolderId: string, fileName: string, source: RowSource<T>, options?: {
+    rowCallback?: RowCallback<T>;
+    sortCallback?: SortCallback<T>;
+    driveOptions?: Options;
+  }) => Promise<import("@googleapis/drive").drive_v3.Schema$File>;
   /**
    * Read a file's contents from Google Drive.
    */
   readFile: (fileId: string, options?: Options) => Promise<{
     body: string;
   }>;
+  /**
+   * Return a file's contents as a stream, without buffering it into memory.
+   */
+  readStream: (fileId: string, options?: Options) => Promise<Readable>;
+  /**
+   * Read a JSON array file back, parsing it incrementally without buffering the full document.
+   */
+  readJsonStream: <T = any>(fileId: string, options?: {
+    rowCallback?: RowCallback<T>;
+    sortCallback?: SortCallback<T>;
+    driveOptions?: Options;
+  }) => Promise<T[]>;
+  readJsonGzipStream: <T = any>(fileId: string, options?: {
+    rowCallback?: RowCallback<T>;
+    sortCallback?: SortCallback<T>;
+    driveOptions?: Options;
+  }) => Promise<T[]>;
   readJson: (fileId: string) => Promise<{
     body: string;
   }>;

package/dist/googleCloud/drive/server/GoogleCloudDrive.mjs

@@ -1,7 +1,11 @@
 import { BaseNamespace } from "../../../_class/_shared/BaseNamespace.mjs";
+import { readJsonGzipStream, readJsonStream } from "../../../stream/server/json/read.mjs";
+import { writeJsonGzipStream, writeJsonStream } from "../../../stream/server/json/write.mjs";
+import "../../../stream/server/index.mjs";
 import { getGzipDecompressed } from "../../../util/server/getGzipDecompressed.mjs";
 import { getGzipCompressed } from "../../../util/server/getGzipCompressed.mjs";
 import "../../../util/server/index.mjs";
+import { PassThrough } from "stream";
 //#region src/googleCloud/drive/server/GoogleCloudDrive.ts
 var GoogleCloudDrive = class extends BaseNamespace {
 	/**
@@ -171,6 +175,57 @@ var GoogleCloudDrive = class extends BaseNamespace {
 		return await this.writeFile(parentFolderId, fileName, finalData, driveOptions);
 	};
 	/**
+	* Write a row source (stream, array, or async iterable) as a JSON array to Drive,
+	* without buffering the full payload in memory. `media.body` being a stream makes
+	* @googleapis/drive perform a resumable upload automatically.
+	*/
+	writeJsonStream = async (parentFolderId, fileName, source, options = {}) => {
+		const googleApisDrive = await import("@googleapis/drive");
+		const destination = new PassThrough();
+		const createPromise = googleApisDrive.drive("v3").files.create({
+			auth: this.client,
+			fields: "id, name",
+			media: {
+				body: destination,
+				mimeType: options.driveOptions?.mimeType ?? "application/json"
+			},
+			requestBody: {
+				name: fileName,
+				parents: [parentFolderId]
+			}
+		});
+		const [, createResponse] = await Promise.all([writeJsonStream({
+			source,
+			destination,
+			rowCallback: options.rowCallback,
+			sortCallback: options.sortCallback
+		}), createPromise]);
+		return createResponse.data;
+	};
+	writeJsonGzipStream = async (parentFolderId, fileName, source, options = {}) => {
+		const googleApisDrive = await import("@googleapis/drive");
+		const destination = new PassThrough();
+		const createPromise = googleApisDrive.drive("v3").files.create({
+			auth: this.client,
+			fields: "id, name",
+			media: {
+				body: destination,
+				mimeType: options.driveOptions?.mimeType ?? "application/gzip"
+			},
+			requestBody: {
+				name: fileName,
+				parents: [parentFolderId]
+			}
+		});
+		const [, createResponse] = await Promise.all([writeJsonGzipStream({
+			source,
+			destination,
+			rowCallback: options.rowCallback,
+			sortCallback: options.sortCallback
+		}), createPromise]);
+		return createResponse.data;
+	};
+	/**
 	* Read a file's contents from Google Drive.
 	*/
 	readFile = async (fileId, options = {}) => {
@@ -181,6 +236,36 @@ var GoogleCloudDrive = class extends BaseNamespace {
 			...options
 		}, { responseType: "text" })).data };
 	};
+	/**
+	* Return a file's contents as a stream, without buffering it into memory.
+	*/
+	readStream = async (fileId, options = {}) => {
+		return (await (await import("@googleapis/drive")).drive("v3").files.get({
+			auth: this.client,
+			fileId,
+			alt: "media",
+			...options
+		}, { responseType: "stream" })).data;
+	};
+	/**
+	* Read a JSON array file back, parsing it incrementally without buffering the full document.
+	*/
+	readJsonStream = async (fileId, options = {}) => {
+		const source = await this.readStream(fileId, options.driveOptions);
+		return readJsonStream({
+			source,
+			rowCallback: options.rowCallback,
+			sortCallback: options.sortCallback
+		});
+	};
+	readJsonGzipStream = async (fileId, options = {}) => {
+		const source = await this.readStream(fileId, options.driveOptions);
+		return readJsonGzipStream({
+			source,
+			rowCallback: options.rowCallback,
+			sortCallback: options.sortCallback
+		});
+	};
 	readJson = async (fileId) => {
 		const apiResponse = await this.readFile(fileId);
 		apiResponse.body = JSON.parse(apiResponse.body) || null;

package/dist/stream/server/index.cjs

@@ -1,3 +1,14 @@
 Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
 const require_stream_server_sort = require("./sort.cjs");
+const require_stream_server_json_logic = require("./json/logic.cjs");
+const require_stream_server_json_read = require("./json/read.cjs");
+const require_stream_server_json_write = require("./json/write.cjs");
+exports.JsonNewLineStringifier = require_stream_server_json_write.JsonNewLineStringifier;
+exports.jsonNewLineStringifier = require_stream_server_json_write.jsonNewLineStringifier;
+exports.readJsonGzipStream = require_stream_server_json_read.readJsonGzipStream;
+exports.readJsonStream = require_stream_server_json_read.readJsonStream;
+exports.rowCallbackTransform = require_stream_server_json_logic.rowCallbackTransform;
 exports.sort = require_stream_server_sort.sort;
+exports.toRowReadable = require_stream_server_json_logic.toRowReadable;
+exports.writeJsonGzipStream = require_stream_server_json_write.writeJsonGzipStream;
+exports.writeJsonStream = require_stream_server_json_write.writeJsonStream;

package/dist/stream/server/index.d.cts

@@ -1,2 +1,5 @@
 import { sort } from "./sort.cjs";
-export { sort };
+import { ReadJsonStreamProps, RowCallback, RowSource, SortCallback, WriteJsonStreamProps, rowCallbackTransform, toRowReadable } from "./json/logic.cjs";
+import { readJsonGzipStream, readJsonStream } from "./json/read.cjs";
+import { JsonNewLineStringifier, jsonNewLineStringifier, writeJsonGzipStream, writeJsonStream } from "./json/write.cjs";
+export { JsonNewLineStringifier, ReadJsonStreamProps, RowCallback, RowSource, SortCallback, WriteJsonStreamProps, jsonNewLineStringifier, readJsonGzipStream, readJsonStream, rowCallbackTransform, sort, toRowReadable, writeJsonGzipStream, writeJsonStream };

package/dist/stream/server/index.d.mts

@@ -1,2 +1,5 @@
 import { sort } from "./sort.mjs";
-export { sort };
+import { ReadJsonStreamProps, RowCallback, RowSource, SortCallback, WriteJsonStreamProps, rowCallbackTransform, toRowReadable } from "./json/logic.mjs";
+import { readJsonGzipStream, readJsonStream } from "./json/read.mjs";
+import { JsonNewLineStringifier, jsonNewLineStringifier, writeJsonGzipStream, writeJsonStream } from "./json/write.mjs";
+export { JsonNewLineStringifier, ReadJsonStreamProps, RowCallback, RowSource, SortCallback, WriteJsonStreamProps, jsonNewLineStringifier, readJsonGzipStream, readJsonStream, rowCallbackTransform, sort, toRowReadable, writeJsonGzipStream, writeJsonStream };

package/dist/stream/server/index.mjs

@@ -1,2 +1,5 @@
 import { sort } from "./sort.mjs";
-export { sort };
+import { rowCallbackTransform, toRowReadable } from "./json/logic.mjs";
+import { readJsonGzipStream, readJsonStream } from "./json/read.mjs";
+import { JsonNewLineStringifier, jsonNewLineStringifier, writeJsonGzipStream, writeJsonStream } from "./json/write.mjs";
+export { JsonNewLineStringifier, jsonNewLineStringifier, readJsonGzipStream, readJsonStream, rowCallbackTransform, sort, toRowReadable, writeJsonGzipStream, writeJsonStream };

package/dist/stream/server/json/logic.cjs

@@ -0,0 +1,18 @@
+Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
+let stream = require("stream");
+//#region src/stream/server/json/logic.ts
+const toRowReadable = (source) => {
+	return source instanceof stream.Readable ? source : stream.Readable.from(source, { objectMode: true });
+};
+/**
+* Applies the optional per-row callback, dropping rows for which it resolves null/undefined.
+*/
+const rowCallbackTransform = (rowCallback) => new stream.Transform({
+	objectMode: true,
+	transform(row, _encoding, callback) {
+		Promise.resolve(rowCallback(row)).then((result) => callback(null, null === result || void 0 === result ? void 0 : result)).catch((error) => callback(error));
+	}
+});
+//#endregion
+exports.rowCallbackTransform = rowCallbackTransform;
+exports.toRowReadable = toRowReadable;

package/dist/stream/server/json/logic.d.cts

@@ -0,0 +1,27 @@
+import { Readable, Transform, Writable } from "stream";
+//#region src/stream/server/json/logic.d.ts
+/** Per-row hook run during write or read; returning null/undefined drops the row. */
+export type RowCallback<T = any> = (row: T) => T | null | undefined | Promise<T | null | undefined>;
+/** Comparator for the optional sort stage, same shape as `Array.prototype.sort`'s compare function. */
+export type SortCallback<T = any> = (a: T, b: T) => number;
+/** Anything a row can be streamed from: a Readable, an async iterable, or a plain array. */
+export type RowSource<T = any> = Readable | AsyncIterable<T> | T[];
+/** Input for writeJsonStream/writeJsonGzipStream: where rows come from and where the JSON goes. */
+export type WriteJsonStreamProps<T = any> = {
+  source: RowSource<T>;
+  destination: Writable;
+  rowCallback?: RowCallback<T>;
+  sortCallback?: SortCallback<T>;
+};
+/** Input for readJsonStream/readJsonGzipStream: the raw JSON byte source to parse. */
+export type ReadJsonStreamProps<T = any> = {
+  source: Readable;
+  rowCallback?: RowCallback<T>;
+  sortCallback?: SortCallback<T>;
+};
+export declare const toRowReadable: <T>(source: RowSource<T>) => Readable;
+/**
+ * Applies the optional per-row callback, dropping rows for which it resolves null/undefined.
+ */
+export declare const rowCallbackTransform: <T>(rowCallback: RowCallback<T>) => Transform;
+//#endregion

package/dist/stream/server/json/logic.d.mts

@@ -0,0 +1,27 @@
+import { Readable, Transform, Writable } from "stream";
+//#region src/stream/server/json/logic.d.ts
+/** Per-row hook run during write or read; returning null/undefined drops the row. */
+export type RowCallback<T = any> = (row: T) => T | null | undefined | Promise<T | null | undefined>;
+/** Comparator for the optional sort stage, same shape as `Array.prototype.sort`'s compare function. */
+export type SortCallback<T = any> = (a: T, b: T) => number;
+/** Anything a row can be streamed from: a Readable, an async iterable, or a plain array. */
+export type RowSource<T = any> = Readable | AsyncIterable<T> | T[];
+/** Input for writeJsonStream/writeJsonGzipStream: where rows come from and where the JSON goes. */
+export type WriteJsonStreamProps<T = any> = {
+  source: RowSource<T>;
+  destination: Writable;
+  rowCallback?: RowCallback<T>;
+  sortCallback?: SortCallback<T>;
+};
+/** Input for readJsonStream/readJsonGzipStream: the raw JSON byte source to parse. */
+export type ReadJsonStreamProps<T = any> = {
+  source: Readable;
+  rowCallback?: RowCallback<T>;
+  sortCallback?: SortCallback<T>;
+};
+export declare const toRowReadable: <T>(source: RowSource<T>) => Readable;
+/**
+ * Applies the optional per-row callback, dropping rows for which it resolves null/undefined.
+ */
+export declare const rowCallbackTransform: <T>(rowCallback: RowCallback<T>) => Transform;
+//#endregion