@8ms/helpers 3.2.7 → 3.2.8

package/dist/file/_shared/fileMimeType.mjs

@@ -9,9 +9,11 @@ const fileMimeType = {
 	[fileType.JPEG]: "image/jpeg",
 	[fileType.JPG]: "image/jpeg",
 	[fileType.JSON]: "application/json",
+	[fileType.JSON_NEWLINE]: "application/x-ndjson",
 	[fileType.MKV]: "video/x-matroska",
 	[fileType.MOV]: "video/quicktime",
 	[fileType.MP4]: "video/mp4",
+	[fileType.PARQUET]: "application/vnd.apache.parquet",
 	[fileType.PDF]: "application/pdf",
 	[fileType.PNG]: "image/png",
 	[fileType.PPT]: "application/vnd.ms-powerpoint",

package/dist/file/_shared/fileType.d.cts

@@ -2,6 +2,8 @@
 export declare const fileType: {
   readonly ZIP: "zip";
   readonly JSON: "json";
+  readonly JSON_NEWLINE: "jsonl";
+  readonly PARQUET: "parquet";
   readonly XML: "xml";
   readonly DOC: "doc";
   readonly DOCX: "docx";

package/dist/file/_shared/fileType.d.mts

@@ -2,6 +2,8 @@
 export declare const fileType: {
   readonly ZIP: "zip";
   readonly JSON: "json";
+  readonly JSON_NEWLINE: "jsonl";
+  readonly PARQUET: "parquet";
   readonly XML: "xml";
   readonly DOC: "doc";
   readonly DOCX: "docx";

package/dist/googleCloud/bigquery/_shared/index.d.cts

@@ -17,5 +17,5 @@ import { GoogleCloudBQRow, googleCloudBQRow } from "./lookup/googleCloud.cjs";
 import { LanguageBQRow, languageBQRow } from "./lookup/language.cjs";
 import { PartitionType, partitionType } from "./partitionType.cjs";
 import { BQField, zodToBQSchema } from "./zodToBQDefinition.cjs";
-import { TableDefinition } from "./types.cjs";
-export { BQBool, BQBytes, BQDate, BQDateTime, BQDatetime, BQField, BQFieldMode, BQFieldModeSchema, BQFieldType, BQFieldTypeSchema, BQFloat, BQInt64, BQNumeric, BQString, BQTime, BQTimestamp, CountryBQRow, GoogleAdsAccountBQRow, GoogleCloudBQRow, LanguageBQRow, PartitionType, TableDefinition, countryBQRow, decodeBQDate, decodeBQDateTime, encodeBQDate, encodeBQDateTime, googleAdsAccountBQRow, googleCloudBQRow, languageBQRow, partitionType, zBQBytes, zBQInt, zBQInt64, zBQNumeric, zBQTime, zLuxonDate, zLuxonDateTime, zodToBQSchema };
+import { BigqueryLoadFormat, Dataset, LoadData, LoadStreamTableProps, LoadTableProps, MergeTableProps, QueryInput, Table, TableDefinition } from "./types.cjs";
+export { BQBool, BQBytes, BQDate, BQDateTime, BQDatetime, BQField, BQFieldMode, BQFieldModeSchema, BQFieldType, BQFieldTypeSchema, BQFloat, BQInt64, BQNumeric, BQString, BQTime, BQTimestamp, BigqueryLoadFormat, CountryBQRow, Dataset, GoogleAdsAccountBQRow, GoogleCloudBQRow, LanguageBQRow, LoadData, LoadStreamTableProps, LoadTableProps, MergeTableProps, PartitionType, QueryInput, Table, TableDefinition, countryBQRow, decodeBQDate, decodeBQDateTime, encodeBQDate, encodeBQDateTime, googleAdsAccountBQRow, googleCloudBQRow, languageBQRow, partitionType, zBQBytes, zBQInt, zBQInt64, zBQNumeric, zBQTime, zLuxonDate, zLuxonDateTime, zodToBQSchema };

package/dist/googleCloud/bigquery/_shared/index.d.mts

@@ -17,5 +17,5 @@ import { GoogleCloudBQRow, googleCloudBQRow } from "./lookup/googleCloud.mjs";
 import { LanguageBQRow, languageBQRow } from "./lookup/language.mjs";
 import { PartitionType, partitionType } from "./partitionType.mjs";
 import { BQField, zodToBQSchema } from "./zodToBQDefinition.mjs";
-import { TableDefinition } from "./types.mjs";
-export { BQBool, BQBytes, BQDate, BQDateTime, BQDatetime, BQField, BQFieldMode, BQFieldModeSchema, BQFieldType, BQFieldTypeSchema, BQFloat, BQInt64, BQNumeric, BQString, BQTime, BQTimestamp, CountryBQRow, GoogleAdsAccountBQRow, GoogleCloudBQRow, LanguageBQRow, PartitionType, TableDefinition, countryBQRow, decodeBQDate, decodeBQDateTime, encodeBQDate, encodeBQDateTime, googleAdsAccountBQRow, googleCloudBQRow, languageBQRow, partitionType, zBQBytes, zBQInt, zBQInt64, zBQNumeric, zBQTime, zLuxonDate, zLuxonDateTime, zodToBQSchema };
+import { BigqueryLoadFormat, Dataset, LoadData, LoadStreamTableProps, LoadTableProps, MergeTableProps, QueryInput, Table, TableDefinition } from "./types.mjs";
+export { BQBool, BQBytes, BQDate, BQDateTime, BQDatetime, BQField, BQFieldMode, BQFieldModeSchema, BQFieldType, BQFieldTypeSchema, BQFloat, BQInt64, BQNumeric, BQString, BQTime, BQTimestamp, BigqueryLoadFormat, CountryBQRow, Dataset, GoogleAdsAccountBQRow, GoogleCloudBQRow, LanguageBQRow, LoadData, LoadStreamTableProps, LoadTableProps, MergeTableProps, PartitionType, QueryInput, Table, TableDefinition, countryBQRow, decodeBQDate, decodeBQDateTime, encodeBQDate, encodeBQDateTime, googleAdsAccountBQRow, googleCloudBQRow, languageBQRow, partitionType, zBQBytes, zBQInt, zBQInt64, zBQNumeric, zBQTime, zLuxonDate, zLuxonDateTime, zodToBQSchema };

package/dist/googleCloud/bigquery/_shared/types.d.cts

@@ -1,6 +1,12 @@
+import { RowSource } from "../../../stream/server/json/logic.cjs";
+import { fileType } from "../../../file/_shared/fileType.cjs";
 import { PartitionType } from "./partitionType.cjs";
 import { BQField } from "./zodToBQDefinition.cjs";
+import { GoogleCloudStorage } from "../../storage/server/GoogleCloudStorage.cjs";
+import { QueryOptions } from "@google-cloud/bigquery";
 //#region src/googleCloud/bigquery/_shared/types.d.ts
+/** The only load formats this library can actually serialise/pass through for loadTable/loadStreamTable. */
+export type BigqueryLoadFormat = typeof fileType.CSV | typeof fileType.JSON_NEWLINE | typeof fileType.PARQUET;
 export type TableDefinition = {
   projectId?: string;
   datasetId: string;
@@ -17,4 +23,52 @@ export type TableDefinition = {
     };
   };
 };
+export type QueryInput = string | {
+  query: string;
+  location?: string;
+  params: QueryOptions;
+};
+export type Dataset = {
+  id: string;
+  location: string;
+  projectId: string;
+};
+export type Table = {
+  id: string;
+};
+export type LoadData<TData> = {
+  googleCloudStorage: GoogleCloudStorage;
+  file: {
+    bucket: string;
+    key: string;
+    data: TData;
+    sourceFormat?: BigqueryLoadFormat;
+    region?: string;
+  };
+  table: {
+    datasetId: string;
+    tableId?: string;
+    region?: string;
+    definition: TableDefinition;
+    options?: TableDefinition["options"];
+  };
+};
+export type LoadTableProps = LoadData<any[]>;
+export type LoadStreamTableProps = LoadData<RowSource<any>>;
+export type MergeTableProps = {
+  googleCloudStorage: GoogleCloudStorage;
+  file: {
+    bucket: string;
+    key: string;
+    data: any[];
+  };
+  table: {
+    datasetId: string;
+    tableId?: string;
+    region?: string;
+    definition: TableDefinition;
+    mergeKey: string;
+    updateColumns: string[];
+  };
+};
 //#endregion

package/dist/googleCloud/bigquery/_shared/types.d.mts

@@ -1,6 +1,12 @@
+import { RowSource } from "../../../stream/server/json/logic.mjs";
+import { fileType } from "../../../file/_shared/fileType.mjs";
 import { PartitionType } from "./partitionType.mjs";
 import { BQField } from "./zodToBQDefinition.mjs";
+import { GoogleCloudStorage } from "../../storage/server/GoogleCloudStorage.mjs";
+import { QueryOptions } from "@google-cloud/bigquery";
 //#region src/googleCloud/bigquery/_shared/types.d.ts
+/** The only load formats this library can actually serialise/pass through for loadTable/loadStreamTable. */
+export type BigqueryLoadFormat = typeof fileType.CSV | typeof fileType.JSON_NEWLINE | typeof fileType.PARQUET;
 export type TableDefinition = {
   projectId?: string;
   datasetId: string;
@@ -17,4 +23,52 @@ export type TableDefinition = {
     };
   };
 };
+export type QueryInput = string | {
+  query: string;
+  location?: string;
+  params: QueryOptions;
+};
+export type Dataset = {
+  id: string;
+  location: string;
+  projectId: string;
+};
+export type Table = {
+  id: string;
+};
+export type LoadData<TData> = {
+  googleCloudStorage: GoogleCloudStorage;
+  file: {
+    bucket: string;
+    key: string;
+    data: TData;
+    sourceFormat?: BigqueryLoadFormat;
+    region?: string;
+  };
+  table: {
+    datasetId: string;
+    tableId?: string;
+    region?: string;
+    definition: TableDefinition;
+    options?: TableDefinition["options"];
+  };
+};
+export type LoadTableProps = LoadData<any[]>;
+export type LoadStreamTableProps = LoadData<RowSource<any>>;
+export type MergeTableProps = {
+  googleCloudStorage: GoogleCloudStorage;
+  file: {
+    bucket: string;
+    key: string;
+    data: any[];
+  };
+  table: {
+    datasetId: string;
+    tableId?: string;
+    region?: string;
+    definition: TableDefinition;
+    mergeKey: string;
+    updateColumns: string[];
+  };
+};
 //#endregion

package/dist/googleCloud/bigquery/client/index.d.cts

@@ -17,6 +17,6 @@ import { GoogleCloudBQRow, googleCloudBQRow } from "../_shared/lookup/googleClou
 import { LanguageBQRow, languageBQRow } from "../_shared/lookup/language.cjs";
 import { PartitionType, partitionType } from "../_shared/partitionType.cjs";
 import { BQField, zodToBQSchema } from "../_shared/zodToBQDefinition.cjs";
-import { TableDefinition } from "../_shared/types.cjs";
+import { BigqueryLoadFormat, Dataset, LoadData, LoadStreamTableProps, LoadTableProps, MergeTableProps, QueryInput, Table, TableDefinition } from "../_shared/types.cjs";
 import "../_shared/index.cjs";
-export { BQBool, BQBytes, BQDate, BQDateTime, BQDatetime, BQField, BQFieldMode, BQFieldModeSchema, BQFieldType, BQFieldTypeSchema, BQFloat, BQInt64, BQNumeric, BQString, BQTime, BQTimestamp, CountryBQRow, GoogleAdsAccountBQRow, GoogleCloudBQRow, LanguageBQRow, PartitionType, TableDefinition, countryBQRow, decodeBQDate, decodeBQDateTime, encodeBQDate, encodeBQDateTime, googleAdsAccountBQRow, googleCloudBQRow, languageBQRow, partitionType, zBQBytes, zBQInt, zBQInt64, zBQNumeric, zBQTime, zLuxonDate, zLuxonDateTime, zodToBQSchema };
+export { BQBool, BQBytes, BQDate, BQDateTime, BQDatetime, BQField, BQFieldMode, BQFieldModeSchema, BQFieldType, BQFieldTypeSchema, BQFloat, BQInt64, BQNumeric, BQString, BQTime, BQTimestamp, BigqueryLoadFormat, CountryBQRow, Dataset, GoogleAdsAccountBQRow, GoogleCloudBQRow, LanguageBQRow, LoadData, LoadStreamTableProps, LoadTableProps, MergeTableProps, PartitionType, QueryInput, Table, TableDefinition, countryBQRow, decodeBQDate, decodeBQDateTime, encodeBQDate, encodeBQDateTime, googleAdsAccountBQRow, googleCloudBQRow, languageBQRow, partitionType, zBQBytes, zBQInt, zBQInt64, zBQNumeric, zBQTime, zLuxonDate, zLuxonDateTime, zodToBQSchema };

package/dist/googleCloud/bigquery/client/index.d.mts

@@ -17,6 +17,6 @@ import { GoogleCloudBQRow, googleCloudBQRow } from "../_shared/lookup/googleClou
 import { LanguageBQRow, languageBQRow } from "../_shared/lookup/language.mjs";
 import { PartitionType, partitionType } from "../_shared/partitionType.mjs";
 import { BQField, zodToBQSchema } from "../_shared/zodToBQDefinition.mjs";
-import { TableDefinition } from "../_shared/types.mjs";
+import { BigqueryLoadFormat, Dataset, LoadData, LoadStreamTableProps, LoadTableProps, MergeTableProps, QueryInput, Table, TableDefinition } from "../_shared/types.mjs";
 import "../_shared/index.mjs";
-export { BQBool, BQBytes, BQDate, BQDateTime, BQDatetime, BQField, BQFieldMode, BQFieldModeSchema, BQFieldType, BQFieldTypeSchema, BQFloat, BQInt64, BQNumeric, BQString, BQTime, BQTimestamp, CountryBQRow, GoogleAdsAccountBQRow, GoogleCloudBQRow, LanguageBQRow, PartitionType, TableDefinition, countryBQRow, decodeBQDate, decodeBQDateTime, encodeBQDate, encodeBQDateTime, googleAdsAccountBQRow, googleCloudBQRow, languageBQRow, partitionType, zBQBytes, zBQInt, zBQInt64, zBQNumeric, zBQTime, zLuxonDate, zLuxonDateTime, zodToBQSchema };
+export { BQBool, BQBytes, BQDate, BQDateTime, BQDatetime, BQField, BQFieldMode, BQFieldModeSchema, BQFieldType, BQFieldTypeSchema, BQFloat, BQInt64, BQNumeric, BQString, BQTime, BQTimestamp, BigqueryLoadFormat, CountryBQRow, Dataset, GoogleAdsAccountBQRow, GoogleCloudBQRow, LanguageBQRow, LoadData, LoadStreamTableProps, LoadTableProps, MergeTableProps, PartitionType, QueryInput, Table, TableDefinition, countryBQRow, decodeBQDate, decodeBQDateTime, encodeBQDate, encodeBQDateTime, googleAdsAccountBQRow, googleCloudBQRow, languageBQRow, partitionType, zBQBytes, zBQInt, zBQInt64, zBQNumeric, zBQTime, zLuxonDate, zLuxonDateTime, zodToBQSchema };

package/dist/googleCloud/bigquery/server/GoogleCloudBigquery.cjs

@@ -1,7 +1,23 @@
 Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
 const require_lodash__shared_isObject = require("../../../lodash/_shared/isObject.cjs");
 const require__class__shared_BaseNamespace = require("../../../_class/_shared/BaseNamespace.cjs");
+const require_date__shared_format = require("../../../date/_shared/format.cjs");
+require("../../../date/_shared/index.cjs");
+const require_file__shared_fileType = require("../../../file/_shared/fileType.cjs");
+const require_file__shared_fileMimeType = require("../../../file/_shared/fileMimeType.cjs");
+require("../../../file/_shared/index.cjs");
+const require_json__shared_getJsonNewline = require("../../../json/_shared/getJsonNewline.cjs");
+const require_stream_server_json_logic = require("../../../stream/server/json/logic.cjs");
+const require_stream_server_json_write = require("../../../stream/server/json/write.cjs");
+require("../../../stream/server/index.cjs");
+let node_stream_promises = require("node:stream/promises");
 //#region src/googleCloud/bigquery/server/GoogleCloudBigquery.ts
+/** Maps this library's generic file-type catalog onto BigQuery's own load-job sourceFormat strings. */
+const bigQuerySourceFormat = {
+	[require_file__shared_fileType.fileType.CSV]: "CSV",
+	[require_file__shared_fileType.fileType.JSON_NEWLINE]: "NEWLINE_DELIMITED_JSON",
+	[require_file__shared_fileType.fileType.PARQUET]: "PARQUET"
+};
 var GoogleCloudBigquery = class extends require__class__shared_BaseNamespace.BaseNamespace {
 	/**
 	* Either a Google Cloud JSON Config or V10 Auth helper.
@@ -65,7 +81,7 @@ var GoogleCloudBigquery = class extends require__class__shared_BaseNamespace.Bas
 		});
 		return response;
 	};
-	query = async (props) => {
+	buildQueryOptions = (props) => {
 		let options = { query: "" };
 		if ("string" === typeof props) {
 			options.query = props;
@@ -76,7 +92,19 @@ var GoogleCloudBigquery = class extends require__class__shared_BaseNamespace.Bas
 			if (resolvedLocation) options.location = resolvedLocation;
 			if (props?.params) options["params"] = props.params;
 		}
-		return await this.client.query(options);
+		return options;
+	};
+	/**
+	* Make a request to the database and return the object per row.
+	*/
+	query = async (props) => {
+		return await this.client.query(this.buildQueryOptions(props));
+	};
+	/**
+	* Return the raw row stream for a query, without buffering the full result set in memory.
+	*/
+	queryStream = (props) => {
+		return this.client.createQueryStream(this.buildQueryOptions(props));
 	};
 	/**
 	* Create a Table if it doesn't already exist.
@@ -113,14 +141,137 @@ var GoogleCloudBigquery = class extends require__class__shared_BaseNamespace.Bas
 		return true;
 	};
 	/**
-	* Load data into this table using Google Cloud Storage.
+	* Load a Google Cloud Storage file into a table. BigQuery load jobs default to
+	* createDisposition "CREATE_IF_NEEDED"; paired with autodetect, this creates the table itself
+	* from the source file's inferred schema if it doesn't already exist, rather than this library
+	* pre-checking existence and silently skipping the load if it isn't there.
 	*/
-	loadData = async ({ file, googleCloudStorageClient, table }) => {
-		if (await this.tableExists(table.datasetId, table.tableId)) await this.client.dataset(table.datasetId, { location: table?.region }).table(table.tableId).load(googleCloudStorageClient.client.bucket(file.bucket).file(file.key), {
-			...file?.metadata ?? {},
-			location: file?.region
+	loadData = async (props) => {
+		const finalTableId = props.table.tableId;
+		await this.createTable(props.table.datasetId, {}, finalTableId, props.table.options);
+		await this.client.dataset(props.table.datasetId, { location: props.table.region }).table(finalTableId).load(props.googleCloudStorage.client.bucket(props.file.bucket).file(props.file.key), {
+			autodetect: true,
+			sourceFormat: bigQuerySourceFormat[props.file.sourceFormat ?? require_file__shared_fileType.fileType.JSON_NEWLINE],
+			schemaUpdateOptions: ["ALLOW_FIELD_ADDITION"],
+			location: props.table.region
 		});
 	};
+	/**
+	* Write rows to Google Cloud Storage, then load them into the table - creating the table first
+	* if it doesn't already exist. Only JSON_NEWLINE (the default) is actually serialised by this
+	* library (getJsonNewline); for PARQUET, `data` is written as-is and must already be encoded -
+	* this library has no Parquet encoder.
+	*/
+	loadTable = async (props) => {
+		const finalTableId = props.table.tableId ?? props.table.definition.tableId;
+		const fileSourceFormat = props.file.sourceFormat ?? require_file__shared_fileType.fileType.JSON_NEWLINE;
+		const finalData = require_file__shared_fileType.fileType.JSON_NEWLINE === fileSourceFormat ? require_json__shared_getJsonNewline.getJsonNewline(props.file.data) : props.file.data;
+		await props.googleCloudStorage.writeFile(props.file.bucket, props.file.key, finalData);
+		await this.loadData({
+			...props,
+			table: {
+				...props.table,
+				tableId: finalTableId
+			}
+		});
+	};
+	/**
+	* Same as loadTable, but streams a row source into Google Cloud Storage instead of buffering the
+	* full collection in memory first. Writes directly against the Storage client's own write stream
+	* (GoogleCloudStorage.writeFile can't stream a Readable source) - creates the table first if it
+	* doesn't already exist. For JSON_NEWLINE (the default) `source` is a stream of row objects,
+	* stringified one at a time; for PARQUET there is no stringify stage - `source` must already be a
+	* byte-level Readable of encoded Parquet content, piped straight through.
+	*/
+	loadStreamTable = async (props) => {
+		const finalTableId = props.table.tableId ?? props.table.definition.tableId;
+		const fileSourceFormat = props.file.sourceFormat ?? require_file__shared_fileType.fileType.JSON_NEWLINE;
+		const destination = props.googleCloudStorage.client.bucket(props.file.bucket).file(props.file.key).createWriteStream({
+			contentType: require_file__shared_fileMimeType.fileMimeType[fileSourceFormat],
+			resumable: false
+		});
+		const source = require_stream_server_json_logic.toRowReadable(props.file.data);
+		if (require_file__shared_fileType.fileType.JSON_NEWLINE === fileSourceFormat) await (0, node_stream_promises.pipeline)([
+			source,
+			require_stream_server_json_write.jsonNewLineStringifier(),
+			destination
+		]);
+		else await (0, node_stream_promises.pipeline)([source, destination]);
+		await this.loadData({
+			...props,
+			table: {
+				...props.table,
+				tableId: finalTableId
+			}
+		});
+	};
+	/**
+	* Brings an existing table's clustering in line with `definition.options.clustering`, if it differs.
+	* Partitioning is immutable on an existing BigQuery table - changing it requires recreating the
+	* table and backfilling data, which this method deliberately does not attempt.
+	*/
+	syncTableClustering = async (datasetId, tableId, clustering) => {
+		const [metadata] = await this.client.dataset(datasetId).table(tableId).getMetadata();
+		const currentFields = metadata?.clustering?.fields ?? [];
+		const desiredFields = clustering?.fields ?? [];
+		if (currentFields.length === desiredFields.length && currentFields.every((field, index) => field === desiredFields[index])) return;
+		await this.client.dataset(datasetId).table(tableId).setMetadata(desiredFields.length > 0 ? { clustering: { fields: desiredFields } } : { clustering: void 0 });
+	};
+	/**
+	* Upsert rows into a table via a staging table + a single MERGE statement.
+	*
+	* BigQuery has no efficient row-level UPDATE/DELETE - both rewrite the affected partitions and
+	* are subject to daily DML mutation quotas. Loading new rows into a disposable staging table and
+	* running one MERGE is the standard, quota-friendly way to upsert in BigQuery.
+	*/
+	mergeTable = async (props) => {
+		if (0 === props.file.data.length) return;
+		const finalTableId = props.table.tableId ?? props.table.definition.tableId;
+		if (await this.tableExists(props.table.datasetId, finalTableId)) await this.syncTableClustering(props.table.datasetId, finalTableId, props.table.definition.options?.clustering);
+		const stagingTableId = `${finalTableId}_staging_${require_date__shared_format.getYmdHisNumber()}`;
+		const stagingTableRef = `\`${props.table.datasetId}.${stagingTableId}\``;
+		try {
+			const stagingDefinition = {
+				...props.table.definition,
+				tableId: stagingTableId,
+				options: props.table.definition.options ? {
+					...props.table.definition.options,
+					timePartitioning: void 0,
+					clustering: void 0
+				} : void 0
+			};
+			await this.loadTable({
+				googleCloudStorage: props.googleCloudStorage,
+				file: props.file,
+				table: {
+					datasetId: props.table.datasetId,
+					definition: stagingDefinition,
+					tableId: stagingTableId,
+					region: props.table.region
+				}
+			});
+			const mainTableRef = `\`${props.table.datasetId}.${finalTableId}\``;
+			const updateSet = props.table.updateColumns.map((column) => `${column} = S.${column}`).join(", ");
+			const allColumns = props.table.definition.options?.schema.map((field) => field.name) ?? [];
+			const mergeQuery = `
+				MERGE ${mainTableRef} AS T
+				USING ${stagingTableRef} AS S
+				ON T.${props.table.mergeKey} = S.${props.table.mergeKey}
+				WHEN MATCHED THEN
+					UPDATE SET ${updateSet}
+				WHEN NOT MATCHED THEN
+					INSERT (${allColumns.join(", ")})
+					VALUES (${allColumns.map((column) => `S.${column}`).join(", ")})
+			`;
+			await this.query(mergeQuery);
+		} finally {
+			try {
+				await this.query(`DROP TABLE IF EXISTS ${stagingTableRef}`);
+			} catch (error) {
+				console.error(`mergeTable: failed to drop staging table ${stagingTableRef} - it must be dropped manually.`, error);
+			}
+		}
+	};
 };
 //#endregion
 exports.GoogleCloudBigquery = GoogleCloudBigquery;

package/dist/googleCloud/bigquery/server/GoogleCloudBigquery.d.cts

@@ -1,34 +1,9 @@
 import { BaseNamespace } from "../../../_class/_shared/BaseNamespace.cjs";
-import { GoogleCloudStorage } from "../../storage/server/GoogleCloudStorage.cjs";
-import { BigQuery, DatasetResource, QueryOptions } from "@google-cloud/bigquery";
+import { Dataset, LoadStreamTableProps, LoadTableProps, MergeTableProps, QueryInput, Table } from "../_shared/types.cjs";
+import "../_shared/index.cjs";
+import { ResourceStream } from "../../../node_modules/@google-cloud/paginator/build/src/resource-stream.cjs";
+import { BigQuery, DatasetResource } from "@google-cloud/bigquery";
 //#region src/googleCloud/bigquery/server/GoogleCloudBigquery.d.ts
-type LoadData = {
-  googleCloudStorageClient: GoogleCloudStorage;
-  file: {
-    bucket: string;
-    key: string;
-    metadata?: object;
-    region?: string;
-  };
-  table: {
-    datasetId: string;
-    region?: string;
-    tableId: string;
-  };
-};
-export type QueryInput = string | {
-  query: string;
-  location?: string;
-  params: QueryOptions;
-};
-type Dataset = {
-  id: string;
-  location: string;
-  projectId: string;
-};
-type Table = {
-  id: string;
-};
 export declare class GoogleCloudBigquery extends BaseNamespace {
   client: BigQuery;
   config: any;
@@ -53,7 +28,15 @@ export declare class GoogleCloudBigquery extends BaseNamespace {
    * Retrieve all the Datasets
    */
   getDatasets: () => Promise<Dataset[]>;
+  private buildQueryOptions;
+  /**
+   * Make a request to the database and return the object per row.
+   */
   query: (props: QueryInput) => Promise<any>;
+  /**
+   * Return the raw row stream for a query, without buffering the full result set in memory.
+   */
+  queryStream: (props: QueryInput) => ResourceStream<any>;
   /**
    * Create a Table if it doesn't already exist.
    */
@@ -69,8 +52,41 @@ export declare class GoogleCloudBigquery extends BaseNamespace {
    */
   tablesExist: (datasetId: string, tableIds: string[]) => Promise<boolean>;
   /**
-   * Load data into this table using Google Cloud Storage.
+   * Load a Google Cloud Storage file into a table. BigQuery load jobs default to
+   * createDisposition "CREATE_IF_NEEDED"; paired with autodetect, this creates the table itself
+   * from the source file's inferred schema if it doesn't already exist, rather than this library
+   * pre-checking existence and silently skipping the load if it isn't there.
+   */
+  private loadData;
+  /**
+   * Write rows to Google Cloud Storage, then load them into the table - creating the table first
+   * if it doesn't already exist. Only JSON_NEWLINE (the default) is actually serialised by this
+   * library (getJsonNewline); for PARQUET, `data` is written as-is and must already be encoded -
+   * this library has no Parquet encoder.
+   */
+  loadTable: (props: LoadTableProps) => Promise<void>;
+  /**
+   * Same as loadTable, but streams a row source into Google Cloud Storage instead of buffering the
+   * full collection in memory first. Writes directly against the Storage client's own write stream
+   * (GoogleCloudStorage.writeFile can't stream a Readable source) - creates the table first if it
+   * doesn't already exist. For JSON_NEWLINE (the default) `source` is a stream of row objects,
+   * stringified one at a time; for PARQUET there is no stringify stage - `source` must already be a
+   * byte-level Readable of encoded Parquet content, piped straight through.
+   */
+  loadStreamTable: (props: LoadStreamTableProps) => Promise<void>;
+  /**
+   * Brings an existing table's clustering in line with `definition.options.clustering`, if it differs.
+   * Partitioning is immutable on an existing BigQuery table - changing it requires recreating the
+   * table and backfilling data, which this method deliberately does not attempt.
+   */
+  private syncTableClustering;
+  /**
+   * Upsert rows into a table via a staging table + a single MERGE statement.
+   *
+   * BigQuery has no efficient row-level UPDATE/DELETE - both rewrite the affected partitions and
+   * are subject to daily DML mutation quotas. Loading new rows into a disposable staging table and
+   * running one MERGE is the standard, quota-friendly way to upsert in BigQuery.
    */
-  loadData: ({ file, googleCloudStorageClient, table }: LoadData) => Promise<void>;
+  mergeTable: (props: MergeTableProps) => Promise<void>;
 }
 //#endregion

package/dist/googleCloud/bigquery/server/GoogleCloudBigquery.d.mts

@@ -1,34 +1,9 @@
 import { BaseNamespace } from "../../../_class/_shared/BaseNamespace.mjs";
-import { GoogleCloudStorage } from "../../storage/server/GoogleCloudStorage.mjs";
-import { BigQuery, DatasetResource, QueryOptions } from "@google-cloud/bigquery";
+import { Dataset, LoadStreamTableProps, LoadTableProps, MergeTableProps, QueryInput, Table } from "../_shared/types.mjs";
+import "../_shared/index.mjs";
+import { ResourceStream } from "../../../node_modules/@google-cloud/paginator/build/src/resource-stream.mjs";
+import { BigQuery, DatasetResource } from "@google-cloud/bigquery";
 //#region src/googleCloud/bigquery/server/GoogleCloudBigquery.d.ts
-type LoadData = {
-  googleCloudStorageClient: GoogleCloudStorage;
-  file: {
-    bucket: string;
-    key: string;
-    metadata?: object;
-    region?: string;
-  };
-  table: {
-    datasetId: string;
-    region?: string;
-    tableId: string;
-  };
-};
-export type QueryInput = string | {
-  query: string;
-  location?: string;
-  params: QueryOptions;
-};
-type Dataset = {
-  id: string;
-  location: string;
-  projectId: string;
-};
-type Table = {
-  id: string;
-};
 export declare class GoogleCloudBigquery extends BaseNamespace {
   client: BigQuery;
   config: any;
@@ -53,7 +28,15 @@ export declare class GoogleCloudBigquery extends BaseNamespace {
    * Retrieve all the Datasets
    */
   getDatasets: () => Promise<Dataset[]>;
+  private buildQueryOptions;
+  /**
+   * Make a request to the database and return the object per row.
+   */
   query: (props: QueryInput) => Promise<any>;
+  /**
+   * Return the raw row stream for a query, without buffering the full result set in memory.
+   */
+  queryStream: (props: QueryInput) => ResourceStream<any>;
   /**
    * Create a Table if it doesn't already exist.
    */
@@ -69,8 +52,41 @@ export declare class GoogleCloudBigquery extends BaseNamespace {
    */
   tablesExist: (datasetId: string, tableIds: string[]) => Promise<boolean>;
   /**
-   * Load data into this table using Google Cloud Storage.
+   * Load a Google Cloud Storage file into a table. BigQuery load jobs default to
+   * createDisposition "CREATE_IF_NEEDED"; paired with autodetect, this creates the table itself
+   * from the source file's inferred schema if it doesn't already exist, rather than this library
+   * pre-checking existence and silently skipping the load if it isn't there.
+   */
+  private loadData;
+  /**
+   * Write rows to Google Cloud Storage, then load them into the table - creating the table first
+   * if it doesn't already exist. Only JSON_NEWLINE (the default) is actually serialised by this
+   * library (getJsonNewline); for PARQUET, `data` is written as-is and must already be encoded -
+   * this library has no Parquet encoder.
+   */
+  loadTable: (props: LoadTableProps) => Promise<void>;
+  /**
+   * Same as loadTable, but streams a row source into Google Cloud Storage instead of buffering the
+   * full collection in memory first. Writes directly against the Storage client's own write stream
+   * (GoogleCloudStorage.writeFile can't stream a Readable source) - creates the table first if it
+   * doesn't already exist. For JSON_NEWLINE (the default) `source` is a stream of row objects,
+   * stringified one at a time; for PARQUET there is no stringify stage - `source` must already be a
+   * byte-level Readable of encoded Parquet content, piped straight through.
+   */
+  loadStreamTable: (props: LoadStreamTableProps) => Promise<void>;
+  /**
+   * Brings an existing table's clustering in line with `definition.options.clustering`, if it differs.
+   * Partitioning is immutable on an existing BigQuery table - changing it requires recreating the
+   * table and backfilling data, which this method deliberately does not attempt.
+   */
+  private syncTableClustering;
+  /**
+   * Upsert rows into a table via a staging table + a single MERGE statement.
+   *
+   * BigQuery has no efficient row-level UPDATE/DELETE - both rewrite the affected partitions and
+   * are subject to daily DML mutation quotas. Loading new rows into a disposable staging table and
+   * running one MERGE is the standard, quota-friendly way to upsert in BigQuery.
    */
-  loadData: ({ file, googleCloudStorageClient, table }: LoadData) => Promise<void>;
+  mergeTable: (props: MergeTableProps) => Promise<void>;
 }
 //#endregion