@flytedesk/app-kit 4.0.0 → 5.0.0

package/dist/bigquery/client.d.ts

@@ -1,4 +1,4 @@
-import { type LoadRowsOptions } from "./load.js";
+import { type LoadRowSource, type LoadRowsOptions, type LoadRowsResult } from "./load.js";
 import type { BigQueryLike, DryRunEstimate, EstimateQueryBytesOptions, ExtractJobHandle, ExtractJobStatus, ExtractTableToGCSInput, JobLabelInput, QueryPage, RunQueryOptions, WaitForExtractJobOptions } from "./types.js";
 export interface CreateBigQueryReadClientOptions {
     /** Merged under any per-call `labels`/`extra` — e.g. `{app: "sms-app"}`. */
@@ -11,9 +11,9 @@ export interface BigQueryReadClient {
     extractTableToGCS(input: ExtractTableToGCSInput): Promise<ExtractJobHandle>;
     getExtractJobStatus(jobId: string): Promise<ExtractJobStatus>;
     waitForExtractJob(jobId: string, options?: WaitForExtractJobOptions): Promise<ExtractJobStatus>;
-    /** Truncate-and-reload (or append) a table from an array of rows — see
-     *  `loadRowsWithRetry`. */
-    loadRows<T extends object>(options: LoadRowsOptions, rows: T[]): Promise<void>;
+    /** Truncate-and-reload (or append) a table from a stream of rows, in memory
+     *  bounded by the chunk size — see `loadRowsWithRetry`. */
+    loadRows<T extends object>(options: LoadRowsOptions, source: LoadRowSource<T>): Promise<LoadRowsResult>;
     /** Builds a `{actor, purpose, ...extra}` label set, pre-merged with this
      *  client's `defaultLabels` — pass the result as `labels` to `query`/
      *  `extractTableToGCS`. */

package/dist/bigquery/client.js

@@ -8,7 +8,7 @@
  */
 import { buildJobLabels } from "./labels.js";
 import { extractTableToGCS, getExtractJobStatus, waitForExtractJob, } from "./extract.js";
-import { loadRowsWithRetry } from "./load.js";
+import { loadRowsWithRetry, } from "./load.js";
 import { estimateQueryBytes, runQuery, runQueryAllPages } from "./query.js";
 export function createBigQueryReadClient(bq, options = {}) {
     const defaultLabels = options.defaultLabels ?? {};
@@ -29,7 +29,7 @@ export function createBigQueryReadClient(bq, options = {}) {
         extractTableToGCS: (input) => extractTableToGCS(bq, { ...input, labels: withDefaults(input.labels) }),
         getExtractJobStatus: (jobId) => getExtractJobStatus(bq, jobId),
         waitForExtractJob: (jobId, waitOptions) => waitForExtractJob(bq, jobId, waitOptions),
-        loadRows: (loadOptions, rows) => loadRowsWithRetry(bq, { ...loadOptions, labels: withDefaults(loadOptions.labels) }, rows),
+        loadRows: (loadOptions, source) => loadRowsWithRetry(bq, { ...loadOptions, labels: withDefaults(loadOptions.labels) }, source),
         buildJobLabels: (input) => ({
             ...defaultLabels,
             ...buildJobLabels(input),

package/dist/bigquery/client.js.map

@@ -1 +1 @@
-{"version":3,"file":"client.js","sourceRoot":"","sources":["../../src/bigquery/client.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EACL,iBAAiB,EACjB,mBAAmB,EACnB,iBAAiB,GAClB,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,iBAAiB,EAAwB,MAAM,WAAW,CAAC;AACpE,OAAO,EAAE,kBAAkB,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AA+C5E,MAAM,UAAU,wBAAwB,CACtC,EAAgB,EAChB,UAA2C,EAAE;IAE7C,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,EAAE,CAAC;IAClD,MAAM,YAAY,GAAG,CAAC,MAA+B,EAAE,EAAE,CAAC,CAAC;QACzD,GAAG,aAAa;QAChB,GAAG,MAAM;KACV,CAAC,CAAC;IAEH,OAAO;QACL,KAAK,EAAE,CAAC,YAAY,EAAE,EAAE,CACtB,QAAQ,CAAC,EAAE,EAAE;YACX,GAAG,YAAY;YACf,MAAM,EAAE,YAAY,CAAC,YAAY,CAAC,MAAM,CAAC;SAC1C,CAAC;QACJ,aAAa,EAAE,CAAC,YAAY,EAAE,EAAE,CAC9B,gBAAgB,CAAC,EAAE,EAAE;YACnB,GAAG,YAAY;YACf,MAAM,EAAE,YAAY,CAAC,YAAY,CAAC,MAAM,CAAC;SAC1C,CAAC;QACJ,kBAAkB,EAAE,CAAC,eAAe,EAAE,EAAE,CACtC,kBAAkB,CAAC,EAAE,EAAE,eAAe,CAAC;QACzC,iBAAiB,EAAE,CAAC,KAAK,EAAE,EAAE,CAC3B,iBAAiB,CAAC,EAAE,EAAE,EAAE,GAAG,KAAK,EAAE,MAAM,EAAE,YAAY,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;QACzE,mBAAmB,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,mBAAmB,CAAC,EAAE,EAAE,KAAK,CAAC;QAC9D,iBAAiB,EAAE,CAAC,KAAK,EAAE,WAAW,EAAE,EAAE,CACxC,iBAAiB,CAAC,EAAE,EAAE,KAAK,EAAE,WAAW,CAAC;QAC3C,QAAQ,EAAE,CAAC,WAAW,EAAE,IAAI,EAAE,EAAE,CAC9B,iBAAiB,CAAC,EAAE,EAAE,EAAE,GAAG,WAAW,EAAE,MAAM,EAAE,YAAY,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE,EAAE,IAAI,CAAC;QAC3F,cAAc,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YAC1B,GAAG,aAAa;YAChB,GAAG,cAAc,CAAC,KAAK,CAAC;SACzB,CAAC;KACH,CAAC;AACJ,CAAC"}
+{"version":3,"file":"client.js","sourceRoot":"","sources":["../../src/bigquery/client.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EACL,iBAAiB,EACjB,mBAAmB,EACnB,iBAAiB,GAClB,MAAM,cAAc,CAAC;AACtB,OAAO,EACL,iBAAiB,GAIlB,MAAM,WAAW,CAAC;AACnB,OAAO,EAAE,kBAAkB,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AA+C5E,MAAM,UAAU,wBAAwB,CACtC,EAAgB,EAChB,UAA2C,EAAE;IAE7C,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,EAAE,CAAC;IAClD,MAAM,YAAY,GAAG,CAAC,MAA+B,EAAE,EAAE,CAAC,CAAC;QACzD,GAAG,aAAa;QAChB,GAAG,MAAM;KACV,CAAC,CAAC;IAEH,OAAO;QACL,KAAK,EAAE,CAAC,YAAY,EAAE,EAAE,CACtB,QAAQ,CAAC,EAAE,EAAE;YACX,GAAG,YAAY;YACf,MAAM,EAAE,YAAY,CAAC,YAAY,CAAC,MAAM,CAAC;SAC1C,CAAC;QACJ,aAAa,EAAE,CAAC,YAAY,EAAE,EAAE,CAC9B,gBAAgB,CAAC,EAAE,EAAE;YACnB,GAAG,YAAY;YACf,MAAM,EAAE,YAAY,CAAC,YAAY,CAAC,MAAM,CAAC;SAC1C,CAAC;QACJ,kBAAkB,EAAE,CAAC,eAAe,EAAE,EAAE,CACtC,kBAAkB,CAAC,EAAE,EAAE,eAAe,CAAC;QACzC,iBAAiB,EAAE,CAAC,KAAK,EAAE,EAAE,CAC3B,iBAAiB,CAAC,EAAE,EAAE,EAAE,GAAG,KAAK,EAAE,MAAM,EAAE,YAAY,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;QACzE,mBAAmB,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,mBAAmB,CAAC,EAAE,EAAE,KAAK,CAAC;QAC9D,iBAAiB,EAAE,CAAC,KAAK,EAAE,WAAW,EAAE,EAAE,CACxC,iBAAiB,CAAC,EAAE,EAAE,KAAK,EAAE,WAAW,CAAC;QAC3C,QAAQ,EAAE,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE,CAChC,iBAAiB,CACf,EAAE,EACF,EAAE,GAAG,WAAW,EAAE,MAAM,EAAE,YAAY,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE,EAC5D,MAAM,CACP;QACH,cAAc,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YAC1B,GAAG,aAAa;YAChB,GAAG,cAAc,CAAC,KAAK,CAAC;SACzB,CAAC;KACH,CAAC;AACJ,CAAC"}

package/dist/bigquery/index.d.ts

@@ -45,8 +45,8 @@ export type { BigQueryReadClient, CreateBigQueryReadClientOptions, } from "./cli
 export { runQuery, runQueryAllPages, estimateQueryBytes } from "./query.js";
 export { extractTableToGCS, getExtractJobStatus, waitForExtractJob, } from "./extract.js";
 export { loadRowsWithRetry } from "./load.js";
-export type { LoadRowsOptions } from "./load.js";
+export type { LoadRowSource, LoadRowsOptions, LoadRowsResult } from "./load.js";
 export { buildJobLabels, sanitizeLabelValue } from "./labels.js";
 export { unwrapBigQueryCount, unwrapBigQueryValue } from "./values.js";
 export { BadQueryError, BigQueryReadError, DryRunBudgetExceededError, MaximumBytesBilledExceededError, PermissionDeniedError, RateLimitExceededError, classifyBigQueryError, isRetryableBigQueryError, } from "./errors.js";
-export type { BigQueryApiErrorDetail, BigQueryCreateQueryJobOptions, BigQueryDatasetLike, BigQueryExtractOptions, BigQueryJobLike, BigQueryJobMetadata, BigQueryLike, BigQueryLoadOptions, BigQueryNextQuery, BigQueryQueryResultsOptions, BigQuerySchemaField, BigQueryTableLike, BigQueryTableReference, BigQueryWriteStreamLike, DryRunEstimate, EstimateQueryBytesOptions, ExtractJobHandle, ExtractJobState, ExtractJobStatus, ExtractTableToGCSInput, GcsFileLike, JobLabelInput, QueryPage, RunQueryOptions, WaitForExtractJobOptions, } from "./types.js";
+export type { BigQueryApiErrorDetail, BigQueryCreateQueryJobOptions, BigQueryDatasetLike, BigQueryExtractOptions, BigQueryJobLike, BigQueryJobMetadata, BigQueryLike, BigQueryLoadOptions, BigQueryNextQuery, BigQueryQueryResultsOptions, BigQuerySchemaField, BigQueryTableLike, BigQueryTableReference, DryRunEstimate, EstimateQueryBytesOptions, ExtractJobHandle, ExtractJobState, ExtractJobStatus, ExtractTableToGCSInput, GcsFileLike, JobLabelInput, QueryPage, RunQueryOptions, WaitForExtractJobOptions, } from "./types.js";

package/dist/bigquery/load.d.ts

@@ -1,4 +1,12 @@
 import type { BigQueryLike, BigQueryLoadOptions } from "./types.js";
+/**
+ * Opens the rows for one load attempt. Called once per attempt: a retry after
+ * a rate-limit error re-uploads everything, so it calls this again and must
+ * get the same rows. That is why the source is a function and not an
+ * iterable — a generator object can only be consumed once, so a retry handed
+ * the same one would silently load nothing.
+ */
+export type LoadRowSource<T extends object> = () => AsyncIterable<T> | Iterable<T>;
 export interface LoadRowsOptions extends BigQueryLoadOptions {
     datasetId: string;
     tableId: string;
@@ -6,17 +14,27 @@ export interface LoadRowsOptions extends BigQueryLoadOptions {
     attempts?: number;
     /** Delay before the first retry; doubles every subsequent retry. Default 1000ms. */
     initialDelayMs?: number;
+    /** Target size of each NDJSON chunk written to the load job, in bytes. The
+     *  helper holds at most about one chunk being built plus what the write
+     *  stream itself buffers. A single row larger than this is written as its
+     *  own chunk. Default 1 MiB. */
+    chunkBytes?: number;
     /** Injectable for tests — real code always uses a real timer. */
     sleep?: (ms: number) => Promise<void>;
 }
+export interface LoadRowsResult {
+    /** Rows the successful attempt uploaded. */
+    rowCount: number;
+}
 /**
- * Loads `rows` into `datasetId.tableId` in one load job, retrying with
- * exponential backoff ONLY when the failure is BigQuery's
+ * Streams the rows `source` yields into `datasetId.tableId` in one load job,
+ * retrying with exponential backoff ONLY when the failure is BigQuery's
  * rateLimitExceeded/quotaExceeded class of error (see
- * `isRetryableBigQueryError`). Any other error (bad schema, auth failure,
- * etc.) is classified and rethrown on the first attempt.
+ * `isRetryableBigQueryError`). Any other error (bad schema, auth failure, a
+ * throw from the source, etc.) is classified and rethrown on the first
+ * attempt. Each attempt calls `source` afresh — see `LoadRowSource`.
  *
- * Pass `writeDisposition: "WRITE_TRUNCATE"` for a full-refresh reload (the
- * IPEDS institutions loader's use case); omit it for an append load.
+ * Pass `writeDisposition: "WRITE_TRUNCATE"` for a full-refresh reload; omit it
+ * for an append load.
  */
-export declare function loadRowsWithRetry<T extends object>(bq: BigQueryLike, options: LoadRowsOptions, rows: T[]): Promise<void>;
+export declare function loadRowsWithRetry<T extends object>(bq: BigQueryLike, options: LoadRowsOptions, source: LoadRowSource<T>): Promise<LoadRowsResult>;

package/dist/bigquery/load.js

@@ -1,61 +1,91 @@
 /**
- * Load-job capability (AK-14 follow-on, MP-200/PLN-2): truncate-and-reload a
- * BigQuery table from an array of rows, with retry-with-backoff limited to
+ * Load-job capability (AK-14 follow-on, MP-200/PLN-2): loads a stream of rows
+ * into a BigQuery table in one load job, with retry-with-backoff limited to
  * BigQuery's own rate-limit/quota class of error — generalized out of
  * media-planner's `apps/api/src/lib/bigqueryLoad.ts` (via flytedesk-id's
  * verbatim port at `src/lib/bigqueryLoad.ts`, MP-200), which this module
- * replaces. Callers needing a periodic full-refresh load (e.g. the IPEDS
- * institutions loader) use this instead of keeping their own copy of the
- * write-stream + retry plumbing.
+ * replaces.
+ *
+ * Memory is bounded by `chunkBytes`, never by the row count (AK-28): rows are
+ * pulled from the caller's source one at a time, serialized into NDJSON chunks
+ * of about `chunkBytes`, and piped into the load job's write stream with
+ * `stream.pipeline`, which stops pulling from the source whenever the write
+ * stream's buffer is full and resumes on its `drain`. A caller that reads its
+ * rows page by page (a keyset-paged or cursor read) can therefore load any
+ * number of rows in a fixed amount of memory.
  */
-import { BigQueryReadError, classifyBigQueryError, isRetryableBigQueryError } from "./errors.js";
+import { once } from "node:events";
+import { pipeline } from "node:stream/promises";
+import { BigQueryReadError, classifyBigQueryError, isRetryableBigQueryError, } from "./errors.js";
 const DEFAULT_ATTEMPTS = 5;
 const DEFAULT_INITIAL_DELAY_MS = 1_000;
+const DEFAULT_CHUNK_BYTES = 1024 * 1024;
 const DEFAULT_SOURCE_FORMAT = "NEWLINE_DELIMITED_JSON";
 function defaultSleep(ms) {
     return new Promise((resolve) => setTimeout(resolve, ms));
 }
-async function runLoadJob(bq, options, rows) {
+/** Serializes `rows` into NDJSON, one `Buffer` of about `chunkBytes` at a
+ *  time, counting rows into `counter` as they are consumed. */
+async function* ndjsonChunks(rows, chunkBytes, counter) {
+    let lines = [];
+    let size = 0;
+    for await (const row of rows) {
+        const line = `${JSON.stringify(row)}\n`;
+        lines.push(line);
+        size += Buffer.byteLength(line);
+        counter.rows += 1;
+        if (size >= chunkBytes) {
+            yield Buffer.from(lines.join(""));
+            lines = [];
+            size = 0;
+        }
+    }
+    if (lines.length > 0)
+        yield Buffer.from(lines.join(""));
+}
+async function runLoadJob(bq, options, source) {
     const table = bq.dataset(options.datasetId).table(options.tableId);
     const createWriteStream = table.createWriteStream;
     if (!createWriteStream) {
         throw new BigQueryReadError(`dataset(${options.datasetId}).table(${options.tableId}) does not implement createWriteStream — this BigQueryLike was not constructed for load jobs`);
     }
-    const ndjson = rows.map((row) => `${JSON.stringify(row)}\n`);
-    await new Promise((resolve, reject) => {
-        const stream = createWriteStream.call(table, {
-            sourceFormat: options.sourceFormat ?? DEFAULT_SOURCE_FORMAT,
-            schema: options.schema,
-            writeDisposition: options.writeDisposition,
-            createDisposition: options.createDisposition,
-            labels: options.labels,
-            jobId: options.jobId,
-        });
-        stream.on("error", reject);
-        stream.on("complete", () => resolve());
-        for (const line of ndjson)
-            stream.write(line);
-        stream.end();
+    const stream = createWriteStream.call(table, {
+        sourceFormat: options.sourceFormat ?? DEFAULT_SOURCE_FORMAT,
+        schema: options.schema,
+        writeDisposition: options.writeDisposition,
+        createDisposition: options.createDisposition,
+        labels: options.labels,
+        jobId: options.jobId,
     });
+    const counter = { rows: 0 };
+    // The load is done only when BigQuery reports the job "complete" AND the
+    // pipeline has flushed every chunk. `once` rejects on the stream's "error",
+    // which is how a failed job surfaces; `pipeline` rejects (and destroys the
+    // stream, aborting the upload) when the source itself throws.
+    await Promise.all([
+        once(stream, "complete"),
+        pipeline(ndjsonChunks(source(), options.chunkBytes ?? DEFAULT_CHUNK_BYTES, counter), stream),
+    ]);
+    return { rowCount: counter.rows };
 }
 /**
- * Loads `rows` into `datasetId.tableId` in one load job, retrying with
- * exponential backoff ONLY when the failure is BigQuery's
+ * Streams the rows `source` yields into `datasetId.tableId` in one load job,
+ * retrying with exponential backoff ONLY when the failure is BigQuery's
  * rateLimitExceeded/quotaExceeded class of error (see
- * `isRetryableBigQueryError`). Any other error (bad schema, auth failure,
- * etc.) is classified and rethrown on the first attempt.
+ * `isRetryableBigQueryError`). Any other error (bad schema, auth failure, a
+ * throw from the source, etc.) is classified and rethrown on the first
+ * attempt. Each attempt calls `source` afresh — see `LoadRowSource`.
  *
- * Pass `writeDisposition: "WRITE_TRUNCATE"` for a full-refresh reload (the
- * IPEDS institutions loader's use case); omit it for an append load.
+ * Pass `writeDisposition: "WRITE_TRUNCATE"` for a full-refresh reload; omit it
+ * for an append load.
  */
-export async function loadRowsWithRetry(bq, options, rows) {
+export async function loadRowsWithRetry(bq, options, source) {
     const attempts = options.attempts ?? DEFAULT_ATTEMPTS;
     const initialDelayMs = options.initialDelayMs ?? DEFAULT_INITIAL_DELAY_MS;
     const sleep = options.sleep ?? defaultSleep;
     for (let attempt = 1;; attempt += 1) {
         try {
-            await runLoadJob(bq, options, rows);
-            return;
+            return await runLoadJob(bq, options, source);
         }
         catch (err) {
             if (attempt >= attempts || !isRetryableBigQueryError(err)) {

package/dist/bigquery/load.js.map

@@ -1 +1 @@
-{"version":3,"file":"load.js","sourceRoot":"","sources":["../../src/bigquery/load.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AACH,OAAO,EAAE,iBAAiB,EAAE,qBAAqB,EAAE,wBAAwB,EAAE,MAAM,aAAa,CAAC;AAcjG,MAAM,gBAAgB,GAAG,CAAC,CAAC;AAC3B,MAAM,wBAAwB,GAAG,KAAK,CAAC;AACvC,MAAM,qBAAqB,GAAG,wBAAwB,CAAC;AAEvD,SAAS,YAAY,CAAC,EAAU;IAC9B,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED,KAAK,UAAU,UAAU,CACvB,EAAgB,EAChB,OAAwB,EACxB,IAAS;IAET,MAAM,KAAK,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IACnE,MAAM,iBAAiB,GAAG,KAAK,CAAC,iBAAiB,CAAC;IAClD,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACvB,MAAM,IAAI,iBAAiB,CACzB,WAAW,OAAO,CAAC,SAAS,WAAW,OAAO,CAAC,OAAO,8FAA8F,CACrJ,CAAC;IACJ,CAAC;IACD,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAE7D,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC1C,MAAM,MAAM,GAAG,iBAAiB,CAAC,IAAI,CAAC,KAAK,EAAE;YAC3C,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,qBAAqB;YAC3D,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;YAC1C,iBAAiB,EAAE,OAAO,CAAC,iBAAiB;YAC5C,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,KAAK,EAAE,OAAO,CAAC,KAAK;SACrB,CAAC,CAAC;QACH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC3B,MAAM,CAAC,EAAE,CAAC,UAAU,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC;QACvC,KAAK,MAAM,IAAI,IAAI,MAAM;YAAE,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC9C,MAAM,CAAC,GAAG,EAAE,CAAC;IACf,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,EAAgB,EAChB,OAAwB,EACxB,IAAS;IAET,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,gBAAgB,CAAC;IACtD,MAAM,cAAc,GAAG,OAAO,CAAC,cAAc,IAAI,wBAAwB,CAAC;IAC1E,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,YAAY,CAAC;IAE5C,KAAK,IAAI,OAAO,GAAG,CAAC,GAAI,OAAO,IAAI,CAAC,EAAE,CAAC;QACrC,IAAI,CAAC;YACH,MAAM,UAAU,CAAC,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;YACpC,OAAO;QACT,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,OAAO,IAAI,QAAQ,IAAI,CAAC,wBAAwB,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC1D,MAAM,qBAAqB,CAAC,GAAG,CAAC,CAAC;YACnC,CAAC;YACD,MAAM,OAAO,GAAG,cAAc,GAAG,CAAC,IAAI,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC;YACpD,OAAO,CAAC,IAAI,CACV,mBAAmB,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC,OAAO,6BAA6B,OAAO,IAAI,QAAQ,iBAAiB,OAAO,IAAI,CACpI,CAAC;YACF,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC;QACvB,CAAC;IACH,CAAC;AACH,CAAC"}
+{"version":3,"file":"load.js","sourceRoot":"","sources":["../../src/bigquery/load.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AACH,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AACnC,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAChD,OAAO,EACL,iBAAiB,EACjB,qBAAqB,EACrB,wBAAwB,GACzB,MAAM,aAAa,CAAC;AAkCrB,MAAM,gBAAgB,GAAG,CAAC,CAAC;AAC3B,MAAM,wBAAwB,GAAG,KAAK,CAAC;AACvC,MAAM,mBAAmB,GAAG,IAAI,GAAG,IAAI,CAAC;AACxC,MAAM,qBAAqB,GAAG,wBAAwB,CAAC;AAEvD,SAAS,YAAY,CAAC,EAAU;IAC9B,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED;+DAC+D;AAC/D,KAAK,SAAS,CAAC,CAAC,YAAY,CAC1B,IAAoC,EACpC,UAAkB,EAClB,OAAyB;IAEzB,IAAI,KAAK,GAAa,EAAE,CAAC;IACzB,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QAC7B,MAAM,IAAI,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC;QACxC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjB,IAAI,IAAI,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;QAChC,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC;QAClB,IAAI,IAAI,IAAI,UAAU,EAAE,CAAC;YACvB,MAAM,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;YAClC,KAAK,GAAG,EAAE,CAAC;YACX,IAAI,GAAG,CAAC,CAAC;QACX,CAAC;IACH,CAAC;IACD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;QAAE,MAAM,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;AAC1D,CAAC;AAED,KAAK,UAAU,UAAU,CACvB,EAAgB,EAChB,OAAwB,EACxB,MAAwB;IAExB,MAAM,KAAK,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IACnE,MAAM,iBAAiB,GAAG,KAAK,CAAC,iBAAiB,CAAC;IAClD,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACvB,MAAM,IAAI,iBAAiB,CACzB,WAAW,OAAO,CAAC,SAAS,WAAW,OAAO,CAAC,OAAO,8FAA8F,CACrJ,CAAC;IACJ,CAAC;IACD,MAAM,MAAM,GAAG,iBAAiB,CAAC,IAAI,CAAC,KAAK,EAAE;QAC3C,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,qBAAqB;QAC3D,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;QAC1C,iBAAiB,EAAE,OAAO,CAAC,iBAAiB;QAC5C,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,KAAK,EAAE,OAAO,CAAC,KAAK;KACrB,CAAC,CAAC;IACH,MAAM,OAAO,GAAG,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;IAC5B,yEAAyE;IACzE,4EAA4E;IAC5E,2EAA2E;IAC3E,8DAA8D;IAC9D,MAAM,OAAO,CAAC,GAAG,CAAC;QAChB,IAAI,CAAC,MAAM,EAAE,UAAU,CAAC;QACxB,QAAQ,CACN,YAAY,CACV,MAAM,EAAE,EACR,OAAO,CAAC,UAAU,IAAI,mBAAmB,EACzC,OAAO,CACR,EACD,MAAM,CACP;KACF,CAAC,CAAC;IACH,OAAO,EAAE,QAAQ,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC;AACpC,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,EAAgB,EAChB,OAAwB,EACxB,MAAwB;IAExB,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,gBAAgB,CAAC;IACtD,MAAM,cAAc,GAAG,OAAO,CAAC,cAAc,IAAI,wBAAwB,CAAC;IAC1E,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,YAAY,CAAC;IAE5C,KAAK,IAAI,OAAO,GAAG,CAAC,GAAI,OAAO,IAAI,CAAC,EAAE,CAAC;QACrC,IAAI,CAAC;YACH,OAAO,MAAM,UAAU,CAAC,EAAE,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;QAC/C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,OAAO,IAAI,QAAQ,IAAI,CAAC,wBAAwB,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC1D,MAAM,qBAAqB,CAAC,GAAG,CAAC,CAAC;YACnC,CAAC;YACD,MAAM,OAAO,GAAG,cAAc,GAAG,CAAC,IAAI,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC;YACpD,OAAO,CAAC,IAAI,CACV,mBAAmB,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC,OAAO,6BAA6B,OAAO,IAAI,QAAQ,iBAAiB,OAAO,IAAI,CACpI,CAAC;YACF,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC;QACvB,CAAC;IACH,CAAC;AACH,CAAC"}

package/dist/bigquery/types.d.ts

@@ -11,6 +11,7 @@
  * `@google-cloud/bigquery` calls it made; inventory-specific SQL stays behind
  * in media-planner.
  */
+import type { Writable } from "node:stream";
 /** One structured error entry, same shape the Google APIs client library
  *  attaches to a rejected job promise's `.errors` array. */
 export interface BigQueryApiErrorDetail {
@@ -109,15 +110,6 @@ export interface GcsFileLike {
         readonly name: string;
     };
 }
-/** Structural subset of Node's own Writable that `Table#createWriteStream`
- *  needs to expose — the "error"/"complete" events a load-job caller pipes
- *  rows into and waits on. */
-export interface BigQueryWriteStreamLike {
-    on(event: "error", listener: (err: unknown) => void): this;
-    on(event: "complete", listener: () => void): this;
-    write(chunk: string): boolean;
-    end(): void;
-}
 /** One BigQuery table-schema field, as `createWriteStream`'s `schema` option
  *  (and the REST API's `tables.insert`) accepts it. `fields` recurses for
  *  RECORD/STRUCT columns. */
@@ -143,12 +135,15 @@ export interface BigQueryLoadOptions {
 }
 export interface BigQueryTableLike {
     createExtractJob(destination: GcsFileLike, options?: BigQueryExtractOptions): Promise<[BigQueryJobLike, BigQueryJobMetadata]>;
-    /** Opens a load-job write stream — the caller pipes newline-delimited rows
-     *  (or other `sourceFormat`) into it, then awaits the stream's "complete"/
-     *  "error" event. Mirrors `@google-cloud/bigquery`'s `Table#createWriteStream`.
-     *  Optional: read-only callers (query/extract) never need it — only
-     *  `loadRowsWithRetry` (./load.js) does, and it fails loudly if missing. */
-    createWriteStream?(options: BigQueryLoadOptions): BigQueryWriteStreamLike;
+    /** Opens a load-job write stream — a real Node `Writable`, since the loader
+     *  pipes NDJSON into it with `stream.pipeline` and relies on its
+     *  backpressure (`write()` returning false, then "drain"). The stream emits
+     *  "complete" once BigQuery finishes the job, or "error" if the job fails.
+     *  Mirrors `@google-cloud/bigquery`'s `Table#createWriteStream`, which
+     *  returns a `Writable`. Optional: read-only callers (query/extract) never
+     *  need it — only `loadRowsWithRetry` (./load.js) does, and it fails loudly
+     *  if missing. */
+    createWriteStream?(options: BigQueryLoadOptions): Writable;
 }
 export interface BigQueryDatasetLike {
     table(id: string): BigQueryTableLike;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@flytedesk/app-kit",
-  "version": "4.0.0",
+  "version": "5.0.0",
   "description": "Shared platform kit for flytedesk apps: flytedesk-id auth (BFF/OIDC client) and a Postgres-native trace/audit layer.",
   "type": "module",
   "license": "UNLICENSED",