@flytedesk/app-kit 3.2.3 → 5.0.0

package/dist/bigquery/errors.js

@@ -42,6 +42,20 @@ export class DryRunBudgetExceededError extends BigQueryReadError {
         this.name = "DryRunBudgetExceededError";
     }
 }
+/** BigQuery itself refused to run a real job because it would bill more than
+ *  the `maximumBytesBilled` passed to `runQuery` — distinct from
+ *  `DryRunBudgetExceededError`, which is this module's own client-side check
+ *  against a *dry-run* estimate. BigQuery reports this as an `invalidQuery`-
+ *  reasoned error whose message names "bytes billed" (see
+ *  `MAXIMUM_BYTES_BILLED_REASON_TEXT` below); this class exists so a caller
+ *  can branch on the billing cap specifically instead of catching a generic
+ *  BadQueryError. */
+export class MaximumBytesBilledExceededError extends BigQueryReadError {
+    constructor(message, cause) {
+        super(message, cause);
+        this.name = "MaximumBytesBilledExceededError";
+    }
+}
 /** Reason codes the Google APIs client library attaches to a rejected job
  *  promise's `.errors[]` — see
  *  https://cloud.google.com/bigquery/docs/error-messages for the full list.
@@ -63,6 +77,35 @@ const BAD_QUERY_REASONS = new Set([
  *  almost always succeeds seconds later, so callers (see `loadRowsWithRetry`
  *  in ./load.js) retry on this and only this. */
 const RATE_LIMIT_REASONS = new Set(["rateLimitExceeded", "quotaExceeded"]);
+const RATE_LIMIT_REASON_TEXT = new RegExp([...RATE_LIMIT_REASONS].join("|"));
+/** BigQuery reports a `maximumBytesBilled` refusal as an `invalidQuery`-
+ *  reasoned error whose message reads e.g. "Query exceeded limit for bytes
+ *  billed: 123456789. Limit: 100000000." — text-matched (case-insensitively)
+ *  since it shares its `reason` with every other bad-query case. */
+const MAXIMUM_BYTES_BILLED_REASON_TEXT = /bytes billed/i;
+/** The structured `errors[]` entry that carries a rate-limit reason, wherever it
+ *  sits in the list — so a rate limit reported as a later entry is both detected
+ *  and described by its own message, not an unrelated sibling's. */
+function rateLimitCause(apiErr) {
+    return apiErr?.errors?.find((cause) => cause.reason !== undefined && RATE_LIMIT_REASONS.has(cause.reason));
+}
+/**
+ * The one definition of "BigQuery rate-limited this call", shared by
+ * `classifyBigQueryError` and `isRetryableBigQueryError` so that what gets
+ * retried and how a retry-exhausted failure is reported can never disagree.
+ * True for any structured `errors[].reason` in RATE_LIMIT_REASONS or, for a raw
+ * REST error with no `errors[]`, that reason text in its `message`.
+ */
+function isRateLimited(err) {
+    if (err instanceof RateLimitExceededError)
+        return true;
+    if (!err || typeof err !== "object")
+        return false;
+    const apiErr = err;
+    if (rateLimitCause(apiErr))
+        return true;
+    return typeof apiErr.message === "string" && RATE_LIMIT_REASON_TEXT.test(apiErr.message);
+}
 /**
  * Normalizes anything a BigQueryLike call can throw/reject with (a Google API
  * client error, a job's `status.errorResult`/`status.errors`, or an already-
@@ -73,12 +116,14 @@ export function classifyBigQueryError(err) {
     if (err instanceof BigQueryReadError)
         return err;
     const apiErr = err;
-    const detail = apiErr?.errors?.[0];
+    // The entry that explains the classification: the rate-limit cause when there
+    // is one (it decides the class below), otherwise the first reported error.
+    const detail = rateLimitCause(apiErr) ?? apiErr?.errors?.[0];
     const reason = detail?.reason;
     const message = detail?.message ??
         apiErr?.message ??
         (err instanceof Error ? err.message : String(err));
-    if (reason && RATE_LIMIT_REASONS.has(reason)) {
+    if (isRateLimited(err)) {
         return new RateLimitExceededError(message, err);
     }
     if (reason && PERMISSION_DENIED_REASONS.has(reason)) {
@@ -87,6 +132,9 @@ export function classifyBigQueryError(err) {
     if (apiErr?.code === 403) {
         return new PermissionDeniedError(message, err);
     }
+    if (MAXIMUM_BYTES_BILLED_REASON_TEXT.test(message)) {
+        return new MaximumBytesBilledExceededError(message, err);
+    }
     if (reason && BAD_QUERY_REASONS.has(reason)) {
         return new BadQueryError(message, err);
     }
@@ -98,18 +146,11 @@ export function classifyBigQueryError(err) {
 /**
  * True only for BigQuery's own rate-limit/quota class of error. Deliberately
  * narrow: a bad schema, a permissions/auth failure, or any other error must
- * keep failing immediately, not get masked behind a retry loop. Checks both
- * the structured `errors[].reason` the client library attaches and, as a
- * fallback, the reason text embedded in a raw REST error's `message`.
+ * keep failing immediately, not get masked behind a retry loop. Retryable
+ * exactly when `classifyBigQueryError` would report a RateLimitExceededError
+ * (see `isRateLimited`).
  */
 export function isRetryableBigQueryError(err) {
-    if (!err || typeof err !== "object")
-        return false;
-    const apiErr = err;
-    if (apiErr.errors?.some((cause) => cause.reason !== undefined && RATE_LIMIT_REASONS.has(cause.reason))) {
-        return true;
-    }
-    return (typeof apiErr.message === "string" &&
-        /rateLimitExceeded|quotaExceeded/.test(apiErr.message));
+    return isRateLimited(err);
 }
 //# sourceMappingURL=errors.js.map

package/dist/bigquery/errors.js.map

@@ -1 +1 @@
-{"version":3,"file":"errors.js","sourceRoot":"","sources":["../../src/bigquery/errors.ts"],"names":[],"mappings":"AAQA,MAAM,OAAO,iBAAkB,SAAQ,KAAK;IAGxB;IAFlB,YACE,OAAe,EACC,KAAe;QAE/B,KAAK,CAAC,OAAO,CAAC,CAAC;QAFC,UAAK,GAAL,KAAK,CAAU;QAG/B,IAAI,CAAC,IAAI,GAAG,mBAAmB,CAAC;IAClC,CAAC;CACF;AAED,+DAA+D;AAC/D,MAAM,OAAO,aAAc,SAAQ,iBAAiB;IAClD,YAAY,OAAe,EAAE,KAAe;QAC1C,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QACtB,IAAI,CAAC,IAAI,GAAG,eAAe,CAAC;IAC9B,CAAC;CACF;AAED,2EAA2E;AAC3E,MAAM,OAAO,qBAAsB,SAAQ,iBAAiB;IAC1D,YAAY,OAAe,EAAE,KAAe;QAC1C,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QACtB,IAAI,CAAC,IAAI,GAAG,uBAAuB,CAAC;IACtC,CAAC;CACF;AAED;;;yEAGyE;AACzE,MAAM,OAAO,sBAAuB,SAAQ,iBAAiB;IAC3D,YAAY,OAAe,EAAE,KAAe;QAC1C,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QACtB,IAAI,CAAC,IAAI,GAAG,wBAAwB,CAAC;IACvC,CAAC;CACF;AAED;+CAC+C;AAC/C,MAAM,OAAO,yBAA0B,SAAQ,iBAAiB;IAE5C;IACA;IAFlB,YACkB,mBAA2B,EAC3B,cAAsB;QAEtC,KAAK,CACH,uBAAuB,mBAAmB,sBAAsB,cAAc,cAAc,CAC7F,CAAC;QALc,wBAAmB,GAAnB,mBAAmB,CAAQ;QAC3B,mBAAc,GAAd,cAAc,CAAQ;QAKtC,IAAI,CAAC,IAAI,GAAG,2BAA2B,CAAC;IAC1C,CAAC;CACF;AAQD;;;;0DAI0D;AAC1D,MAAM,yBAAyB,GAAG,IAAI,GAAG,CAAC;IACxC,cAAc;IACd,WAAW;IACX,yBAAyB;CAC1B,CAAC,CAAC;AACH,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC;IAChC,cAAc;IACd,SAAS;IACT,YAAY;IACZ,uBAAuB;CACxB,CAAC,CAAC;AACH;;;iDAGiD;AACjD,MAAM,kBAAkB,GAAG,IAAI,GAAG,CAAC,CAAC,mBAAmB,EAAE,eAAe,CAAC,CAAC,CAAC;AAE3E;;;;;GAKG;AACH,MAAM,UAAU,qBAAqB,CAAC,GAAY;IAChD,IAAI,GAAG,YAAY,iBAAiB;QAAE,OAAO,GAAG,CAAC;IAEjD,MAAM,MAAM,GAAG,GAAyB,CAAC;IACzC,MAAM,MAAM,GAAG,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC;IACnC,MAAM,MAAM,GAAG,MAAM,EAAE,MAAM,CAAC;IAC9B,MAAM,OAAO,GACX,MAAM,EAAE,OAAO;QACf,MAAM,EAAE,OAAO;QACf,CAAC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;IAErD,IAAI,MAAM,IAAI,kBAAkB,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;QAC7C,OAAO,IAAI,sBAAsB,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IAClD,CAAC;IACD,IAAI,MAAM,IAAI,yBAAyB,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;QACpD,OAAO,IAAI,qBAAqB,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IACjD,CAAC;IACD,IAAI,MAAM,EAAE,IAAI,KAAK,GAAG,EAAE,CAAC;QACzB,OAAO,IAAI,qBAAqB,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IACjD,CAAC;IACD,IAAI,MAAM,IAAI,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;QAC5C,OAAO,IAAI,aAAa,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IACzC,CAAC;IACD,IAAI,MAAM,EAAE,IAAI,KAAK,GAAG,EAAE,CAAC;QACzB,OAAO,IAAI,aAAa,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IACzC,CAAC;IACD,OAAO,IAAI,iBAAiB,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;AAC7C,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,wBAAwB,CAAC,GAAY;IACnD,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IAClD,MAAM,MAAM,GAAG,GAAyB,CAAC;IAEzC,IACE,MAAM,CAAC,MAAM,EAAE,IAAI,CACjB,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,SAAS,IAAI,kBAAkB,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAC9E,EACD,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO,CACL,OAAO,MAAM,CAAC,OAAO,KAAK,QAAQ;QAClC,iCAAiC,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CACvD,CAAC;AACJ,CAAC"}
+{"version":3,"file":"errors.js","sourceRoot":"","sources":["../../src/bigquery/errors.ts"],"names":[],"mappings":"AAQA,MAAM,OAAO,iBAAkB,SAAQ,KAAK;IAGxB;IAFlB,YACE,OAAe,EACC,KAAe;QAE/B,KAAK,CAAC,OAAO,CAAC,CAAC;QAFC,UAAK,GAAL,KAAK,CAAU;QAG/B,IAAI,CAAC,IAAI,GAAG,mBAAmB,CAAC;IAClC,CAAC;CACF;AAED,+DAA+D;AAC/D,MAAM,OAAO,aAAc,SAAQ,iBAAiB;IAClD,YAAY,OAAe,EAAE,KAAe;QAC1C,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QACtB,IAAI,CAAC,IAAI,GAAG,eAAe,CAAC;IAC9B,CAAC;CACF;AAED,2EAA2E;AAC3E,MAAM,OAAO,qBAAsB,SAAQ,iBAAiB;IAC1D,YAAY,OAAe,EAAE,KAAe;QAC1C,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QACtB,IAAI,CAAC,IAAI,GAAG,uBAAuB,CAAC;IACtC,CAAC;CACF;AAED;;;yEAGyE;AACzE,MAAM,OAAO,sBAAuB,SAAQ,iBAAiB;IAC3D,YAAY,OAAe,EAAE,KAAe;QAC1C,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QACtB,IAAI,CAAC,IAAI,GAAG,wBAAwB,CAAC;IACvC,CAAC;CACF;AAED;+CAC+C;AAC/C,MAAM,OAAO,yBAA0B,SAAQ,iBAAiB;IAE5C;IACA;IAFlB,YACkB,mBAA2B,EAC3B,cAAsB;QAEtC,KAAK,CACH,uBAAuB,mBAAmB,sBAAsB,cAAc,cAAc,CAC7F,CAAC;QALc,wBAAmB,GAAnB,mBAAmB,CAAQ;QAC3B,mBAAc,GAAd,cAAc,CAAQ;QAKtC,IAAI,CAAC,IAAI,GAAG,2BAA2B,CAAC;IAC1C,CAAC;CACF;AAED;;;;;;;qBAOqB;AACrB,MAAM,OAAO,+BAAgC,SAAQ,iBAAiB;IACpE,YAAY,OAAe,EAAE,KAAe;QAC1C,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QACtB,IAAI,CAAC,IAAI,GAAG,iCAAiC,CAAC;IAChD,CAAC;CACF;AAQD;;;;0DAI0D;AAC1D,MAAM,yBAAyB,GAAG,IAAI,GAAG,CAAC;IACxC,cAAc;IACd,WAAW;IACX,yBAAyB;CAC1B,CAAC,CAAC;AACH,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC;IAChC,cAAc;IACd,SAAS;IACT,YAAY;IACZ,uBAAuB;CACxB,CAAC,CAAC;AACH;;;iDAGiD;AACjD,MAAM,kBAAkB,GAAG,IAAI,GAAG,CAAC,CAAC,mBAAmB,EAAE,eAAe,CAAC,CAAC,CAAC;AAC3E,MAAM,sBAAsB,GAAG,IAAI,MAAM,CAAC,CAAC,GAAG,kBAAkB,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAE7E;;;oEAGoE;AACpE,MAAM,gCAAgC,GAAG,eAAe,CAAC;AAEzD;;oEAEoE;AACpE,SAAS,cAAc,CAAC,MAA6C;IACnE,OAAO,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,SAAS,IAAI,kBAAkB,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;AAC7G,CAAC;AAED;;;;;;GAMG;AACH,SAAS,aAAa,CAAC,GAAY;IACjC,IAAI,GAAG,YAAY,sBAAsB;QAAE,OAAO,IAAI,CAAC;IACvD,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IAClD,MAAM,MAAM,GAAG,GAAyB,CAAC;IACzC,IAAI,cAAc,CAAC,MAAM,CAAC;QAAE,OAAO,IAAI,CAAC;IACxC,OAAO,OAAO,MAAM,CAAC,OAAO,KAAK,QAAQ,IAAI,sBAAsB,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;AAC3F,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,qBAAqB,CAAC,GAAY;IAChD,IAAI,GAAG,YAAY,iBAAiB;QAAE,OAAO,GAAG,CAAC;IAEjD,MAAM,MAAM,GAAG,GAAyB,CAAC;IACzC,8EAA8E;IAC9E,2EAA2E;IAC3E,MAAM,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,IAAI,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC;IAC7D,MAAM,MAAM,GAAG,MAAM,EAAE,MAAM,CAAC;IAC9B,MAAM,OAAO,GACX,MAAM,EAAE,OAAO;QACf,MAAM,EAAE,OAAO;QACf,CAAC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;IAErD,IAAI,aAAa,CAAC,GAAG,CAAC,EAAE,CAAC;QACvB,OAAO,IAAI,sBAAsB,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IAClD,CAAC;IACD,IAAI,MAAM,IAAI,yBAAyB,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;QACpD,OAAO,IAAI,qBAAqB,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IACjD,CAAC;IACD,IAAI,MAAM,EAAE,IAAI,KAAK,GAAG,EAAE,CAAC;QACzB,OAAO,IAAI,qBAAqB,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IACjD,CAAC;IACD,IAAI,gCAAgC,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QACnD,OAAO,IAAI,+BAA+B,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IAC3D,CAAC;IACD,IAAI,MAAM,IAAI,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;QAC5C,OAAO,IAAI,aAAa,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IACzC,CAAC;IACD,IAAI,MAAM,EAAE,IAAI,KAAK,GAAG,EAAE,CAAC;QACzB,OAAO,IAAI,aAAa,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IACzC,CAAC;IACD,OAAO,IAAI,iBAAiB,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;AAC7C,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,wBAAwB,CAAC,GAAY;IACnD,OAAO,aAAa,CAAC,GAAG,CAAC,CAAC;AAC5B,CAAC"}

package/dist/bigquery/index.d.ts

@@ -28,7 +28,9 @@
  *     labels: client.buildJobLabels({ actor: userId, purpose: "audience-export" }),
  *     pageSize: 500,
  *   });
- *   // page.nextPageToken, if present, resumes with { ...same options, pageToken }
+ *   // page.nextPageToken, if present, resumes on the SAME job with
+ *   // { ...same options, pageToken: page.nextPageToken, jobId: page.jobId }
+ *   // (a page token is only valid against the job that produced it)
  *
  *   const { jobId } = await client.extractTableToGCS({
  *     datasetId: "dataset",
@@ -43,8 +45,8 @@ export type { BigQueryReadClient, CreateBigQueryReadClientOptions, } from "./cli
 export { runQuery, runQueryAllPages, estimateQueryBytes } from "./query.js";
 export { extractTableToGCS, getExtractJobStatus, waitForExtractJob, } from "./extract.js";
 export { loadRowsWithRetry } from "./load.js";
-export type { LoadRowsOptions } from "./load.js";
+export type { LoadRowSource, LoadRowsOptions, LoadRowsResult } from "./load.js";
 export { buildJobLabels, sanitizeLabelValue } from "./labels.js";
 export { unwrapBigQueryCount, unwrapBigQueryValue } from "./values.js";
-export { BadQueryError, BigQueryReadError, DryRunBudgetExceededError, PermissionDeniedError, RateLimitExceededError, classifyBigQueryError, isRetryableBigQueryError, } from "./errors.js";
-export type { BigQueryApiErrorDetail, BigQueryCreateQueryJobOptions, BigQueryDatasetLike, BigQueryExtractOptions, BigQueryJobLike, BigQueryJobMetadata, BigQueryLike, BigQueryLoadOptions, BigQueryNextQuery, BigQueryQueryResultsOptions, BigQuerySchemaField, BigQueryTableLike, BigQueryWriteStreamLike, DryRunEstimate, EstimateQueryBytesOptions, ExtractJobHandle, ExtractJobState, ExtractJobStatus, ExtractTableToGCSInput, GcsFileLike, JobLabelInput, QueryPage, RunQueryOptions, WaitForExtractJobOptions, } from "./types.js";
+export { BadQueryError, BigQueryReadError, DryRunBudgetExceededError, MaximumBytesBilledExceededError, PermissionDeniedError, RateLimitExceededError, classifyBigQueryError, isRetryableBigQueryError, } from "./errors.js";
+export type { BigQueryApiErrorDetail, BigQueryCreateQueryJobOptions, BigQueryDatasetLike, BigQueryExtractOptions, BigQueryJobLike, BigQueryJobMetadata, BigQueryLike, BigQueryLoadOptions, BigQueryNextQuery, BigQueryQueryResultsOptions, BigQuerySchemaField, BigQueryTableLike, BigQueryTableReference, DryRunEstimate, EstimateQueryBytesOptions, ExtractJobHandle, ExtractJobState, ExtractJobStatus, ExtractTableToGCSInput, GcsFileLike, JobLabelInput, QueryPage, RunQueryOptions, WaitForExtractJobOptions, } from "./types.js";

package/dist/bigquery/index.js

@@ -28,7 +28,9 @@
  *     labels: client.buildJobLabels({ actor: userId, purpose: "audience-export" }),
  *     pageSize: 500,
  *   });
- *   // page.nextPageToken, if present, resumes with { ...same options, pageToken }
+ *   // page.nextPageToken, if present, resumes on the SAME job with
+ *   // { ...same options, pageToken: page.nextPageToken, jobId: page.jobId }
+ *   // (a page token is only valid against the job that produced it)
  *
  *   const { jobId } = await client.extractTableToGCS({
  *     datasetId: "dataset",
@@ -44,5 +46,5 @@ export { extractTableToGCS, getExtractJobStatus, waitForExtractJob, } from "./ex
 export { loadRowsWithRetry } from "./load.js";
 export { buildJobLabels, sanitizeLabelValue } from "./labels.js";
 export { unwrapBigQueryCount, unwrapBigQueryValue } from "./values.js";
-export { BadQueryError, BigQueryReadError, DryRunBudgetExceededError, PermissionDeniedError, RateLimitExceededError, classifyBigQueryError, isRetryableBigQueryError, } from "./errors.js";
+export { BadQueryError, BigQueryReadError, DryRunBudgetExceededError, MaximumBytesBilledExceededError, PermissionDeniedError, RateLimitExceededError, classifyBigQueryError, isRetryableBigQueryError, } from "./errors.js";
 //# sourceMappingURL=index.js.map

package/dist/bigquery/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/bigquery/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuCG;AACH,OAAO,EAAE,wBAAwB,EAAE,MAAM,aAAa,CAAC;AAMvD,OAAO,EAAE,QAAQ,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAE5E,OAAO,EACL,iBAAiB,EACjB,mBAAmB,EACnB,iBAAiB,GAClB,MAAM,cAAc,CAAC;AAEtB,OAAO,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAC;AAG9C,OAAO,EAAE,cAAc,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAEjE,OAAO,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAEvE,OAAO,EACL,aAAa,EACb,iBAAiB,EACjB,yBAAyB,EACzB,qBAAqB,EACrB,sBAAsB,EACtB,qBAAqB,EACrB,wBAAwB,GACzB,MAAM,aAAa,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/bigquery/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AACH,OAAO,EAAE,wBAAwB,EAAE,MAAM,aAAa,CAAC;AAMvD,OAAO,EAAE,QAAQ,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAE5E,OAAO,EACL,iBAAiB,EACjB,mBAAmB,EACnB,iBAAiB,GAClB,MAAM,cAAc,CAAC;AAEtB,OAAO,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAC;AAG9C,OAAO,EAAE,cAAc,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAEjE,OAAO,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAEvE,OAAO,EACL,aAAa,EACb,iBAAiB,EACjB,yBAAyB,EACzB,+BAA+B,EAC/B,qBAAqB,EACrB,sBAAsB,EACtB,qBAAqB,EACrB,wBAAwB,GACzB,MAAM,aAAa,CAAC"}

package/dist/bigquery/load.d.ts

@@ -1,4 +1,12 @@
 import type { BigQueryLike, BigQueryLoadOptions } from "./types.js";
+/**
+ * Opens the rows for one load attempt. Called once per attempt: a retry after
+ * a rate-limit error re-uploads everything, so it calls this again and must
+ * get the same rows. That is why the source is a function and not an
+ * iterable — a generator object can only be consumed once, so a retry handed
+ * the same one would silently load nothing.
+ */
+export type LoadRowSource<T extends object> = () => AsyncIterable<T> | Iterable<T>;
 export interface LoadRowsOptions extends BigQueryLoadOptions {
     datasetId: string;
     tableId: string;
@@ -6,17 +14,27 @@ export interface LoadRowsOptions extends BigQueryLoadOptions {
     attempts?: number;
     /** Delay before the first retry; doubles every subsequent retry. Default 1000ms. */
     initialDelayMs?: number;
+    /** Target size of each NDJSON chunk written to the load job, in bytes. The
+     *  helper holds at most about one chunk being built plus what the write
+     *  stream itself buffers. A single row larger than this is written as its
+     *  own chunk. Default 1 MiB. */
+    chunkBytes?: number;
     /** Injectable for tests — real code always uses a real timer. */
     sleep?: (ms: number) => Promise<void>;
 }
+export interface LoadRowsResult {
+    /** Rows the successful attempt uploaded. */
+    rowCount: number;
+}
 /**
- * Loads `rows` into `datasetId.tableId` in one load job, retrying with
- * exponential backoff ONLY when the failure is BigQuery's
+ * Streams the rows `source` yields into `datasetId.tableId` in one load job,
+ * retrying with exponential backoff ONLY when the failure is BigQuery's
  * rateLimitExceeded/quotaExceeded class of error (see
- * `isRetryableBigQueryError`). Any other error (bad schema, auth failure,
- * etc.) is classified and rethrown on the first attempt.
+ * `isRetryableBigQueryError`). Any other error (bad schema, auth failure, a
+ * throw from the source, etc.) is classified and rethrown on the first
+ * attempt. Each attempt calls `source` afresh — see `LoadRowSource`.
  *
- * Pass `writeDisposition: "WRITE_TRUNCATE"` for a full-refresh reload (the
- * IPEDS institutions loader's use case); omit it for an append load.
+ * Pass `writeDisposition: "WRITE_TRUNCATE"` for a full-refresh reload; omit it
+ * for an append load.
  */
-export declare function loadRowsWithRetry<T extends object>(bq: BigQueryLike, options: LoadRowsOptions, rows: T[]): Promise<void>;
+export declare function loadRowsWithRetry<T extends object>(bq: BigQueryLike, options: LoadRowsOptions, source: LoadRowSource<T>): Promise<LoadRowsResult>;

package/dist/bigquery/load.js

@@ -1,61 +1,91 @@
 /**
- * Load-job capability (AK-14 follow-on, MP-200/PLN-2): truncate-and-reload a
- * BigQuery table from an array of rows, with retry-with-backoff limited to
+ * Load-job capability (AK-14 follow-on, MP-200/PLN-2): loads a stream of rows
+ * into a BigQuery table in one load job, with retry-with-backoff limited to
  * BigQuery's own rate-limit/quota class of error — generalized out of
  * media-planner's `apps/api/src/lib/bigqueryLoad.ts` (via flytedesk-id's
  * verbatim port at `src/lib/bigqueryLoad.ts`, MP-200), which this module
- * replaces. Callers needing a periodic full-refresh load (e.g. the IPEDS
- * institutions loader) use this instead of keeping their own copy of the
- * write-stream + retry plumbing.
+ * replaces.
+ *
+ * Memory is bounded by `chunkBytes`, never by the row count (AK-28): rows are
+ * pulled from the caller's source one at a time, serialized into NDJSON chunks
+ * of about `chunkBytes`, and piped into the load job's write stream with
+ * `stream.pipeline`, which stops pulling from the source whenever the write
+ * stream's buffer is full and resumes on its `drain`. A caller that reads its
+ * rows page by page (a keyset-paged or cursor read) can therefore load any
+ * number of rows in a fixed amount of memory.
  */
-import { BigQueryReadError, classifyBigQueryError, isRetryableBigQueryError } from "./errors.js";
+import { once } from "node:events";
+import { pipeline } from "node:stream/promises";
+import { BigQueryReadError, classifyBigQueryError, isRetryableBigQueryError, } from "./errors.js";
 const DEFAULT_ATTEMPTS = 5;
 const DEFAULT_INITIAL_DELAY_MS = 1_000;
+const DEFAULT_CHUNK_BYTES = 1024 * 1024;
 const DEFAULT_SOURCE_FORMAT = "NEWLINE_DELIMITED_JSON";
 function defaultSleep(ms) {
     return new Promise((resolve) => setTimeout(resolve, ms));
 }
-async function runLoadJob(bq, options, rows) {
+/** Serializes `rows` into NDJSON, one `Buffer` of about `chunkBytes` at a
+ *  time, counting rows into `counter` as they are consumed. */
+async function* ndjsonChunks(rows, chunkBytes, counter) {
+    let lines = [];
+    let size = 0;
+    for await (const row of rows) {
+        const line = `${JSON.stringify(row)}\n`;
+        lines.push(line);
+        size += Buffer.byteLength(line);
+        counter.rows += 1;
+        if (size >= chunkBytes) {
+            yield Buffer.from(lines.join(""));
+            lines = [];
+            size = 0;
+        }
+    }
+    if (lines.length > 0)
+        yield Buffer.from(lines.join(""));
+}
+async function runLoadJob(bq, options, source) {
     const table = bq.dataset(options.datasetId).table(options.tableId);
     const createWriteStream = table.createWriteStream;
     if (!createWriteStream) {
         throw new BigQueryReadError(`dataset(${options.datasetId}).table(${options.tableId}) does not implement createWriteStream — this BigQueryLike was not constructed for load jobs`);
     }
-    const ndjson = rows.map((row) => `${JSON.stringify(row)}\n`);
-    await new Promise((resolve, reject) => {
-        const stream = createWriteStream.call(table, {
-            sourceFormat: options.sourceFormat ?? DEFAULT_SOURCE_FORMAT,
-            schema: options.schema,
-            writeDisposition: options.writeDisposition,
-            createDisposition: options.createDisposition,
-            labels: options.labels,
-            jobId: options.jobId,
-        });
-        stream.on("error", reject);
-        stream.on("complete", () => resolve());
-        for (const line of ndjson)
-            stream.write(line);
-        stream.end();
+    const stream = createWriteStream.call(table, {
+        sourceFormat: options.sourceFormat ?? DEFAULT_SOURCE_FORMAT,
+        schema: options.schema,
+        writeDisposition: options.writeDisposition,
+        createDisposition: options.createDisposition,
+        labels: options.labels,
+        jobId: options.jobId,
     });
+    const counter = { rows: 0 };
+    // The load is done only when BigQuery reports the job "complete" AND the
+    // pipeline has flushed every chunk. `once` rejects on the stream's "error",
+    // which is how a failed job surfaces; `pipeline` rejects (and destroys the
+    // stream, aborting the upload) when the source itself throws.
+    await Promise.all([
+        once(stream, "complete"),
+        pipeline(ndjsonChunks(source(), options.chunkBytes ?? DEFAULT_CHUNK_BYTES, counter), stream),
+    ]);
+    return { rowCount: counter.rows };
 }
 /**
- * Loads `rows` into `datasetId.tableId` in one load job, retrying with
- * exponential backoff ONLY when the failure is BigQuery's
+ * Streams the rows `source` yields into `datasetId.tableId` in one load job,
+ * retrying with exponential backoff ONLY when the failure is BigQuery's
  * rateLimitExceeded/quotaExceeded class of error (see
- * `isRetryableBigQueryError`). Any other error (bad schema, auth failure,
- * etc.) is classified and rethrown on the first attempt.
+ * `isRetryableBigQueryError`). Any other error (bad schema, auth failure, a
+ * throw from the source, etc.) is classified and rethrown on the first
+ * attempt. Each attempt calls `source` afresh — see `LoadRowSource`.
  *
- * Pass `writeDisposition: "WRITE_TRUNCATE"` for a full-refresh reload (the
- * IPEDS institutions loader's use case); omit it for an append load.
+ * Pass `writeDisposition: "WRITE_TRUNCATE"` for a full-refresh reload; omit it
+ * for an append load.
  */
-export async function loadRowsWithRetry(bq, options, rows) {
+export async function loadRowsWithRetry(bq, options, source) {
     const attempts = options.attempts ?? DEFAULT_ATTEMPTS;
     const initialDelayMs = options.initialDelayMs ?? DEFAULT_INITIAL_DELAY_MS;
     const sleep = options.sleep ?? defaultSleep;
     for (let attempt = 1;; attempt += 1) {
         try {
-            await runLoadJob(bq, options, rows);
-            return;
+            return await runLoadJob(bq, options, source);
         }
         catch (err) {
             if (attempt >= attempts || !isRetryableBigQueryError(err)) {

package/dist/bigquery/load.js.map

@@ -1 +1 @@
-{"version":3,"file":"load.js","sourceRoot":"","sources":["../../src/bigquery/load.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AACH,OAAO,EAAE,iBAAiB,EAAE,qBAAqB,EAAE,wBAAwB,EAAE,MAAM,aAAa,CAAC;AAcjG,MAAM,gBAAgB,GAAG,CAAC,CAAC;AAC3B,MAAM,wBAAwB,GAAG,KAAK,CAAC;AACvC,MAAM,qBAAqB,GAAG,wBAAwB,CAAC;AAEvD,SAAS,YAAY,CAAC,EAAU;IAC9B,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED,KAAK,UAAU,UAAU,CACvB,EAAgB,EAChB,OAAwB,EACxB,IAAS;IAET,MAAM,KAAK,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IACnE,MAAM,iBAAiB,GAAG,KAAK,CAAC,iBAAiB,CAAC;IAClD,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACvB,MAAM,IAAI,iBAAiB,CACzB,WAAW,OAAO,CAAC,SAAS,WAAW,OAAO,CAAC,OAAO,8FAA8F,CACrJ,CAAC;IACJ,CAAC;IACD,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAE7D,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC1C,MAAM,MAAM,GAAG,iBAAiB,CAAC,IAAI,CAAC,KAAK,EAAE;YAC3C,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,qBAAqB;YAC3D,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;YAC1C,iBAAiB,EAAE,OAAO,CAAC,iBAAiB;YAC5C,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,KAAK,EAAE,OAAO,CAAC,KAAK;SACrB,CAAC,CAAC;QACH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC3B,MAAM,CAAC,EAAE,CAAC,UAAU,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC;QACvC,KAAK,MAAM,IAAI,IAAI,MAAM;YAAE,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC9C,MAAM,CAAC,GAAG,EAAE,CAAC;IACf,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,EAAgB,EAChB,OAAwB,EACxB,IAAS;IAET,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,gBAAgB,CAAC;IACtD,MAAM,cAAc,GAAG,OAAO,CAAC,cAAc,IAAI,wBAAwB,CAAC;IAC1E,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,YAAY,CAAC;IAE5C,KAAK,IAAI,OAAO,GAAG,CAAC,GAAI,OAAO,IAAI,CAAC,EAAE,CAAC;QACrC,IAAI,CAAC;YACH,MAAM,UAAU,CAAC,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;YACpC,OAAO;QACT,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,OAAO,IAAI,QAAQ,IAAI,CAAC,wBAAwB,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC1D,MAAM,qBAAqB,CAAC,GAAG,CAAC,CAAC;YACnC,CAAC;YACD,MAAM,OAAO,GAAG,cAAc,GAAG,CAAC,IAAI,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC;YACpD,OAAO,CAAC,IAAI,CACV,mBAAmB,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC,OAAO,6BAA6B,OAAO,IAAI,QAAQ,iBAAiB,OAAO,IAAI,CACpI,CAAC;YACF,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC;QACvB,CAAC;IACH,CAAC;AACH,CAAC"}
+{"version":3,"file":"load.js","sourceRoot":"","sources":["../../src/bigquery/load.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AACH,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AACnC,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAChD,OAAO,EACL,iBAAiB,EACjB,qBAAqB,EACrB,wBAAwB,GACzB,MAAM,aAAa,CAAC;AAkCrB,MAAM,gBAAgB,GAAG,CAAC,CAAC;AAC3B,MAAM,wBAAwB,GAAG,KAAK,CAAC;AACvC,MAAM,mBAAmB,GAAG,IAAI,GAAG,IAAI,CAAC;AACxC,MAAM,qBAAqB,GAAG,wBAAwB,CAAC;AAEvD,SAAS,YAAY,CAAC,EAAU;IAC9B,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED;+DAC+D;AAC/D,KAAK,SAAS,CAAC,CAAC,YAAY,CAC1B,IAAoC,EACpC,UAAkB,EAClB,OAAyB;IAEzB,IAAI,KAAK,GAAa,EAAE,CAAC;IACzB,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QAC7B,MAAM,IAAI,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC;QACxC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjB,IAAI,IAAI,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;QAChC,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC;QAClB,IAAI,IAAI,IAAI,UAAU,EAAE,CAAC;YACvB,MAAM,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;YAClC,KAAK,GAAG,EAAE,CAAC;YACX,IAAI,GAAG,CAAC,CAAC;QACX,CAAC;IACH,CAAC;IACD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;QAAE,MAAM,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;AAC1D,CAAC;AAED,KAAK,UAAU,UAAU,CACvB,EAAgB,EAChB,OAAwB,EACxB,MAAwB;IAExB,MAAM,KAAK,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IACnE,MAAM,iBAAiB,GAAG,KAAK,CAAC,iBAAiB,CAAC;IAClD,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACvB,MAAM,IAAI,iBAAiB,CACzB,WAAW,OAAO,CAAC,SAAS,WAAW,OAAO,CAAC,OAAO,8FAA8F,CACrJ,CAAC;IACJ,CAAC;IACD,MAAM,MAAM,GAAG,iBAAiB,CAAC,IAAI,CAAC,KAAK,EAAE;QAC3C,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,qBAAqB;QAC3D,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;QAC1C,iBAAiB,EAAE,OAAO,CAAC,iBAAiB;QAC5C,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,KAAK,EAAE,OAAO,CAAC,KAAK;KACrB,CAAC,CAAC;IACH,MAAM,OAAO,GAAG,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;IAC5B,yEAAyE;IACzE,4EAA4E;IAC5E,2EAA2E;IAC3E,8DAA8D;IAC9D,MAAM,OAAO,CAAC,GAAG,CAAC;QAChB,IAAI,CAAC,MAAM,EAAE,UAAU,CAAC;QACxB,QAAQ,CACN,YAAY,CACV,MAAM,EAAE,EACR,OAAO,CAAC,UAAU,IAAI,mBAAmB,EACzC,OAAO,CACR,EACD,MAAM,CACP;KACF,CAAC,CAAC;IACH,OAAO,EAAE,QAAQ,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC;AACpC,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,EAAgB,EAChB,OAAwB,EACxB,MAAwB;IAExB,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,gBAAgB,CAAC;IACtD,MAAM,cAAc,GAAG,OAAO,CAAC,cAAc,IAAI,wBAAwB,CAAC;IAC1E,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,YAAY,CAAC;IAE5C,KAAK,IAAI,OAAO,GAAG,CAAC,GAAI,OAAO,IAAI,CAAC,EAAE,CAAC;QACrC,IAAI,CAAC;YACH,OAAO,MAAM,UAAU,CAAC,EAAE,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;QAC/C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,OAAO,IAAI,QAAQ,IAAI,CAAC,wBAAwB,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC1D,MAAM,qBAAqB,CAAC,GAAG,CAAC,CAAC;YACnC,CAAC;YACD,MAAM,OAAO,GAAG,cAAc,GAAG,CAAC,IAAI,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC;YACpD,OAAO,CAAC,IAAI,CACV,mBAAmB,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC,OAAO,6BAA6B,OAAO,IAAI,QAAQ,iBAAiB,OAAO,IAAI,CACpI,CAAC;YACF,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC;QACvB,CAAC;IACH,CAAC;AACH,CAAC"}

package/dist/bigquery/query.d.ts

@@ -1,10 +1,17 @@
 import type { BigQueryLike, DryRunEstimate, EstimateQueryBytesOptions, QueryPage, RunQueryOptions } from "./types.js";
 /** Runs one page of a parameterised, read-only query. Always bind
  *  user-supplied values through `options.params`/`options.types` — never
- *  string-interpolate them into `options.sql`. */
+ *  string-interpolate them into `options.sql`.
+ *
+ *  A BigQuery page token is only valid against the job that produced it
+ *  (AK-26) — passing `pageToken` therefore REQUIRES `jobId` (the `jobId` a
+ *  prior `QueryPage` returned) so this fetches the *same* job's next page
+ *  instead of creating a fresh job and handing it a token it never issued. */
 export declare function runQuery<T = Record<string, unknown>>(bq: BigQueryLike, options: RunQueryOptions): Promise<QueryPage<T>>;
 /** Pages through every result of a query, yielding one row-array per page.
- *  Stops once BigQuery reports no further `nextPageToken`. */
+ *  Stops once BigQuery reports no further `nextPageToken`. Carries the
+ *  producing job's id forward across pages (AK-26) so every page after the
+ *  first resumes the *same* job instead of starting a new one. */
 export declare function runQueryAllPages<T = Record<string, unknown>>(bq: BigQueryLike, options: RunQueryOptions): AsyncGenerator<T[], void, void>;
 /**
  * Dry-runs a query to get its billed-byte estimate without actually running

package/dist/bigquery/query.js

@@ -4,25 +4,41 @@
  * (AK-11). Inventory-specific SQL stays in media-planner; this module only
  * ever takes SQL + params from the caller and runs it.
  */
-import { classifyBigQueryError } from "./errors.js";
+import { BadQueryError, classifyBigQueryError } from "./errors.js";
 import { DryRunBudgetExceededError } from "./errors.js";
 /** Runs one page of a parameterised, read-only query. Always bind
  *  user-supplied values through `options.params`/`options.types` — never
- *  string-interpolate them into `options.sql`. */
+ *  string-interpolate them into `options.sql`.
+ *
+ *  A BigQuery page token is only valid against the job that produced it
+ *  (AK-26) — passing `pageToken` therefore REQUIRES `jobId` (the `jobId` a
+ *  prior `QueryPage` returned) so this fetches the *same* job's next page
+ *  instead of creating a fresh job and handing it a token it never issued. */
 export async function runQuery(bq, options) {
     let job;
-    try {
-        [job] = await bq.createQueryJob({
-            query: options.sql,
-            params: options.params,
-            types: options.types,
-            labels: options.labels,
-            location: options.location,
-            maxResults: options.pageSize,
-        });
+    if (options.pageToken !== undefined) {
+        if (!options.jobId) {
+            throw new BadQueryError("runQuery: pageToken requires the jobId of the job that produced it (QueryPage.jobId)");
+        }
+        job = bq.job(options.jobId);
     }
-    catch (err) {
-        throw classifyBigQueryError(err);
+    else {
+        try {
+            [job] = await bq.createQueryJob({
+                query: options.sql,
+                params: options.params,
+                types: options.types,
+                labels: options.labels,
+                location: options.location,
+                maxResults: options.pageSize,
+                maximumBytesBilled: options.maximumBytesBilled !== undefined
+                    ? String(options.maximumBytesBilled)
+                    : undefined,
+            });
+        }
+        catch (err) {
+            throw classifyBigQueryError(err);
+        }
     }
     try {
         const [rows, next] = await job.getQueryResults({
@@ -31,7 +47,7 @@ export async function runQuery(bq, options) {
         });
         return {
             rows,
-            jobId: job.id ?? "",
+            jobId: job.id ?? options.jobId ?? "",
             nextPageToken: next?.pageToken,
         };
     }
@@ -40,15 +56,19 @@ export async function runQuery(bq, options) {
     }
 }
 /** Pages through every result of a query, yielding one row-array per page.
- *  Stops once BigQuery reports no further `nextPageToken`. */
+ *  Stops once BigQuery reports no further `nextPageToken`. Carries the
+ *  producing job's id forward across pages (AK-26) so every page after the
+ *  first resumes the *same* job instead of starting a new one. */
 export async function* runQueryAllPages(bq, options) {
     let pageToken = options.pageToken;
+    let jobId = options.jobId;
     for (;;) {
-        const page = await runQuery(bq, { ...options, pageToken });
+        const page = await runQuery(bq, { ...options, pageToken, jobId });
         yield page.rows;
         if (!page.nextPageToken)
             return;
         pageToken = page.nextPageToken;
+        jobId = page.jobId;
     }
 }
 /**
@@ -76,6 +96,8 @@ export async function estimateQueryBytes(bq, options) {
     const estimate = {
         totalBytesProcessed: Number(stats?.totalBytesProcessed ?? 0),
         cacheHit: stats?.cacheHit ?? false,
+        statementType: stats?.statementType,
+        referencedTables: stats?.referencedTables,
     };
     if (options.maxBytesBilled !== undefined &&
         estimate.totalBytesProcessed > options.maxBytesBilled) {

package/dist/bigquery/query.js.map

@@ -1 +1 @@
-{"version":3,"file":"query.js","sourceRoot":"","sources":["../../src/bigquery/query.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,OAAO,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AACpD,OAAO,EAAE,yBAAyB,EAAE,MAAM,aAAa,CAAC;AASxD;;kDAEkD;AAClD,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,EAAgB,EAChB,OAAwB;IAExB,IAAI,GAAG,CAAC;IACR,IAAI,CAAC;QACH,CAAC,GAAG,CAAC,GAAG,MAAM,EAAE,CAAC,cAAc,CAAC;YAC9B,KAAK,EAAE,OAAO,CAAC,GAAG;YAClB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,UAAU,EAAE,OAAO,CAAC,QAAQ;SAC7B,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,qBAAqB,CAAC,GAAG,CAAC,CAAC;IACnC,CAAC;IAED,IAAI,CAAC;QACH,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,MAAM,GAAG,CAAC,eAAe,CAAI;YAChD,UAAU,EAAE,OAAO,CAAC,QAAQ;YAC5B,SAAS,EAAE,OAAO,CAAC,SAAS;SAC7B,CAAC,CAAC;QACH,OAAO;YACL,IAAI;YACJ,KAAK,EAAE,GAAG,CAAC,EAAE,IAAI,EAAE;YACnB,aAAa,EAAE,IAAI,EAAE,SAAS;SAC/B,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,qBAAqB,CAAC,GAAG,CAAC,CAAC;IACnC,CAAC;AACH,CAAC;AAED;8DAC8D;AAC9D,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,gBAAgB,CACrC,EAAgB,EAChB,OAAwB;IAExB,IAAI,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;IAClC,SAAS,CAAC;QACR,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAI,EAAE,EAAE,EAAE,GAAG,OAAO,EAAE,SAAS,EAAE,CAAC,CAAC;QAC9D,MAAM,IAAI,CAAC,IAAI,CAAC;QAChB,IAAI,CAAC,IAAI,CAAC,aAAa;YAAE,OAAO;QAChC,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC;IACjC,CAAC;AACH,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,EAAgB,EAChB,OAAkC;IAElC,IAAI,QAAQ,CAAC;IACb,IAAI,CAAC;QACH,CAAC,EAAE,QAAQ,CAAC,GAAG,MAAM,EAAE,CAAC,cAAc,CAAC;YACrC,KAAK,EAAE,OAAO,CAAC,GAAG;YAClB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,MAAM,EAAE,IAAI;SACb,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,qBAAqB,CAAC,GAAG,CAAC,CAAC;IACnC,CAAC;IAED,MAAM,KAAK,GAAG,QAAQ,CAAC,UAAU,EAAE,KAAK,CAAC;IACzC,MAAM,QAAQ,GAAmB;QAC/B,mBAAmB,EAAE,MAAM,CAAC,KAAK,EAAE,mBAAmB,IAAI,CAAC,CAAC;QAC5D,QAAQ,EAAE,KAAK,EAAE,QAAQ,IAAI,KAAK;KACnC,CAAC;IAEF,IACE,OAAO,CAAC,cAAc,KAAK,SAAS;QACpC,QAAQ,CAAC,mBAAmB,GAAG,OAAO,CAAC,cAAc,EACrD,CAAC;QACD,MAAM,IAAI,yBAAyB,CACjC,QAAQ,CAAC,mBAAmB,EAC5B,OAAO,CAAC,cAAc,CACvB,CAAC;IACJ,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC"}
+{"version":3,"file":"query.js","sourceRoot":"","sources":["../../src/bigquery/query.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,OAAO,EAAE,aAAa,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AACnE,OAAO,EAAE,yBAAyB,EAAE,MAAM,aAAa,CAAC;AASxD;;;;;;;8EAO8E;AAC9E,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,EAAgB,EAChB,OAAwB;IAExB,IAAI,GAAG,CAAC;IACR,IAAI,OAAO,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;QACpC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YACnB,MAAM,IAAI,aAAa,CACrB,sFAAsF,CACvF,CAAC;QACJ,CAAC;QACD,GAAG,GAAG,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAC9B,CAAC;SAAM,CAAC;QACN,IAAI,CAAC;YACH,CAAC,GAAG,CAAC,GAAG,MAAM,EAAE,CAAC,cAAc,CAAC;gBAC9B,KAAK,EAAE,OAAO,CAAC,GAAG;gBAClB,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,QAAQ,EAAE,OAAO,CAAC,QAAQ;gBAC1B,UAAU,EAAE,OAAO,CAAC,QAAQ;gBAC5B,kBAAkB,EAChB,OAAO,CAAC,kBAAkB,KAAK,SAAS;oBACtC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,kBAAkB,CAAC;oBACpC,CAAC,CAAC,SAAS;aAChB,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,qBAAqB,CAAC,GAAG,CAAC,CAAC;QACnC,CAAC;IACH,CAAC;IAED,IAAI,CAAC;QACH,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,MAAM,GAAG,CAAC,eAAe,CAAI;YAChD,UAAU,EAAE,OAAO,CAAC,QAAQ;YAC5B,SAAS,EAAE,OAAO,CAAC,SAAS;SAC7B,CAAC,CAAC;QACH,OAAO;YACL,IAAI;YACJ,KAAK,EAAE,GAAG,CAAC,EAAE,IAAI,OAAO,CAAC,KAAK,IAAI,EAAE;YACpC,aAAa,EAAE,IAAI,EAAE,SAAS;SAC/B,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,qBAAqB,CAAC,GAAG,CAAC,CAAC;IACnC,CAAC;AACH,CAAC;AAED;;;kEAGkE;AAClE,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,gBAAgB,CACrC,EAAgB,EAChB,OAAwB;IAExB,IAAI,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;IAClC,IAAI,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;IAC1B,SAAS,CAAC;QACR,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAI,EAAE,EAAE,EAAE,GAAG,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;QACrE,MAAM,IAAI,CAAC,IAAI,CAAC;QAChB,IAAI,CAAC,IAAI,CAAC,aAAa;YAAE,OAAO;QAChC,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC;QAC/B,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;IACrB,CAAC;AACH,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,EAAgB,EAChB,OAAkC;IAElC,IAAI,QAAQ,CAAC;IACb,IAAI,CAAC;QACH,CAAC,EAAE,QAAQ,CAAC,GAAG,MAAM,EAAE,CAAC,cAAc,CAAC;YACrC,KAAK,EAAE,OAAO,CAAC,GAAG;YAClB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,MAAM,EAAE,IAAI;SACb,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,qBAAqB,CAAC,GAAG,CAAC,CAAC;IACnC,CAAC;IAED,MAAM,KAAK,GAAG,QAAQ,CAAC,UAAU,EAAE,KAAK,CAAC;IACzC,MAAM,QAAQ,GAAmB;QAC/B,mBAAmB,EAAE,MAAM,CAAC,KAAK,EAAE,mBAAmB,IAAI,CAAC,CAAC;QAC5D,QAAQ,EAAE,KAAK,EAAE,QAAQ,IAAI,KAAK;QAClC,aAAa,EAAE,KAAK,EAAE,aAAa;QACnC,gBAAgB,EAAE,KAAK,EAAE,gBAAgB;KAC1C,CAAC;IAEF,IACE,OAAO,CAAC,cAAc,KAAK,SAAS;QACpC,QAAQ,CAAC,mBAAmB,GAAG,OAAO,CAAC,cAAc,EACrD,CAAC;QACD,MAAM,IAAI,yBAAyB,CACjC,QAAQ,CAAC,mBAAmB,EAC5B,OAAO,CAAC,cAAc,CACvB,CAAC;IACJ,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC"}

package/dist/bigquery/types.d.ts

@@ -11,6 +11,7 @@
  * `@google-cloud/bigquery` calls it made; inventory-specific SQL stays behind
  * in media-planner.
  */
+import type { Writable } from "node:stream";
 /** One structured error entry, same shape the Google APIs client library
  *  attaches to a rejected job promise's `.errors` array. */
 export interface BigQueryApiErrorDetail {
@@ -23,11 +24,24 @@ export interface BigQueryJobStatus {
     errorResult?: BigQueryApiErrorDetail | null;
     errors?: BigQueryApiErrorDetail[];
 }
+/** A table a dry-run job reports the query as touching — the SDK's
+ *  `ITableReference` (bigquery.d.ts), all fields optional per the REST API
+ *  even though BigQuery always populates them in practice. */
+export interface BigQueryTableReference {
+    projectId?: string;
+    datasetId?: string;
+    tableId?: string;
+}
 export interface BigQueryJobStatistics {
     query?: {
         totalBytesProcessed?: string | number;
         totalBytesBilled?: string | number;
         cacheHit?: boolean;
+        /** The dry-run job's classified statement kind, e.g. `"SELECT"` —
+         *  `IJobStatistics2.statementType` in the real SDK. */
+        statementType?: string;
+        /** Every table the query touches — `IJobStatistics2.referencedTables`. */
+        referencedTables?: BigQueryTableReference[];
     };
 }
 export interface BigQueryJobMetadata {
@@ -72,6 +86,12 @@ export interface BigQueryCreateQueryJobOptions {
     location?: string;
     maxResults?: number;
     jobId?: string;
+    /** BigQuery's own hard billing cap on the real job — the SDK's
+     *  `IJobConfigurationQuery.maximumBytesBilled` is a decimal string; a job
+     *  that would bill more than this is refused by BigQuery itself rather
+     *  than run. Distinct from `EstimateQueryBytesOptions.maxBytesBilled`
+     *  (a pre-flight dry-run check this module enforces client-side). */
+    maximumBytesBilled?: string;
 }
 export interface BigQueryExtractOptions {
     /** `@google-cloud/bigquery`'s own extract-job vocabulary — note this is
@@ -90,15 +110,6 @@ export interface GcsFileLike {
         readonly name: string;
     };
 }
-/** Structural subset of Node's own Writable that `Table#createWriteStream`
- *  needs to expose — the "error"/"complete" events a load-job caller pipes
- *  rows into and waits on. */
-export interface BigQueryWriteStreamLike {
-    on(event: "error", listener: (err: unknown) => void): this;
-    on(event: "complete", listener: () => void): this;
-    write(chunk: string): boolean;
-    end(): void;
-}
 /** One BigQuery table-schema field, as `createWriteStream`'s `schema` option
  *  (and the REST API's `tables.insert`) accepts it. `fields` recurses for
  *  RECORD/STRUCT columns. */
@@ -124,12 +135,15 @@ export interface BigQueryLoadOptions {
 }
 export interface BigQueryTableLike {
     createExtractJob(destination: GcsFileLike, options?: BigQueryExtractOptions): Promise<[BigQueryJobLike, BigQueryJobMetadata]>;
-    /** Opens a load-job write stream — the caller pipes newline-delimited rows
-     *  (or other `sourceFormat`) into it, then awaits the stream's "complete"/
-     *  "error" event. Mirrors `@google-cloud/bigquery`'s `Table#createWriteStream`.
-     *  Optional: read-only callers (query/extract) never need it — only
-     *  `loadRowsWithRetry` (./load.js) does, and it fails loudly if missing. */
-    createWriteStream?(options: BigQueryLoadOptions): BigQueryWriteStreamLike;
+    /** Opens a load-job write stream — a real Node `Writable`, since the loader
+     *  pipes NDJSON into it with `stream.pipeline` and relies on its
+     *  backpressure (`write()` returning false, then "drain"). The stream emits
+     *  "complete" once BigQuery finishes the job, or "error" if the job fails.
+     *  Mirrors `@google-cloud/bigquery`'s `Table#createWriteStream`, which
+     *  returns a `Writable`. Optional: read-only callers (query/extract) never
+     *  need it — only `loadRowsWithRetry` (./load.js) does, and it fails loudly
+     *  if missing. */
+    createWriteStream?(options: BigQueryLoadOptions): Writable;
 }
 export interface BigQueryDatasetLike {
     table(id: string): BigQueryTableLike;
@@ -151,8 +165,23 @@ export interface RunQueryOptions {
     location?: string;
     /** Page size. Omit to let BigQuery pick a default. */
     pageSize?: number;
-    /** Resume a prior page — from `QueryPage.nextPageToken`. */
+    /** Resume a prior page — from `QueryPage.nextPageToken`. A page token is
+     *  only valid against the job that produced it, so `jobId` (from that same
+     *  `QueryPage.jobId`) is REQUIRED whenever this is set — `runQuery` throws
+     *  a `BadQueryError` otherwise rather than creating a new job and handing
+     *  it a token it never issued (AK-26). */
     pageToken?: string;
+    /** The job that produced `pageToken` — required together with it to fetch
+     *  the next page of that same job instead of starting a new one. */
+    jobId?: string;
+    /** Hard cap BigQuery itself enforces on the real job — a job that would
+     *  bill more than this many bytes is refused by BigQuery before running,
+     *  classified as `MaximumBytesBilledExceededError` (see errors.ts).
+     *  Distinct from `estimateQueryBytes`'s `maxBytesBilled`, which is a
+     *  client-side pre-flight check on a *dry-run* estimate — this is the
+     *  belt to that check's suspenders, since data can grow between a dry
+     *  run and the real job, or a caller can skip the dry run entirely. */
+    maximumBytesBilled?: number;
 }
 export interface QueryPage<T = Record<string, unknown>> {
     rows: T[];
@@ -172,6 +201,16 @@ export interface EstimateQueryBytesOptions {
 export interface DryRunEstimate {
     totalBytesProcessed: number;
     cacheHit: boolean;
+    /** The dry run's classified statement kind, e.g. `"SELECT"` — `undefined`
+     *  when BigQuery's dry-run statistics omitted it. A caller enforcing
+     *  read-only access should reject anything other than `"SELECT"`. */
+    statementType?: string;
+    /** Every table the query touches, per BigQuery's own dry-run analysis —
+     *  `undefined` when the statistics omitted it, `[]` when the query
+     *  genuinely touches none. The allowlist-enforcement surface: a caller
+     *  checks every entry against its own read-only table allowlist before
+     *  ever submitting the real job. */
+    referencedTables?: BigQueryTableReference[];
 }
 export interface ExtractTableToGCSInput {
     datasetId: string;