@adcp/sdk 15.8.0 → 15.9.0

package/dist/lib/reporting/bigquery/index.d.ts

@@ -16,6 +16,8 @@ import type { PostgresReportingLedgerStore, ReportingLedgerRevisionMetadataV1, R
  * BigQuery refuses to run twice. A failed load job is retried under a new
  * attempt suffix only after BigQuery reports that job finished with an error,
  * and each table keeps its own attempt count, so no batch lands twice.
+ * Concurrent runs claim one durable plan with compare-and-swap. Replays of
+ * that plan share job IDs; stale runs cannot change its attempts or cursor.
  */
 /** Apply after `REPORTING_LEDGER_CHANGES_MIGRATION`. Holds one row per sink. */
 export declare const REPORTING_WAREHOUSE_SINK_MIGRATION: string;
@@ -30,9 +32,12 @@ export interface ReportingBigQueryClientV1 {
         };
     };
     job(jobId: string, options?: {
+        projectId?: string;
         location?: string;
     }): {
         getMetadata(): Promise<unknown>;
+        /** Request cancellation. Optional so wrappers written before the sink used it still type-check. */
+        cancel?(): Promise<unknown>;
     };
     query(options: {
         query: string;
@@ -45,12 +50,25 @@ export interface ReportingWarehouseRowContextV1 {
 }
 export interface CreateBigQueryReportingWarehouseSinkOptionsV1 {
     /** Ledger store with `changeFeed: true`. */
-    store: Pick<PostgresReportingLedgerStore, 'changesAfter' | 'saveFeedConsumerCursor' | 'getRevisionMetadata' | 'readRevisionRows'>;
+    store: Pick<PostgresReportingLedgerStore, 'changesAfter' | 'getRevisionMetadata' | 'readRevisionRows'> & {
+        /**
+         * The sink ignores the result, so wrappers written when this resolved
+         * `void` keep compiling.
+         */
+        saveFeedConsumerCursor(name: string, cursor: string, options?: {
+            refreshUnchanged?: boolean;
+        }): Promise<unknown>;
+    } & /** Needed only by `reset()`; custom store wrappers that predate it keep compiling. */ Partial<Pick<PostgresReportingLedgerStore, 'removeFeedConsumer'>>;
     /** PostgreSQL pool holding `REPORTING_WAREHOUSE_SINK_MIGRATION`. */
     db: Pick<ReportingPgPool, 'query'>;
     /** Host-constructed official BigQuery client. */
     bigquery: ReportingBigQueryClientV1;
-    /** Stable sink name; also the registered change-feed consumer name. */
+    /**
+     * Stable sink and consumer name. All replicas sharing it must use the same
+     * ledger database, feed scope, destination, location and row mapping. Use
+     * a new name when that configuration changes. Names must also be unique
+     * across independent ledgers sharing a BigQuery project.
+     */
     name: string;
     projectId?: string;
     datasetId: string;
@@ -69,19 +87,38 @@ export interface CreateBigQueryReportingWarehouseSinkOptionsV1 {
     mapRow?(row: Record<string, unknown>, context: ReportingWarehouseRowContextV1): Record<string, unknown>;
     /** Revisions per batch. Default 200. */
     maxRevisionsPerBatch?: number;
-    /** Job labels for cost attribution. */
+    /**
+     * @deprecated Has no effect, and the sink warns when it is set. The official
+     * client submits a local-file load by uploading a job resource whose
+     * `configuration.load` is built from the load options, and offers no way to
+     * set `configuration.labels`; BigQuery silently drops `labels` found inside
+     * `configuration.load`. For cost attribution, filter
+     * `INFORMATION_SCHEMA.JOBS` on `destination_table` or on the `adcp_rr_`
+     * job ID prefix instead.
+     */
     labels?: Readonly<Record<string, string>>;
     /**
-     * Deadline for one load (and its status check). Default 10 minutes. It is
-     * also sent as the job's `jobTimeoutMs`. A run that times out leaves its
-     * plan in place; the next run reuses the same job ID and waits for, or
-     * accepts, the job BigQuery is still running.
+     * Deadline for one load (and for each status check). Default 10 minutes.
+     * When a load exceeds it, the sink asks BigQuery to cancel that job by its
+     * deterministic ID (in `projectId` and `location`) and rejects, leaving the
+     * plan in place. The next run finds the cancelled job finished with an
+     * error, advances that table's attempt and rejects; the run after that
+     * loads under the new attempt. If the cancel has not taken effect yet the
+     * next run waits for the job. Each attempt doubles the load deadline (up to
+     * 6 hours), so a load that is slow but healthy converges; set this above
+     * your slowest expected load to avoid the cancelled attempts. A status
+     * check that times out is not cancelled. This is a client-side deadline:
+     * BigQuery's server-side `jobTimeoutMs` cannot be set on a local-file load
+     * through the official client, and cancellation is best effort.
      */
     loadTimeoutMilliseconds?: number;
 }
 export interface BigQueryReportingWarehouseSinkV1 {
     /**
-     * Load at most one batch. Resolves with what was loaded; `idle` when nothing new committed.
+     * Load at most one batch. Resolves with what was loaded; `idle` when no
+     * revision changes were selected. Idle ticks can advance past other feed changes.
+     * An overlapping run that loses its plan or cursor claim rejects. Retry
+     * `runOnce()` to resume the current durable plan or committed cursor.
      *
      * `missing` lists revision IDs the change feed announced that no longer
      * exist in the ledger because their period was retired before this batch
@@ -100,6 +137,33 @@ export interface BigQueryReportingWarehouseSinkV1 {
         rows: number;
         missing: string[];
     }>;
+    /**
+     * Clear the sink's saved state so the next `runOnce()` starts again from the
+     * oldest retained change, and unregister its feed consumer so the old
+     * checkpoint stops holding back retention. This is the way out of a sink
+     * that keeps rejecting with `CURSOR_EXPIRED`: a brand-new sink whose first
+     * read lost a startup race with pruning, or a sink that fell behind
+     * retention and has been resynchronized from current revisions. Stop every
+     * worker using this sink name first.
+     *
+     * It refuses while a batch plan is retained, because that batch's load jobs
+     * may already have succeeded; pass `discardRetainedPlan: true` once you have
+     * reconciled it against BigQuery. The warehouse is not touched: a sink that
+     * already loaded history will load the retained revisions again, so empty
+     * the destination tables first, or use a new sink name against new tables.
+     * Each reset starts a new job ID generation, so reloaded batches submit
+     * fresh BigQuery jobs instead of matching the finished jobs of the earlier
+     * history. Nothing holds change retention between `reset()` and the first
+     * committed batch. If `reset()` rejects after clearing the sink's state
+     * (the consumer could not be unregistered), call it again before restarting
+     * workers; it is idempotent. Needs a store with `removeFeedConsumer`.
+     * Resolves with whether a saved sink state existed.
+     */
+    reset(options?: {
+        discardRetainedPlan?: boolean;
+    }): Promise<{
+        cleared: boolean;
+    }>;
     /** DDL for the default rows and revisions tables (partitioned and clustered). */
     readonly defaultTablesSql: string;
     /** DDL for the reference `current_rows` view. */
@@ -107,7 +171,7 @@ export interface BigQueryReportingWarehouseSinkV1 {
 }
 export declare function createBigQueryReportingWarehouseSinkV1(options: CreateBigQueryReportingWarehouseSinkOptionsV1): BigQueryReportingWarehouseSinkV1;
 /** Deterministic load job ID for one planned change-feed range. */
-export declare function reportingWarehouseJobIdV1(name: string, from: string | null, to: string): string;
+export declare function reportingWarehouseJobIdV1(name: string, from: string | null, to: string, generation?: number): string;
 interface WarehouseTablesV1 {
     projectId?: string;
     datasetId: string;

package/dist/lib/reporting/bigquery/index.js

@@ -39,6 +39,7 @@ var import_crypto = require("crypto");
 var import_fs = require("fs");
 var import_os = require("os");
 var path = __toESM(require("path"));
+var import_change_feed = require('../ledger/change-feed.js');
 const REPORTING_WAREHOUSE_SINK_MIGRATION = `
 CREATE TABLE IF NOT EXISTS adcp_reporting_warehouse_sinks (
   sink_name TEXT PRIMARY KEY CHECK (sink_name ~ '^[A-Za-z0-9_.:-]{1,128}$'),
@@ -48,10 +49,15 @@ CREATE TABLE IF NOT EXISTS adcp_reporting_warehouse_sinks (
   planned_revisions JSONB,
   planned_rows_attempt INTEGER NOT NULL DEFAULT 0 CHECK (planned_rows_attempt >= 0),
   planned_revisions_attempt INTEGER NOT NULL DEFAULT 0 CHECK (planned_revisions_attempt >= 0),
+  reset_generation INTEGER NOT NULL DEFAULT 0 CHECK (reset_generation >= 0),
   updated_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
   CHECK ((planned_to_cursor IS NULL) = (planned_revisions IS NULL))
 );
+ALTER TABLE adcp_reporting_warehouse_sinks
+  ADD COLUMN IF NOT EXISTS reset_generation INTEGER NOT NULL DEFAULT 0 CHECK (reset_generation >= 0);
 `.trim();
+const MAX_LOAD_TIMEOUT = 6 * 36e5;
+let warnedLabelsIgnored = false;
 const PROJECT = /^[a-z][a-z0-9-]{4,61}[a-z0-9]$/;
 const DATASET_OR_TABLE = /^[A-Za-z_][A-Za-z0-9_]{0,1023}$/;
 const NAME = /^[A-Za-z0-9_.:-]{1,128}$/;
@@ -68,13 +74,20 @@ function createBigQueryReportingWarehouseSinkV1(options) {
     if (!DATASET_OR_TABLE.test(value)) throw new TypeError(`BigQuery ${label} is invalid`);
   }
   const loadTimeout = options.loadTimeoutMilliseconds ?? 6e5;
-  if (!Number.isSafeInteger(loadTimeout) || loadTimeout < 1e3 || loadTimeout > 6 * 36e5) {
+  if (!Number.isSafeInteger(loadTimeout) || loadTimeout < 1e3 || loadTimeout > MAX_LOAD_TIMEOUT) {
     throw new RangeError("loadTimeoutMilliseconds must be between 1 second and 6 hours");
   }
   const maxRevisions = options.maxRevisionsPerBatch ?? 200;
   if (!Number.isSafeInteger(maxRevisions) || maxRevisions < 1 || maxRevisions > 1e4) {
     throw new RangeError("maxRevisionsPerBatch must be between 1 and 10000");
   }
+  if (options.labels && Object.keys(options.labels).length > 0 && !warnedLabelsIgnored) {
+    warnedLabelsIgnored = true;
+    process.emitWarning(
+      "BigQuery warehouse sink option `labels` has no effect: BigQuery ignores labels inside a load job's configuration.load and the official client cannot set configuration.labels for local-file loads. Attribute cost through INFORMATION_SCHEMA.JOBS by destination table or the adcp_rr_ job ID prefix.",
+      { code: "ADCP_BIGQUERY_SINK_LABELS_IGNORED", type: "DeprecationWarning" }
+    );
+  }
   const tables = {
     projectId: options.projectId,
     datasetId: options.datasetId,
@@ -85,29 +98,55 @@ function createBigQueryReportingWarehouseSinkV1(options) {
     ...options.projectId ? { projectId: options.projectId } : {},
     ...options.location ? { location: options.location } : {}
   });
-  async function loadOnce(tableId, jobId, lines, signal) {
+  const matchesPlan = `WHERE sink_name = $1
+    AND committed_cursor IS NOT DISTINCT FROM $2
+    AND planned_from_cursor IS NOT DISTINCT FROM $2
+    AND planned_to_cursor = $3 AND planned_revisions = $4::jsonb
+    AND planned_rows_attempt = $5 AND planned_revisions_attempt = $6
+    AND reset_generation = $7`;
+  const stalePlan = () => new Error("Warehouse sink plan changed during an overlapping run; retry runOnce()");
+  const jobOptions = options.projectId || options.location ? {
+    ...options.projectId ? { projectId: options.projectId } : {},
+    ...options.location ? { location: options.location } : {}
+  } : void 0;
+  async function cancelLoad(jobId, signal) {
+    try {
+      const job = options.bigquery.job(jobId, jobOptions);
+      if (typeof job.cancel !== "function") return;
+      const cancel = job.cancel.bind(job);
+      await withinDeadline(cancel, Math.min(loadTimeout, 3e4), signal);
+    } catch (error) {
+      const code = error?.code;
+      if (code === 404 || signal?.aborted) return;
+      process.emitWarning(
+        `Cancelling BigQuery load job ${jobId} failed (${typeof code === "number" || typeof code === "string" ? code : "no status code"}); the job may still be running`,
+        { code: "ADCP_BIGQUERY_SINK_CANCEL_FAILED" }
+      );
+    }
+  }
+  async function loadOnce(tableId, jobId, lines, attempt, signal) {
+    const loadDeadline = Math.min(loadTimeout * 2 ** Math.min(attempt, 16), MAX_LOAD_TIMEOUT);
     signal?.throwIfAborted();
     const directory = await import_fs.promises.mkdtemp(path.join((0, import_os.tmpdir)(), "adcp-warehouse-"));
     const file = path.join(directory, `${jobId}.ndjson`);
     try {
-      await import_fs.promises.writeFile(file, lines.length ? lines.join("\n") + "\n" : "");
+      await import_fs.promises.writeFile(file, lines.length ? lines.join("\n") + "\n" : "", { mode: 384 });
       try {
         await withinDeadline(
           () => dataset.table(tableId).load(file, {
             jobId,
             sourceFormat: "NEWLINE_DELIMITED_JSON",
             writeDisposition: "WRITE_APPEND",
-            jobTimeoutMs: String(loadTimeout),
-            ...options.location ? { location: options.location } : {},
-            ...options.labels ? { labels: { ...options.labels } } : {}
+            ...options.location ? { location: options.location } : {}
           }),
-          loadTimeout,
+          loadDeadline,
           signal
         );
       } catch (error) {
+        if (isDeadlineExceeded(error)) await cancelLoad(jobId, signal);
         if (!isAlreadyExists(error)) throw error;
         const metadata = await withinDeadline(
-          () => options.bigquery.job(jobId, options.location ? { location: options.location } : void 0).getMetadata(),
+          () => options.bigquery.job(jobId, jobOptions).getMetadata(),
           loadTimeout,
           signal
         );
@@ -120,6 +159,40 @@ function createBigQueryReportingWarehouseSinkV1(options) {
   return {
     defaultTablesSql: reportingWarehouseDefaultTablesSqlV1(tables),
     currentRowsViewSql: reportingWarehouseCurrentRowsViewSqlV1(tables),
+    async reset(resetOptions = {}) {
+      const removeConsumer = options.store.removeFeedConsumer?.bind(options.store);
+      if (!removeConsumer) {
+        throw new TypeError("reset() needs a store with removeFeedConsumer (PostgresReportingLedgerStore has it)");
+      }
+      const cleared = await options.db.query(
+        `UPDATE adcp_reporting_warehouse_sinks
+            SET committed_cursor = NULL, planned_from_cursor = NULL, planned_to_cursor = NULL,
+                planned_revisions = NULL, planned_rows_attempt = 0, planned_revisions_attempt = 0,
+                reset_generation = reset_generation + 1, updated_at = clock_timestamp()
+          WHERE sink_name = $1 AND ($2::boolean OR planned_to_cursor IS NULL)
+          RETURNING sink_name`,
+        [options.name, resetOptions.discardRetainedPlan === true]
+      ).catch(missingMigration);
+      if (cleared.rows.length === 0) {
+        const exists = await options.db.query(`SELECT 1 FROM adcp_reporting_warehouse_sinks WHERE sink_name = $1`, [
+          options.name
+        ]);
+        if (exists.rows.length > 0) {
+          throw new Error(
+            "Warehouse sink has a retained batch plan whose load jobs may have succeeded; reconcile it against BigQuery, then call reset({ discardRetainedPlan: true })"
+          );
+        }
+      }
+      try {
+        await removeConsumer(options.name);
+      } catch (cause) {
+        throw new Error(
+          "Warehouse sink state was cleared but its feed consumer could not be unregistered; call reset() again before restarting workers",
+          { cause }
+        );
+      }
+      return { cleared: cleared.rows.length > 0 };
+    },
     async runOnce(runOptions = {}) {
       const signal = runOptions.signal;
       await options.db.query(
@@ -128,10 +201,20 @@ function createBigQueryReportingWarehouseSinkV1(options) {
       );
       const state = (await options.db.query(
         `SELECT committed_cursor, planned_from_cursor, planned_to_cursor, planned_revisions,
-                  planned_rows_attempt, planned_revisions_attempt
+                  planned_rows_attempt, planned_revisions_attempt, reset_generation
              FROM adcp_reporting_warehouse_sinks WHERE sink_name = $1`,
         [options.name]
-      )).rows[0];
+      ).catch(missingMigration)).rows[0];
+      if (state.planned_to_cursor !== null && state.planned_from_cursor !== state.committed_cursor) {
+        throw new Error(
+          "Warehouse sink has an inconsistent retained plan; reconcile the plan and BigQuery jobs before resuming"
+        );
+      }
+      if (options.account_id && state.planned_revisions?.some((planned) => planned.account_id !== options.account_id)) {
+        throw new Error(
+          "Warehouse sink has a retained plan for another account; this sink name is shared across feed scopes"
+        );
+      }
       let plan = state.planned_to_cursor && state.planned_revisions ? {
         from: state.planned_from_cursor,
         to: state.planned_to_cursor,
@@ -140,31 +223,74 @@ function createBigQueryReportingWarehouseSinkV1(options) {
         revisionsAttempt: state.planned_revisions_attempt
       } : void 0;
       if (!plan) {
-        const page = await options.store.changesAfter({
+        const position = {
           ...state.committed_cursor ? { cursor: state.committed_cursor } : {},
           ...options.account_id ? { account_id: options.account_id } : {},
-          kinds: ["revision"],
           limit: maxRevisions
-        });
-        if (page.records.length === 0) {
-          if (page.cursor !== state.committed_cursor) await commit(page.cursor);
+        };
+        let page;
+        let readCursor = position.cursor;
+        try {
+          page = await options.store.changesAfter({ ...position, kinds: ["revision"] });
+          if (page.records.length === 0) {
+            readCursor = page.cursor;
+            page = await options.store.changesAfter({ ...position, cursor: readCursor });
+          }
+        } catch (error) {
+          if (error?.code === "CURSOR_EXPIRED" && state.committed_cursor === null) {
+            const cursor = readCursor ?? error.cursor;
+            if (cursor !== void 0 && typeof cursor !== "string") throw error;
+            if (cursor !== void 0) {
+              try {
+                (0, import_change_feed.decodeReportingChangeCursorV1)(cursor, position.account_id ?? null);
+              } catch {
+                throw error;
+              }
+              await options.db.query(
+                `UPDATE adcp_reporting_warehouse_sinks SET committed_cursor = $2, updated_at = clock_timestamp()
+                  WHERE sink_name = $1 AND committed_cursor IS NULL AND reset_generation = $3
+                    AND planned_from_cursor IS NULL AND planned_to_cursor IS NULL AND planned_revisions IS NULL`,
+                [options.name, cursor, state.reset_generation]
+              );
+            }
+          }
+          throw error;
+        }
+        if (page.records.length === 0 && page.cursor === state.committed_cursor) {
+          if (state.committed_cursor) {
+            await options.store.saveFeedConsumerCursor(options.name, state.committed_cursor, {
+              refreshUnchanged: false
+            });
+          }
           return { idle: true, revisions: 0, rows: 0, missing: [] };
         }
         plan = {
           from: state.committed_cursor,
           to: page.cursor,
-          revisions: page.records.map((record) => ({ id: record.record_id, account_id: record.account_id })),
+          revisions: page.records.filter((record) => record.kind === "revision").map((record) => ({ id: record.record_id, account_id: record.account_id })),
           rowsAttempt: 0,
           revisionsAttempt: 0
         };
-        await options.db.query(
+        const claimed = await options.db.query(
           `UPDATE adcp_reporting_warehouse_sinks
               SET planned_from_cursor = $2, planned_to_cursor = $3, planned_revisions = $4::jsonb,
                   planned_rows_attempt = 0, planned_revisions_attempt = 0, updated_at = clock_timestamp()
-            WHERE sink_name = $1`,
-          [options.name, plan.from, plan.to, JSON.stringify(plan.revisions)]
+            WHERE sink_name = $1 AND committed_cursor IS NOT DISTINCT FROM $2
+              AND planned_to_cursor IS NULL AND planned_revisions IS NULL AND reset_generation = $5
+            RETURNING sink_name`,
+          [options.name, plan.from, plan.to, JSON.stringify(plan.revisions), state.reset_generation]
         );
+        if (claimed.rows.length === 0) throw stalePlan();
       }
+      const planValues = [
+        options.name,
+        plan.from,
+        plan.to,
+        JSON.stringify(plan.revisions),
+        plan.rowsAttempt,
+        plan.revisionsAttempt,
+        state.reset_generation
+      ];
       const rowLines = [];
       const revisionLines = [];
       const missing = [];
@@ -218,7 +344,7 @@ function createBigQueryReportingWarehouseSinkV1(options) {
           })
         );
       }
-      const baseJobId = reportingWarehouseJobIdV1(options.name, plan.from, plan.to);
+      const baseJobId = reportingWarehouseJobIdV1(options.name, plan.from, plan.to, state.reset_generation);
       const suffix = (attempt) => attempt ? `_a${attempt}` : "";
       const steps = [
         { table: options.rowsTableId, kind: "rows", lines: rowLines, attempt: plan.rowsAttempt },
@@ -227,37 +353,50 @@ function createBigQueryReportingWarehouseSinkV1(options) {
       for (const step of steps) {
         if (!step.lines.length) continue;
         try {
-          await loadOnce(step.table, `${baseJobId}_${step.kind}${suffix(step.attempt)}`, [...step.lines], signal);
+          await loadOnce(
+            step.table,
+            `${baseJobId}_${step.kind}${suffix(step.attempt)}`,
+            [...step.lines],
+            step.attempt,
+            signal
+          );
         } catch (error) {
           if (isFinishedWithError(error)) {
             const column = step.kind === "rows" ? "planned_rows_attempt" : "planned_revisions_attempt";
             await options.db.query(
               `UPDATE adcp_reporting_warehouse_sinks SET ${column} = ${column} + 1,
-                      updated_at = clock_timestamp() WHERE sink_name = $1`,
-              [options.name]
+                      updated_at = clock_timestamp() ${matchesPlan}`,
+              planValues
             );
           }
           throw error;
         }
       }
-      await commit(plan.to);
-      return { idle: false, revisions: revisionLines.length, rows: rowLines.length, missing };
+      const committed = await options.db.query(
+        `UPDATE adcp_reporting_warehouse_sinks
+            SET committed_cursor = $3, planned_from_cursor = NULL, planned_to_cursor = NULL,
+                planned_revisions = NULL, planned_rows_attempt = 0, planned_revisions_attempt = 0,
+                updated_at = clock_timestamp()
+          ${matchesPlan} RETURNING sink_name`,
+        planValues
+      );
+      if (committed.rows.length === 0) throw stalePlan();
+      await options.store.saveFeedConsumerCursor(options.name, plan.to);
+      return { idle: plan.revisions.length === 0, revisions: revisionLines.length, rows: rowLines.length, missing };
     }
   };
-  async function commit(cursor) {
-    await options.db.query(
-      `UPDATE adcp_reporting_warehouse_sinks
-          SET committed_cursor = $2, planned_from_cursor = NULL, planned_to_cursor = NULL,
-              planned_revisions = NULL, planned_rows_attempt = 0, planned_revisions_attempt = 0,
-              updated_at = clock_timestamp()
-        WHERE sink_name = $1`,
-      [options.name, cursor]
+}
+function missingMigration(error) {
+  if (error?.code === "42703") {
+    throw new Error(
+      "The adcp_reporting_warehouse_sinks table is missing columns; apply REPORTING_WAREHOUSE_SINK_MIGRATION again (it is idempotent)",
+      { cause: error }
     );
-    await options.store.saveFeedConsumerCursor(options.name, cursor);
   }
+  throw error;
 }
-function reportingWarehouseJobIdV1(name, from, to) {
-  return `adcp_rr_${(0, import_crypto.createHash)("sha256").update(JSON.stringify([name, from, to])).digest("hex").slice(0, 40)}`;
+function reportingWarehouseJobIdV1(name, from, to, generation = 0) {
+  return `adcp_rr_${(0, import_crypto.createHash)("sha256").update(JSON.stringify(generation > 0 ? [name, from, to, generation] : [name, from, to])).digest("hex").slice(0, 40)}`;
 }
 function qualified(tables, table) {
   return `\`${tables.projectId ? `${tables.projectId}.` : ""}${tables.datasetId}.${table}\``;
@@ -336,10 +475,11 @@ async function withinDeadline(operation, milliseconds, signal) {
     return await Promise.race([
       operation(),
       new Promise((_, reject) => {
-        timer = setTimeout(
-          () => reject(new Error("BigQuery call exceeded its deadline; the batch will be retried")),
-          milliseconds
-        );
+        timer = setTimeout(() => {
+          const error = new Error("BigQuery call exceeded its deadline; the batch will be retried");
+          Object.defineProperty(error, "adcpDeadlineExceeded", { value: true });
+          reject(error);
+        }, milliseconds);
         if (signal) {
           onAbort = () => reject(signal.reason ?? new Error("aborted"));
           signal.addEventListener("abort", onAbort, { once: true });
@@ -351,6 +491,9 @@ async function withinDeadline(operation, milliseconds, signal) {
     if (signal && onAbort) signal.removeEventListener("abort", onAbort);
   }
 }
+function isDeadlineExceeded(error) {
+  return error?.adcpDeadlineExceeded === true;
+}
 function isAlreadyExists(error) {
   const record = error;
   return record?.code === 409 || !!record?.errors?.some((entry) => entry.reason === "duplicate");