@adcp/sdk 15.8.1 → 15.9.1

package/dist/lib/reporting/bigquery/index.mjs

@@ -2,6 +2,7 @@ import { createHash } from "crypto";
 import { promises as fs } from "fs";
 import { tmpdir } from "os";
 import * as path from "path";
+import { decodeReportingChangeCursorV1 } from "../ledger/change-feed.mjs";
 const REPORTING_WAREHOUSE_SINK_MIGRATION = `
 CREATE TABLE IF NOT EXISTS adcp_reporting_warehouse_sinks (
   sink_name TEXT PRIMARY KEY CHECK (sink_name ~ '^[A-Za-z0-9_.:-]{1,128}$'),
@@ -11,10 +12,22 @@ CREATE TABLE IF NOT EXISTS adcp_reporting_warehouse_sinks (
   planned_revisions JSONB,
   planned_rows_attempt INTEGER NOT NULL DEFAULT 0 CHECK (planned_rows_attempt >= 0),
   planned_revisions_attempt INTEGER NOT NULL DEFAULT 0 CHECK (planned_revisions_attempt >= 0),
+  planned_rows_timeouts INTEGER NOT NULL DEFAULT 0 CHECK (planned_rows_timeouts >= 0),
+  planned_revisions_timeouts INTEGER NOT NULL DEFAULT 0 CHECK (planned_revisions_timeouts >= 0),
+  reset_generation INTEGER NOT NULL DEFAULT 0 CHECK (reset_generation >= 0),
   updated_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
   CHECK ((planned_to_cursor IS NULL) = (planned_revisions IS NULL))
 );
+ALTER TABLE adcp_reporting_warehouse_sinks
+  ADD COLUMN IF NOT EXISTS reset_generation INTEGER NOT NULL DEFAULT 0 CHECK (reset_generation >= 0);
+ALTER TABLE adcp_reporting_warehouse_sinks
+  ADD COLUMN IF NOT EXISTS planned_rows_timeouts INTEGER NOT NULL DEFAULT 0 CHECK (planned_rows_timeouts >= 0);
+ALTER TABLE adcp_reporting_warehouse_sinks
+  ADD COLUMN IF NOT EXISTS planned_revisions_timeouts INTEGER NOT NULL DEFAULT 0 CHECK (planned_revisions_timeouts >= 0);
 `.trim();
+const MAX_LOAD_TIMEOUT = 6 * 36e5;
+const MAX_TIMEOUT_DOUBLINGS = 16;
+let warnedLabelsIgnored = false;
 const PROJECT = /^[a-z][a-z0-9-]{4,61}[a-z0-9]$/;
 const DATASET_OR_TABLE = /^[A-Za-z_][A-Za-z0-9_]{0,1023}$/;
 const NAME = /^[A-Za-z0-9_.:-]{1,128}$/;
@@ -31,13 +44,20 @@ function createBigQueryReportingWarehouseSinkV1(options) {
     if (!DATASET_OR_TABLE.test(value)) throw new TypeError(`BigQuery ${label} is invalid`);
   }
   const loadTimeout = options.loadTimeoutMilliseconds ?? 6e5;
-  if (!Number.isSafeInteger(loadTimeout) || loadTimeout < 1e3 || loadTimeout > 6 * 36e5) {
+  if (!Number.isSafeInteger(loadTimeout) || loadTimeout < 1e3 || loadTimeout > MAX_LOAD_TIMEOUT) {
     throw new RangeError("loadTimeoutMilliseconds must be between 1 second and 6 hours");
   }
   const maxRevisions = options.maxRevisionsPerBatch ?? 200;
   if (!Number.isSafeInteger(maxRevisions) || maxRevisions < 1 || maxRevisions > 1e4) {
     throw new RangeError("maxRevisionsPerBatch must be between 1 and 10000");
   }
+  if (options.labels && Object.keys(options.labels).length > 0 && !warnedLabelsIgnored) {
+    warnedLabelsIgnored = true;
+    process.emitWarning(
+      "BigQuery warehouse sink option `labels` has no effect: BigQuery ignores labels inside a load job's configuration.load and the official client cannot set configuration.labels for local-file loads. Attribute cost through INFORMATION_SCHEMA.JOBS by destination table or the adcp_rr_ job ID prefix.",
+      { code: "ADCP_BIGQUERY_SINK_LABELS_IGNORED", type: "DeprecationWarning" }
+    );
+  }
   const tables = {
     projectId: options.projectId,
     datasetId: options.datasetId,
@@ -48,41 +68,155 @@ function createBigQueryReportingWarehouseSinkV1(options) {
     ...options.projectId ? { projectId: options.projectId } : {},
     ...options.location ? { location: options.location } : {}
   });
-  async function loadOnce(tableId, jobId, lines, signal) {
-    signal?.throwIfAborted();
-    const directory = await fs.mkdtemp(path.join(tmpdir(), "adcp-warehouse-"));
-    const file = path.join(directory, `${jobId}.ndjson`);
+  const matchesPlan = `WHERE sink_name = $1
+    AND committed_cursor IS NOT DISTINCT FROM $2
+    AND planned_from_cursor IS NOT DISTINCT FROM $2
+    AND planned_to_cursor = $3 AND planned_revisions = $4::jsonb
+    AND planned_rows_attempt = $5 AND planned_revisions_attempt = $6
+    AND reset_generation = $7`;
+  const stalePlan = () => new Error("Warehouse sink plan changed during an overlapping run; retry runOnce()");
+  const jobOptions = options.projectId || options.location ? {
+    ...options.projectId ? { projectId: options.projectId } : {},
+    ...options.location ? { location: options.location } : {}
+  } : void 0;
+  const uploads = /* @__PURE__ */ new Map();
+  function startUpload(tableId, jobId, lines) {
+    const entry = {
+      startedAt: Date.now(),
+      done: Promise.resolve()
+    };
+    const cleanup = async () => {
+      if (uploads.get(jobId) === entry) uploads.delete(jobId);
+      if (entry.directory) await fs.rm(entry.directory, { recursive: true, force: true }).catch(() => {
+      });
+    };
+    entry.done = (async () => {
+      const directory = await fs.mkdtemp(path.join(tmpdir(), "adcp-warehouse-"));
+      entry.directory = directory;
+      const file = path.join(directory, `${jobId}.ndjson`);
+      await fs.writeFile(file, lines.length ? lines.join("\n") + "\n" : "", { mode: 384 });
+      return dataset.table(tableId).load(file, {
+        jobId,
+        sourceFormat: "NEWLINE_DELIMITED_JSON",
+        writeDisposition: "WRITE_APPEND",
+        ...options.location ? { location: options.location } : {}
+      });
+    })();
+    const replaced = uploads.get(jobId);
+    uploads.set(jobId, entry);
+    void entry.done.then(cleanup, cleanup);
+    if (replaced?.directory) void fs.rm(replaced.directory, { recursive: true, force: true }).catch(() => {
+    });
+    return entry.done;
+  }
+  async function jobStatus(jobId, signal, boundMilliseconds = loadTimeout, context) {
+    let metadata;
     try {
-      await fs.writeFile(file, lines.length ? lines.join("\n") + "\n" : "");
+      metadata = await withinDeadline(
+        () => options.bigquery.job(jobId, jobOptions).getMetadata(),
+        boundMilliseconds,
+        signal,
+        context
+      );
+    } catch (error) {
+      if (error?.code === 404) return "missing";
+      throw error;
+    }
+    const record = Array.isArray(metadata) ? metadata[0] : metadata;
+    if (record?.status?.state !== "DONE") return "running";
+    if (record.status.errorResult) {
+      const failure = new Error("A BigQuery load job for this batch failed; it will be retried as a new attempt");
+      Object.defineProperty(failure, "adcpJobFailed", { value: true });
+      throw failure;
+    }
+    return "done";
+  }
+  async function loadOnce(role, tableId, jobId, lines, timeouts, signal) {
+    signal?.throwIfAborted();
+    const table = `the ${role} table "${tableId}" (timeout count ${timeouts})`;
+    let status = await jobStatus(jobId, signal, loadTimeout, table);
+    if (status === "done") return;
+    const loadDeadline = reportingWarehouseLoadDeadlineV1(loadTimeout, timeouts);
+    const deadlineAt = Date.now() + loadDeadline;
+    const timedOut = (what) => {
+      const error = new Error(
+        `${what} exceeded this run's ${loadDeadline} ms deadline for ${table}; the next run checks the job again`
+      );
+      Object.defineProperty(error, "adcpDeadlineExceeded", { value: true });
+      Object.defineProperty(error, "adcpLoadTimedOut", { value: true });
+      return error;
+    };
+    if (status === "missing") {
       try {
-        await withinDeadline(
-          () => dataset.table(tableId).load(file, {
-            jobId,
-            sourceFormat: "NEWLINE_DELIMITED_JSON",
-            writeDisposition: "WRITE_APPEND",
-            jobTimeoutMs: String(loadTimeout),
-            ...options.location ? { location: options.location } : {},
-            ...options.labels ? { labels: { ...options.labels } } : {}
-          }),
-          loadTimeout,
-          signal
-        );
+        const running = uploads.get(jobId);
+        const upload = running && Date.now() - running.startedAt < MAX_LOAD_TIMEOUT ? running.done : startUpload(tableId, jobId, lines);
+        await withinDeadline(() => upload, Math.max(deadlineAt - Date.now(), 1), signal);
+        return;
       } catch (error) {
+        if (isDeadlineExceeded(error)) throw timedOut("Uploading the load");
         if (!isAlreadyExists(error)) throw error;
-        const metadata = await withinDeadline(
-          () => options.bigquery.job(jobId, options.location ? { location: options.location } : void 0).getMetadata(),
-          loadTimeout,
-          signal
-        );
-        assertJobSucceeded(metadata);
+        status = await jobStatus(jobId, signal, loadTimeout, table);
+        if (status === "missing") {
+          throw new Error(
+            "BigQuery refused the load job ID as already existing, but the job cannot be found. Check that projectId and location match the dataset (location is required for datasets outside the US and EU multi-regions)",
+            { cause: error }
+          );
+        }
       }
-    } finally {
-      await fs.rm(directory, { recursive: true, force: true });
+    }
+    const pollMilliseconds = Math.min(5e3, Math.max(250, Math.floor(loadDeadline / 10)));
+    while (status !== "done") {
+      const remaining = deadlineAt - Date.now();
+      if (remaining <= 0) throw timedOut("Waiting for the running load job");
+      await sleep(Math.min(pollMilliseconds, remaining), signal);
+      const bound = Math.max(Math.min(loadTimeout, deadlineAt - Date.now()), 1);
+      try {
+        status = await jobStatus(jobId, signal, bound, table);
+      } catch (error) {
+        if (isDeadlineExceeded(error) && bound < loadTimeout) throw timedOut("Waiting for the running load job");
+        throw error;
+      }
+      if (status === "missing") throw new Error("A BigQuery load job for this batch disappeared while it was awaited");
     }
   }
   return {
     defaultTablesSql: reportingWarehouseDefaultTablesSqlV1(tables),
     currentRowsViewSql: reportingWarehouseCurrentRowsViewSqlV1(tables),
+    async reset(resetOptions = {}) {
+      const removeConsumer = options.store.removeFeedConsumer?.bind(options.store);
+      if (!removeConsumer) {
+        throw new TypeError("reset() needs a store with removeFeedConsumer (PostgresReportingLedgerStore has it)");
+      }
+      const cleared = await options.db.query(
+        `UPDATE adcp_reporting_warehouse_sinks
+            SET committed_cursor = NULL, planned_from_cursor = NULL, planned_to_cursor = NULL,
+                planned_revisions = NULL, planned_rows_attempt = 0, planned_revisions_attempt = 0,
+                planned_rows_timeouts = 0, planned_revisions_timeouts = 0,
+                reset_generation = reset_generation + 1, updated_at = clock_timestamp()
+          WHERE sink_name = $1 AND ($2::boolean OR planned_to_cursor IS NULL)
+          RETURNING sink_name`,
+        [options.name, resetOptions.discardRetainedPlan === true]
+      ).catch(missingMigration);
+      if (cleared.rows.length === 0) {
+        const exists = await options.db.query(`SELECT 1 FROM adcp_reporting_warehouse_sinks WHERE sink_name = $1`, [
+          options.name
+        ]);
+        if (exists.rows.length > 0) {
+          throw new Error(
+            "Warehouse sink has a retained batch plan whose load jobs may have succeeded; reconcile it against BigQuery, then call reset({ discardRetainedPlan: true })"
+          );
+        }
+      }
+      try {
+        await removeConsumer(options.name);
+      } catch (cause) {
+        throw new Error(
+          "Warehouse sink state was cleared but its feed consumer could not be unregistered; call reset() again before restarting workers",
+          { cause }
+        );
+      }
+      return { cleared: cleared.rows.length > 0 };
+    },
     async runOnce(runOptions = {}) {
       const signal = runOptions.signal;
       await options.db.query(
@@ -91,43 +225,102 @@ function createBigQueryReportingWarehouseSinkV1(options) {
       );
       const state = (await options.db.query(
         `SELECT committed_cursor, planned_from_cursor, planned_to_cursor, planned_revisions,
-                  planned_rows_attempt, planned_revisions_attempt
+                  planned_rows_attempt, planned_revisions_attempt,
+                  planned_rows_timeouts, planned_revisions_timeouts, reset_generation
              FROM adcp_reporting_warehouse_sinks WHERE sink_name = $1`,
         [options.name]
-      )).rows[0];
+      ).catch(missingMigration)).rows[0];
+      if (state.planned_to_cursor !== null && state.planned_from_cursor !== state.committed_cursor) {
+        throw new Error(
+          "Warehouse sink has an inconsistent retained plan; reconcile the plan and BigQuery jobs before resuming"
+        );
+      }
+      if (options.account_id && state.planned_revisions?.some((planned) => planned.account_id !== options.account_id)) {
+        throw new Error(
+          "Warehouse sink has a retained plan for another account; this sink name is shared across feed scopes"
+        );
+      }
       let plan = state.planned_to_cursor && state.planned_revisions ? {
         from: state.planned_from_cursor,
         to: state.planned_to_cursor,
         revisions: state.planned_revisions,
         rowsAttempt: state.planned_rows_attempt,
-        revisionsAttempt: state.planned_revisions_attempt
+        revisionsAttempt: state.planned_revisions_attempt,
+        rowsTimeouts: state.planned_rows_timeouts,
+        revisionsTimeouts: state.planned_revisions_timeouts
       } : void 0;
       if (!plan) {
-        const page = await options.store.changesAfter({
+        const position = {
           ...state.committed_cursor ? { cursor: state.committed_cursor } : {},
           ...options.account_id ? { account_id: options.account_id } : {},
-          kinds: ["revision"],
           limit: maxRevisions
-        });
-        if (page.records.length === 0) {
-          if (page.cursor !== state.committed_cursor) await commit(page.cursor);
+        };
+        let page;
+        let readCursor = position.cursor;
+        try {
+          page = await options.store.changesAfter({ ...position, kinds: ["revision"] });
+          if (page.records.length === 0) {
+            readCursor = page.cursor;
+            page = await options.store.changesAfter({ ...position, cursor: readCursor });
+          }
+        } catch (error) {
+          if (error?.code === "CURSOR_EXPIRED" && state.committed_cursor === null) {
+            const cursor = readCursor ?? error.cursor;
+            if (cursor !== void 0 && typeof cursor !== "string") throw error;
+            if (cursor !== void 0) {
+              try {
+                decodeReportingChangeCursorV1(cursor, position.account_id ?? null);
+              } catch {
+                throw error;
+              }
+              await options.db.query(
+                `UPDATE adcp_reporting_warehouse_sinks SET committed_cursor = $2, updated_at = clock_timestamp()
+                  WHERE sink_name = $1 AND committed_cursor IS NULL AND reset_generation = $3
+                    AND planned_from_cursor IS NULL AND planned_to_cursor IS NULL AND planned_revisions IS NULL`,
+                [options.name, cursor, state.reset_generation]
+              );
+            }
+          }
+          throw error;
+        }
+        if (page.records.length === 0 && page.cursor === state.committed_cursor) {
+          if (state.committed_cursor) {
+            await options.store.saveFeedConsumerCursor(options.name, state.committed_cursor, {
+              refreshUnchanged: false
+            });
+          }
           return { idle: true, revisions: 0, rows: 0, missing: [] };
         }
         plan = {
           from: state.committed_cursor,
           to: page.cursor,
-          revisions: page.records.map((record) => ({ id: record.record_id, account_id: record.account_id })),
+          revisions: page.records.filter((record) => record.kind === "revision").map((record) => ({ id: record.record_id, account_id: record.account_id })),
           rowsAttempt: 0,
-          revisionsAttempt: 0
+          revisionsAttempt: 0,
+          rowsTimeouts: 0,
+          revisionsTimeouts: 0
         };
-        await options.db.query(
+        const claimed = await options.db.query(
           `UPDATE adcp_reporting_warehouse_sinks
               SET planned_from_cursor = $2, planned_to_cursor = $3, planned_revisions = $4::jsonb,
-                  planned_rows_attempt = 0, planned_revisions_attempt = 0, updated_at = clock_timestamp()
-            WHERE sink_name = $1`,
-          [options.name, plan.from, plan.to, JSON.stringify(plan.revisions)]
+                  planned_rows_attempt = 0, planned_revisions_attempt = 0,
+                  planned_rows_timeouts = 0, planned_revisions_timeouts = 0, updated_at = clock_timestamp()
+            WHERE sink_name = $1 AND committed_cursor IS NOT DISTINCT FROM $2
+              AND planned_to_cursor IS NULL AND planned_revisions IS NULL AND reset_generation = $5
+            RETURNING sink_name`,
+          [options.name, plan.from, plan.to, JSON.stringify(plan.revisions), state.reset_generation]
         );
+        if (claimed.rows.length === 0) throw stalePlan();
       }
+      const planValues = [
+        options.name,
+        plan.from,
+        plan.to,
+        JSON.stringify(plan.revisions),
+        plan.rowsAttempt,
+        plan.revisionsAttempt,
+        state.reset_generation
+      ];
       const rowLines = [];
       const revisionLines = [];
       const missing = [];
@@ -181,46 +374,87 @@ function createBigQueryReportingWarehouseSinkV1(options) {
           })
         );
       }
-      const baseJobId = reportingWarehouseJobIdV1(options.name, plan.from, plan.to);
+      const baseJobId = reportingWarehouseJobIdV1(options.name, plan.from, plan.to, state.reset_generation);
       const suffix = (attempt) => attempt ? `_a${attempt}` : "";
       const steps = [
-        { table: options.rowsTableId, kind: "rows", lines: rowLines, attempt: plan.rowsAttempt },
-        { table: options.revisionsTableId, kind: "revs", lines: revisionLines, attempt: plan.revisionsAttempt }
+        {
+          table: options.rowsTableId,
+          role: "rows",
+          kind: "rows",
+          lines: rowLines,
+          attempt: plan.rowsAttempt,
+          timeouts: plan.rowsTimeouts,
+          timeoutColumn: "planned_rows_timeouts",
+          attemptColumn: "planned_rows_attempt"
+        },
+        {
+          table: options.revisionsTableId,
+          role: "revisions",
+          kind: "revs",
+          lines: revisionLines,
+          attempt: plan.revisionsAttempt,
+          timeouts: plan.revisionsTimeouts,
+          timeoutColumn: "planned_revisions_timeouts",
+          attemptColumn: "planned_revisions_attempt"
+        }
       ];
       for (const step of steps) {
         if (!step.lines.length) continue;
         try {
-          await loadOnce(step.table, `${baseJobId}_${step.kind}${suffix(step.attempt)}`, [...step.lines], signal);
+          await loadOnce(
+            step.role,
+            step.table,
+            `${baseJobId}_${step.kind}${suffix(step.attempt)}`,
+            [...step.lines],
+            step.timeouts,
+            signal
+          );
         } catch (error) {
           if (isFinishedWithError(error)) {
-            const column = step.kind === "rows" ? "planned_rows_attempt" : "planned_revisions_attempt";
             await options.db.query(
-              `UPDATE adcp_reporting_warehouse_sinks SET ${column} = ${column} + 1,
-                      updated_at = clock_timestamp() WHERE sink_name = $1`,
-              [options.name]
+              `UPDATE adcp_reporting_warehouse_sinks SET ${step.attemptColumn} = ${step.attemptColumn} + 1,
+                      updated_at = clock_timestamp() ${matchesPlan}`,
+              planValues
+            );
+          } else if (isLoadTimedOut(error)) {
+            await options.db.query(
+              `UPDATE adcp_reporting_warehouse_sinks
+                  SET ${step.timeoutColumn} = $8::integer + 1, updated_at = clock_timestamp()
+                ${matchesPlan} AND ${step.timeoutColumn} = $8::integer`,
+              [...planValues, step.timeouts]
             );
           }
           throw error;
         }
       }
-      await commit(plan.to);
-      return { idle: false, revisions: revisionLines.length, rows: rowLines.length, missing };
+      const committed = await options.db.query(
+        `UPDATE adcp_reporting_warehouse_sinks
+            SET committed_cursor = $3, planned_from_cursor = NULL, planned_to_cursor = NULL,
+                planned_revisions = NULL, planned_rows_attempt = 0, planned_revisions_attempt = 0,
+                planned_rows_timeouts = 0, planned_revisions_timeouts = 0, updated_at = clock_timestamp()
+          ${matchesPlan} RETURNING sink_name`,
+        planValues
+      );
+      if (committed.rows.length === 0) throw stalePlan();
+      await options.store.saveFeedConsumerCursor(options.name, plan.to);
+      return { idle: plan.revisions.length === 0, revisions: revisionLines.length, rows: rowLines.length, missing };
     }
   };
-  async function commit(cursor) {
-    await options.db.query(
-      `UPDATE adcp_reporting_warehouse_sinks
-          SET committed_cursor = $2, planned_from_cursor = NULL, planned_to_cursor = NULL,
-              planned_revisions = NULL, planned_rows_attempt = 0, planned_revisions_attempt = 0,
-              updated_at = clock_timestamp()
-        WHERE sink_name = $1`,
-      [options.name, cursor]
+}
+function missingMigration(error) {
+  if (error?.code === "42703") {
+    throw new Error(
+      "The adcp_reporting_warehouse_sinks table is missing columns; apply REPORTING_WAREHOUSE_SINK_MIGRATION again (it is idempotent)",
+      { cause: error }
     );
-    await options.store.saveFeedConsumerCursor(options.name, cursor);
   }
+  throw error;
+}
+function reportingWarehouseLoadDeadlineV1(baseMilliseconds, timeouts) {
+  return Math.min(baseMilliseconds * 2 ** Math.min(Math.max(timeouts, 0), MAX_TIMEOUT_DOUBLINGS), MAX_LOAD_TIMEOUT);
 }
-function reportingWarehouseJobIdV1(name, from, to) {
-  return `adcp_rr_${createHash("sha256").update(JSON.stringify([name, from, to])).digest("hex").slice(0, 40)}`;
+function reportingWarehouseJobIdV1(name, from, to, generation = 0) {
+  return `adcp_rr_${createHash("sha256").update(JSON.stringify(generation > 0 ? [name, from, to, generation] : [name, from, to])).digest("hex").slice(0, 40)}`;
 }
 function qualified(tables, table) {
   return `\`${tables.projectId ? `${tables.projectId}.` : ""}${tables.datasetId}.${table}\``;
@@ -291,7 +525,7 @@ function sourceLocalDate(start, timeZone) {
     return instant.toISOString().slice(0, 10);
   }
 }
-async function withinDeadline(operation, milliseconds, signal) {
+async function withinDeadline(operation, milliseconds, signal, context) {
   signal?.throwIfAborted();
   let timer;
   let onAbort;
@@ -299,10 +533,13 @@ async function withinDeadline(operation, milliseconds, signal) {
     return await Promise.race([
       operation(),
       new Promise((_, reject) => {
-        timer = setTimeout(
-          () => reject(new Error("BigQuery call exceeded its deadline; the batch will be retried")),
-          milliseconds
-        );
+        timer = setTimeout(() => {
+          const error = new Error(
+            `BigQuery call exceeded its deadline${context ? ` of ${milliseconds} ms for ${context}` : ""}; the batch will be retried`
+          );
+          Object.defineProperty(error, "adcpDeadlineExceeded", { value: true });
+          reject(error);
+        }, milliseconds);
         if (signal) {
           onAbort = () => reject(signal.reason ?? new Error("aborted"));
           signal.addEventListener("abort", onAbort, { once: true });
@@ -314,28 +551,38 @@ async function withinDeadline(operation, milliseconds, signal) {
     if (signal && onAbort) signal.removeEventListener("abort", onAbort);
   }
 }
+function isDeadlineExceeded(error) {
+  return error?.adcpDeadlineExceeded === true;
+}
 function isAlreadyExists(error) {
   const record = error;
   return record?.code === 409 || !!record?.errors?.some((entry) => entry.reason === "duplicate");
 }
+function isLoadTimedOut(error) {
+  return error?.adcpLoadTimedOut === true;
+}
+function sleep(milliseconds, signal) {
+  signal?.throwIfAborted();
+  return new Promise((resolve, reject) => {
+    const onAbort = () => {
+      clearTimeout(timer);
+      reject(signal?.reason ?? new Error("aborted"));
+    };
+    const timer = setTimeout(() => {
+      signal?.removeEventListener("abort", onAbort);
+      resolve();
+    }, milliseconds);
+    signal?.addEventListener("abort", onAbort, { once: true });
+  });
+}
 function isFinishedWithError(error) {
   return error?.adcpJobFailed === true;
 }
-function assertJobSucceeded(metadata) {
-  const record = Array.isArray(metadata) ? metadata[0] : metadata;
-  if (record?.status?.state !== "DONE") {
-    throw new Error("A BigQuery load job for this batch is still running; retry later");
-  }
-  if (record.status.errorResult) {
-    const failure = new Error("A BigQuery load job for this batch failed; it will be retried as a new attempt");
-    Object.defineProperty(failure, "adcpJobFailed", { value: true });
-    throw failure;
-  }
-}
 export {
   REPORTING_WAREHOUSE_SINK_MIGRATION,
   createBigQueryReportingWarehouseSinkV1,
   reportingWarehouseCurrentRowsViewSqlV1,
   reportingWarehouseDefaultTablesSqlV1,
-  reportingWarehouseJobIdV1
+  reportingWarehouseJobIdV1,
+  reportingWarehouseLoadDeadlineV1
 };

package/dist/lib/reporting/ledger/change-feed.d.mts

@@ -7,7 +7,9 @@
  * adjustment and retirement in the same transaction as the record, taking the
  * sequence value only after the account lock, so per-account sequence order is
  * commit order. Deployment-wide readers order by `(xid, seq)` and stop below
- * the reading snapshot's `xmin`, so a slow commit is never skipped.
+ * the reading snapshot's `xmin`, so a slow commit is never skipped. Account
+ * readers order by `seq` and end the page before the first change at or above
+ * that `xmin`. Use exactly one writer SDK per ledger database.
  */
 export declare const REPORTING_LEDGER_CHANGES_MIGRATION: string;
 export type ReportingChangeKindV1 = 'obligation' | 'revision' | 'adjustment' | 'retirement';
@@ -35,7 +37,12 @@ export declare function encodeReportingChangeCursorV1(cursor: ChangeCursorV1): s
 export declare function decodeReportingChangeCursorV1(value: string, account: string | null): ChangeCursorV1;
 /** Thrown when a change cursor is older than the retained change history. */
 export declare class ReportingChangeCursorExpiredError extends Error {
+    readonly cursor?: string | undefined;
     readonly code: "CURSOR_EXPIRED";
-    constructor();
+    /**
+     * Original feed position, when available, so a cold-start consumer can retain its recovery
+     * boundary. It encodes the feed's account scope: do not serialize it to other tenants.
+     */
+    constructor(cursor?: string | undefined);
 }
 export {};

package/dist/lib/reporting/ledger/change-feed.d.ts

@@ -7,7 +7,9 @@
  * adjustment and retirement in the same transaction as the record, taking the
  * sequence value only after the account lock, so per-account sequence order is
  * commit order. Deployment-wide readers order by `(xid, seq)` and stop below
- * the reading snapshot's `xmin`, so a slow commit is never skipped.
+ * the reading snapshot's `xmin`, so a slow commit is never skipped. Account
+ * readers order by `seq` and end the page before the first change at or above
+ * that `xmin`. Use exactly one writer SDK per ledger database.
  */
 export declare const REPORTING_LEDGER_CHANGES_MIGRATION: string;
 export type ReportingChangeKindV1 = 'obligation' | 'revision' | 'adjustment' | 'retirement';
@@ -35,7 +37,12 @@ export declare function encodeReportingChangeCursorV1(cursor: ChangeCursorV1): s
 export declare function decodeReportingChangeCursorV1(value: string, account: string | null): ChangeCursorV1;
 /** Thrown when a change cursor is older than the retained change history. */
 export declare class ReportingChangeCursorExpiredError extends Error {
+    readonly cursor?: string | undefined;
     readonly code: "CURSOR_EXPIRED";
-    constructor();
+    /**
+     * Original feed position, when available, so a cold-start consumer can retain its recovery
+     * boundary. It encodes the feed's account scope: do not serialize it to other tenants.
+     */
+    constructor(cursor?: string | undefined);
 }
 export {};

package/dist/lib/reporting/ledger/change-feed.js

@@ -86,11 +86,17 @@ function decodeReportingChangeCursorV1(value, account) {
   return cursor;
 }
 class ReportingChangeCursorExpiredError extends Error {
-  code = "CURSOR_EXPIRED";
-  constructor() {
+  /**
+   * Original feed position, when available, so a cold-start consumer can retain its recovery
+   * boundary. It encodes the feed's account scope: do not serialize it to other tenants.
+   */
+  constructor(cursor) {
     super("Reporting change cursor is older than the retained change history; resynchronize from current revisions");
+    this.cursor = cursor;
     this.name = "ReportingChangeCursorExpiredError";
   }
+  cursor;
+  code = "CURSOR_EXPIRED";
 }
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {

package/dist/lib/reporting/ledger/change-feed.mjs

@@ -60,11 +60,17 @@ function decodeReportingChangeCursorV1(value, account) {
   return cursor;
 }
 class ReportingChangeCursorExpiredError extends Error {
-  code = "CURSOR_EXPIRED";
-  constructor() {
+  /**
+   * Original feed position, when available, so a cold-start consumer can retain its recovery
+   * boundary. It encodes the feed's account scope: do not serialize it to other tenants.
+   */
+  constructor(cursor) {
     super("Reporting change cursor is older than the retained change history; resynchronize from current revisions");
+    this.cursor = cursor;
     this.name = "ReportingChangeCursorExpiredError";
   }
+  cursor;
+  code = "CURSOR_EXPIRED";
 }
 export {
   REPORTING_LEDGER_CHANGES_MIGRATION,