@adcp/sdk 15.8.1 → 16.0.0-beta.1

package/dist/lib/reporting/bigquery/index.mjs

@@ -2,6 +2,8 @@ import { createHash } from "crypto";
 import { promises as fs } from "fs";
 import { tmpdir } from "os";
 import * as path from "path";
+import { decodeReportingChangeCursorV1 } from "../ledger/change-feed.mjs";
+import { reportingPeriodDate } from "../ledger/schedule.mjs";
 const REPORTING_WAREHOUSE_SINK_MIGRATION = `
 CREATE TABLE IF NOT EXISTS adcp_reporting_warehouse_sinks (
   sink_name TEXT PRIMARY KEY CHECK (sink_name ~ '^[A-Za-z0-9_.:-]{1,128}$'),
@@ -11,10 +13,22 @@ CREATE TABLE IF NOT EXISTS adcp_reporting_warehouse_sinks (
   planned_revisions JSONB,
   planned_rows_attempt INTEGER NOT NULL DEFAULT 0 CHECK (planned_rows_attempt >= 0),
   planned_revisions_attempt INTEGER NOT NULL DEFAULT 0 CHECK (planned_revisions_attempt >= 0),
+  planned_rows_timeouts INTEGER NOT NULL DEFAULT 0 CHECK (planned_rows_timeouts >= 0),
+  planned_revisions_timeouts INTEGER NOT NULL DEFAULT 0 CHECK (planned_revisions_timeouts >= 0),
+  reset_generation INTEGER NOT NULL DEFAULT 0 CHECK (reset_generation >= 0),
   updated_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
   CHECK ((planned_to_cursor IS NULL) = (planned_revisions IS NULL))
 );
+ALTER TABLE adcp_reporting_warehouse_sinks
+  ADD COLUMN IF NOT EXISTS reset_generation INTEGER NOT NULL DEFAULT 0 CHECK (reset_generation >= 0);
+ALTER TABLE adcp_reporting_warehouse_sinks
+  ADD COLUMN IF NOT EXISTS planned_rows_timeouts INTEGER NOT NULL DEFAULT 0 CHECK (planned_rows_timeouts >= 0);
+ALTER TABLE adcp_reporting_warehouse_sinks
+  ADD COLUMN IF NOT EXISTS planned_revisions_timeouts INTEGER NOT NULL DEFAULT 0 CHECK (planned_revisions_timeouts >= 0);
 `.trim();
+const MAX_LOAD_TIMEOUT = 6 * 36e5;
+const MAX_TIMEOUT_DOUBLINGS = 16;
+let warnedLabelsIgnored = false;
 const PROJECT = /^[a-z][a-z0-9-]{4,61}[a-z0-9]$/;
 const DATASET_OR_TABLE = /^[A-Za-z_][A-Za-z0-9_]{0,1023}$/;
 const NAME = /^[A-Za-z0-9_.:-]{1,128}$/;
@@ -31,13 +45,20 @@ function createBigQueryReportingWarehouseSinkV1(options) {
     if (!DATASET_OR_TABLE.test(value)) throw new TypeError(`BigQuery ${label} is invalid`);
   }
   const loadTimeout = options.loadTimeoutMilliseconds ?? 6e5;
-  if (!Number.isSafeInteger(loadTimeout) || loadTimeout < 1e3 || loadTimeout > 6 * 36e5) {
+  if (!Number.isSafeInteger(loadTimeout) || loadTimeout < 1e3 || loadTimeout > MAX_LOAD_TIMEOUT) {
     throw new RangeError("loadTimeoutMilliseconds must be between 1 second and 6 hours");
   }
   const maxRevisions = options.maxRevisionsPerBatch ?? 200;
   if (!Number.isSafeInteger(maxRevisions) || maxRevisions < 1 || maxRevisions > 1e4) {
     throw new RangeError("maxRevisionsPerBatch must be between 1 and 10000");
   }
+  if (options.labels && Object.keys(options.labels).length > 0 && !warnedLabelsIgnored) {
+    warnedLabelsIgnored = true;
+    process.emitWarning(
+      "BigQuery warehouse sink option `labels` has no effect: BigQuery ignores labels inside a load job's configuration.load and the official client cannot set configuration.labels for local-file loads. Attribute cost through INFORMATION_SCHEMA.JOBS by destination table or the adcp_rr_ job ID prefix.",
+      { code: "ADCP_BIGQUERY_SINK_LABELS_IGNORED", type: "DeprecationWarning" }
+    );
+  }
   const tables = {
     projectId: options.projectId,
     datasetId: options.datasetId,
@@ -48,41 +69,155 @@ function createBigQueryReportingWarehouseSinkV1(options) {
     ...options.projectId ? { projectId: options.projectId } : {},
     ...options.location ? { location: options.location } : {}
   });
-  async function loadOnce(tableId, jobId, lines, signal) {
-    signal?.throwIfAborted();
-    const directory = await fs.mkdtemp(path.join(tmpdir(), "adcp-warehouse-"));
-    const file = path.join(directory, `${jobId}.ndjson`);
+  const matchesPlan = `WHERE sink_name = $1
+    AND committed_cursor IS NOT DISTINCT FROM $2
+    AND planned_from_cursor IS NOT DISTINCT FROM $2
+    AND planned_to_cursor = $3 AND planned_revisions = $4::jsonb
+    AND planned_rows_attempt = $5 AND planned_revisions_attempt = $6
+    AND reset_generation = $7`;
+  const stalePlan = () => new Error("Warehouse sink plan changed during an overlapping run; retry runOnce()");
+  const jobOptions = options.projectId || options.location ? {
+    ...options.projectId ? { projectId: options.projectId } : {},
+    ...options.location ? { location: options.location } : {}
+  } : void 0;
+  const uploads = /* @__PURE__ */ new Map();
+  function startUpload(tableId, jobId, lines) {
+    const entry = {
+      startedAt: Date.now(),
+      done: Promise.resolve()
+    };
+    const cleanup = async () => {
+      if (uploads.get(jobId) === entry) uploads.delete(jobId);
+      if (entry.directory) await fs.rm(entry.directory, { recursive: true, force: true }).catch(() => {
+      });
+    };
+    entry.done = (async () => {
+      const directory = await fs.mkdtemp(path.join(tmpdir(), "adcp-warehouse-"));
+      entry.directory = directory;
+      const file = path.join(directory, `${jobId}.ndjson`);
+      await fs.writeFile(file, lines.length ? lines.join("\n") + "\n" : "", { mode: 384 });
+      return dataset.table(tableId).load(file, {
+        jobId,
+        sourceFormat: "NEWLINE_DELIMITED_JSON",
+        writeDisposition: "WRITE_APPEND",
+        ...options.location ? { location: options.location } : {}
+      });
+    })();
+    const replaced = uploads.get(jobId);
+    uploads.set(jobId, entry);
+    void entry.done.then(cleanup, cleanup);
+    if (replaced?.directory) void fs.rm(replaced.directory, { recursive: true, force: true }).catch(() => {
+    });
+    return entry.done;
+  }
+  async function jobStatus(jobId, signal, boundMilliseconds = loadTimeout, context) {
+    let metadata;
     try {
-      await fs.writeFile(file, lines.length ? lines.join("\n") + "\n" : "");
+      metadata = await withinDeadline(
+        () => options.bigquery.job(jobId, jobOptions).getMetadata(),
+        boundMilliseconds,
+        signal,
+        context
+      );
+    } catch (error) {
+      if (error?.code === 404) return "missing";
+      throw error;
+    }
+    const record = Array.isArray(metadata) ? metadata[0] : metadata;
+    if (record?.status?.state !== "DONE") return "running";
+    if (record.status.errorResult) {
+      const failure = new Error("A BigQuery load job for this batch failed; it will be retried as a new attempt");
+      Object.defineProperty(failure, "adcpJobFailed", { value: true });
+      throw failure;
+    }
+    return "done";
+  }
+  async function loadOnce(role, tableId, jobId, lines, timeouts, signal) {
+    signal?.throwIfAborted();
+    const table = `the ${role} table "${tableId}" (timeout count ${timeouts})`;
+    let status = await jobStatus(jobId, signal, loadTimeout, table);
+    if (status === "done") return;
+    const loadDeadline = reportingWarehouseLoadDeadlineV1(loadTimeout, timeouts);
+    const deadlineAt = Date.now() + loadDeadline;
+    const timedOut = (what) => {
+      const error = new Error(
+        `${what} exceeded this run's ${loadDeadline} ms deadline for ${table}; the next run checks the job again`
+      );
+      Object.defineProperty(error, "adcpDeadlineExceeded", { value: true });
+      Object.defineProperty(error, "adcpLoadTimedOut", { value: true });
+      return error;
+    };
+    if (status === "missing") {
       try {
-        await withinDeadline(
-          () => dataset.table(tableId).load(file, {
-            jobId,
-            sourceFormat: "NEWLINE_DELIMITED_JSON",
-            writeDisposition: "WRITE_APPEND",
-            jobTimeoutMs: String(loadTimeout),
-            ...options.location ? { location: options.location } : {},
-            ...options.labels ? { labels: { ...options.labels } } : {}
-          }),
-          loadTimeout,
-          signal
-        );
+        const running = uploads.get(jobId);
+        const upload = running && Date.now() - running.startedAt < MAX_LOAD_TIMEOUT ? running.done : startUpload(tableId, jobId, lines);
+        await withinDeadline(() => upload, Math.max(deadlineAt - Date.now(), 1), signal);
+        return;
       } catch (error) {
+        if (isDeadlineExceeded(error)) throw timedOut("Uploading the load");
         if (!isAlreadyExists(error)) throw error;
-        const metadata = await withinDeadline(
-          () => options.bigquery.job(jobId, options.location ? { location: options.location } : void 0).getMetadata(),
-          loadTimeout,
-          signal
-        );
-        assertJobSucceeded(metadata);
+        status = await jobStatus(jobId, signal, loadTimeout, table);
+        if (status === "missing") {
+          throw new Error(
+            "BigQuery refused the load job ID as already existing, but the job cannot be found. Check that projectId and location match the dataset (location is required for datasets outside the US and EU multi-regions)",
+            { cause: error }
+          );
+        }
       }
-    } finally {
-      await fs.rm(directory, { recursive: true, force: true });
+    }
+    const pollMilliseconds = Math.min(5e3, Math.max(250, Math.floor(loadDeadline / 10)));
+    while (status !== "done") {
+      const remaining = deadlineAt - Date.now();
+      if (remaining <= 0) throw timedOut("Waiting for the running load job");
+      await sleep(Math.min(pollMilliseconds, remaining), signal);
+      const bound = Math.max(Math.min(loadTimeout, deadlineAt - Date.now()), 1);
+      try {
+        status = await jobStatus(jobId, signal, bound, table);
+      } catch (error) {
+        if (isDeadlineExceeded(error) && bound < loadTimeout) throw timedOut("Waiting for the running load job");
+        throw error;
+      }
+      if (status === "missing") throw new Error("A BigQuery load job for this batch disappeared while it was awaited");
     }
   }
   return {
     defaultTablesSql: reportingWarehouseDefaultTablesSqlV1(tables),
     currentRowsViewSql: reportingWarehouseCurrentRowsViewSqlV1(tables),
+    async reset(resetOptions = {}) {
+      const removeConsumer = options.store.removeFeedConsumer?.bind(options.store);
+      if (!removeConsumer) {
+        throw new TypeError("reset() needs a store with removeFeedConsumer (PostgresReportingLedgerStore has it)");
+      }
+      const cleared = await options.db.query(
+        `UPDATE adcp_reporting_warehouse_sinks
+            SET committed_cursor = NULL, planned_from_cursor = NULL, planned_to_cursor = NULL,
+                planned_revisions = NULL, planned_rows_attempt = 0, planned_revisions_attempt = 0,
+                planned_rows_timeouts = 0, planned_revisions_timeouts = 0,
+                reset_generation = reset_generation + 1, updated_at = clock_timestamp()
+          WHERE sink_name = $1 AND ($2::boolean OR planned_to_cursor IS NULL)
+          RETURNING sink_name`,
+        [options.name, resetOptions.discardRetainedPlan === true]
+      ).catch(missingMigration);
+      if (cleared.rows.length === 0) {
+        const exists = await options.db.query(`SELECT 1 FROM adcp_reporting_warehouse_sinks WHERE sink_name = $1`, [
+          options.name
+        ]);
+        if (exists.rows.length > 0) {
+          throw new Error(
+            "Warehouse sink has a retained batch plan whose load jobs may have succeeded; reconcile it against BigQuery, then call reset({ discardRetainedPlan: true })"
+          );
+        }
+      }
+      try {
+        await removeConsumer(options.name);
+      } catch (cause) {
+        throw new Error(
+          "Warehouse sink state was cleared but its feed consumer could not be unregistered; call reset() again before restarting workers",
+          { cause }
+        );
+      }
+      return { cleared: cleared.rows.length > 0 };
+    },
     async runOnce(runOptions = {}) {
       const signal = runOptions.signal;
       await options.db.query(
@@ -91,46 +226,106 @@ function createBigQueryReportingWarehouseSinkV1(options) {
       );
       const state = (await options.db.query(
         `SELECT committed_cursor, planned_from_cursor, planned_to_cursor, planned_revisions,
-                  planned_rows_attempt, planned_revisions_attempt
+                  planned_rows_attempt, planned_revisions_attempt,
+                  planned_rows_timeouts, planned_revisions_timeouts, reset_generation
              FROM adcp_reporting_warehouse_sinks WHERE sink_name = $1`,
         [options.name]
-      )).rows[0];
+      ).catch(missingMigration)).rows[0];
+      if (state.planned_to_cursor !== null && state.planned_from_cursor !== state.committed_cursor) {
+        throw new Error(
+          "Warehouse sink has an inconsistent retained plan; reconcile the plan and BigQuery jobs before resuming"
+        );
+      }
+      if (options.account_id && state.planned_revisions?.some((planned) => planned.account_id !== options.account_id)) {
+        throw new Error(
+          "Warehouse sink has a retained plan for another account; this sink name is shared across feed scopes"
+        );
+      }
       let plan = state.planned_to_cursor && state.planned_revisions ? {
         from: state.planned_from_cursor,
         to: state.planned_to_cursor,
         revisions: state.planned_revisions,
         rowsAttempt: state.planned_rows_attempt,
-        revisionsAttempt: state.planned_revisions_attempt
+        revisionsAttempt: state.planned_revisions_attempt,
+        rowsTimeouts: state.planned_rows_timeouts,
+        revisionsTimeouts: state.planned_revisions_timeouts
       } : void 0;
       if (!plan) {
-        const page = await options.store.changesAfter({
+        const position = {
           ...state.committed_cursor ? { cursor: state.committed_cursor } : {},
           ...options.account_id ? { account_id: options.account_id } : {},
-          kinds: ["revision"],
           limit: maxRevisions
-        });
-        if (page.records.length === 0) {
-          if (page.cursor !== state.committed_cursor) await commit(page.cursor);
+        };
+        let page;
+        let readCursor = position.cursor;
+        try {
+          page = await options.store.changesAfter({ ...position, kinds: ["revision"] });
+          if (page.records.length === 0) {
+            readCursor = page.cursor;
+            page = await options.store.changesAfter({ ...position, cursor: readCursor });
+          }
+        } catch (error) {
+          if (error?.code === "CURSOR_EXPIRED" && state.committed_cursor === null) {
+            const cursor = readCursor ?? error.cursor;
+            if (cursor !== void 0 && typeof cursor !== "string") throw error;
+            if (cursor !== void 0) {
+              try {
+                decodeReportingChangeCursorV1(cursor, position.account_id ?? null);
+              } catch {
+                throw error;
+              }
+              await options.db.query(
+                `UPDATE adcp_reporting_warehouse_sinks SET committed_cursor = $2, updated_at = clock_timestamp()
+                  WHERE sink_name = $1 AND committed_cursor IS NULL AND reset_generation = $3
+                    AND planned_from_cursor IS NULL AND planned_to_cursor IS NULL AND planned_revisions IS NULL`,
+                [options.name, cursor, state.reset_generation]
+              );
+            }
+          }
+          throw error;
+        }
+        if (page.records.length === 0 && page.cursor === state.committed_cursor) {
+          if (state.committed_cursor) {
+            await options.store.saveFeedConsumerCursor(options.name, state.committed_cursor, {
+              refreshUnchanged: false
+            });
+          }
           return { idle: true, revisions: 0, rows: 0, missing: [] };
         }
         plan = {
           from: state.committed_cursor,
           to: page.cursor,
-          revisions: page.records.map((record) => ({ id: record.record_id, account_id: record.account_id })),
+          revisions: page.records.filter((record) => record.kind === "revision").map((record) => ({ id: record.record_id, account_id: record.account_id })),
           rowsAttempt: 0,
-          revisionsAttempt: 0
+          revisionsAttempt: 0,
+          rowsTimeouts: 0,
+          revisionsTimeouts: 0
         };
-        await options.db.query(
+        const claimed = await options.db.query(
           `UPDATE adcp_reporting_warehouse_sinks
               SET planned_from_cursor = $2, planned_to_cursor = $3, planned_revisions = $4::jsonb,
-                  planned_rows_attempt = 0, planned_revisions_attempt = 0, updated_at = clock_timestamp()
-            WHERE sink_name = $1`,
-          [options.name, plan.from, plan.to, JSON.stringify(plan.revisions)]
+                  planned_rows_attempt = 0, planned_revisions_attempt = 0,
+                  planned_rows_timeouts = 0, planned_revisions_timeouts = 0, updated_at = clock_timestamp()
+            WHERE sink_name = $1 AND committed_cursor IS NOT DISTINCT FROM $2
+              AND planned_to_cursor IS NULL AND planned_revisions IS NULL AND reset_generation = $5
+            RETURNING sink_name`,
+          [options.name, plan.from, plan.to, JSON.stringify(plan.revisions), state.reset_generation]
         );
+        if (claimed.rows.length === 0) throw stalePlan();
       }
+      const planValues = [
+        options.name,
+        plan.from,
+        plan.to,
+        JSON.stringify(plan.revisions),
+        plan.rowsAttempt,
+        plan.revisionsAttempt,
+        state.reset_generation
+      ];
       const rowLines = [];
       const revisionLines = [];
       const missing = [];
+      const resolved = [];
       for (const planned of plan.revisions) {
         signal?.throwIfAborted();
         const revision = await options.store.getRevisionMetadata(planned.id, planned.account_id);
@@ -138,10 +333,10 @@ function createBigQueryReportingWarehouseSinkV1(options) {
           missing.push(planned.id);
           continue;
         }
-        const periodDate = sourceLocalDate(
-          revision.wireRevision.period?.start,
-          revision.wireRevision.period?.source_timezone
-        );
+        resolved.push({ planned, revision, periodDate: periodDateOf(revision) });
+      }
+      for (const { planned, revision, periodDate } of resolved) {
+        signal?.throwIfAborted();
         const base = {
           reporting_revision_id: revision.reporting_revision_id,
           reporting_obligation_id: revision.reporting_obligation_id,
@@ -170,9 +365,9 @@ function createBigQueryReportingWarehouseSinkV1(options) {
           JSON.stringify({
             ...base,
             supersedes_reporting_revision_id: revision.supersedes_reporting_revision_id ?? null,
-            period_start: revision.wireRevision.period?.start ?? null,
-            period_end: revision.wireRevision.period?.end ?? null,
-            source_timezone: revision.wireRevision.period?.source_timezone ?? null,
+            period_start: revision.wireRevision.period.start,
+            period_end: revision.wireRevision.period.end,
+            source_timezone: revision.wireRevision.period.source_timezone,
             observed_at: revision.observedAt,
             data_through: revision.dataThrough,
             revision_content_sha256: revision.binding.sha256,
@@ -181,46 +376,87 @@ function createBigQueryReportingWarehouseSinkV1(options) {
           })
         );
       }
-      const baseJobId = reportingWarehouseJobIdV1(options.name, plan.from, plan.to);
+      const baseJobId = reportingWarehouseJobIdV1(options.name, plan.from, plan.to, state.reset_generation);
       const suffix = (attempt) => attempt ? `_a${attempt}` : "";
       const steps = [
-        { table: options.rowsTableId, kind: "rows", lines: rowLines, attempt: plan.rowsAttempt },
-        { table: options.revisionsTableId, kind: "revs", lines: revisionLines, attempt: plan.revisionsAttempt }
+        {
+          table: options.rowsTableId,
+          role: "rows",
+          kind: "rows",
+          lines: rowLines,
+          attempt: plan.rowsAttempt,
+          timeouts: plan.rowsTimeouts,
+          timeoutColumn: "planned_rows_timeouts",
+          attemptColumn: "planned_rows_attempt"
+        },
+        {
+          table: options.revisionsTableId,
+          role: "revisions",
+          kind: "revs",
+          lines: revisionLines,
+          attempt: plan.revisionsAttempt,
+          timeouts: plan.revisionsTimeouts,
+          timeoutColumn: "planned_revisions_timeouts",
+          attemptColumn: "planned_revisions_attempt"
+        }
       ];
       for (const step of steps) {
         if (!step.lines.length) continue;
         try {
-          await loadOnce(step.table, `${baseJobId}_${step.kind}${suffix(step.attempt)}`, [...step.lines], signal);
+          await loadOnce(
+            step.role,
+            step.table,
+            `${baseJobId}_${step.kind}${suffix(step.attempt)}`,
+            [...step.lines],
+            step.timeouts,
+            signal
+          );
         } catch (error) {
           if (isFinishedWithError(error)) {
-            const column = step.kind === "rows" ? "planned_rows_attempt" : "planned_revisions_attempt";
             await options.db.query(
-              `UPDATE adcp_reporting_warehouse_sinks SET ${column} = ${column} + 1,
-                      updated_at = clock_timestamp() WHERE sink_name = $1`,
-              [options.name]
+              `UPDATE adcp_reporting_warehouse_sinks SET ${step.attemptColumn} = ${step.attemptColumn} + 1,
+                      updated_at = clock_timestamp() ${matchesPlan}`,
+              planValues
+            );
+          } else if (isLoadTimedOut(error)) {
+            await options.db.query(
+              `UPDATE adcp_reporting_warehouse_sinks
+                  SET ${step.timeoutColumn} = $8::integer + 1, updated_at = clock_timestamp()
+                ${matchesPlan} AND ${step.timeoutColumn} = $8::integer`,
+              [...planValues, step.timeouts]
             );
           }
           throw error;
         }
       }
-      await commit(plan.to);
-      return { idle: false, revisions: revisionLines.length, rows: rowLines.length, missing };
+      const committed = await options.db.query(
+        `UPDATE adcp_reporting_warehouse_sinks
+            SET committed_cursor = $3, planned_from_cursor = NULL, planned_to_cursor = NULL,
+                planned_revisions = NULL, planned_rows_attempt = 0, planned_revisions_attempt = 0,
+                planned_rows_timeouts = 0, planned_revisions_timeouts = 0, updated_at = clock_timestamp()
+          ${matchesPlan} RETURNING sink_name`,
+        planValues
+      );
+      if (committed.rows.length === 0) throw stalePlan();
+      await options.store.saveFeedConsumerCursor(options.name, plan.to);
+      return { idle: plan.revisions.length === 0, revisions: revisionLines.length, rows: rowLines.length, missing };
     }
   };
-  async function commit(cursor) {
-    await options.db.query(
-      `UPDATE adcp_reporting_warehouse_sinks
-          SET committed_cursor = $2, planned_from_cursor = NULL, planned_to_cursor = NULL,
-              planned_revisions = NULL, planned_rows_attempt = 0, planned_revisions_attempt = 0,
-              updated_at = clock_timestamp()
-        WHERE sink_name = $1`,
-      [options.name, cursor]
+}
+function missingMigration(error) {
+  if (error?.code === "42703") {
+    throw new Error(
+      "The adcp_reporting_warehouse_sinks table is missing columns; apply REPORTING_WAREHOUSE_SINK_MIGRATION again (it is idempotent)",
+      { cause: error }
     );
-    await options.store.saveFeedConsumerCursor(options.name, cursor);
   }
+  throw error;
+}
+function reportingWarehouseLoadDeadlineV1(baseMilliseconds, timeouts) {
+  return Math.min(baseMilliseconds * 2 ** Math.min(Math.max(timeouts, 0), MAX_TIMEOUT_DOUBLINGS), MAX_LOAD_TIMEOUT);
 }
-function reportingWarehouseJobIdV1(name, from, to) {
-  return `adcp_rr_${createHash("sha256").update(JSON.stringify([name, from, to])).digest("hex").slice(0, 40)}`;
+function reportingWarehouseJobIdV1(name, from, to, generation = 0) {
+  return `adcp_rr_${createHash("sha256").update(JSON.stringify(generation > 0 ? [name, from, to, generation] : [name, from, to])).digest("hex").slice(0, 40)}`;
 }
 function qualified(tables, table) {
   return `\`${tables.projectId ? `${tables.projectId}.` : ""}${tables.datasetId}.${table}\``;
@@ -275,23 +511,19 @@ SELECT row_data.*
   ) AS current_revision
  USING (reporting_obligation_id, reporting_revision_id);`.trim();
 }
-function sourceLocalDate(start, timeZone) {
-  if (!start) return null;
-  const instant = new Date(start);
-  if (Number.isNaN(instant.getTime())) return null;
+function periodDateOf(revision) {
+  const period = revision.wireRevision.period;
   try {
-    const parts = new Intl.DateTimeFormat("en-CA", {
-      timeZone: timeZone ?? "UTC",
-      year: "numeric",
-      month: "2-digit",
-      day: "2-digit"
-    }).format(instant);
-    return parts;
-  } catch {
-    return instant.toISOString().slice(0, 10);
+    if (!period) throw new TypeError("the revision carries no period");
+    return reportingPeriodDate(period.start, period.source_timezone);
+  } catch (error) {
+    throw new Error(
+      `Cannot compute period_date for revision ${revision.reporting_revision_id}: ${error instanceof Error ? error.message : String(error)}. The batch stays retained and every run fails until the revision's period is repaired (or, if this runtime's timezone data is older than the writer's, until it is upgraded); reset({ discardRetainedPlan: true }) restarts from the beginning, so empty the destination tables first.`,
+      { cause: error }
+    );
   }
 }
-async function withinDeadline(operation, milliseconds, signal) {
+async function withinDeadline(operation, milliseconds, signal, context) {
   signal?.throwIfAborted();
   let timer;
   let onAbort;
@@ -299,10 +531,13 @@ async function withinDeadline(operation, milliseconds, signal) {
     return await Promise.race([
       operation(),
       new Promise((_, reject) => {
-        timer = setTimeout(
-          () => reject(new Error("BigQuery call exceeded its deadline; the batch will be retried")),
-          milliseconds
-        );
+        timer = setTimeout(() => {
+          const error = new Error(
+            `BigQuery call exceeded its deadline${context ? ` of ${milliseconds} ms for ${context}` : ""}; the batch will be retried`
+          );
+          Object.defineProperty(error, "adcpDeadlineExceeded", { value: true });
+          reject(error);
+        }, milliseconds);
         if (signal) {
           onAbort = () => reject(signal.reason ?? new Error("aborted"));
           signal.addEventListener("abort", onAbort, { once: true });
@@ -314,28 +549,38 @@ async function withinDeadline(operation, milliseconds, signal) {
     if (signal && onAbort) signal.removeEventListener("abort", onAbort);
   }
 }
+function isDeadlineExceeded(error) {
+  return error?.adcpDeadlineExceeded === true;
+}
 function isAlreadyExists(error) {
   const record = error;
   return record?.code === 409 || !!record?.errors?.some((entry) => entry.reason === "duplicate");
 }
+function isLoadTimedOut(error) {
+  return error?.adcpLoadTimedOut === true;
+}
+function sleep(milliseconds, signal) {
+  signal?.throwIfAborted();
+  return new Promise((resolve, reject) => {
+    const onAbort = () => {
+      clearTimeout(timer);
+      reject(signal?.reason ?? new Error("aborted"));
+    };
+    const timer = setTimeout(() => {
+      signal?.removeEventListener("abort", onAbort);
+      resolve();
+    }, milliseconds);
+    signal?.addEventListener("abort", onAbort, { once: true });
+  });
+}
 function isFinishedWithError(error) {
   return error?.adcpJobFailed === true;
 }
-function assertJobSucceeded(metadata) {
-  const record = Array.isArray(metadata) ? metadata[0] : metadata;
-  if (record?.status?.state !== "DONE") {
-    throw new Error("A BigQuery load job for this batch is still running; retry later");
-  }
-  if (record.status.errorResult) {
-    const failure = new Error("A BigQuery load job for this batch failed; it will be retried as a new attempt");
-    Object.defineProperty(failure, "adcpJobFailed", { value: true });
-    throw failure;
-  }
-}
 export {
   REPORTING_WAREHOUSE_SINK_MIGRATION,
   createBigQueryReportingWarehouseSinkV1,
   reportingWarehouseCurrentRowsViewSqlV1,
   reportingWarehouseDefaultTablesSqlV1,
-  reportingWarehouseJobIdV1
+  reportingWarehouseJobIdV1,
+  reportingWarehouseLoadDeadlineV1
 };

package/dist/lib/reporting/gcs/adapter.d.mts

@@ -5,6 +5,9 @@ import type { ReportingLedgerStore } from '../ledger/types.mjs';
 import { type ReportingObjectWriteAuthorityStoreV1 } from '../ledger/object-writes.mjs';
 import { type GcsReportingReaderOptionsV1 } from './reader.mjs';
 type DeliveryInput = Parameters<ReportingManagedDeliveryAdapterV1['deliver']>[0];
+/**
+ * @experimental Reliable Reporting storage surface: breaking changes may ship in minor releases. See https://github.com/adcontextprotocol/adcp-client/blob/main/docs/guides/REPORTING-STABILITY.md
+ */
 export interface CreateGcsReportingManagedDeliveryAdapterOptionsV1 {
     storage: Storage;
     coreStore: ReportingLedgerStore;
@@ -24,7 +27,11 @@ export interface CreateGcsReportingManagedDeliveryAdapterOptionsV1 {
     operationDeadlineMilliseconds?: number;
     minimumResourceRetentionDays?: number;
 }
-/** Complete file-transfer adapter. Availability and cleanup lease commits remain with the managed runtime. */
+/**
+ * Complete file-transfer adapter. Availability and cleanup lease commits remain with the managed runtime.
+ *
+ * @experimental Reliable Reporting storage surface: breaking changes may ship in minor releases. See https://github.com/adcontextprotocol/adcp-client/blob/main/docs/guides/REPORTING-STABILITY.md
+ */
 export declare function createGcsReportingManagedDeliveryAdapterV1(options: CreateGcsReportingManagedDeliveryAdapterOptionsV1, context?: {
     signal?: AbortSignal;
 }): Promise<ReportingManagedDeliveryAdapterV1>;

package/dist/lib/reporting/gcs/adapter.d.ts

@@ -5,6 +5,9 @@ import type { ReportingLedgerStore } from '../ledger/types';
 import { type ReportingObjectWriteAuthorityStoreV1 } from '../ledger/object-writes';
 import { type GcsReportingReaderOptionsV1 } from './reader';
 type DeliveryInput = Parameters<ReportingManagedDeliveryAdapterV1['deliver']>[0];
+/**
+ * @experimental Reliable Reporting storage surface: breaking changes may ship in minor releases. See https://github.com/adcontextprotocol/adcp-client/blob/main/docs/guides/REPORTING-STABILITY.md
+ */
 export interface CreateGcsReportingManagedDeliveryAdapterOptionsV1 {
     storage: Storage;
     coreStore: ReportingLedgerStore;
@@ -24,7 +27,11 @@ export interface CreateGcsReportingManagedDeliveryAdapterOptionsV1 {
     operationDeadlineMilliseconds?: number;
     minimumResourceRetentionDays?: number;
 }
-/** Complete file-transfer adapter. Availability and cleanup lease commits remain with the managed runtime. */
+/**
+ * Complete file-transfer adapter. Availability and cleanup lease commits remain with the managed runtime.
+ *
+ * @experimental Reliable Reporting storage surface: breaking changes may ship in minor releases. See https://github.com/adcontextprotocol/adcp-client/blob/main/docs/guides/REPORTING-STABILITY.md
+ */
 export declare function createGcsReportingManagedDeliveryAdapterV1(options: CreateGcsReportingManagedDeliveryAdapterOptionsV1, context?: {
     signal?: AbortSignal;
 }): Promise<ReportingManagedDeliveryAdapterV1>;

package/dist/lib/reporting/gcs/errors.d.mts

@@ -1,5 +1,11 @@
+/**
+ * @experimental Reliable Reporting storage surface: breaking changes may ship in minor releases. See https://github.com/adcontextprotocol/adcp-client/blob/main/docs/guides/REPORTING-STABILITY.md
+ */
 export declare class ReportingGcsFenceError extends Error {
     readonly code: 'INVALID_INPUT' | 'UNSAFE_BUCKET' | 'REVOKED' | 'NOT_REVOKED' | 'CONTENT_CONFLICT' | 'ABORTED' | 'PROVIDER_UNAVAILABLE' | 'DEADLINE_EXCEEDED' | 'HOST_CALLBACK_FAILED' | 'STATE_UNAVAILABLE';
     constructor(code: 'INVALID_INPUT' | 'UNSAFE_BUCKET' | 'REVOKED' | 'NOT_REVOKED' | 'CONTENT_CONFLICT' | 'ABORTED' | 'PROVIDER_UNAVAILABLE' | 'DEADLINE_EXCEEDED' | 'HOST_CALLBACK_FAILED' | 'STATE_UNAVAILABLE');
 }
+/**
+ * @experimental Reliable Reporting storage surface: breaking changes may ship in minor releases. See https://github.com/adcontextprotocol/adcp-client/blob/main/docs/guides/REPORTING-STABILITY.md
+ */
 export declare function isReportingGcsFenceError(error: unknown): error is ReportingGcsFenceError;