@adcp/sdk 15.9.0 → 16.0.0-beta.1

package/dist/lib/reporting/bigquery/index.js

@@ -32,7 +32,8 @@ __export(bigquery_exports, {
   createBigQueryReportingWarehouseSinkV1: () => createBigQueryReportingWarehouseSinkV1,
   reportingWarehouseCurrentRowsViewSqlV1: () => reportingWarehouseCurrentRowsViewSqlV1,
   reportingWarehouseDefaultTablesSqlV1: () => reportingWarehouseDefaultTablesSqlV1,
-  reportingWarehouseJobIdV1: () => reportingWarehouseJobIdV1
+  reportingWarehouseJobIdV1: () => reportingWarehouseJobIdV1,
+  reportingWarehouseLoadDeadlineV1: () => reportingWarehouseLoadDeadlineV1
 });
 module.exports = __toCommonJS(bigquery_exports);
 var import_crypto = require("crypto");
@@ -40,6 +41,7 @@ var import_fs = require("fs");
 var import_os = require("os");
 var path = __toESM(require("path"));
 var import_change_feed = require('../ledger/change-feed.js');
+var import_schedule = require('../ledger/schedule.js');
 const REPORTING_WAREHOUSE_SINK_MIGRATION = `
 CREATE TABLE IF NOT EXISTS adcp_reporting_warehouse_sinks (
   sink_name TEXT PRIMARY KEY CHECK (sink_name ~ '^[A-Za-z0-9_.:-]{1,128}$'),
@@ -49,14 +51,21 @@ CREATE TABLE IF NOT EXISTS adcp_reporting_warehouse_sinks (
   planned_revisions JSONB,
   planned_rows_attempt INTEGER NOT NULL DEFAULT 0 CHECK (planned_rows_attempt >= 0),
   planned_revisions_attempt INTEGER NOT NULL DEFAULT 0 CHECK (planned_revisions_attempt >= 0),
+  planned_rows_timeouts INTEGER NOT NULL DEFAULT 0 CHECK (planned_rows_timeouts >= 0),
+  planned_revisions_timeouts INTEGER NOT NULL DEFAULT 0 CHECK (planned_revisions_timeouts >= 0),
   reset_generation INTEGER NOT NULL DEFAULT 0 CHECK (reset_generation >= 0),
   updated_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
   CHECK ((planned_to_cursor IS NULL) = (planned_revisions IS NULL))
 );
 ALTER TABLE adcp_reporting_warehouse_sinks
   ADD COLUMN IF NOT EXISTS reset_generation INTEGER NOT NULL DEFAULT 0 CHECK (reset_generation >= 0);
+ALTER TABLE adcp_reporting_warehouse_sinks
+  ADD COLUMN IF NOT EXISTS planned_rows_timeouts INTEGER NOT NULL DEFAULT 0 CHECK (planned_rows_timeouts >= 0);
+ALTER TABLE adcp_reporting_warehouse_sinks
+  ADD COLUMN IF NOT EXISTS planned_revisions_timeouts INTEGER NOT NULL DEFAULT 0 CHECK (planned_revisions_timeouts >= 0);
 `.trim();
 const MAX_LOAD_TIMEOUT = 6 * 36e5;
+const MAX_TIMEOUT_DOUBLINGS = 16;
 let warnedLabelsIgnored = false;
 const PROJECT = /^[a-z][a-z0-9-]{4,61}[a-z0-9]$/;
 const DATASET_OR_TABLE = /^[A-Za-z_][A-Za-z0-9_]{0,1023}$/;
@@ -109,51 +118,104 @@ function createBigQueryReportingWarehouseSinkV1(options) {
     ...options.projectId ? { projectId: options.projectId } : {},
     ...options.location ? { location: options.location } : {}
   } : void 0;
-  async function cancelLoad(jobId, signal) {
+  const uploads = /* @__PURE__ */ new Map();
+  function startUpload(tableId, jobId, lines) {
+    const entry = {
+      startedAt: Date.now(),
+      done: Promise.resolve()
+    };
+    const cleanup = async () => {
+      if (uploads.get(jobId) === entry) uploads.delete(jobId);
+      if (entry.directory) await import_fs.promises.rm(entry.directory, { recursive: true, force: true }).catch(() => {
+      });
+    };
+    entry.done = (async () => {
+      const directory = await import_fs.promises.mkdtemp(path.join((0, import_os.tmpdir)(), "adcp-warehouse-"));
+      entry.directory = directory;
+      const file = path.join(directory, `${jobId}.ndjson`);
+      await import_fs.promises.writeFile(file, lines.length ? lines.join("\n") + "\n" : "", { mode: 384 });
+      return dataset.table(tableId).load(file, {
+        jobId,
+        sourceFormat: "NEWLINE_DELIMITED_JSON",
+        writeDisposition: "WRITE_APPEND",
+        ...options.location ? { location: options.location } : {}
+      });
+    })();
+    const replaced = uploads.get(jobId);
+    uploads.set(jobId, entry);
+    void entry.done.then(cleanup, cleanup);
+    if (replaced?.directory) void import_fs.promises.rm(replaced.directory, { recursive: true, force: true }).catch(() => {
+    });
+    return entry.done;
+  }
+  async function jobStatus(jobId, signal, boundMilliseconds = loadTimeout, context) {
+    let metadata;
     try {
-      const job = options.bigquery.job(jobId, jobOptions);
-      if (typeof job.cancel !== "function") return;
-      const cancel = job.cancel.bind(job);
-      await withinDeadline(cancel, Math.min(loadTimeout, 3e4), signal);
-    } catch (error) {
-      const code = error?.code;
-      if (code === 404 || signal?.aborted) return;
-      process.emitWarning(
-        `Cancelling BigQuery load job ${jobId} failed (${typeof code === "number" || typeof code === "string" ? code : "no status code"}); the job may still be running`,
-        { code: "ADCP_BIGQUERY_SINK_CANCEL_FAILED" }
+      metadata = await withinDeadline(
+        () => options.bigquery.job(jobId, jobOptions).getMetadata(),
+        boundMilliseconds,
+        signal,
+        context
       );
+    } catch (error) {
+      if (error?.code === 404) return "missing";
+      throw error;
+    }
+    const record = Array.isArray(metadata) ? metadata[0] : metadata;
+    if (record?.status?.state !== "DONE") return "running";
+    if (record.status.errorResult) {
+      const failure = new Error("A BigQuery load job for this batch failed; it will be retried as a new attempt");
+      Object.defineProperty(failure, "adcpJobFailed", { value: true });
+      throw failure;
     }
+    return "done";
   }
-  async function loadOnce(tableId, jobId, lines, attempt, signal) {
-    const loadDeadline = Math.min(loadTimeout * 2 ** Math.min(attempt, 16), MAX_LOAD_TIMEOUT);
+  async function loadOnce(role, tableId, jobId, lines, timeouts, signal) {
     signal?.throwIfAborted();
-    const directory = await import_fs.promises.mkdtemp(path.join((0, import_os.tmpdir)(), "adcp-warehouse-"));
-    const file = path.join(directory, `${jobId}.ndjson`);
-    try {
-      await import_fs.promises.writeFile(file, lines.length ? lines.join("\n") + "\n" : "", { mode: 384 });
+    const table = `the ${role} table "${tableId}" (timeout count ${timeouts})`;
+    let status = await jobStatus(jobId, signal, loadTimeout, table);
+    if (status === "done") return;
+    const loadDeadline = reportingWarehouseLoadDeadlineV1(loadTimeout, timeouts);
+    const deadlineAt = Date.now() + loadDeadline;
+    const timedOut = (what) => {
+      const error = new Error(
+        `${what} exceeded this run's ${loadDeadline} ms deadline for ${table}; the next run checks the job again`
+      );
+      Object.defineProperty(error, "adcpDeadlineExceeded", { value: true });
+      Object.defineProperty(error, "adcpLoadTimedOut", { value: true });
+      return error;
+    };
+    if (status === "missing") {
       try {
-        await withinDeadline(
-          () => dataset.table(tableId).load(file, {
-            jobId,
-            sourceFormat: "NEWLINE_DELIMITED_JSON",
-            writeDisposition: "WRITE_APPEND",
-            ...options.location ? { location: options.location } : {}
-          }),
-          loadDeadline,
-          signal
-        );
+        const running = uploads.get(jobId);
+        const upload = running && Date.now() - running.startedAt < MAX_LOAD_TIMEOUT ? running.done : startUpload(tableId, jobId, lines);
+        await withinDeadline(() => upload, Math.max(deadlineAt - Date.now(), 1), signal);
+        return;
       } catch (error) {
-        if (isDeadlineExceeded(error)) await cancelLoad(jobId, signal);
+        if (isDeadlineExceeded(error)) throw timedOut("Uploading the load");
         if (!isAlreadyExists(error)) throw error;
-        const metadata = await withinDeadline(
-          () => options.bigquery.job(jobId, jobOptions).getMetadata(),
-          loadTimeout,
-          signal
-        );
-        assertJobSucceeded(metadata);
+        status = await jobStatus(jobId, signal, loadTimeout, table);
+        if (status === "missing") {
+          throw new Error(
+            "BigQuery refused the load job ID as already existing, but the job cannot be found. Check that projectId and location match the dataset (location is required for datasets outside the US and EU multi-regions)",
+            { cause: error }
+          );
+        }
+      }
+    }
+    const pollMilliseconds = Math.min(5e3, Math.max(250, Math.floor(loadDeadline / 10)));
+    while (status !== "done") {
+      const remaining = deadlineAt - Date.now();
+      if (remaining <= 0) throw timedOut("Waiting for the running load job");
+      await sleep(Math.min(pollMilliseconds, remaining), signal);
+      const bound = Math.max(Math.min(loadTimeout, deadlineAt - Date.now()), 1);
+      try {
+        status = await jobStatus(jobId, signal, bound, table);
+      } catch (error) {
+        if (isDeadlineExceeded(error) && bound < loadTimeout) throw timedOut("Waiting for the running load job");
+        throw error;
       }
-    } finally {
-      await import_fs.promises.rm(directory, { recursive: true, force: true });
+      if (status === "missing") throw new Error("A BigQuery load job for this batch disappeared while it was awaited");
     }
   }
   return {
@@ -168,6 +230,7 @@ function createBigQueryReportingWarehouseSinkV1(options) {
         `UPDATE adcp_reporting_warehouse_sinks
             SET committed_cursor = NULL, planned_from_cursor = NULL, planned_to_cursor = NULL,
                 planned_revisions = NULL, planned_rows_attempt = 0, planned_revisions_attempt = 0,
+                planned_rows_timeouts = 0, planned_revisions_timeouts = 0,
                 reset_generation = reset_generation + 1, updated_at = clock_timestamp()
           WHERE sink_name = $1 AND ($2::boolean OR planned_to_cursor IS NULL)
           RETURNING sink_name`,
@@ -201,7 +264,8 @@ function createBigQueryReportingWarehouseSinkV1(options) {
       );
       const state = (await options.db.query(
         `SELECT committed_cursor, planned_from_cursor, planned_to_cursor, planned_revisions,
-                  planned_rows_attempt, planned_revisions_attempt, reset_generation
+                  planned_rows_attempt, planned_revisions_attempt,
+                  planned_rows_timeouts, planned_revisions_timeouts, reset_generation
              FROM adcp_reporting_warehouse_sinks WHERE sink_name = $1`,
         [options.name]
       ).catch(missingMigration)).rows[0];
@@ -220,7 +284,9 @@ function createBigQueryReportingWarehouseSinkV1(options) {
         to: state.planned_to_cursor,
         revisions: state.planned_revisions,
         rowsAttempt: state.planned_rows_attempt,
-        revisionsAttempt: state.planned_revisions_attempt
+        revisionsAttempt: state.planned_revisions_attempt,
+        rowsTimeouts: state.planned_rows_timeouts,
+        revisionsTimeouts: state.planned_revisions_timeouts
       } : void 0;
       if (!plan) {
         const position = {
@@ -269,12 +335,15 @@ function createBigQueryReportingWarehouseSinkV1(options) {
           to: page.cursor,
           revisions: page.records.filter((record) => record.kind === "revision").map((record) => ({ id: record.record_id, account_id: record.account_id })),
           rowsAttempt: 0,
-          revisionsAttempt: 0
+          revisionsAttempt: 0,
+          rowsTimeouts: 0,
+          revisionsTimeouts: 0
         };
         const claimed = await options.db.query(
           `UPDATE adcp_reporting_warehouse_sinks
               SET planned_from_cursor = $2, planned_to_cursor = $3, planned_revisions = $4::jsonb,
-                  planned_rows_attempt = 0, planned_revisions_attempt = 0, updated_at = clock_timestamp()
+                  planned_rows_attempt = 0, planned_revisions_attempt = 0,
+                  planned_rows_timeouts = 0, planned_revisions_timeouts = 0, updated_at = clock_timestamp()
             WHERE sink_name = $1 AND committed_cursor IS NOT DISTINCT FROM $2
               AND planned_to_cursor IS NULL AND planned_revisions IS NULL AND reset_generation = $5
             RETURNING sink_name`,
@@ -294,6 +363,7 @@ function createBigQueryReportingWarehouseSinkV1(options) {
       const rowLines = [];
       const revisionLines = [];
       const missing = [];
+      const resolved = [];
       for (const planned of plan.revisions) {
         signal?.throwIfAborted();
         const revision = await options.store.getRevisionMetadata(planned.id, planned.account_id);
@@ -301,10 +371,10 @@ function createBigQueryReportingWarehouseSinkV1(options) {
           missing.push(planned.id);
           continue;
         }
-        const periodDate = sourceLocalDate(
-          revision.wireRevision.period?.start,
-          revision.wireRevision.period?.source_timezone
-        );
+        resolved.push({ planned, revision, periodDate: periodDateOf(revision) });
+      }
+      for (const { planned, revision, periodDate } of resolved) {
+        signal?.throwIfAborted();
         const base = {
           reporting_revision_id: revision.reporting_revision_id,
           reporting_obligation_id: revision.reporting_obligation_id,
@@ -333,9 +403,9 @@ function createBigQueryReportingWarehouseSinkV1(options) {
           JSON.stringify({
             ...base,
             supersedes_reporting_revision_id: revision.supersedes_reporting_revision_id ?? null,
-            period_start: revision.wireRevision.period?.start ?? null,
-            period_end: revision.wireRevision.period?.end ?? null,
-            source_timezone: revision.wireRevision.period?.source_timezone ?? null,
+            period_start: revision.wireRevision.period.start,
+            period_end: revision.wireRevision.period.end,
+            source_timezone: revision.wireRevision.period.source_timezone,
             observed_at: revision.observedAt,
             data_through: revision.dataThrough,
             revision_content_sha256: revision.binding.sha256,
@@ -347,27 +417,52 @@ function createBigQueryReportingWarehouseSinkV1(options) {
       const baseJobId = reportingWarehouseJobIdV1(options.name, plan.from, plan.to, state.reset_generation);
       const suffix = (attempt) => attempt ? `_a${attempt}` : "";
       const steps = [
-        { table: options.rowsTableId, kind: "rows", lines: rowLines, attempt: plan.rowsAttempt },
-        { table: options.revisionsTableId, kind: "revs", lines: revisionLines, attempt: plan.revisionsAttempt }
+        {
+          table: options.rowsTableId,
+          role: "rows",
+          kind: "rows",
+          lines: rowLines,
+          attempt: plan.rowsAttempt,
+          timeouts: plan.rowsTimeouts,
+          timeoutColumn: "planned_rows_timeouts",
+          attemptColumn: "planned_rows_attempt"
+        },
+        {
+          table: options.revisionsTableId,
+          role: "revisions",
+          kind: "revs",
+          lines: revisionLines,
+          attempt: plan.revisionsAttempt,
+          timeouts: plan.revisionsTimeouts,
+          timeoutColumn: "planned_revisions_timeouts",
+          attemptColumn: "planned_revisions_attempt"
+        }
       ];
       for (const step of steps) {
         if (!step.lines.length) continue;
         try {
           await loadOnce(
+            step.role,
             step.table,
             `${baseJobId}_${step.kind}${suffix(step.attempt)}`,
             [...step.lines],
-            step.attempt,
+            step.timeouts,
             signal
           );
         } catch (error) {
           if (isFinishedWithError(error)) {
-            const column = step.kind === "rows" ? "planned_rows_attempt" : "planned_revisions_attempt";
             await options.db.query(
-              `UPDATE adcp_reporting_warehouse_sinks SET ${column} = ${column} + 1,
+              `UPDATE adcp_reporting_warehouse_sinks SET ${step.attemptColumn} = ${step.attemptColumn} + 1,
                       updated_at = clock_timestamp() ${matchesPlan}`,
               planValues
             );
+          } else if (isLoadTimedOut(error)) {
+            await options.db.query(
+              `UPDATE adcp_reporting_warehouse_sinks
+                  SET ${step.timeoutColumn} = $8::integer + 1, updated_at = clock_timestamp()
+                ${matchesPlan} AND ${step.timeoutColumn} = $8::integer`,
+              [...planValues, step.timeouts]
+            );
           }
           throw error;
         }
@@ -376,7 +471,7 @@ function createBigQueryReportingWarehouseSinkV1(options) {
         `UPDATE adcp_reporting_warehouse_sinks
             SET committed_cursor = $3, planned_from_cursor = NULL, planned_to_cursor = NULL,
                 planned_revisions = NULL, planned_rows_attempt = 0, planned_revisions_attempt = 0,
-                updated_at = clock_timestamp()
+                planned_rows_timeouts = 0, planned_revisions_timeouts = 0, updated_at = clock_timestamp()
           ${matchesPlan} RETURNING sink_name`,
         planValues
       );
@@ -395,6 +490,9 @@ function missingMigration(error) {
   }
   throw error;
 }
+function reportingWarehouseLoadDeadlineV1(baseMilliseconds, timeouts) {
+  return Math.min(baseMilliseconds * 2 ** Math.min(Math.max(timeouts, 0), MAX_TIMEOUT_DOUBLINGS), MAX_LOAD_TIMEOUT);
+}
 function reportingWarehouseJobIdV1(name, from, to, generation = 0) {
   return `adcp_rr_${(0, import_crypto.createHash)("sha256").update(JSON.stringify(generation > 0 ? [name, from, to, generation] : [name, from, to])).digest("hex").slice(0, 40)}`;
 }
@@ -451,23 +549,19 @@ SELECT row_data.*
   ) AS current_revision
  USING (reporting_obligation_id, reporting_revision_id);`.trim();
 }
-function sourceLocalDate(start, timeZone) {
-  if (!start) return null;
-  const instant = new Date(start);
-  if (Number.isNaN(instant.getTime())) return null;
+function periodDateOf(revision) {
+  const period = revision.wireRevision.period;
   try {
-    const parts = new Intl.DateTimeFormat("en-CA", {
-      timeZone: timeZone ?? "UTC",
-      year: "numeric",
-      month: "2-digit",
-      day: "2-digit"
-    }).format(instant);
-    return parts;
-  } catch {
-    return instant.toISOString().slice(0, 10);
+    if (!period) throw new TypeError("the revision carries no period");
+    return (0, import_schedule.reportingPeriodDate)(period.start, period.source_timezone);
+  } catch (error) {
+    throw new Error(
+      `Cannot compute period_date for revision ${revision.reporting_revision_id}: ${error instanceof Error ? error.message : String(error)}. The batch stays retained and every run fails until the revision's period is repaired (or, if this runtime's timezone data is older than the writer's, until it is upgraded); reset({ discardRetainedPlan: true }) restarts from the beginning, so empty the destination tables first.`,
+      { cause: error }
+    );
   }
 }
-async function withinDeadline(operation, milliseconds, signal) {
+async function withinDeadline(operation, milliseconds, signal, context) {
   signal?.throwIfAborted();
   let timer;
   let onAbort;
@@ -476,7 +570,9 @@ async function withinDeadline(operation, milliseconds, signal) {
       operation(),
       new Promise((_, reject) => {
         timer = setTimeout(() => {
-          const error = new Error("BigQuery call exceeded its deadline; the batch will be retried");
+          const error = new Error(
+            `BigQuery call exceeded its deadline${context ? ` of ${milliseconds} ms for ${context}` : ""}; the batch will be retried`
+          );
           Object.defineProperty(error, "adcpDeadlineExceeded", { value: true });
           reject(error);
         }, milliseconds);
@@ -498,25 +594,32 @@ function isAlreadyExists(error) {
   const record = error;
   return record?.code === 409 || !!record?.errors?.some((entry) => entry.reason === "duplicate");
 }
+function isLoadTimedOut(error) {
+  return error?.adcpLoadTimedOut === true;
+}
+function sleep(milliseconds, signal) {
+  signal?.throwIfAborted();
+  return new Promise((resolve, reject) => {
+    const onAbort = () => {
+      clearTimeout(timer);
+      reject(signal?.reason ?? new Error("aborted"));
+    };
+    const timer = setTimeout(() => {
+      signal?.removeEventListener("abort", onAbort);
+      resolve();
+    }, milliseconds);
+    signal?.addEventListener("abort", onAbort, { once: true });
+  });
+}
 function isFinishedWithError(error) {
   return error?.adcpJobFailed === true;
 }
-function assertJobSucceeded(metadata) {
-  const record = Array.isArray(metadata) ? metadata[0] : metadata;
-  if (record?.status?.state !== "DONE") {
-    throw new Error("A BigQuery load job for this batch is still running; retry later");
-  }
-  if (record.status.errorResult) {
-    const failure = new Error("A BigQuery load job for this batch failed; it will be retried as a new attempt");
-    Object.defineProperty(failure, "adcpJobFailed", { value: true });
-    throw failure;
-  }
-}
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   REPORTING_WAREHOUSE_SINK_MIGRATION,
   createBigQueryReportingWarehouseSinkV1,
   reportingWarehouseCurrentRowsViewSqlV1,
   reportingWarehouseDefaultTablesSqlV1,
-  reportingWarehouseJobIdV1
+  reportingWarehouseJobIdV1,
+  reportingWarehouseLoadDeadlineV1
 });