@adcp/sdk 15.5.0 → 15.6.0

package/dist/lib/reporting/bigquery/index.d.ts

@@ -0,0 +1,127 @@
+import type { PostgresReportingLedgerStore, ReportingLedgerRevisionMetadataV1, ReportingPgPool } from '../ledger';
+/**
+ * BigQuery warehouse sink for Reliable Reporting (shared SDK persistence spec,
+ * adcontextprotocol/adcp#7996 §3.6).
+ *
+ * The sink is a derived, non-authoritative copy of committed revisions. It
+ * follows the ledger's host change feed, reads every revision's rows through
+ * the store's verified reader, maps them to the adopter's columns, and loads
+ * one batch per tick with a deterministic job ID. It is never read for
+ * serving or verification, so its tables, columns, partitioning and retention
+ * are the adopter's.
+ *
+ * Exactly-once loading: a batch (change-feed range, revision list, job IDs) is
+ * planned durably in PostgreSQL before any load. A crash between the load and
+ * the cursor commit replays the same batch under the same job IDs, which
+ * BigQuery refuses to run twice. A failed load job is retried under a new
+ * attempt suffix only after BigQuery reports that job finished with an error,
+ * and each table keeps its own attempt count, so no batch lands twice.
+ */
+/** Apply after `REPORTING_LEDGER_CHANGES_MIGRATION`. Holds one row per sink. */
+export declare const REPORTING_WAREHOUSE_SINK_MIGRATION: string;
+/** Structural subset of the official `@google-cloud/bigquery` client the sink uses. */
+export interface ReportingBigQueryClientV1 {
+    dataset(datasetId: string, options?: {
+        projectId?: string;
+        location?: string;
+    }): {
+        table(tableId: string): {
+            load(source: string, metadata: Record<string, unknown>): Promise<unknown>;
+        };
+    };
+    job(jobId: string, options?: {
+        location?: string;
+    }): {
+        getMetadata(): Promise<unknown>;
+    };
+    query(options: {
+        query: string;
+        location?: string;
+    }): Promise<unknown>;
+}
+export interface ReportingWarehouseRowContextV1 {
+    revision: ReportingLedgerRevisionMetadataV1;
+    ordinal: number;
+}
+export interface CreateBigQueryReportingWarehouseSinkOptionsV1 {
+    /** Ledger store with `changeFeed: true`. */
+    store: Pick<PostgresReportingLedgerStore, 'changesAfter' | 'saveFeedConsumerCursor' | 'getRevisionMetadata' | 'readRevisionRows'>;
+    /** PostgreSQL pool holding `REPORTING_WAREHOUSE_SINK_MIGRATION`. */
+    db: Pick<ReportingPgPool, 'query'>;
+    /** Host-constructed official BigQuery client. */
+    bigquery: ReportingBigQueryClientV1;
+    /** Stable sink name; also the registered change-feed consumer name. */
+    name: string;
+    projectId?: string;
+    datasetId: string;
+    rowsTableId: string;
+    revisionsTableId: string;
+    /** Dataset location, so jobs run in the dataset's region. */
+    location?: string;
+    /** Restrict the sink to one account; omit for the deployment-wide feed. */
+    account_id?: string;
+    /**
+     * Map one verified report row to the adopter's columns. The sink always adds
+     * `reporting_revision_id`, `reporting_obligation_id`, `account_id`,
+     * `revision_number`, `finality`, `period_date` and `ordinal`. Default: the
+     * row as a `row` JSON column.
+     */
+    mapRow?(row: Record<string, unknown>, context: ReportingWarehouseRowContextV1): Record<string, unknown>;
+    /** Revisions per batch. Default 200. */
+    maxRevisionsPerBatch?: number;
+    /** Job labels for cost attribution. */
+    labels?: Readonly<Record<string, string>>;
+    /**
+     * Deadline for one load (and its status check). Default 10 minutes. It is
+     * also sent as the job's `jobTimeoutMs`. A run that times out leaves its
+     * plan in place; the next run reuses the same job ID and waits for, or
+     * accepts, the job BigQuery is still running.
+     */
+    loadTimeoutMilliseconds?: number;
+}
+export interface BigQueryReportingWarehouseSinkV1 {
+    /**
+     * Load at most one batch. Resolves with what was loaded; `idle` when nothing new committed.
+     *
+     * `missing` lists revision IDs the change feed announced that no longer
+     * exist in the ledger because their period was retired before this batch
+     * ran. That is a data-loss signal: the sink fell behind retention (it was
+     * down longer than `maxFeedHoldDays`, so retention stopped waiting for it)
+     * and the warehouse lacks those revisions. Backfill them from another source
+     * or accept the gap. The cursor still advances so the sink is not stuck.
+     * Alert on a non-empty `missing`. After a crash-and-replay of a planned
+     * batch, a listed revision may already have been loaded by the first attempt.
+     */
+    runOnce(options?: {
+        signal?: AbortSignal;
+    }): Promise<{
+        idle: boolean;
+        revisions: number;
+        rows: number;
+        missing: string[];
+    }>;
+    /** DDL for the default rows and revisions tables (partitioned and clustered). */
+    readonly defaultTablesSql: string;
+    /** DDL for the reference `current_rows` view. */
+    readonly currentRowsViewSql: string;
+}
+export declare function createBigQueryReportingWarehouseSinkV1(options: CreateBigQueryReportingWarehouseSinkOptionsV1): BigQueryReportingWarehouseSinkV1;
+/** Deterministic load job ID for one planned change-feed range. */
+export declare function reportingWarehouseJobIdV1(name: string, from: string | null, to: string): string;
+interface WarehouseTablesV1 {
+    projectId?: string;
+    datasetId: string;
+    rowsTableId: string;
+    revisionsTableId: string;
+}
+/** DDL for the default warehouse tables: partitioned by period date, clustered by account and revision. */
+export declare function reportingWarehouseDefaultTablesSqlV1(tables: WarehouseTablesV1): string;
+/**
+ * DDL for the reference `current_rows` view: only each obligation's current
+ * revision (its official revision, otherwise its latest snapshot). Summing
+ * cumulative snapshots across revisions double-counts; query this view.
+ */
+export declare function reportingWarehouseCurrentRowsViewSqlV1(tables: WarehouseTablesV1 & {
+    viewId?: string;
+}): string;
+export {};

package/dist/lib/reporting/bigquery/index.js

@@ -0,0 +1,379 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var bigquery_exports = {};
+__export(bigquery_exports, {
+  REPORTING_WAREHOUSE_SINK_MIGRATION: () => REPORTING_WAREHOUSE_SINK_MIGRATION,
+  createBigQueryReportingWarehouseSinkV1: () => createBigQueryReportingWarehouseSinkV1,
+  reportingWarehouseCurrentRowsViewSqlV1: () => reportingWarehouseCurrentRowsViewSqlV1,
+  reportingWarehouseDefaultTablesSqlV1: () => reportingWarehouseDefaultTablesSqlV1,
+  reportingWarehouseJobIdV1: () => reportingWarehouseJobIdV1
+});
+module.exports = __toCommonJS(bigquery_exports);
+var import_crypto = require("crypto");
+var import_fs = require("fs");
+var import_os = require("os");
+var path = __toESM(require("path"));
+const REPORTING_WAREHOUSE_SINK_MIGRATION = `
+CREATE TABLE IF NOT EXISTS adcp_reporting_warehouse_sinks (
+  sink_name TEXT PRIMARY KEY CHECK (sink_name ~ '^[A-Za-z0-9_.:-]{1,128}$'),
+  committed_cursor TEXT,
+  planned_from_cursor TEXT,
+  planned_to_cursor TEXT,
+  planned_revisions JSONB,
+  planned_rows_attempt INTEGER NOT NULL DEFAULT 0 CHECK (planned_rows_attempt >= 0),
+  planned_revisions_attempt INTEGER NOT NULL DEFAULT 0 CHECK (planned_revisions_attempt >= 0),
+  updated_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
+  CHECK ((planned_to_cursor IS NULL) = (planned_revisions IS NULL))
+);
+`.trim();
+const PROJECT = /^[a-z][a-z0-9-]{4,61}[a-z0-9]$/;
+const DATASET_OR_TABLE = /^[A-Za-z_][A-Za-z0-9_]{0,1023}$/;
+const NAME = /^[A-Za-z0-9_.:-]{1,128}$/;
+function createBigQueryReportingWarehouseSinkV1(options) {
+  if (!NAME.test(options.name)) throw new TypeError("Warehouse sink name must be a bounded non-secret identifier");
+  if (options.projectId !== void 0 && !PROJECT.test(options.projectId)) {
+    throw new TypeError("BigQuery projectId is invalid");
+  }
+  for (const [label, value] of [
+    ["datasetId", options.datasetId],
+    ["rowsTableId", options.rowsTableId],
+    ["revisionsTableId", options.revisionsTableId]
+  ]) {
+    if (!DATASET_OR_TABLE.test(value)) throw new TypeError(`BigQuery ${label} is invalid`);
+  }
+  const loadTimeout = options.loadTimeoutMilliseconds ?? 6e5;
+  if (!Number.isSafeInteger(loadTimeout) || loadTimeout < 1e3 || loadTimeout > 6 * 36e5) {
+    throw new RangeError("loadTimeoutMilliseconds must be between 1 second and 6 hours");
+  }
+  const maxRevisions = options.maxRevisionsPerBatch ?? 200;
+  if (!Number.isSafeInteger(maxRevisions) || maxRevisions < 1 || maxRevisions > 1e4) {
+    throw new RangeError("maxRevisionsPerBatch must be between 1 and 10000");
+  }
+  const tables = {
+    projectId: options.projectId,
+    datasetId: options.datasetId,
+    rowsTableId: options.rowsTableId,
+    revisionsTableId: options.revisionsTableId
+  };
+  const dataset = options.bigquery.dataset(options.datasetId, {
+    ...options.projectId ? { projectId: options.projectId } : {},
+    ...options.location ? { location: options.location } : {}
+  });
+  async function loadOnce(tableId, jobId, lines, signal) {
+    signal?.throwIfAborted();
+    const directory = await import_fs.promises.mkdtemp(path.join((0, import_os.tmpdir)(), "adcp-warehouse-"));
+    const file = path.join(directory, `${jobId}.ndjson`);
+    try {
+      await import_fs.promises.writeFile(file, lines.length ? lines.join("\n") + "\n" : "");
+      try {
+        await withinDeadline(
+          () => dataset.table(tableId).load(file, {
+            jobId,
+            sourceFormat: "NEWLINE_DELIMITED_JSON",
+            writeDisposition: "WRITE_APPEND",
+            jobTimeoutMs: String(loadTimeout),
+            ...options.location ? { location: options.location } : {},
+            ...options.labels ? { labels: { ...options.labels } } : {}
+          }),
+          loadTimeout,
+          signal
+        );
+      } catch (error) {
+        if (!isAlreadyExists(error)) throw error;
+        const metadata = await withinDeadline(
+          () => options.bigquery.job(jobId, options.location ? { location: options.location } : void 0).getMetadata(),
+          loadTimeout,
+          signal
+        );
+        assertJobSucceeded(metadata);
+      }
+    } finally {
+      await import_fs.promises.rm(directory, { recursive: true, force: true });
+    }
+  }
+  return {
+    defaultTablesSql: reportingWarehouseDefaultTablesSqlV1(tables),
+    currentRowsViewSql: reportingWarehouseCurrentRowsViewSqlV1(tables),
+    async runOnce(runOptions = {}) {
+      const signal = runOptions.signal;
+      await options.db.query(
+        `INSERT INTO adcp_reporting_warehouse_sinks (sink_name) VALUES ($1) ON CONFLICT DO NOTHING`,
+        [options.name]
+      );
+      const state = (await options.db.query(
+        `SELECT committed_cursor, planned_from_cursor, planned_to_cursor, planned_revisions,
+                  planned_rows_attempt, planned_revisions_attempt
+             FROM adcp_reporting_warehouse_sinks WHERE sink_name = $1`,
+        [options.name]
+      )).rows[0];
+      let plan = state.planned_to_cursor && state.planned_revisions ? {
+        from: state.planned_from_cursor,
+        to: state.planned_to_cursor,
+        revisions: state.planned_revisions,
+        rowsAttempt: state.planned_rows_attempt,
+        revisionsAttempt: state.planned_revisions_attempt
+      } : void 0;
+      if (!plan) {
+        const page = await options.store.changesAfter({
+          ...state.committed_cursor ? { cursor: state.committed_cursor } : {},
+          ...options.account_id ? { account_id: options.account_id } : {},
+          kinds: ["revision"],
+          limit: maxRevisions
+        });
+        if (page.records.length === 0) {
+          if (page.cursor !== state.committed_cursor) await commit(page.cursor);
+          return { idle: true, revisions: 0, rows: 0, missing: [] };
+        }
+        plan = {
+          from: state.committed_cursor,
+          to: page.cursor,
+          revisions: page.records.map((record) => ({ id: record.record_id, account_id: record.account_id })),
+          rowsAttempt: 0,
+          revisionsAttempt: 0
+        };
+        await options.db.query(
+          `UPDATE adcp_reporting_warehouse_sinks
+              SET planned_from_cursor = $2, planned_to_cursor = $3, planned_revisions = $4::jsonb,
+                  planned_rows_attempt = 0, planned_revisions_attempt = 0, updated_at = clock_timestamp()
+            WHERE sink_name = $1`,
+          [options.name, plan.from, plan.to, JSON.stringify(plan.revisions)]
+        );
+      }
+      const rowLines = [];
+      const revisionLines = [];
+      const missing = [];
+      for (const planned of plan.revisions) {
+        signal?.throwIfAborted();
+        const revision = await options.store.getRevisionMetadata(planned.id, planned.account_id);
+        if (!revision) {
+          missing.push(planned.id);
+          continue;
+        }
+        const periodDate = sourceLocalDate(
+          revision.wireRevision.period?.start,
+          revision.wireRevision.period?.source_timezone
+        );
+        const base = {
+          reporting_revision_id: revision.reporting_revision_id,
+          reporting_obligation_id: revision.reporting_obligation_id,
+          account_id: planned.account_id,
+          revision_number: revision.revisionNumber,
+          finality: revision.finality,
+          period_date: periodDate
+        };
+        for (let offset = 0; offset < revision.binding.rowCount; ) {
+          const page = await options.store.readRevisionRows({
+            reporting_revision_id: revision.reporting_revision_id,
+            account_id: planned.account_id,
+            offset,
+            limit: 5e3,
+            ...signal ? { signal } : {}
+          });
+          if (!page || page.rows.length === 0) throw new Error("Reporting revision rows ended early");
+          page.rows.forEach((row, index) => {
+            const ordinal = offset + index;
+            const mapped = options.mapRow ? options.mapRow(row, { revision, ordinal }) : { row };
+            rowLines.push(JSON.stringify({ ...mapped, ...base, ordinal }));
+          });
+          offset += page.rows.length;
+        }
+        revisionLines.push(
+          JSON.stringify({
+            ...base,
+            supersedes_reporting_revision_id: revision.supersedes_reporting_revision_id ?? null,
+            period_start: revision.wireRevision.period?.start ?? null,
+            period_end: revision.wireRevision.period?.end ?? null,
+            source_timezone: revision.wireRevision.period?.source_timezone ?? null,
+            observed_at: revision.observedAt,
+            data_through: revision.dataThrough,
+            revision_content_sha256: revision.binding.sha256,
+            row_count: revision.binding.rowCount,
+            control_totals: revision.wireRevision.control_totals ?? []
+          })
+        );
+      }
+      const baseJobId = reportingWarehouseJobIdV1(options.name, plan.from, plan.to);
+      const suffix = (attempt) => attempt ? `_a${attempt}` : "";
+      const steps = [
+        { table: options.rowsTableId, kind: "rows", lines: rowLines, attempt: plan.rowsAttempt },
+        { table: options.revisionsTableId, kind: "revs", lines: revisionLines, attempt: plan.revisionsAttempt }
+      ];
+      for (const step of steps) {
+        if (!step.lines.length) continue;
+        try {
+          await loadOnce(step.table, `${baseJobId}_${step.kind}${suffix(step.attempt)}`, [...step.lines], signal);
+        } catch (error) {
+          if (isFinishedWithError(error)) {
+            const column = step.kind === "rows" ? "planned_rows_attempt" : "planned_revisions_attempt";
+            await options.db.query(
+              `UPDATE adcp_reporting_warehouse_sinks SET ${column} = ${column} + 1,
+                      updated_at = clock_timestamp() WHERE sink_name = $1`,
+              [options.name]
+            );
+          }
+          throw error;
+        }
+      }
+      await commit(plan.to);
+      return { idle: false, revisions: revisionLines.length, rows: rowLines.length, missing };
+    }
+  };
+  async function commit(cursor) {
+    await options.db.query(
+      `UPDATE adcp_reporting_warehouse_sinks
+          SET committed_cursor = $2, planned_from_cursor = NULL, planned_to_cursor = NULL,
+              planned_revisions = NULL, planned_rows_attempt = 0, planned_revisions_attempt = 0,
+              updated_at = clock_timestamp()
+        WHERE sink_name = $1`,
+      [options.name, cursor]
+    );
+    await options.store.saveFeedConsumerCursor(options.name, cursor);
+  }
+}
+function reportingWarehouseJobIdV1(name, from, to) {
+  return `adcp_rr_${(0, import_crypto.createHash)("sha256").update(JSON.stringify([name, from, to])).digest("hex").slice(0, 40)}`;
+}
+function qualified(tables, table) {
+  return `\`${tables.projectId ? `${tables.projectId}.` : ""}${tables.datasetId}.${table}\``;
+}
+function reportingWarehouseDefaultTablesSqlV1(tables) {
+  return `
+CREATE TABLE IF NOT EXISTS ${qualified(tables, tables.rowsTableId)} (
+  reporting_revision_id STRING NOT NULL,
+  reporting_obligation_id STRING NOT NULL,
+  account_id STRING NOT NULL,
+  revision_number INT64 NOT NULL,
+  finality STRING NOT NULL,
+  period_date DATE,
+  ordinal INT64 NOT NULL,
+  row JSON
+)
+PARTITION BY period_date
+CLUSTER BY account_id, reporting_revision_id;
+
+CREATE TABLE IF NOT EXISTS ${qualified(tables, tables.revisionsTableId)} (
+  reporting_revision_id STRING NOT NULL,
+  reporting_obligation_id STRING NOT NULL,
+  account_id STRING NOT NULL,
+  revision_number INT64 NOT NULL,
+  finality STRING NOT NULL,
+  period_date DATE,
+  supersedes_reporting_revision_id STRING,
+  period_start TIMESTAMP,
+  period_end TIMESTAMP,
+  source_timezone STRING,
+  observed_at TIMESTAMP,
+  data_through TIMESTAMP,
+  revision_content_sha256 STRING NOT NULL,
+  row_count INT64 NOT NULL,
+  control_totals JSON
+)
+PARTITION BY period_date
+CLUSTER BY account_id, reporting_obligation_id;`.trim();
+}
+function reportingWarehouseCurrentRowsViewSqlV1(tables) {
+  const view = tables.viewId ?? "current_rows";
+  if (!DATASET_OR_TABLE.test(view)) throw new TypeError("BigQuery viewId is invalid");
+  return `
+CREATE OR REPLACE VIEW ${qualified(tables, view)} AS
+SELECT row_data.*
+  FROM ${qualified(tables, tables.rowsTableId)} AS row_data
+  JOIN (
+    SELECT reporting_obligation_id,
+           ARRAY_AGG(reporting_revision_id ORDER BY revision_number DESC LIMIT 1)[OFFSET(0)] AS reporting_revision_id
+      FROM ${qualified(tables, tables.revisionsTableId)}
+     GROUP BY reporting_obligation_id
+  ) AS current_revision
+ USING (reporting_obligation_id, reporting_revision_id);`.trim();
+}
+function sourceLocalDate(start, timeZone) {
+  if (!start) return null;
+  const instant = new Date(start);
+  if (Number.isNaN(instant.getTime())) return null;
+  try {
+    const parts = new Intl.DateTimeFormat("en-CA", {
+      timeZone: timeZone ?? "UTC",
+      year: "numeric",
+      month: "2-digit",
+      day: "2-digit"
+    }).format(instant);
+    return parts;
+  } catch {
+    return instant.toISOString().slice(0, 10);
+  }
+}
+async function withinDeadline(operation, milliseconds, signal) {
+  signal?.throwIfAborted();
+  let timer;
+  let onAbort;
+  try {
+    return await Promise.race([
+      operation(),
+      new Promise((_, reject) => {
+        timer = setTimeout(
+          () => reject(new Error("BigQuery call exceeded its deadline; the batch will be retried")),
+          milliseconds
+        );
+        if (signal) {
+          onAbort = () => reject(signal.reason ?? new Error("aborted"));
+          signal.addEventListener("abort", onAbort, { once: true });
+        }
+      })
+    ]);
+  } finally {
+    if (timer) clearTimeout(timer);
+    if (signal && onAbort) signal.removeEventListener("abort", onAbort);
+  }
+}
+function isAlreadyExists(error) {
+  const record = error;
+  return record?.code === 409 || !!record?.errors?.some((entry) => entry.reason === "duplicate");
+}
+function isFinishedWithError(error) {
+  return error?.adcpJobFailed === true;
+}
+function assertJobSucceeded(metadata) {
+  const record = Array.isArray(metadata) ? metadata[0] : metadata;
+  if (record?.status?.state !== "DONE") {
+    throw new Error("A BigQuery load job for this batch is still running; retry later");
+  }
+  if (record.status.errorResult) {
+    const failure = new Error("A BigQuery load job for this batch failed; it will be retried as a new attempt");
+    Object.defineProperty(failure, "adcpJobFailed", { value: true });
+    throw failure;
+  }
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  REPORTING_WAREHOUSE_SINK_MIGRATION,
+  createBigQueryReportingWarehouseSinkV1,
+  reportingWarehouseCurrentRowsViewSqlV1,
+  reportingWarehouseDefaultTablesSqlV1,
+  reportingWarehouseJobIdV1
+});