@adcp/sdk 15.8.1 → 15.9.1

package/dist/lib/reporting/bigquery/index.d.ts

@@ -11,11 +11,18 @@ import type { PostgresReportingLedgerStore, ReportingLedgerRevisionMetadataV1, R
  * are the adopter's.
  *
  * Exactly-once loading: a batch (change-feed range, revision list, job IDs) is
- * planned durably in PostgreSQL before any load. A crash between the load and
- * the cursor commit replays the same batch under the same job IDs, which
- * BigQuery refuses to run twice. A failed load job is retried under a new
- * attempt suffix only after BigQuery reports that job finished with an error,
- * and each table keeps its own attempt count, so no batch lands twice.
+ * planned durably in PostgreSQL before any load. Before every submission the
+ * sink looks the deterministic job ID up in the destination project and
+ * location: a job that finished cleanly counts as loaded, a job still running
+ * is waited for (never uploaded again), and only a missing job is uploaded.
+ * A crash or timeout between the load and the cursor commit therefore replays
+ * the same batch under the same job IDs without a second upload. A failed load
+ * job is retried under a new attempt suffix only after BigQuery reports that
+ * job finished with an error, and each table keeps its own attempt count, so
+ * no batch lands twice. A load that outlasts the deadline never changes the job
+ * ID: the sink grows that table's deadline instead (persisted with the plan).
+ * Concurrent runs claim one durable plan with compare-and-swap. Replays of
+ * that plan share job IDs; stale runs cannot change its attempts or cursor.
  */
 /** Apply after `REPORTING_LEDGER_CHANGES_MIGRATION`. Holds one row per sink. */
 export declare const REPORTING_WAREHOUSE_SINK_MIGRATION: string;
@@ -30,9 +37,12 @@ export interface ReportingBigQueryClientV1 {
         };
     };
     job(jobId: string, options?: {
+        projectId?: string;
         location?: string;
     }): {
         getMetadata(): Promise<unknown>;
+        /** @deprecated Unused: BigQuery does not stop a running load, so the sink never cancels. Kept so wrappers that declare it still type-check. */
+        cancel?(): Promise<unknown>;
     };
     query(options: {
         query: string;
@@ -45,18 +55,35 @@ export interface ReportingWarehouseRowContextV1 {
 }
 export interface CreateBigQueryReportingWarehouseSinkOptionsV1 {
     /** Ledger store with `changeFeed: true`. */
-    store: Pick<PostgresReportingLedgerStore, 'changesAfter' | 'saveFeedConsumerCursor' | 'getRevisionMetadata' | 'readRevisionRows'>;
+    store: Pick<PostgresReportingLedgerStore, 'changesAfter' | 'getRevisionMetadata' | 'readRevisionRows'> & {
+        /**
+         * The sink ignores the result, so wrappers written when this resolved
+         * `void` keep compiling.
+         */
+        saveFeedConsumerCursor(name: string, cursor: string, options?: {
+            refreshUnchanged?: boolean;
+        }): Promise<unknown>;
+    } & /** Needed only by `reset()`; custom store wrappers that predate it keep compiling. */ Partial<Pick<PostgresReportingLedgerStore, 'removeFeedConsumer'>>;
     /** PostgreSQL pool holding `REPORTING_WAREHOUSE_SINK_MIGRATION`. */
     db: Pick<ReportingPgPool, 'query'>;
     /** Host-constructed official BigQuery client. */
     bigquery: ReportingBigQueryClientV1;
-    /** Stable sink name; also the registered change-feed consumer name. */
+    /**
+     * Stable sink and consumer name. All replicas sharing it must use the same
+     * ledger database, feed scope, destination, location and row mapping. Use
+     * a new name when that configuration changes. Names must also be unique
+     * across independent ledgers sharing a BigQuery project.
+     */
     name: string;
     projectId?: string;
     datasetId: string;
     rowsTableId: string;
     revisionsTableId: string;
-    /** Dataset location, so jobs run in the dataset's region. */
+    /**
+     * Dataset location, so jobs run in the dataset's region. Set it for any dataset outside the US and EU
+     * multi-regions: BigQuery answers a job lookup without the right location with 404, and the sink then
+     * cannot find the jobs it created.
+     */
     location?: string;
     /** Restrict the sink to one account; omit for the deployment-wide feed. */
     account_id?: string;
@@ -69,19 +96,44 @@ export interface CreateBigQueryReportingWarehouseSinkOptionsV1 {
     mapRow?(row: Record<string, unknown>, context: ReportingWarehouseRowContextV1): Record<string, unknown>;
     /** Revisions per batch. Default 200. */
     maxRevisionsPerBatch?: number;
-    /** Job labels for cost attribution. */
+    /**
+     * @deprecated Has no effect, and the sink warns when it is set. The official
+     * client submits a local-file load by uploading a job resource whose
+     * `configuration.load` is built from the load options, and offers no way to
+     * set `configuration.labels`; BigQuery silently drops `labels` found inside
+     * `configuration.load`. For cost attribution, filter
+     * `INFORMATION_SCHEMA.JOBS` on `destination_table` or on the `adcp_rr_`
+     * job ID prefix instead.
+     */
     labels?: Readonly<Record<string, string>>;
     /**
-     * Deadline for one load (and its status check). Default 10 minutes. It is
-     * also sent as the job's `jobTimeoutMs`. A run that times out leaves its
-     * plan in place; the next run reuses the same job ID and waits for, or
-     * accepts, the job BigQuery is still running.
+     * Base deadline for one run's work on one load. Default 10 minutes. It bounds
+     * how long a run uploads a file or waits for a running job; it does not bound
+     * the job. BigQuery does not stop a load when the client gives up (a cancel
+     * is accepted and the job finishes anyway), so the sink never cancels.
+     * Before every submission it looks the deterministic job ID up in `projectId`
+     * and `location`: a job that finished cleanly is accepted, a job that is
+     * pending or running is waited for (polled within the deadline) instead of
+     * being uploaded again, and only a missing job is uploaded. When the
+     * deadline passes, the run rejects with the plan retained and nothing
+     * advanced, and the next run checks the job again. Each such timeout doubles
+     * that table's deadline for later runs (up to 6 hours, persisted with the
+     * plan, and never changing the job ID), so an upload slower than the
+     * deadline eventually gets enough time. Set it above your slowest expected
+     * load to avoid the extra runs. Only a job that finished with an error
+     * advances the attempt. A status check that times out is bounded by this base
+     * deadline and does not grow it. This is a client-side deadline: BigQuery's
+     * server-side `jobTimeoutMs` cannot be set on a local-file load through the
+     * official client.
      */
     loadTimeoutMilliseconds?: number;
 }
 export interface BigQueryReportingWarehouseSinkV1 {
     /**
-     * Load at most one batch. Resolves with what was loaded; `idle` when nothing new committed.
+     * Load at most one batch. Resolves with what was loaded; `idle` when no
+     * revision changes were selected. Idle ticks can advance past other feed changes.
+     * An overlapping run that loses its plan or cursor claim rejects. Retry
+     * `runOnce()` to resume the current durable plan or committed cursor.
      *
      * `missing` lists revision IDs the change feed announced that no longer
      * exist in the ledger because their period was retired before this batch
@@ -100,14 +152,47 @@ export interface BigQueryReportingWarehouseSinkV1 {
         rows: number;
         missing: string[];
     }>;
+    /**
+     * Clear the sink's saved state so the next `runOnce()` starts again from the
+     * oldest retained change, and unregister its feed consumer so the old
+     * checkpoint stops holding back retention. This is the way out of a sink
+     * that keeps rejecting with `CURSOR_EXPIRED`: a brand-new sink whose first
+     * read lost a startup race with pruning, or a sink that fell behind
+     * retention and has been resynchronized from current revisions. Stop every
+     * worker using this sink name first.
+     *
+     * It refuses while a batch plan is retained, because that batch's load jobs
+     * may already have succeeded; pass `discardRetainedPlan: true` once you have
+     * reconciled it against BigQuery. The warehouse is not touched: a sink that
+     * already loaded history will load the retained revisions again, so empty
+     * the destination tables first, or use a new sink name against new tables.
+     * Each reset starts a new job ID generation, so reloaded batches submit
+     * fresh BigQuery jobs instead of matching the finished jobs of the earlier
+     * history. The sink never cancels a load, so stop every worker and let its
+     * in-flight uploads and jobs finish before emptying the destination tables. Nothing holds change retention between `reset()` and the first
+     * committed batch. If `reset()` rejects after clearing the sink's state
+     * (the consumer could not be unregistered), call it again before restarting
+     * workers; it is idempotent. Needs a store with `removeFeedConsumer`.
+     * Resolves with whether a saved sink state existed.
+     */
+    reset(options?: {
+        discardRetainedPlan?: boolean;
+    }): Promise<{
+        cleared: boolean;
+    }>;
     /** DDL for the default rows and revisions tables (partitioned and clustered). */
     readonly defaultTablesSql: string;
     /** DDL for the reference `current_rows` view. */
     readonly currentRowsViewSql: string;
 }
 export declare function createBigQueryReportingWarehouseSinkV1(options: CreateBigQueryReportingWarehouseSinkOptionsV1): BigQueryReportingWarehouseSinkV1;
+/**
+ * How long one run may upload a file or wait for a running job: the configured
+ * base doubled once per earlier timeout of that table's load, capped at 6 hours.
+ */
+export declare function reportingWarehouseLoadDeadlineV1(baseMilliseconds: number, timeouts: number): number;
 /** Deterministic load job ID for one planned change-feed range. */
-export declare function reportingWarehouseJobIdV1(name: string, from: string | null, to: string): string;
+export declare function reportingWarehouseJobIdV1(name: string, from: string | null, to: string, generation?: number): string;
 interface WarehouseTablesV1 {
     projectId?: string;
     datasetId: string;