@firebase-function-kits/firestore-bigquery-export 0.0.1 → 0.0.2-rc.1

package/src/events.ts

@@ -0,0 +1,146 @@
+/*
+ * Copyright 2019 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import * as eventArc from "firebase-admin/eventarc";
+
+const { getEventarc } = eventArc;
+
+/**
+ * Builds the Eventarc event type for this extension.
+ *
+ * @param eventName The name of the event (e.g., "onStart", "onError", etc.)
+ * @returns The event type string.
+ */
+const getEventTypes = (eventName: string) => [
+  `firebase.extensions.firestore-bigquery-export.v1.${eventName}`,
+];
+
+let eventChannel: eventArc.Channel | undefined;
+
+/**
+ * Sets up the Eventarc channel.
+ *
+ * This function retrieves the Eventarc channel based on the environment variables:
+ * - `EVENTARC_CHANNEL` specifies the channel to use for publishing events.
+ * - `EXT_SELECTED_EVENTS` defines the allowed event types.
+ *
+ * @function setupEventChannel
+ */
+export const setupEventChannel = () => {
+  eventChannel = process.env.EVENTARC_CHANNEL
+    ? getEventarc().channel(process.env.EVENTARC_CHANNEL, {
+        allowedEventTypes: process.env.EXT_SELECTED_EVENTS,
+      })
+    : undefined;
+};
+
+/**
+ * Publishes a "start" event using both OLD and NEW event types.
+ *
+ * @param data The payload to send with the event. Can be a string or an object.
+ * @returns A Promise resolving when both events are published.
+ */
+export const recordStartEvent = async (data: string | object) => {
+  if (!eventChannel) return Promise.resolve();
+
+  const eventTypes = getEventTypes("onStart");
+
+  // Publish events for both OLD and NEW event types
+  return Promise.all(
+    eventTypes.map((type) =>
+      eventChannel.publish({
+        type,
+        data,
+      })
+    )
+  );
+};
+
+/**
+ * Publishes an "error" event using both OLD and NEW event types.
+ *
+ * @param err The Error object containing the error message.
+ * @param subject (Optional) Subject identifier related to the error event.
+ * @returns A Promise resolving when both events are published.
+ */
+export const recordErrorEvent = async (err: Error, subject?: string) => {
+  if (!eventChannel) return Promise.resolve();
+
+  const eventTypes = getEventTypes("onError");
+
+  // Publish events for both OLD and NEW event types
+  return Promise.all(
+    eventTypes.map((type) =>
+      eventChannel.publish({
+        type,
+        data: { message: err.message },
+        subject,
+      })
+    )
+  );
+};
+
+/**
+ * Publishes a "success" event using both OLD and NEW event types.
+ *
+ * @param params An object containing the subject and the event data.
+ * @param params.subject A string representing the subject of the event.
+ * @param params.data The payload to send with the event.
+ * @returns A Promise resolving when both events are published.
+ */
+export const recordSuccessEvent = async ({
+  subject,
+  data,
+}: {
+  subject: string;
+  data: string | object;
+}) => {
+  if (!eventChannel) return Promise.resolve();
+
+  const eventTypes = getEventTypes("onSuccess");
+
+  // Publish events for both OLD and NEW event types
+  return Promise.all(
+    eventTypes.map((type) =>
+      eventChannel.publish({
+        type,
+        subject,
+        data,
+      })
+    )
+  );
+};
+
+/**
+ * Publishes a "completion" event using both OLD and NEW event types.
+ *
+ * @param data The payload to send with the event. Can be a string or an object.
+ * @returns A Promise resolving when both events are published.
+ */
+export const recordCompletionEvent = async (data: string | object) => {
+  if (!eventChannel) return Promise.resolve();
+
+  const eventTypes = getEventTypes("onCompletion");
+
+  // Publish events for both OLD and NEW event types
+  return Promise.all(
+    eventTypes.map((type) =>
+      eventChannel.publish({
+        type,
+        data,
+      })
+    )
+  );
+};

package/src/export-config.ts

@@ -0,0 +1,205 @@
+/*
+ * Copyright 2019 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import type {
+  ChangeTrackerConfig,
+  LogLevel,
+} from "@firebaseextensions/firestore-bigquery-change-tracker";
+import type { Expression } from "firebase-functions/params";
+
+type TrackerLogLevel = "debug" | "info" | "warn" | "error" | "silent";
+type ConfigValue<T extends string | number | boolean | string[]> =
+  | T
+  | Expression<T>;
+
+/** How the BigQuery changelog view is materialized. */
+export type ViewType =
+  | "view"
+  | "materialized_incremental"
+  | "materialized_non_incremental";
+
+/**
+ * The export configuration. The main entry point builds it from deploy-time
+ * params; handler consumers can construct it directly.
+ *
+ * `collectionPath`, `datasetId`, `tableId`, `location`, and `projectId` are
+ * required; everything else has a sensible default.
+ */
+export interface ExportConfig {
+  /** Firestore collection (or collection group) path to mirror, e.g. `users`
+   *  or `regions/{regionId}/users`. */
+  collectionPath: ConfigValue<string>;
+  /** BigQuery dataset id to write into. */
+  datasetId: ConfigValue<string>;
+  /** BigQuery changelog table id. */
+  tableId: ConfigValue<string>;
+
+  /** Region for the trigger and task queue. */
+  location: ConfigValue<string>;
+  /** BigQuery dataset location, e.g. `us`, `eu`. Defaults to `us`. */
+  datasetLocation?: ConfigValue<string>;
+  /** Project that owns the BigQuery dataset, if different from the function's
+   *  project. */
+  bqProjectId?: ConfigValue<string>;
+  /** GCP project id. */
+  projectId: ConfigValue<string>;
+  /** Firestore database id. Defaults to `(default)`. */
+  databaseId?: ConfigValue<string>;
+
+  /** Include path-param values as columns. Defaults to `false`. */
+  wildcardIds?: ConfigValue<boolean>;
+  /** Skip writing the previous document state on updates. Defaults to `false`. */
+  excludeOldData?: ConfigValue<boolean>;
+  /** Use the newer snapshot view query syntax. Defaults to `false`. */
+  useNewSnapshotQuerySyntax?: ConfigValue<boolean>;
+  /** Changelog view strategy. Defaults to `view`. */
+  viewType?: ConfigValue<ViewType>;
+
+  /** Partitioning strategy for the changelog table. */
+  partitioning?: ChangeTrackerConfig["partitioning"];
+  /** Clustering columns (max 4). */
+  clustering?: string[] | null;
+  /** Materialized-view max staleness interval, e.g. `0-0 0 4:0:0`. */
+  maxStaleness?: ConfigValue<string>;
+  /** Incremental materialized-view refresh interval in minutes. */
+  refreshIntervalMinutes?: ConfigValue<number>;
+
+  /** Collection id used for the changelog backup table. */
+  backupCollectionId?: ConfigValue<string>;
+  /** Name of a transform function to apply before writing. */
+  transformFunction?: ConfigValue<string>;
+  /** Customer-managed encryption key for the dataset. */
+  kmsKeyName?: ConfigValue<string>;
+
+  /** Log verbosity. Defaults to `info`. */
+  logLevel?: ConfigValue<TrackerLogLevel | LogLevel>;
+}
+
+/** {@link ExportConfig} with all defaults applied. */
+export interface ResolvedExportConfig {
+  collectionPath: string;
+  datasetId: string;
+  tableId: string;
+  location: string;
+  datasetLocation: string;
+  bqProjectId?: string;
+  projectId: string;
+  databaseId: string;
+  wildcardIds: boolean;
+  excludeOldData: boolean;
+  useNewSnapshotQuerySyntax: boolean;
+  viewType: ViewType;
+  partitioning?: ChangeTrackerConfig["partitioning"];
+  clustering: string[] | null;
+  maxStaleness?: string;
+  refreshIntervalMinutes?: number;
+  backupCollectionId?: string;
+  transformFunction?: string;
+  kmsKeyName?: string;
+  logLevel: TrackerLogLevel;
+}
+
+function isExpression<T extends string | number | boolean | string[]>(
+  value: ConfigValue<T>
+): value is Expression<T> {
+  return typeof value === "object" && value !== null && "value" in value;
+}
+
+function resolveConfigValue<T extends string | number | boolean | string[]>(
+  value: ConfigValue<T>
+): T {
+  return isExpression(value) ? value.value() : value;
+}
+
+function resolveOptionalConfigValue<
+  T extends string | number | boolean | string[]
+>(value: ConfigValue<T> | undefined): T | undefined {
+  return value === undefined ? undefined : resolveConfigValue(value);
+}
+
+/**
+ * Applies defaults to an {@link ExportConfig}.
+ *
+ * @param config - The user-supplied configuration.
+ * @returns The configuration with every field populated.
+ */
+export function resolveExportConfig(
+  config: ExportConfig
+): ResolvedExportConfig {
+  const projectId = resolveConfigValue(config.projectId);
+  const viewType = resolveOptionalConfigValue(config.viewType);
+  const logLevel = resolveOptionalConfigValue(config.logLevel);
+
+  return {
+    collectionPath: resolveConfigValue(config.collectionPath),
+    datasetId: resolveConfigValue(config.datasetId),
+    tableId: resolveConfigValue(config.tableId),
+    location: resolveConfigValue(config.location),
+    datasetLocation: resolveOptionalConfigValue(config.datasetLocation) ?? "us",
+    bqProjectId: resolveOptionalConfigValue(config.bqProjectId),
+    projectId,
+    databaseId: resolveOptionalConfigValue(config.databaseId) ?? "(default)",
+    wildcardIds: resolveOptionalConfigValue(config.wildcardIds) ?? false,
+    excludeOldData: resolveOptionalConfigValue(config.excludeOldData) ?? false,
+    useNewSnapshotQuerySyntax:
+      resolveOptionalConfigValue(config.useNewSnapshotQuerySyntax) ?? false,
+    viewType: viewType ?? "view",
+    partitioning: config.partitioning,
+    clustering: config.clustering ?? null,
+    maxStaleness: resolveOptionalConfigValue(config.maxStaleness),
+    refreshIntervalMinutes: resolveOptionalConfigValue(
+      config.refreshIntervalMinutes
+    ),
+    backupCollectionId: resolveOptionalConfigValue(config.backupCollectionId),
+    transformFunction: resolveOptionalConfigValue(config.transformFunction),
+    kmsKeyName: resolveOptionalConfigValue(config.kmsKeyName),
+    logLevel: (logLevel as TrackerLogLevel) ?? "info",
+  };
+}
+
+/**
+ * Maps a resolved export config onto the change-tracker's config shape.
+ *
+ * @param config - The resolved export configuration.
+ * @returns The {@link ChangeTrackerConfig} for the event history tracker.
+ */
+export function toTrackerConfig(
+  config: ResolvedExportConfig
+): ChangeTrackerConfig {
+  return {
+    firestoreInstanceId: config.databaseId,
+    tableId: config.tableId,
+    datasetId: config.datasetId,
+    datasetLocation: config.datasetLocation,
+    backupTableId: config.backupCollectionId,
+    transformFunction: config.transformFunction,
+    partitioning: config.partitioning,
+    databaseId: config.databaseId,
+    clustering: config.clustering,
+    wildcardIds: config.wildcardIds,
+    bqProjectId: config.bqProjectId ?? config.projectId,
+    useNewSnapshotQuerySyntax: config.useNewSnapshotQuerySyntax,
+    skipInit: true,
+    kmsKeyName: config.kmsKeyName,
+    useMaterializedView:
+      config.viewType === "materialized_incremental" ||
+      config.viewType === "materialized_non_incremental",
+    useIncrementalMaterializedView:
+      config.viewType === "materialized_incremental",
+    maxStaleness: config.maxStaleness,
+    refreshIntervalMinutes: config.refreshIntervalMinutes,
+    logLevel: config.logLevel,
+  };
+}

package/src/handlers.ts

@@ -0,0 +1,211 @@
+/*
+ * Copyright 2019 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {
+  ChangeType,
+  type FirestoreBigQueryEventHistoryTracker,
+  type FirestoreDocumentChangeEvent,
+} from "@firebaseextensions/firestore-bigquery-change-tracker";
+import type {
+  Change,
+  DocumentSnapshot,
+  FirestoreEvent,
+} from "firebase-functions/firestore";
+import * as events from "./events";
+import type { ResolvedExportConfig } from "./export-config";
+import * as logs from "./logs";
+import { getChangeType, getDocumentId } from "./util";
+
+/** Serialized Firestore change ready to write to BigQuery. */
+interface SerializedDocumentChange {
+  timestamp: string;
+  eventId: string;
+  fullResourceName: string;
+  changeType: ChangeType;
+  documentId: string;
+  params: FirestoreDocumentChangeEvent["pathParams"] | null;
+  data: FirestoreDocumentChangeEvent["data"];
+  oldData: FirestoreDocumentChangeEvent["oldData"];
+}
+
+/** The Firestore document-write event passed to {@link handleDocumentWrite}. */
+export type DocumentWriteEvent = FirestoreEvent<
+  Change<DocumentSnapshot> | undefined,
+  Record<string, string>
+>;
+
+/**
+ * Everything a handler needs to do its work, injected by the caller so the
+ * handlers stay free of global state.
+ */
+export interface HandlerContext {
+  tracker: FirestoreBigQueryEventHistoryTracker;
+  config: ResolvedExportConfig;
+  /**
+   * Provisions the BigQuery dataset/table/views once per instance. Only called
+   * after an inline write failure as a self-heal; the hot path relies on
+   * out-of-band provisioning (`initBigQuerySync` / `setupBigQuerySync`).
+   */
+  ensureInitialized: () => Promise<void>;
+}
+
+/**
+ * Records a Firestore document change to BigQuery.
+ *
+ * @param change - Serialized change metadata and payload.
+ * @param tracker - The event history tracker to write through.
+ */
+async function recordEventToBigQuery(
+  change: SerializedDocumentChange,
+  tracker: FirestoreBigQueryEventHistoryTracker
+): Promise<void> {
+  const event: FirestoreDocumentChangeEvent = {
+    timestamp: change.timestamp,
+    operation: change.changeType,
+    documentName: change.fullResourceName,
+    documentId: change.documentId,
+    pathParams: change.params,
+    eventId: change.eventId,
+    data: change.data,
+    oldData: change.oldData,
+  };
+
+  await tracker.record([event]);
+}
+
+/**
+ * Gives a failed inline write one self-heal attempt before surfacing it to the
+ * Firestore trigger retry policy.
+ *
+ * @param change - The serialized change to write.
+ * @param ctx - The handler context.
+ */
+async function retryAfterSelfHeal(
+  change: SerializedDocumentChange,
+  ctx: HandlerContext
+): Promise<void> {
+  try {
+    await ctx.ensureInitialized();
+    await recordEventToBigQuery(change, ctx.tracker);
+  } catch (retryErr) {
+    await events.recordErrorEvent(retryErr as Error);
+
+    logs.logFailedEventAction(
+      "Failed to write event to BigQuery from onWrite handler after self-heal",
+      change.fullResourceName,
+      change.eventId,
+      change.changeType,
+      retryErr as Error
+    );
+
+    throw retryErr;
+  }
+}
+
+/**
+ * Handles a Firestore document write: serializes the change and writes it to
+ * BigQuery. Failed writes are surfaced to the trigger retry policy after one
+ * self-heal attempt.
+ *
+ * @param event - The Firestore document-write event.
+ * @param ctx - The handler context.
+ */
+export async function handleDocumentWrite(
+  event: DocumentWriteEvent,
+  ctx: HandlerContext
+): Promise<void> {
+  const { data, ...context } = event;
+  if (!data) return;
+
+  logs.start();
+
+  // No provisioning on the hot path: BigQuery resources are provisioned
+  // out-of-band (afterFirstDeploy / afterRedeploy tasks). If they are missing,
+  // the inline write fails, self-heals once, then falls back to the trigger
+  // retry policy.
+  const { config, tracker } = ctx;
+  const changeType = getChangeType(data);
+  const documentId = getDocumentId(data);
+  const isCreated = changeType === ChangeType.CREATE;
+  const isDeleted = changeType === ChangeType.DELETE;
+
+  const newData = isDeleted ? undefined : data.after.data();
+  const oldData =
+    isCreated || config.excludeOldData ? undefined : data.before.data();
+
+  const relativeName = context.document;
+  const projectId = config.projectId;
+  const fullResourceName = `projects/${projectId}/databases/${config.databaseId}/documents/${relativeName}`;
+  const eventId = context.id;
+  const operation = changeType;
+
+  logs.logEventAction(
+    "Firestore event received by onDocumentWritten trigger",
+    fullResourceName,
+    eventId,
+    operation
+  );
+
+  let serializedData: FirestoreDocumentChangeEvent["data"];
+  let serializedOldData: FirestoreDocumentChangeEvent["oldData"];
+
+  try {
+    serializedData = tracker.serializeData(newData);
+    serializedOldData = tracker.serializeData(oldData);
+  } catch (err) {
+    logs.logFailedEventAction(
+      "Failed to serialize data",
+      fullResourceName,
+      eventId,
+      operation,
+      err as Error
+    );
+    throw err;
+  }
+
+  try {
+    await events.recordStartEvent({
+      documentId,
+      changeType,
+      before: { data: data.before.data() },
+      after: { data: data.after.data() },
+      context,
+    });
+  } catch (err) {
+    logs.error(false, "Failed to record start event", err);
+    throw err;
+  }
+
+  const change: SerializedDocumentChange = {
+    timestamp: context.time,
+    eventId: context.id,
+    fullResourceName,
+    changeType,
+    documentId,
+    params: config.wildcardIds ? { ...context.params, documentId } : null,
+    data: serializedData,
+    oldData: serializedOldData,
+  };
+
+  try {
+    await recordEventToBigQuery(change, tracker);
+  } catch (err) {
+    logs.failedToWriteToBigQueryImmediately(err as Error);
+    await retryAfterSelfHeal(change, ctx);
+  }
+
+  logs.complete();
+}