@firebase-function-kits/firestore-bigquery-export 0.0.1 → 0.0.2-rc.1

package/src/index.ts

@@ -0,0 +1,174 @@
+/*
+ * Copyright 2019 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Main entry point. Exports the wired functions with deploy-time param
+ * expressions, then resolves concrete config lazily at runtime. Re-export
+ * `fsexportbigquery` and `initBigQuerySync` from your own functions codebase
+ * entry; configuration comes from a `.env` (or
+ * `.env.<projectId>`), which the Firebase CLI loads at deploy.
+ *
+ * Because this module initializes runtime dependencies lazily, deploy discovery
+ * can analyze it without resolving params too early.
+ * For side-effect-free imports (the handlers and config types), import from
+ * `./lib` instead.
+ */
+
+import { FirestoreBigQueryEventHistoryTracker } from "@firebaseextensions/firestore-bigquery-change-tracker";
+import { getApps, initializeApp } from "firebase-admin/app";
+import { onDocumentWritten } from "firebase-functions/firestore";
+import { expr } from "firebase-functions/params";
+import { onTaskDispatched } from "firebase-functions/tasks";
+import type { Role } from "firebase-functions/v2";
+import { requiresAPI, requiresRole } from "firebase-functions/v2";
+import {
+  afterFirstDeploy,
+  afterRedeploy,
+} from "firebase-functions/v2/lifecycle";
+import { CONFIG_EXPRESSIONS, configFromEnv } from "./config";
+import * as events from "./events";
+import { resolveExportConfig, toTrackerConfig } from "./export-config";
+import { type HandlerContext, handleDocumentWrite } from "./handlers";
+import { createEnsureInitialized } from "./init";
+import * as logs from "./logs";
+
+// Re-export the side-effect-free library surface (handlers and config types).
+export * from "./lib";
+
+const INIT_BIGQUERY_SYNC_FUNCTION = "initBigQuerySync";
+const SETUP_BIGQUERY_SYNC_FUNCTION = "setupBigQuerySync";
+const LIFECYCLE_RETRY_CONFIG = {
+  maxAttempts: 15,
+  minBackoffSeconds: 60,
+} as const;
+const REQUIRED_ROLES: ReadonlyArray<Role> = [
+  "roles/bigquery.dataEditor",
+  "roles/datastore.user",
+  "roles/bigquery.user",
+  // Gen2 Firestore triggers need Eventarc receive and run.invoker on the function SA.
+  "roles/eventarc.eventReceiver",
+  "roles/run.invoker",
+];
+const REQUIRED_APIS = [
+  {
+    api: "bigquery.googleapis.com",
+    reason: "Mirrors data from your Cloud Firestore collection in BigQuery.",
+  },
+] as const;
+
+for (const role of REQUIRED_ROLES) {
+  requiresRole(role);
+}
+
+for (const { api, reason } of REQUIRED_APIS) {
+  requiresAPI(api, reason);
+}
+
+afterFirstDeploy({
+  task: { function: INIT_BIGQUERY_SYNC_FUNCTION, body: { data: {} } },
+});
+afterRedeploy({
+  task: { function: SETUP_BIGQUERY_SYNC_FUNCTION, body: { data: {} } },
+});
+
+let ctx: HandlerContext | undefined;
+
+function getHandlerContext(): HandlerContext {
+  if (ctx) {
+    return ctx;
+  }
+
+  const config = resolveExportConfig(configFromEnv());
+  const tracker = new FirestoreBigQueryEventHistoryTracker(
+    toTrackerConfig(config)
+  );
+
+  logs.logger.setLogLevel(config.logLevel);
+  logs.init(config);
+
+  if (getApps().length === 0) {
+    initializeApp();
+  }
+
+  events.setupEventChannel();
+
+  const ensureInitialized = createEnsureInitialized(tracker);
+
+  ctx = {
+    tracker,
+    config,
+    ensureInitialized,
+  };
+
+  return ctx;
+}
+
+const functionOptions = {
+  region: CONFIG_EXPRESSIONS.location,
+};
+
+/**
+ * Firestore trigger: streams document writes on the watched collection into the
+ * BigQuery changelog table. Failed executions are retried by the Firebase
+ * Functions runtime.
+ */
+export const fsexportbigquery = onDocumentWritten(
+  {
+    ...functionOptions,
+    document: expr`${CONFIG_EXPRESSIONS.collectionPath}/{documentId}`,
+    database: CONFIG_EXPRESSIONS.database,
+    retry: true,
+  },
+  (event) => handleDocumentWrite(event, getHandlerContext())
+);
+
+async function handleBigQuerySyncInitialization(): Promise<void> {
+  try {
+    await getHandlerContext().ensureInitialized();
+  } catch (err) {
+    logs.error(false, "Failed to initialize BigQuery resources", err);
+    throw err;
+  }
+}
+
+/**
+ * First-deploy provisioning task (`afterFirstDeploy`). Runs in the function's
+ * own identity, so it has the runtime service account and bound secrets that
+ * creating the dataset/table/views needs (e.g. CMEK). Cloud Tasks retries a
+ * failed initialization on its own schedule, so a transient BigQuery error does
+ * not leave the resources unprovisioned. It is idempotent (`initialize()`
+ * no-ops when resources already exist) and can also be invoked directly as an
+ * authenticated HTTP POST, without queue retries.
+ */
+export const initBigQuerySync = onTaskDispatched(
+  {
+    ...functionOptions,
+    retryConfig: LIFECYCLE_RETRY_CONFIG,
+  },
+  handleBigQuerySyncInitialization
+);
+
+/**
+ * Update/configure lifecycle task. Uses the same idempotent provisioning path
+ * so BigQuery resources are reconciled after parameter changes.
+ */
+export const setupBigQuerySync = onTaskDispatched(
+  {
+    ...functionOptions,
+    retryConfig: LIFECYCLE_RETRY_CONFIG,
+  },
+  handleBigQuerySyncInitialization
+);

package/src/init.ts

@@ -0,0 +1,45 @@
+/*
+ * Copyright 2019 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { FirestoreBigQueryEventHistoryTracker } from "@firebaseextensions/firestore-bigquery-change-tracker";
+
+/**
+ * Builds the provisioning guard used by the `initBigQuerySync` endpoint and the
+ * retry-path self-heal. The hot write path never calls it.
+ *
+ * The returned function runs `tracker.initialize()` at most once per instance:
+ * concurrent invocations on a cold instance share a single in-flight promise. A
+ * failed initialization resets the guard so a later invocation can retry after
+ * a transient error.
+ *
+ * @param tracker - The event history tracker whose `initialize()` provisions
+ *   the BigQuery dataset, table, and views.
+ * @returns A function that resolves once the resources are provisioned.
+ */
+export function createEnsureInitialized(
+  tracker: FirestoreBigQueryEventHistoryTracker
+): () => Promise<void> {
+  let initialization: Promise<void> | null = null;
+  return () => {
+    if (!initialization) {
+      initialization = tracker.initialize().catch((err) => {
+        initialization = null;
+        throw err;
+      });
+    }
+    return initialization;
+  };
+}

package/src/lib.ts

@@ -0,0 +1,55 @@
+/*
+ * Copyright 2019 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Side-effect-free library surface:
+ *
+ * - {@link handleDocumentWrite} — the handler, for consumers who want to own
+ *   trigger registration themselves. It takes an injected
+ *   {@link HandlerContext}.
+ * - Config types and helpers ({@link ExportConfig}, {@link resolveExportConfig},
+ *   {@link toTrackerConfig}) for building that context.
+ *
+ * Importing this module has no side effects (it reads no environment), so it is
+ * safe to import anywhere. The main entry point (`./index`) is the one that
+ * reads env params and exports the wired functions.
+ *
+ * The change-tracker engine is an internal dependency and is deliberately not
+ * re-exported; only the types the handler signatures need are surfaced.
+ */
+
+// Engine types referenced by the handler and config signatures. Type-only —
+// the engine itself is internal. `ChangeType` is exported for handler tests and
+// advanced consumers that build Firestore events directly.
+export {
+  type ChangeTrackerConfig,
+  ChangeType,
+  type FirestoreBigQueryEventHistoryTracker,
+} from "@firebaseextensions/firestore-bigquery-change-tracker";
+// Config types and helpers
+export {
+  type ExportConfig,
+  type ResolvedExportConfig,
+  resolveExportConfig,
+  toTrackerConfig,
+  type ViewType,
+} from "./export-config";
+// Handlers
+export {
+  type DocumentWriteEvent,
+  type HandlerContext,
+  handleDocumentWrite,
+} from "./handlers";

package/src/logs.ts

@@ -0,0 +1,233 @@
+/*
+ * Copyright 2019 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {
+  type ChangeType,
+  Logger,
+} from "@firebaseextensions/firestore-bigquery-change-tracker";
+
+export const logger = new Logger();
+
+export const arrayFieldInvalid = (fieldName: string) => {
+  logger.warn(`Array field '${fieldName}' does not contain an array, skipping`);
+};
+
+export const bigQueryDatasetCreated = (datasetId: string) => {
+  logger.info(`Created BigQuery dataset: ${datasetId}`);
+};
+
+export const bigQueryDatasetCreating = (datasetId: string) => {
+  logger.debug(`Creating BigQuery dataset: ${datasetId}`);
+};
+
+export const bigQueryDatasetExists = (datasetId: string) => {
+  logger.info(`BigQuery dataset already exists: ${datasetId}`);
+};
+
+export const bigQueryErrorRecordingDocumentChange = (e: Error) => {
+  logger.error(`Error recording document changes.`, e);
+};
+
+export const bigQueryLatestSnapshotViewQueryCreated = (query: string) => {
+  logger.info(`BigQuery latest snapshot view query:\n${query}`);
+};
+
+export const bigQueryTableAlreadyExists = (
+  tableName: string,
+  datasetName: string
+) => {
+  logger.info(
+    `BigQuery table with name ${tableName} already ` +
+      `exists in dataset ${datasetName}!`
+  );
+};
+
+export const bigQueryTableCreated = (tableName: string) => {
+  logger.info(`Created BigQuery table: ${tableName}`);
+};
+
+export const bigQueryTableCreating = (tableName: string) => {
+  logger.debug(`Creating BigQuery table: ${tableName}`);
+};
+
+export const bigQueryTableUpdated = (tableName: string) => {
+  logger.info(`Updated existing BigQuery table: ${tableName}`);
+};
+
+export const bigQueryTableUpdating = (tableName: string) => {
+  logger.debug(`Updating existing BigQuery table: ${tableName}`);
+};
+
+export const bigQueryTableUpToDate = (tableName: string) => {
+  logger.info(`BigQuery table: ${tableName} is up to date`);
+};
+
+export const bigQueryTableValidated = (tableName: string) => {
+  logger.info(`Validated existing BigQuery table: ${tableName}`);
+};
+
+export const bigQueryTableValidating = (tableName: string) => {
+  logger.debug(`Validating existing BigQuery table: ${tableName}`);
+};
+
+export const bigQueryUserDefinedFunctionCreating = (
+  functionDefinition: string
+) => {
+  logger.debug(
+    `Creating BigQuery User-defined Function:\n${functionDefinition}`
+  );
+};
+
+export const bigQueryUserDefinedFunctionCreated = (
+  functionDefinition: string
+) => {
+  logger.info(`Created BigQuery User-defined Function:\n${functionDefinition}`);
+};
+
+export const bigQueryViewCreated = (viewName: string) => {
+  logger.info(`Created BigQuery view: ${viewName}`);
+};
+
+export const bigQueryViewCreating = (viewName: string) => {
+  logger.debug(`Creating BigQuery view: ${viewName}`);
+};
+
+export const bigQueryViewAlreadyExists = (
+  viewName: string,
+  datasetName: string
+) => {
+  logger.info(
+    `View with id ${viewName} already exists in dataset ${datasetName}.`
+  );
+};
+
+export const bigQueryViewUpdated = (viewName: string) => {
+  logger.info(`Updated existing BigQuery view: ${viewName}`);
+};
+
+export const bigQueryViewUpdating = (viewName: string) => {
+  logger.debug(`Updating existing BigQuery view: ${viewName}`);
+};
+
+export const bigQueryViewUpToDate = (viewName: string) => {
+  logger.info(`BigQuery view: ${viewName} is up to date`);
+};
+
+export const bigQueryViewValidated = (viewName: string) => {
+  logger.info(`Validated existing BigQuery view: ${viewName}`);
+};
+
+export const bigQueryViewValidating = (viewName: string) => {
+  logger.debug(`Validating existing BigQuery view: ${viewName}`);
+};
+
+export const complete = () => {
+  logger.info("Completed execution of extension");
+};
+
+export const dataInserted = (rowCount: number) => {
+  logger.debug(`Inserted ${rowCount} row(s) of data into BigQuery`);
+};
+
+export const dataInserting = (rowCount: number) => {
+  logger.debug(`Inserting ${rowCount} row(s) of data into BigQuery`);
+};
+
+export const dataTypeInvalid = (
+  fieldName: string,
+  fieldType: string,
+  dataType: string
+) => {
+  logger.warn(
+    `Field '${fieldName}' has invalid data. Expected: ${fieldType}, received: ${dataType}`
+  );
+};
+
+export const error = (
+  includeEvent: boolean,
+  message: string,
+  err: Error,
+  event?: any, // Made optional, as it is not always required
+  eventTrackerConfig?: any // Made optional, as it is not always required
+) => {
+  const logDetails: Record<string, any> = { error: err };
+
+  if (includeEvent && event) {
+    logDetails.event = event;
+  }
+
+  if (includeEvent && eventTrackerConfig) {
+    logDetails.eventTrackerConfig = eventTrackerConfig;
+  }
+
+  logger.error(`Error when mirroring data to BigQuery: ${message}`, logDetails);
+};
+
+export const init = (config?: unknown) => {
+  logger.info("Initializing with configuration", config);
+};
+
+export const start = (config?: unknown) => {
+  logger.info("Started execution with configuration", config);
+};
+
+export const timestampMissingValue = (fieldName: string) => {
+  logger.warn(
+    `Missing value for timestamp field: ${fieldName}, using default timestamp instead.`
+  );
+};
+
+export const logEventAction = (
+  action: string,
+  document_name: string,
+  event_id: string,
+  operation: ChangeType
+) => {
+  logger.info(action, {
+    document_name,
+    event_id,
+    operation,
+  });
+};
+
+export const logFailedEventAction = (
+  action: string,
+  document_name: string,
+  event_id: string,
+  operation: ChangeType,
+  error: Error
+) => {
+  const changeTypeMap = {
+    0: "CREATE",
+    1: "DELETE",
+    2: "UPDATE",
+    3: "IMPORT",
+  };
+
+  logger.error(action, {
+    document_name,
+    event_id,
+    operation: changeTypeMap[operation],
+    error,
+  });
+};
+
+export const failedToWriteToBigQueryImmediately = (error: Error) => {
+  logger.warn(
+    "Failed to write event to BigQuery Immediately. Will attempt to Enqueue to Cloud Tasks.",
+    error
+  );
+};

package/src/util.ts

@@ -0,0 +1,64 @@
+/*
+ * Copyright 2019 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { ChangeType } from "@firebaseextensions/firestore-bigquery-change-tracker";
+import type { Change, DocumentSnapshot } from "firebase-functions/firestore";
+
+/**
+ * Get the change type (CREATE, UPDATE, DELETE) from the Firestore change.
+ * @param change Firestore document change object.
+ * @returns {ChangeType} The type of change.
+ */
+export function getChangeType(change: Change<DocumentSnapshot>): ChangeType {
+  if (!change.after.exists) {
+    return ChangeType.DELETE;
+  }
+  if (!change.before.exists) {
+    return ChangeType.CREATE;
+  }
+  return ChangeType.UPDATE;
+}
+
+/**
+ * Get the document ID from the Firestore change.
+ * @param change Firestore document change object.
+ * @returns {string} The document ID.
+ */
+export function getDocumentId(change: Change<DocumentSnapshot>): string {
+  if (change.after.exists) {
+    return change.after.id;
+  }
+  return change.before.id;
+}
+
+/**
+ *
+ * @param template - eg, regions/{regionId}/countries
+ * @param text - eg, regions/asia/countries
+ *
+ * @return - eg, { regionId: "asia" }
+ */
+export const resolveWildcardIds = (template: string, text: string) => {
+  const textSegments = text.split("/");
+  return template
+    .split("/")
+    .reduce((previousValue, currentValue, currentIndex) => {
+      if (currentValue.startsWith("{") && currentValue.endsWith("}")) {
+        previousValue[currentValue.slice(1, -1)] = textSegments[currentIndex];
+      }
+      return previousValue;
+    }, {});
+};