@firebase-function-kits/bigquery-firestore-export 0.0.1 → 0.0.2-rc.1

package/package.json

@@ -1,12 +1,48 @@
 {
   "name": "@firebase-function-kits/bigquery-firestore-export",
-  "version": "0.0.1",
+  "version": "0.0.2-rc.1",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/firebase/extensions.git",
+    "directory": "kits/bigquery-firestore-export"
+  },
+  "private": false,
   "description": "Reverse-sync rows from BigQuery into Firestore",
   "license": "Apache-2.0",
-  "main": "index.js",
-  "private": false,
-  "repository": "firebase/extensions",
+  "main": "lib/index.js",
+  "types": "lib/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./lib/index.d.ts",
+      "default": "./lib/index.js"
+    },
+    "./lib": {
+      "types": "./lib/lib.d.ts",
+      "default": "./lib/lib.js"
+    }
+  },
   "engines": {
     "node": ">=22"
+  },
+  "scripts": {
+    "build": "tsc -b",
+    "clean": "tsc -b --clean",
+    "test": "vitest run",
+    "deploy": "pnpm build && firebase deploy --only functions",
+    "serve": "firebase emulators:start --only functions"
+  },
+  "dependencies": {
+    "@firebase/app": "^0.16.0",
+    "@google-cloud/bigquery": "^8.3.1",
+    "@google-cloud/bigquery-data-transfer": "^5.0.1",
+    "@google-cloud/pubsub": "^4.11.0",
+    "firebase-admin": "^14.1.0",
+    "firebase-functions": "7.3.2"
+  },
+  "devDependencies": {
+    "vitest": "^4.1.10"
+  },
+  "overrides": {
+    "uuid@9.0.1": "11.1.1"
   }
 }

package/src/config.ts

@@ -0,0 +1,205 @@
+/*
+ * Copyright 2026 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {
+  defineString,
+  expr,
+  projectID,
+  select,
+} from "firebase-functions/params";
+import type {
+  BigqueryFirestoreExportConfig,
+  DeployTimeOptions,
+  LogLevel,
+} from "./export-config";
+
+const LOG_LEVEL_OPTIONS = ["debug", "info", "warn", "error", "silent"] as const;
+const instanceId = defineString("INSTANCE_ID");
+
+const params = {
+  instanceId,
+  bigqueryDatasetLocation: defineString("BIGQUERY_DATASET_LOCATION", {
+    label: "BigQuery Dataset Location",
+    description:
+      "What is the location of the BigQuery dataset referenced in the query?",
+
+    default: "US",
+    input: select({
+      "Columbus, Ohio (us-east5)": "us-east5",
+      "Iowa (us-central1)": "us-central1",
+      "Las Vegas (us-west4)": "us-west4",
+      "Los Angeles (us-west2)": "us-west2",
+      "Montréal (northamerica-northeast1)": "northamerica-northeast1",
+      "Northern Virginia (us-east4)": "us-east4",
+      "Oregon (us-west1)": "us-west1",
+      "Salt Lake City (us-west3)": "us-west3",
+      "São Paulo (southamerica-east1)": "southamerica-east1",
+      "Santiago (southamerica-west1)": "southamerica-west1",
+      "South Carolina (us-east1)": "us-east1",
+      "Toronto (northamerica-northeast2)": "northamerica-northeast2",
+      "Delhi (asia-south2)": "asia-south2",
+      "Hong Kong (asia-east2)": "asia-east2",
+      "Jakarta (asia-southeast2)": "asia-southeast2",
+      "Melbourne (australia-southeast2)": "australia-southeast2",
+      "Mumbai (asia-south1)": "asia-south1",
+      "Osaka (asia-northeast2)": "asia-northeast2",
+      "Seoul (asia-northeast3)": "asia-northeast3",
+      "Singapore (asia-southeast1)": "asia-southeast1",
+      "Sydney (australia-southeast1)": "australia-southeast1",
+      "Taiwan (asia-east1)": "asia-east1",
+      "Tokyo (asia-northeast1)": "asia-northeast1",
+      "Belgium (europe-west1)": "europe-west1",
+      "Finland (europe-north1)": "europe-north1",
+      "Frankfurt (europe-west3)": "europe-west3",
+      "London (europe-west2)": "europe-west2",
+      "Madrid (europe-southwest1)": "europe-southwest1",
+      "Milan (europe-west8)": "europe-west8",
+      "Netherlands (europe-west4)": "europe-west4",
+      "Paris (europe-west9)": "europe-west9",
+      "Warsaw (europe-central2)": "europe-central2",
+      "Zürich (europe-west6)": "europe-west6",
+      "US Multi-Region (US)": "US",
+      "EU Mutli-Region (EU)": "EU",
+    }),
+  }),
+  transferConfigName: defineString("TRANSFER_CONFIG_NAME", { default: "" }),
+  datasetId: defineString("DATASET_ID", {
+    label: "Dataset ID",
+    description:
+      "What's the BigQuery destination dataset you'd like to use? Each transfer run will write to a table in this destination dataset.",
+    input: {
+      text: {
+        nonEmpty: true,
+        example: "customer_data",
+      },
+    },
+  }),
+  tableName: defineString("TABLE_NAME", {
+    label: "Destination Table Name",
+    description:
+      "What's the destination table name prefix you'd like to use? Each transfer run will write to the table with this name, postfixed with the runtime.",
+    input: {
+      text: {
+        nonEmpty: true,
+        example: "transactions",
+      },
+    },
+  }),
+  queryString: defineString("QUERY_STRING", {
+    label: "Query String",
+    description: "What's the BQ query you'd like to execute?",
+    input: {
+      text: {
+        nonEmpty: true,
+
+        example: "SELECT * from <PROJECT_ID>.customer_data.transactions",
+      },
+    },
+  }),
+  displayName: defineString("DISPLAY_NAME", {
+    label: "Display Name",
+    description: "What display name would you like to use?",
+    input: {
+      text: {
+        nonEmpty: true,
+        example: "Daily Rollup - Customer Transactions",
+      },
+    },
+  }),
+  partitioningField: defineString("PARTITIONING_FIELD", {
+    label: "Partitioning Field",
+    description:
+      "What's the partitioning field on the destination table ID? Leave empty if not using a partitioning field.",
+    default: "",
+    input: { text: { example: "timestamp" } },
+  }),
+  schedule: defineString("SCHEDULE", {
+    label: "Schedule",
+    description:
+      "What's the execution schedule you'd like to use for this query?",
+    input: {
+      text: {
+        nonEmpty: true,
+        example: "every 15 minutes",
+      },
+    },
+  }),
+  firestoreCollection: defineString("COLLECTION_PATH", {
+    label: "Firestore Collection",
+    description:
+      "What's the top-level Firestore Collection to store transfer configs, run metadata, and query output?",
+
+    default: "transferConfigs",
+    input: {
+      text: {
+        example: "transferConfigs",
+
+        validationRegex: /^[^\/]+(\/[^\/]+\/[^\/]+)*$/,
+        validationErrorMessage: "Must be a valid Cloud Firestore Collection",
+      },
+    },
+  }),
+  logLevel: defineString("LOG_LEVEL", {
+    label: "Log Level",
+    description: "What's the log level you'd like to use for this extension?",
+
+    default: "info",
+    input: select({
+      DEBUG: "debug",
+      INFO: "info",
+      WARN: "warn",
+      ERROR: "error",
+      SILENT: "silent",
+    }),
+  }),
+};
+
+export const CONFIG_EXPRESSIONS: DeployTimeOptions = {
+  pubSubTopic: expr`kit-${instanceId}-processMessages`,
+};
+
+function optional(value: string): string | undefined {
+  const normalized = value.trim();
+  return normalized.length > 0 ? normalized : undefined;
+}
+
+function normalizeLogLevel(value: string): LogLevel {
+  const normalized = value.toLowerCase();
+  return LOG_LEVEL_OPTIONS.includes(normalized as LogLevel)
+    ? (normalized as LogLevel)
+    : "info";
+}
+
+/** Reads runtime values from Firebase deploy-time parameters. */
+export function configFromEnv(): BigqueryFirestoreExportConfig {
+  const resolvedInstanceId = params.instanceId.value();
+
+  return {
+    bigqueryDatasetLocation: params.bigqueryDatasetLocation.value(),
+    projectId: projectID.value(),
+    instanceId: resolvedInstanceId,
+    transferConfigName: optional(params.transferConfigName.value()),
+    datasetId: params.datasetId.value(),
+    tableName: params.tableName.value(),
+    queryString: params.queryString.value(),
+    displayName: params.displayName.value(),
+    partitioningField: optional(params.partitioningField.value()),
+    schedule: params.schedule.value(),
+    pubSubTopic: `kit-${resolvedInstanceId}-processMessages`,
+    firestoreCollection: params.firestoreCollection.value(),
+    logLevel: normalizeLogLevel(params.logLevel.value()),
+  };
+}

package/src/dts.ts

@@ -0,0 +1,217 @@
+/*
+ * Copyright 2025 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import * as bigqueryDataTransfer from "@google-cloud/bigquery-data-transfer";
+import type { ResolvedBigqueryFirestoreExportConfig } from "./export-config";
+import * as logs from "./logs";
+
+export type DataTransferClient =
+  bigqueryDataTransfer.v1.DataTransferServiceClient;
+export type TransferConfig =
+  bigqueryDataTransfer.protos.google.cloud.bigquery.datatransfer.v1.ITransferConfig;
+
+const GRPC_NOT_FOUND = 5;
+const PACKAGE_PARTITIONING_ERROR =
+  "Cannot remove partitioning_field from an existing transfer config";
+
+export const PARTITIONING_FIELD_REMOVAL_ERROR_PREFIX =
+  PACKAGE_PARTITIONING_ERROR;
+export const PARTITIONING_FIELD_REMOVAL_ERROR = `${PACKAGE_PARTITIONING_ERROR}. The BigQuery Data Transfer API does not support clearing this parameter once it has been set. To change partitioning, create a new transfer config with the desired setting.`;
+
+function isNotFoundError(err: unknown): boolean {
+  return (
+    typeof err === "object" &&
+    err !== null &&
+    "code" in err &&
+    err.code === GRPC_NOT_FOUND
+  );
+}
+
+function transferConfigFields(config: TransferConfig) {
+  const fields = config.params?.fields;
+  if (!fields) {
+    throw new Error(
+      "Transfer config has invalid structure: missing params.fields"
+    );
+  }
+  if (!fields.query) {
+    throw new Error(
+      "Transfer config has invalid structure: missing params.fields.query"
+    );
+  }
+  if (!fields.destination_table_name_template) {
+    throw new Error(
+      "Transfer config has invalid structure: missing params.fields.destination_table_name_template"
+    );
+  }
+
+  return fields;
+}
+
+function stringField(value: string | undefined): { stringValue: string } {
+  return { stringValue: value ?? "" };
+}
+
+/**
+ * Creates the protobuf-shaped request used for a scheduled query.
+ *
+ * The query runs as whichever identity creates it, which is this function. To
+ * name a different account, `serviceAccountName` goes on the request rather
+ * than on the transfer config, and the caller needs `actAs` on the account it
+ * names, including its own.
+ */
+export function createTransferConfigRequest(
+  config: ResolvedBigqueryFirestoreExportConfig
+): bigqueryDataTransfer.protos.google.cloud.bigquery.datatransfer.v1.ICreateTransferConfigRequest {
+  return {
+    parent: `projects/${config.projectId}`,
+    transferConfig: {
+      destinationDatasetId: config.datasetId,
+      displayName: config.displayName,
+      dataSourceId: "scheduled_query",
+      params: {
+        fields: {
+          query: stringField(config.queryString),
+          destination_table_name_template: stringField(
+            `${config.tableName}_{run_time|"%H%M%S"}`
+          ),
+          write_disposition: stringField("WRITE_TRUNCATE"),
+          partitioning_field: stringField(config.partitioningField),
+        },
+      },
+      schedule: config.schedule,
+      notificationPubsubTopic: `projects/${config.projectId}/topics/${config.pubSubTopic}`,
+    },
+  };
+}
+
+export async function getTransferConfig(
+  client: DataTransferClient,
+  transferConfigName: string
+): Promise<TransferConfig | null> {
+  try {
+    const [config] = await client.getTransferConfig({
+      name: transferConfigName,
+    });
+    return config;
+  } catch (err) {
+    if (isNotFoundError(err)) {
+      logs.transferConfigNotFound(transferConfigName);
+      return null;
+    }
+    logs.getTransferConfigFailed(transferConfigName, err);
+    throw err;
+  }
+}
+
+export async function createTransferConfig(
+  client: DataTransferClient,
+  config: ResolvedBigqueryFirestoreExportConfig
+): Promise<TransferConfig> {
+  logs.createTransferConfig();
+  const [created] = await client.createTransferConfig(
+    createTransferConfigRequest(config)
+  );
+  if (!created.name) {
+    throw new Error("BigQuery API returned a transfer config without a name");
+  }
+  logs.transferConfigCreated(created.name);
+  return created;
+}
+
+/** Builds a minimal update mask while retaining unsupported immutable fields. */
+export async function constructUpdateTransferConfigRequest(
+  client: DataTransferClient,
+  transferConfigName: string,
+  config: ResolvedBigqueryFirestoreExportConfig
+): Promise<bigqueryDataTransfer.protos.google.cloud.bigquery.datatransfer.v1.IUpdateTransferConfigRequest> {
+  const transferConfig = await getTransferConfig(client, transferConfigName);
+  if (!transferConfig) throw new Error("Transfer config not found");
+
+  const fields = transferConfigFields(transferConfig);
+  const updatedConfig = JSON.parse(
+    JSON.stringify(transferConfig)
+  ) as TransferConfig;
+  const updatedFields = transferConfigFields(updatedConfig);
+  const updateMask: string[] = [];
+
+  if (config.datasetId !== transferConfig.destinationDatasetId) {
+    updateMask.push("destination_dataset_id");
+    updatedConfig.destinationDatasetId = config.datasetId;
+  }
+  if (config.queryString !== fields.query.stringValue) {
+    updateMask.push("params");
+    updatedFields.query.stringValue = config.queryString;
+  }
+
+  const tableTemplate = `${config.tableName}_{run_time|"%H%M%S"}`;
+  if (tableTemplate !== fields.destination_table_name_template.stringValue) {
+    updateMask.push("params");
+    updatedFields.destination_table_name_template.stringValue = tableTemplate;
+  }
+
+  const existingPartitioningField =
+    fields.partitioning_field?.stringValue ?? "";
+  const newPartitioningField = config.partitioningField ?? "";
+  if (newPartitioningField !== existingPartitioningField) {
+    if (!newPartitioningField) {
+      logs.partitioningFieldRemovalAttempted(
+        transferConfigName,
+        existingPartitioningField
+      );
+      throw new Error(PARTITIONING_FIELD_REMOVAL_ERROR);
+    }
+    updateMask.push("params");
+    updatedFields.partitioning_field ??= {};
+    updatedFields.partitioning_field.stringValue = newPartitioningField;
+  }
+
+  if (config.schedule !== transferConfig.schedule) {
+    updateMask.push("schedule");
+    updatedConfig.schedule = config.schedule;
+  }
+
+  const expectedTopic = `projects/${config.projectId}/topics/${config.pubSubTopic}`;
+  if (expectedTopic !== transferConfig.notificationPubsubTopic) {
+    updateMask.push("notification_pubsub_topic");
+    updatedConfig.notificationPubsubTopic = expectedTopic;
+  }
+
+  return {
+    transferConfig: updatedConfig,
+    updateMask: { paths: [...new Set(updateMask)] },
+  };
+}
+
+export async function updateTransferConfig(
+  client: DataTransferClient,
+  transferConfigName: string,
+  config: ResolvedBigqueryFirestoreExportConfig
+): Promise<TransferConfig> {
+  const request = await constructUpdateTransferConfigRequest(
+    client,
+    transferConfigName,
+    config
+  );
+  logs.updateTransferConfig(transferConfigName);
+  const converted =
+    bigqueryDataTransfer.protos.google.cloud.bigquery.datatransfer.v1.UpdateTransferConfigRequest.fromObject(
+      request
+    );
+  const [updated] = await client.updateTransferConfig(converted);
+  logs.transferConfigUpdated(transferConfigName);
+  return updated;
+}

package/src/export-config.ts

@@ -0,0 +1,119 @@
+/*
+ * Copyright 2026 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { Expression } from "firebase-functions/params";
+
+/** Log levels supported by the original extension. */
+export type LogLevel = "debug" | "info" | "warn" | "error" | "silent";
+
+/** Public configuration for the BigQuery-to-Firestore functions. */
+export interface BigqueryFirestoreExportConfig {
+  /** Location of the BigQuery destination dataset, for example `US`. */
+  bigqueryDatasetLocation: string;
+  /** Google Cloud project id. */
+  projectId: string;
+  /** Stable id used to associate a DTS config with this deployment. */
+  instanceId: string;
+  /** Link this existing DTS config instead of creating one. */
+  transferConfigName?: string;
+  /** BigQuery destination dataset id. */
+  datasetId: string;
+  /** Prefix for per-run destination tables. */
+  tableName: string;
+  /** Scheduled query text. */
+  queryString: string;
+  /** Human-readable DTS scheduled-query name. */
+  displayName: string;
+  /** Optional destination-table partitioning field. */
+  partitioningField?: string;
+  /** BigQuery Data Transfer schedule, for example `every 15 minutes`. */
+  schedule: string;
+  /** Pub/Sub topic receiving DTS completion notifications. */
+  pubSubTopic?: string;
+  /** Root Firestore collection for configs and run output. */
+  firestoreCollection?: string;
+  /** Log verbosity. Defaults to `info`. */
+  logLevel?: LogLevel;
+}
+
+/** Configuration after defaults and validation have been applied. */
+export interface ResolvedBigqueryFirestoreExportConfig {
+  bigqueryDatasetLocation: string;
+  projectId: string;
+  instanceId: string;
+  transferConfigName?: string;
+  datasetId: string;
+  tableName: string;
+  queryString: string;
+  displayName: string;
+  partitioningField?: string;
+  schedule: string;
+  pubSubTopic: string;
+  firestoreCollection: string;
+  logLevel: LogLevel;
+}
+
+/** Deploy-time values used to construct the v2 triggers. */
+export interface DeployTimeOptions {
+  pubSubTopic: string | Expression<string>;
+}
+
+function required(value: string, field: string): string {
+  const normalized = value.trim();
+  if (!normalized) {
+    throw new Error(`${field} must be a non-empty string.`);
+  }
+  return normalized;
+}
+
+function optional(value: string | undefined): string | undefined {
+  if (value === undefined) return undefined;
+  const normalized = value.trim();
+  return normalized || undefined;
+}
+
+/** Applies package defaults and validates a user-supplied configuration. */
+export function resolveConfig(
+  config: BigqueryFirestoreExportConfig
+): ResolvedBigqueryFirestoreExportConfig {
+  const instanceId = required(config.instanceId, "instanceId");
+  const logLevel = config.logLevel ?? "info";
+
+  if (!["debug", "info", "warn", "error", "silent"].includes(logLevel)) {
+    throw new Error(`Unsupported logLevel: ${logLevel}`);
+  }
+
+  return {
+    bigqueryDatasetLocation: required(
+      config.bigqueryDatasetLocation,
+      "bigqueryDatasetLocation"
+    ),
+    projectId: required(config.projectId, "projectId"),
+    instanceId,
+    transferConfigName: optional(config.transferConfigName),
+    datasetId: required(config.datasetId, "datasetId"),
+    tableName: required(config.tableName, "tableName"),
+    queryString: required(config.queryString, "queryString"),
+    displayName: required(config.displayName, "displayName"),
+    partitioningField: optional(config.partitioningField),
+    schedule: required(config.schedule, "schedule"),
+    pubSubTopic:
+      optional(config.pubSubTopic) ?? `kit-${instanceId}-processMessages`,
+    firestoreCollection:
+      optional(config.firestoreCollection) ?? "transferConfigs",
+    logLevel,
+  };
+}