@firebase-function-kits/bigquery-firestore-export 0.0.1 → 0.0.2-rc.0

package/src/lib.ts

@@ -0,0 +1,69 @@
+/*
+ * Copyright 2026 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Side-effect-free library surface. Importing this module registers no
+ * functions, lifecycle hooks, roles, or APIs and reads no environment params.
+ */
+
+export {
+  constructUpdateTransferConfigRequest,
+  createTransferConfig,
+  createTransferConfigRequest,
+  type DataTransferClient,
+  getTransferConfig,
+  PARTITIONING_FIELD_REMOVAL_ERROR,
+  PARTITIONING_FIELD_REMOVAL_ERROR_PREFIX,
+  type TransferConfig,
+  updateTransferConfig,
+} from "./dts";
+export {
+  type BigqueryFirestoreExportConfig,
+  type DeployTimeOptions,
+  type LogLevel,
+  type ResolvedBigqueryFirestoreExportConfig,
+  resolveConfig,
+} from "./export-config";
+export {
+  type HandlerContext,
+  handleMessagePublished,
+  handleUpsertTransferConfig,
+  type TransferRunEvent,
+} from "./handlers";
+export {
+  convertUnsupportedDataTypes,
+  getBigqueryResults,
+  handleTransferRunMessage,
+  type ParsedTransferConfigName,
+  type ParsedTransferRunName,
+  parseTransferConfigName,
+  parseTransferRunName,
+  type ResultHandlerContext,
+  transferConfigAssociatedWithInstance,
+  updateLatestRunDocument,
+  writeRunResultsToFirestore,
+} from "./helper";
+export { metadata } from "./metadata";
+export type {
+  BigQueryRow,
+  BigQueryRowValue,
+  FirestoreRow,
+  FirestoreRowValue,
+  TransferRunMessage,
+  TransferRunParams,
+  TransferRunPayload,
+  TransferRunState,
+} from "./types";

package/src/logs.ts

@@ -0,0 +1,145 @@
+/*
+ * Copyright 2025 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { logger } from "firebase-functions";
+import type { ResolvedBigqueryFirestoreExportConfig } from "./export-config";
+
+export function init(config: ResolvedBigqueryFirestoreExportConfig): void {
+  logger.info("Initializing BigQuery Firestore export", {
+    ...config,
+    queryString: "<omitted>",
+  });
+}
+
+export function start(): void {
+  logger.info("Started BigQuery Firestore export execution");
+}
+
+export function complete(): void {
+  logger.info("Completed BigQuery Firestore export execution");
+}
+
+export function error(err: unknown): void {
+  logger.error("Unhandled error during BigQuery Firestore export", err);
+}
+
+export function errorWritingToFirestore(err: unknown): void {
+  logger.error("Error writing a BigQuery row to Firestore", err);
+}
+
+export function bigqueryJobStarted(jobId: string | undefined): void {
+  logger.info(`BigQuery job ${jobId ?? "<unknown>"} started`);
+}
+
+export function bigqueryResultsRowCount(
+  transferConfigId: string,
+  runId: string,
+  count: number
+): void {
+  logger.debug("Read BigQuery transfer results", {
+    transferConfigId,
+    runId,
+    count,
+  });
+}
+
+export function bigqueryQueryFailed(
+  transferConfigId: string,
+  runId: string,
+  tableName: string,
+  err: unknown
+): void {
+  logger.error("BigQuery result query failed", {
+    transferConfigId,
+    runId,
+    tableName,
+    err,
+  });
+}
+
+export function runResultsWrittenToFirestore(
+  runId: string,
+  successCount: number,
+  totalCount: number
+): void {
+  logger.info("Finished writing BigQuery results to Firestore", {
+    runId,
+    successCount,
+    totalCount,
+  });
+}
+
+export function latestDocUpdateSkipped(
+  transferConfigId: string,
+  runId: string,
+  reason: string
+): void {
+  logger.debug("Skipped updating latest transfer-run document", {
+    transferConfigId,
+    runId,
+    reason,
+  });
+}
+
+export function handlingNonSuccessRun(
+  transferConfigId: string,
+  runId: string,
+  state: string
+): void {
+  logger.info("Recording non-successful BigQuery transfer run", {
+    transferConfigId,
+    runId,
+    state,
+  });
+}
+
+export function createTransferConfig(): void {
+  logger.info("Creating BigQuery Data Transfer config");
+}
+
+export function transferConfigCreated(name: string): void {
+  logger.info("Created BigQuery Data Transfer config", { name });
+}
+
+export function updateTransferConfig(name: string): void {
+  logger.info("Updating BigQuery Data Transfer config", { name });
+}
+
+export function transferConfigUpdated(name: string): void {
+  logger.info("Updated BigQuery Data Transfer config", { name });
+}
+
+export function transferConfigNotFound(name: string): void {
+  logger.error("BigQuery Data Transfer config not found", { name });
+}
+
+export function getTransferConfigFailed(name: string, err: unknown): void {
+  logger.error("Failed to read BigQuery Data Transfer config", { name, err });
+}
+
+export function partitioningFieldRemovalAttempted(
+  name: string,
+  existingField: string
+): void {
+  logger.warn("Cannot remove an existing DTS partitioning field", {
+    name,
+    existingField,
+  });
+}
+
+export function topicCreated(name: string): void {
+  logger.info("Created Pub/Sub topic for transfer notifications", { name });
+}

package/src/metadata.ts

@@ -0,0 +1,31 @@
+/*
+ * Copyright 2026 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export const metadata = {
+  roles: [
+    "datastore.user",
+    "bigquery.admin",
+    "pubsub.admin",
+    "eventarc.eventReceiver",
+    "run.invoker",
+  ],
+  apis: [
+    "bigquery.googleapis.com",
+    "bigquerydatatransfer.googleapis.com",
+    "pubsub.googleapis.com",
+  ],
+  functionNames: ["processMessages", "upsertTransferConfig"],
+} as const;

package/src/types.ts

@@ -0,0 +1,100 @@
+/*
+ * Copyright 2025 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type {
+  BigQueryDate,
+  BigQueryDatetime,
+  BigQueryTime,
+  BigQueryTimestamp,
+  Geography,
+} from "@google-cloud/bigquery";
+import type { Timestamp } from "firebase-admin/firestore";
+
+/** Parameters attached to a BigQuery scheduled-query transfer run. */
+export interface TransferRunParams {
+  destination_table_name_template: string;
+  partitioning_field: string;
+  query: string;
+  write_disposition: string;
+}
+
+/** State of a BigQuery Data Transfer run. */
+export type TransferRunState =
+  | "SUCCEEDED"
+  | "FAILED"
+  | "CANCELLED"
+  | "PENDING"
+  | "RUNNING"
+  | "TRANSFER_STATE_UNSPECIFIED";
+
+/** JSON payload from a BigQuery Data Transfer Pub/Sub notification. */
+export interface TransferRunPayload {
+  name: string;
+  runTime: string;
+  state: TransferRunState;
+  destinationDatasetId: string;
+  dataSourceId: string;
+  schedule: string;
+  scheduleTime: string;
+  startTime: string;
+  endTime: string;
+  updateTime: string;
+  userId: string;
+  notificationPubsubTopic: string;
+  params: TransferRunParams;
+  emailPreferences: Record<string, unknown>;
+  errorStatus: Record<string, unknown>;
+}
+
+/** Message-shaped wrapper retained for the core handler boundary. */
+export interface TransferRunMessage {
+  json: TransferRunPayload;
+}
+
+/** Values returned in a BigQuery row before conversion. */
+export type BigQueryRowValue =
+  | string
+  | number
+  | boolean
+  | null
+  | Date
+  | Buffer
+  | BigQueryTimestamp
+  | BigQueryDate
+  | BigQueryTime
+  | BigQueryDatetime
+  | Geography
+  | BigQueryRow
+  | BigQueryRowValue[];
+
+export interface BigQueryRow {
+  [key: string]: BigQueryRowValue;
+}
+
+/** Firestore-compatible equivalent of a BigQuery row value. */
+export type FirestoreRowValue =
+  | string
+  | number
+  | boolean
+  | null
+  | Timestamp
+  | Uint8Array
+  | FirestoreRow
+  | FirestoreRowValue[];
+
+export interface FirestoreRow {
+  [key: string]: FirestoreRowValue;
+}

package/tests/config.test.ts

@@ -0,0 +1,61 @@
+/*
+ * Copyright 2026 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Expression } from "firebase-functions/params";
+import { afterEach, describe, expect, test, vi } from "vitest";
+import { CONFIG_EXPRESSIONS, configFromEnv } from "../src/config";
+
+afterEach(() => {
+  vi.unstubAllEnvs();
+});
+
+describe("CONFIG_EXPRESSIONS", () => {
+  test("namespaces the Pub/Sub topic with the required instance id", () => {
+    expect(CONFIG_EXPRESSIONS.pubSubTopic).toBeInstanceOf(Expression);
+    expect((CONFIG_EXPRESSIONS.pubSubTopic as Expression<string>).toCEL()).toBe(
+      "kit-{{ params.INSTANCE_ID }}-processMessages"
+    );
+  });
+});
+
+describe("configFromEnv", () => {
+  test("reads runtime parameters and derives the same topic", () => {
+    vi.stubEnv(
+      "FIREBASE_CONFIG",
+      JSON.stringify({ projectId: "test-project" })
+    );
+    vi.stubEnv("INSTANCE_ID", "users-export");
+    vi.stubEnv("BIGQUERY_DATASET_LOCATION", "EU");
+    vi.stubEnv("DATASET_ID", "analytics");
+    vi.stubEnv("TABLE_NAME", "users");
+    vi.stubEnv("QUERY_STRING", "SELECT * FROM source.users");
+    vi.stubEnv("DISPLAY_NAME", "Users export");
+    vi.stubEnv("SCHEDULE", "every 24 hours");
+    vi.stubEnv("COLLECTION_PATH", "transferConfigs");
+    vi.stubEnv("LOG_LEVEL", "info");
+
+    expect(configFromEnv()).toMatchObject({
+      projectId: "test-project",
+      instanceId: "users-export",
+      bigqueryDatasetLocation: "EU",
+      datasetId: "analytics",
+      tableName: "users",
+      pubSubTopic: "kit-users-export-processMessages",
+      firestoreCollection: "transferConfigs",
+      logLevel: "info",
+    });
+  });
+});

package/tests/dts.test.ts

@@ -0,0 +1,124 @@
+/*
+ * Copyright 2026 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { describe, expect, test, vi } from "vitest";
+import {
+  constructUpdateTransferConfigRequest,
+  createTransferConfigRequest,
+  type DataTransferClient,
+  PARTITIONING_FIELD_REMOVAL_ERROR,
+} from "../src/dts";
+import { resolveConfig } from "../src/export-config";
+
+const config = resolveConfig({
+  bigqueryDatasetLocation: "US",
+  projectId: "test-project",
+  instanceId: "users-export",
+  datasetId: "analytics",
+  tableName: "users",
+  queryString: "SELECT * FROM source.users",
+  displayName: "Users export",
+  partitioningField: "created_at",
+  schedule: "every 24 hours",
+});
+
+function clientWithTransferConfig(transferConfig: object): DataTransferClient {
+  return {
+    getTransferConfig: vi.fn().mockResolvedValue([transferConfig]),
+  } as unknown as DataTransferClient;
+}
+
+describe("createTransferConfigRequest", () => {
+  test("creates the scheduled-query request", () => {
+    const request = createTransferConfigRequest(
+      config,
+      "runtime@test-project.iam.gserviceaccount.com"
+    );
+
+    expect(request).toMatchObject({
+      parent: "projects/test-project",
+      transferConfig: {
+        destinationDatasetId: "analytics",
+        displayName: "Users export",
+        dataSourceId: "scheduled_query",
+        schedule: "every 24 hours",
+        notificationPubsubTopic:
+          "projects/test-project/topics/kit-users-export-processMessages",
+        serviceAccountName: "runtime@test-project.iam.gserviceaccount.com",
+      },
+    });
+    expect(
+      request.transferConfig?.params?.fields?.destination_table_name_template
+        ?.stringValue
+    ).toBe('users_{run_time|"%H%M%S"}');
+  });
+});
+
+describe("constructUpdateTransferConfigRequest", () => {
+  test("deduplicates the params update mask", async () => {
+    const client = clientWithTransferConfig({
+      name: "projects/p/locations/us/transferConfigs/c",
+      destinationDatasetId: "analytics",
+      schedule: "every 24 hours",
+      notificationPubsubTopic:
+        "projects/test-project/topics/kit-users-export-processMessages",
+      params: {
+        fields: {
+          query: { stringValue: "SELECT 1" },
+          destination_table_name_template: {
+            stringValue: 'old_{run_time|"%H%M%S"}',
+          },
+          partitioning_field: { stringValue: "created_at" },
+        },
+      },
+    });
+
+    const request = await constructUpdateTransferConfigRequest(
+      client,
+      "projects/p/locations/us/transferConfigs/c",
+      config
+    );
+
+    expect(request.updateMask?.paths).toEqual(["params"]);
+  });
+
+  test("rejects clearing an existing partitioning field", async () => {
+    const client = clientWithTransferConfig({
+      name: "projects/p/locations/us/transferConfigs/c",
+      destinationDatasetId: "analytics",
+      schedule: "every 24 hours",
+      notificationPubsubTopic:
+        "projects/test-project/topics/kit-users-export-processMessages",
+      params: {
+        fields: {
+          query: { stringValue: config.queryString },
+          destination_table_name_template: {
+            stringValue: 'users_{run_time|"%H%M%S"}',
+          },
+          partitioning_field: { stringValue: "created_at" },
+        },
+      },
+    });
+
+    await expect(
+      constructUpdateTransferConfigRequest(
+        client,
+        "projects/p/locations/us/transferConfigs/c",
+        { ...config, partitioningField: undefined }
+      )
+    ).rejects.toThrow(PARTITIONING_FIELD_REMOVAL_ERROR);
+  });
+});

package/tests/export-config.test.ts

@@ -0,0 +1,78 @@
+/*
+ * Copyright 2026 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { describe, expect, test } from "vitest";
+import {
+  type BigqueryFirestoreExportConfig,
+  resolveConfig,
+} from "../src/export-config";
+
+const minimal: BigqueryFirestoreExportConfig = {
+  bigqueryDatasetLocation: "US",
+  projectId: "test-project",
+  instanceId: "users-export",
+  datasetId: "analytics",
+  tableName: "users",
+  queryString: "SELECT * FROM source.users",
+  displayName: "Users export",
+  schedule: "every 24 hours",
+};
+
+describe("resolveConfig", () => {
+  test("applies kit defaults", () => {
+    expect(resolveConfig(minimal)).toEqual({
+      ...minimal,
+      pubSubTopic: "kit-users-export-processMessages",
+      firestoreCollection: "transferConfigs",
+      logLevel: "info",
+    });
+  });
+
+  test("normalizes optional strings", () => {
+    const resolved = resolveConfig({
+      ...minimal,
+      transferConfigName: "  projects/p/locations/us/transferConfigs/c  ",
+      partitioningField: "  created_at  ",
+      serviceAccount: "  runtime@test-project.iam.gserviceaccount.com  ",
+    });
+
+    expect(resolved.transferConfigName).toBe(
+      "projects/p/locations/us/transferConfigs/c"
+    );
+    expect(resolved.partitioningField).toBe("created_at");
+    expect(resolved.serviceAccount).toBe(
+      "runtime@test-project.iam.gserviceaccount.com"
+    );
+  });
+
+  test.each(["instanceId", "datasetId", "tableName", "queryString"] as const)(
+    "rejects an empty %s",
+    (field) => {
+      expect(() => resolveConfig({ ...minimal, [field]: " " })).toThrow(
+        `${field} must be a non-empty string.`
+      );
+    }
+  );
+
+  test("rejects unsupported log levels", () => {
+    expect(() =>
+      resolveConfig({
+        ...minimal,
+        logLevel: "verbose" as BigqueryFirestoreExportConfig["logLevel"],
+      })
+    ).toThrow("Unsupported logLevel: verbose");
+  });
+});