@firebase-function-kits/firestore-bigquery-export 0.0.1 → 0.0.2-rc.0

package/src/logs.ts

@@ -0,0 +1,233 @@
+/*
+ * Copyright 2019 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {
+  type ChangeType,
+  Logger,
+} from "@firebaseextensions/firestore-bigquery-change-tracker";
+
+export const logger = new Logger();
+
+export const arrayFieldInvalid = (fieldName: string) => {
+  logger.warn(`Array field '${fieldName}' does not contain an array, skipping`);
+};
+
+export const bigQueryDatasetCreated = (datasetId: string) => {
+  logger.info(`Created BigQuery dataset: ${datasetId}`);
+};
+
+export const bigQueryDatasetCreating = (datasetId: string) => {
+  logger.debug(`Creating BigQuery dataset: ${datasetId}`);
+};
+
+export const bigQueryDatasetExists = (datasetId: string) => {
+  logger.info(`BigQuery dataset already exists: ${datasetId}`);
+};
+
+export const bigQueryErrorRecordingDocumentChange = (e: Error) => {
+  logger.error(`Error recording document changes.`, e);
+};
+
+export const bigQueryLatestSnapshotViewQueryCreated = (query: string) => {
+  logger.info(`BigQuery latest snapshot view query:\n${query}`);
+};
+
+export const bigQueryTableAlreadyExists = (
+  tableName: string,
+  datasetName: string
+) => {
+  logger.info(
+    `BigQuery table with name ${tableName} already ` +
+      `exists in dataset ${datasetName}!`
+  );
+};
+
+export const bigQueryTableCreated = (tableName: string) => {
+  logger.info(`Created BigQuery table: ${tableName}`);
+};
+
+export const bigQueryTableCreating = (tableName: string) => {
+  logger.debug(`Creating BigQuery table: ${tableName}`);
+};
+
+export const bigQueryTableUpdated = (tableName: string) => {
+  logger.info(`Updated existing BigQuery table: ${tableName}`);
+};
+
+export const bigQueryTableUpdating = (tableName: string) => {
+  logger.debug(`Updating existing BigQuery table: ${tableName}`);
+};
+
+export const bigQueryTableUpToDate = (tableName: string) => {
+  logger.info(`BigQuery table: ${tableName} is up to date`);
+};
+
+export const bigQueryTableValidated = (tableName: string) => {
+  logger.info(`Validated existing BigQuery table: ${tableName}`);
+};
+
+export const bigQueryTableValidating = (tableName: string) => {
+  logger.debug(`Validating existing BigQuery table: ${tableName}`);
+};
+
+export const bigQueryUserDefinedFunctionCreating = (
+  functionDefinition: string
+) => {
+  logger.debug(
+    `Creating BigQuery User-defined Function:\n${functionDefinition}`
+  );
+};
+
+export const bigQueryUserDefinedFunctionCreated = (
+  functionDefinition: string
+) => {
+  logger.info(`Created BigQuery User-defined Function:\n${functionDefinition}`);
+};
+
+export const bigQueryViewCreated = (viewName: string) => {
+  logger.info(`Created BigQuery view: ${viewName}`);
+};
+
+export const bigQueryViewCreating = (viewName: string) => {
+  logger.debug(`Creating BigQuery view: ${viewName}`);
+};
+
+export const bigQueryViewAlreadyExists = (
+  viewName: string,
+  datasetName: string
+) => {
+  logger.info(
+    `View with id ${viewName} already exists in dataset ${datasetName}.`
+  );
+};
+
+export const bigQueryViewUpdated = (viewName: string) => {
+  logger.info(`Updated existing BigQuery view: ${viewName}`);
+};
+
+export const bigQueryViewUpdating = (viewName: string) => {
+  logger.debug(`Updating existing BigQuery view: ${viewName}`);
+};
+
+export const bigQueryViewUpToDate = (viewName: string) => {
+  logger.info(`BigQuery view: ${viewName} is up to date`);
+};
+
+export const bigQueryViewValidated = (viewName: string) => {
+  logger.info(`Validated existing BigQuery view: ${viewName}`);
+};
+
+export const bigQueryViewValidating = (viewName: string) => {
+  logger.debug(`Validating existing BigQuery view: ${viewName}`);
+};
+
+export const complete = () => {
+  logger.info("Completed execution of extension");
+};
+
+export const dataInserted = (rowCount: number) => {
+  logger.debug(`Inserted ${rowCount} row(s) of data into BigQuery`);
+};
+
+export const dataInserting = (rowCount: number) => {
+  logger.debug(`Inserting ${rowCount} row(s) of data into BigQuery`);
+};
+
+export const dataTypeInvalid = (
+  fieldName: string,
+  fieldType: string,
+  dataType: string
+) => {
+  logger.warn(
+    `Field '${fieldName}' has invalid data. Expected: ${fieldType}, received: ${dataType}`
+  );
+};
+
+export const error = (
+  includeEvent: boolean,
+  message: string,
+  err: Error,
+  event?: any, // Made optional, as it is not always required
+  eventTrackerConfig?: any // Made optional, as it is not always required
+) => {
+  const logDetails: Record<string, any> = { error: err };
+
+  if (includeEvent && event) {
+    logDetails.event = event;
+  }
+
+  if (includeEvent && eventTrackerConfig) {
+    logDetails.eventTrackerConfig = eventTrackerConfig;
+  }
+
+  logger.error(`Error when mirroring data to BigQuery: ${message}`, logDetails);
+};
+
+export const init = (config?: unknown) => {
+  logger.info("Initializing with configuration", config);
+};
+
+export const start = (config?: unknown) => {
+  logger.info("Started execution with configuration", config);
+};
+
+export const timestampMissingValue = (fieldName: string) => {
+  logger.warn(
+    `Missing value for timestamp field: ${fieldName}, using default timestamp instead.`
+  );
+};
+
+export const logEventAction = (
+  action: string,
+  document_name: string,
+  event_id: string,
+  operation: ChangeType
+) => {
+  logger.info(action, {
+    document_name,
+    event_id,
+    operation,
+  });
+};
+
+export const logFailedEventAction = (
+  action: string,
+  document_name: string,
+  event_id: string,
+  operation: ChangeType,
+  error: Error
+) => {
+  const changeTypeMap = {
+    0: "CREATE",
+    1: "DELETE",
+    2: "UPDATE",
+    3: "IMPORT",
+  };
+
+  logger.error(action, {
+    document_name,
+    event_id,
+    operation: changeTypeMap[operation],
+    error,
+  });
+};
+
+export const failedToWriteToBigQueryImmediately = (error: Error) => {
+  logger.warn(
+    "Failed to write event to BigQuery Immediately. Will attempt to Enqueue to Cloud Tasks.",
+    error
+  );
+};

package/src/util.ts

@@ -0,0 +1,64 @@
+/*
+ * Copyright 2019 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { ChangeType } from "@firebaseextensions/firestore-bigquery-change-tracker";
+import type { Change, DocumentSnapshot } from "firebase-functions/firestore";
+
+/**
+ * Get the change type (CREATE, UPDATE, DELETE) from the Firestore change.
+ * @param change Firestore document change object.
+ * @returns {ChangeType} The type of change.
+ */
+export function getChangeType(change: Change<DocumentSnapshot>): ChangeType {
+  if (!change.after.exists) {
+    return ChangeType.DELETE;
+  }
+  if (!change.before.exists) {
+    return ChangeType.CREATE;
+  }
+  return ChangeType.UPDATE;
+}
+
+/**
+ * Get the document ID from the Firestore change.
+ * @param change Firestore document change object.
+ * @returns {string} The document ID.
+ */
+export function getDocumentId(change: Change<DocumentSnapshot>): string {
+  if (change.after.exists) {
+    return change.after.id;
+  }
+  return change.before.id;
+}
+
+/**
+ *
+ * @param template - eg, regions/{regionId}/countries
+ * @param text - eg, regions/asia/countries
+ *
+ * @return - eg, { regionId: "asia" }
+ */
+export const resolveWildcardIds = (template: string, text: string) => {
+  const textSegments = text.split("/");
+  return template
+    .split("/")
+    .reduce((previousValue, currentValue, currentIndex) => {
+      if (currentValue.startsWith("{") && currentValue.endsWith("}")) {
+        previousValue[currentValue.slice(1, -1)] = textSegments[currentIndex];
+      }
+      return previousValue;
+    }, {});
+};

package/tests/config.test.ts

@@ -0,0 +1,182 @@
+/**
+ * Copyright 2026 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { describe, expect, test, vi } from "vitest";
+
+vi.mock("firebase-functions/params", () => ({
+  Expression: class Expression<T> {
+    value(): T {
+      return this.runtimeValue();
+    }
+
+    runtimeValue(): T {
+      throw new Error("Not implemented");
+    }
+
+    toCEL(): string {
+      return `{{ ${this.toString()} }}`;
+    }
+
+    toJSON(): string {
+      return this.toString();
+    }
+  },
+  defineString: (
+    _name: string,
+    opts?: { default?: string | { value(): string } }
+  ) => ({
+    value: () =>
+      typeof opts?.default === "string"
+        ? opts.default
+        : opts?.default?.value() ?? "",
+    toString: () => `params.${_name}`,
+  }),
+  defineInt: (_name: string, opts?: { default?: number }) => ({
+    value: () => opts?.default ?? 0,
+    toString: () => `params.${_name}`,
+  }),
+  defineBoolean: (_name: string, opts?: { default?: boolean }) => ({
+    value: () => opts?.default ?? false,
+    toString: () => `params.${_name}`,
+  }),
+  select: (options: unknown) => ({ select: { options } }),
+  expr: (
+    strings: TemplateStringsArray,
+    ...values: ReadonlyArray<{ toString: () => string }>
+  ) => ({
+    value: () =>
+      strings.reduce(
+        (acc, part, index) =>
+          `${acc}${values[index - 1]?.toString() ?? ""}${part}`
+      ),
+    toString: () =>
+      strings.reduce(
+        (acc, part, index) =>
+          `${acc}${values[index - 1]?.toString() ?? ""}${part}`
+      ),
+  }),
+  projectID: { value: () => "test-project" },
+}));
+
+import {
+  buildPartitioningConfig,
+  CONFIG_EXPRESSIONS,
+  clustering,
+  configFromEnv,
+} from "../src/config";
+import { resolveExportConfig } from "../src/export-config";
+
+describe("clustering", () => {
+  test("splits a comma list", () => {
+    expect(clustering("a,b,c")).toEqual(["a", "b", "c"]);
+  });
+
+  test("caps at four columns", () => {
+    expect(clustering("a,b,c,d,e")).toEqual(["a", "b", "c", "d"]);
+  });
+
+  test("empty or undefined becomes null", () => {
+    expect(clustering("")).toBeNull();
+    expect(clustering(undefined)).toBeNull();
+  });
+});
+
+describe("buildPartitioningConfig", () => {
+  const base = {
+    timePartitioningField: undefined,
+    timePartitioningFieldType: undefined,
+    timePartitioningFirestoreField: undefined,
+  };
+
+  test("no granularity and no fields => NONE", () => {
+    expect(
+      buildPartitioningConfig({ ...base, timePartitioning: null })
+    ).toEqual({ granularity: "NONE" });
+  });
+
+  test("granularity only", () => {
+    expect(
+      buildPartitioningConfig({ ...base, timePartitioning: "DAY" })
+    ).toEqual({ granularity: "DAY" });
+  });
+
+  test("timestamp field shorthand", () => {
+    expect(
+      buildPartitioningConfig({
+        ...base,
+        timePartitioning: "DAY",
+        timePartitioningField: "timestamp",
+      })
+    ).toMatchObject({ granularity: "DAY", bigqueryColumnName: "timestamp" });
+  });
+
+  test("full custom field", () => {
+    expect(
+      buildPartitioningConfig({
+        timePartitioning: "DAY",
+        timePartitioningField: "created",
+        timePartitioningFieldType: "TIMESTAMP",
+        timePartitioningFirestoreField: "createdAt",
+      })
+    ).toEqual({
+      granularity: "DAY",
+      bigqueryColumnName: "created",
+      bigqueryColumnType: "TIMESTAMP",
+      firestoreFieldName: "createdAt",
+    });
+  });
+
+  test("throws when fields are set without a granularity", () => {
+    expect(() =>
+      buildPartitioningConfig({
+        ...base,
+        timePartitioning: null,
+        timePartitioningField: "created",
+      })
+    ).toThrow();
+  });
+
+  test("throws on incomplete custom field", () => {
+    expect(() =>
+      buildPartitioningConfig({
+        timePartitioning: "DAY",
+        timePartitioningField: "created",
+        timePartitioningFieldType: undefined,
+        timePartitioningFirestoreField: "createdAt",
+      })
+    ).toThrow();
+  });
+});
+
+describe("configFromEnv", () => {
+  test("maps params without inventing a region default", () => {
+    const config = configFromEnv();
+    expect(config.projectId).toBe("test-project");
+    expect(config.bqProjectId).toBe("test-project");
+    expect(config.databaseId).toBe("(default)");
+    expect(resolveExportConfig(config).location).toBe("");
+    expect(config.viewType).toBe("view");
+  });
+
+  test("exposes deploy-time expressions for trigger metadata", () => {
+    expect(CONFIG_EXPRESSIONS.collectionPath.toString()).toBe(
+      "params.COLLECTION_PATH"
+    );
+    expect(CONFIG_EXPRESSIONS.location.toString()).toBe(
+      "params.DATABASE_REGION"
+    );
+  });
+});

package/tests/events.test.ts

@@ -0,0 +1,84 @@
+/**
+ * Copyright 2026 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";
+
+const { publish, channel } = vi.hoisted(() => {
+  const publish = vi.fn().mockResolvedValue(undefined);
+  const channel = vi.fn(() => ({ publish }));
+  return { publish, channel };
+});
+
+vi.mock("firebase-admin/eventarc", () => ({
+  getEventarc: () => ({ channel }),
+}));
+
+import {
+  recordCompletionEvent,
+  recordErrorEvent,
+  recordStartEvent,
+  recordSuccessEvent,
+  setupEventChannel,
+} from "../src/events";
+
+const ORIGINAL = process.env.EVENTARC_CHANNEL;
+
+beforeEach(() => vi.clearAllMocks());
+afterEach(() => {
+  if (ORIGINAL === undefined) delete process.env.EVENTARC_CHANNEL;
+  else process.env.EVENTARC_CHANNEL = ORIGINAL;
+});
+
+describe("no channel configured", () => {
+  test("publishers are a no-op when EVENTARC_CHANNEL is unset", async () => {
+    delete process.env.EVENTARC_CHANNEL;
+    setupEventChannel();
+
+    await recordStartEvent({ a: 1 });
+    await recordCompletionEvent({ a: 1 });
+    expect(publish).not.toHaveBeenCalled();
+  });
+});
+
+describe("channel configured", () => {
+  beforeEach(() => {
+    process.env.EVENTARC_CHANNEL = "projects/p/locations/l/channels/c";
+    setupEventChannel();
+  });
+
+  test("publishes the firestore-bigquery-export event type only", async () => {
+    await recordStartEvent({ a: 1 });
+    expect(publish).toHaveBeenCalledTimes(1);
+    expect(publish).toHaveBeenCalledWith(
+      expect.objectContaining({
+        type: "firebase.extensions.firestore-bigquery-export.v1.onStart",
+      })
+    );
+  });
+
+  test("error / success / completion map to their event types", async () => {
+    await recordErrorEvent(new Error("boom"));
+    await recordSuccessEvent({ subject: "doc1", data: { a: 1 } });
+    await recordCompletionEvent({ a: 1 });
+
+    const types = publish.mock.calls.map((c) => c[0].type);
+    expect(types).toEqual([
+      "firebase.extensions.firestore-bigquery-export.v1.onError",
+      "firebase.extensions.firestore-bigquery-export.v1.onSuccess",
+      "firebase.extensions.firestore-bigquery-export.v1.onCompletion",
+    ]);
+  });
+});

package/tests/export-config.test.ts

@@ -0,0 +1,114 @@
+/**
+ * Copyright 2026 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { describe, expect, test } from "vitest";
+import { resolveExportConfig, toTrackerConfig } from "../src/export-config";
+
+describe("resolveExportConfig", () => {
+  const minimal = {
+    collectionPath: "users",
+    datasetId: "analytics",
+    tableId: "users",
+    location: "us-central1",
+    projectId: "test-project",
+  };
+
+  test("applies defaults for omitted fields", () => {
+    const resolved = resolveExportConfig(minimal);
+
+    expect(resolved.projectId).toBe("test-project");
+    expect(resolved.datasetLocation).toBe("us");
+    expect(resolved.databaseId).toBe("(default)");
+    expect(resolved.viewType).toBe("view");
+    expect(resolved.wildcardIds).toBe(false);
+    expect(resolved.excludeOldData).toBe(false);
+    expect(resolved.clustering).toBeNull();
+    expect(resolved.logLevel).toBe("info");
+  });
+
+  test("keeps caller-supplied values", () => {
+    const resolved = resolveExportConfig({
+      ...minimal,
+      location: "europe-west2",
+      databaseId: "secondary",
+      wildcardIds: true,
+    });
+
+    expect(resolved.location).toBe("europe-west2");
+    expect(resolved.databaseId).toBe("secondary");
+    expect(resolved.wildcardIds).toBe(true);
+  });
+
+  test("carries required fields through", () => {
+    const resolved = resolveExportConfig(minimal);
+    expect(resolved.collectionPath).toBe("users");
+    expect(resolved.datasetId).toBe("analytics");
+    expect(resolved.tableId).toBe("users");
+  });
+});
+
+describe("toTrackerConfig", () => {
+  const base = {
+    collectionPath: "users",
+    datasetId: "analytics",
+    tableId: "users",
+    location: "us-central1",
+    projectId: "test-project",
+  };
+
+  test("derives materialized view flags from viewType", () => {
+    const plain = toTrackerConfig(resolveExportConfig(base));
+    expect(plain.useMaterializedView).toBe(false);
+    expect(plain.useIncrementalMaterializedView).toBe(false);
+
+    const incremental = toTrackerConfig(
+      resolveExportConfig({ ...base, viewType: "materialized_incremental" })
+    );
+    expect(incremental.useMaterializedView).toBe(true);
+    expect(incremental.useIncrementalMaterializedView).toBe(true);
+
+    const nonIncremental = toTrackerConfig(
+      resolveExportConfig({ ...base, viewType: "materialized_non_incremental" })
+    );
+    expect(nonIncremental.useMaterializedView).toBe(true);
+    expect(nonIncremental.useIncrementalMaterializedView).toBe(false);
+  });
+
+  test("always skips init (initialization is handled lazily)", () => {
+    const tracker = toTrackerConfig(resolveExportConfig(base));
+    expect(tracker.skipInit).toBe(true);
+  });
+
+  test("maps databaseId onto both firestore fields", () => {
+    const tracker = toTrackerConfig(
+      resolveExportConfig({ ...base, databaseId: "secondary" })
+    );
+    expect(tracker.databaseId).toBe("secondary");
+    expect(tracker.firestoreInstanceId).toBe("secondary");
+  });
+
+  test("uses projectId as the default BigQuery project", () => {
+    const tracker = toTrackerConfig(resolveExportConfig(base));
+    expect(tracker.bqProjectId).toBe("test-project");
+  });
+
+  test("keeps an explicitly configured BigQuery project", () => {
+    const tracker = toTrackerConfig(
+      resolveExportConfig({ ...base, bqProjectId: "analytics-project" })
+    );
+    expect(tracker.bqProjectId).toBe("analytics-project");
+  });
+});