@firebase-function-kits/firestore-bigquery-export 0.0.1 → 0.0.2-rc.1

package/tests/config.test.ts

@@ -0,0 +1,182 @@
+/**
+ * Copyright 2026 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { describe, expect, test, vi } from "vitest";
+
+vi.mock("firebase-functions/params", () => ({
+  Expression: class Expression<T> {
+    value(): T {
+      return this.runtimeValue();
+    }
+
+    runtimeValue(): T {
+      throw new Error("Not implemented");
+    }
+
+    toCEL(): string {
+      return `{{ ${this.toString()} }}`;
+    }
+
+    toJSON(): string {
+      return this.toString();
+    }
+  },
+  defineString: (
+    _name: string,
+    opts?: { default?: string | { value(): string } }
+  ) => ({
+    value: () =>
+      typeof opts?.default === "string"
+        ? opts.default
+        : opts?.default?.value() ?? "",
+    toString: () => `params.${_name}`,
+  }),
+  defineInt: (_name: string, opts?: { default?: number }) => ({
+    value: () => opts?.default ?? 0,
+    toString: () => `params.${_name}`,
+  }),
+  defineBoolean: (_name: string, opts?: { default?: boolean }) => ({
+    value: () => opts?.default ?? false,
+    toString: () => `params.${_name}`,
+  }),
+  select: (options: unknown) => ({ select: { options } }),
+  expr: (
+    strings: TemplateStringsArray,
+    ...values: ReadonlyArray<{ toString: () => string }>
+  ) => ({
+    value: () =>
+      strings.reduce(
+        (acc, part, index) =>
+          `${acc}${values[index - 1]?.toString() ?? ""}${part}`
+      ),
+    toString: () =>
+      strings.reduce(
+        (acc, part, index) =>
+          `${acc}${values[index - 1]?.toString() ?? ""}${part}`
+      ),
+  }),
+  projectID: { value: () => "test-project" },
+}));
+
+import {
+  buildPartitioningConfig,
+  CONFIG_EXPRESSIONS,
+  clustering,
+  configFromEnv,
+} from "../src/config";
+import { resolveExportConfig } from "../src/export-config";
+
+describe("clustering", () => {
+  test("splits a comma list", () => {
+    expect(clustering("a,b,c")).toEqual(["a", "b", "c"]);
+  });
+
+  test("caps at four columns", () => {
+    expect(clustering("a,b,c,d,e")).toEqual(["a", "b", "c", "d"]);
+  });
+
+  test("empty or undefined becomes null", () => {
+    expect(clustering("")).toBeNull();
+    expect(clustering(undefined)).toBeNull();
+  });
+});
+
+describe("buildPartitioningConfig", () => {
+  const base = {
+    timePartitioningField: undefined,
+    timePartitioningFieldType: undefined,
+    timePartitioningFirestoreField: undefined,
+  };
+
+  test("no granularity and no fields => NONE", () => {
+    expect(
+      buildPartitioningConfig({ ...base, timePartitioning: null })
+    ).toEqual({ granularity: "NONE" });
+  });
+
+  test("granularity only", () => {
+    expect(
+      buildPartitioningConfig({ ...base, timePartitioning: "DAY" })
+    ).toEqual({ granularity: "DAY" });
+  });
+
+  test("timestamp field shorthand", () => {
+    expect(
+      buildPartitioningConfig({
+        ...base,
+        timePartitioning: "DAY",
+        timePartitioningField: "timestamp",
+      })
+    ).toMatchObject({ granularity: "DAY", bigqueryColumnName: "timestamp" });
+  });
+
+  test("full custom field", () => {
+    expect(
+      buildPartitioningConfig({
+        timePartitioning: "DAY",
+        timePartitioningField: "created",
+        timePartitioningFieldType: "TIMESTAMP",
+        timePartitioningFirestoreField: "createdAt",
+      })
+    ).toEqual({
+      granularity: "DAY",
+      bigqueryColumnName: "created",
+      bigqueryColumnType: "TIMESTAMP",
+      firestoreFieldName: "createdAt",
+    });
+  });
+
+  test("throws when fields are set without a granularity", () => {
+    expect(() =>
+      buildPartitioningConfig({
+        ...base,
+        timePartitioning: null,
+        timePartitioningField: "created",
+      })
+    ).toThrow();
+  });
+
+  test("throws on incomplete custom field", () => {
+    expect(() =>
+      buildPartitioningConfig({
+        timePartitioning: "DAY",
+        timePartitioningField: "created",
+        timePartitioningFieldType: undefined,
+        timePartitioningFirestoreField: "createdAt",
+      })
+    ).toThrow();
+  });
+});
+
+describe("configFromEnv", () => {
+  test("maps params without inventing a region default", () => {
+    const config = configFromEnv();
+    expect(config.projectId).toBe("test-project");
+    expect(config.bqProjectId).toBe("test-project");
+    expect(config.databaseId).toBe("(default)");
+    expect(resolveExportConfig(config).location).toBe("");
+    expect(config.viewType).toBe("view");
+  });
+
+  test("exposes deploy-time expressions for trigger metadata", () => {
+    expect(CONFIG_EXPRESSIONS.collectionPath.toString()).toBe(
+      "params.COLLECTION_PATH"
+    );
+    expect(CONFIG_EXPRESSIONS.location.toString()).toBe(
+      "params.DATABASE_REGION"
+    );
+  });
+});

package/tests/events.test.ts

@@ -0,0 +1,84 @@
+/**
+ * Copyright 2026 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";
+
+const { publish, channel } = vi.hoisted(() => {
+  const publish = vi.fn().mockResolvedValue(undefined);
+  const channel = vi.fn(() => ({ publish }));
+  return { publish, channel };
+});
+
+vi.mock("firebase-admin/eventarc", () => ({
+  getEventarc: () => ({ channel }),
+}));
+
+import {
+  recordCompletionEvent,
+  recordErrorEvent,
+  recordStartEvent,
+  recordSuccessEvent,
+  setupEventChannel,
+} from "../src/events";
+
+const ORIGINAL = process.env.EVENTARC_CHANNEL;
+
+beforeEach(() => vi.clearAllMocks());
+afterEach(() => {
+  if (ORIGINAL === undefined) delete process.env.EVENTARC_CHANNEL;
+  else process.env.EVENTARC_CHANNEL = ORIGINAL;
+});
+
+describe("no channel configured", () => {
+  test("publishers are a no-op when EVENTARC_CHANNEL is unset", async () => {
+    delete process.env.EVENTARC_CHANNEL;
+    setupEventChannel();
+
+    await recordStartEvent({ a: 1 });
+    await recordCompletionEvent({ a: 1 });
+    expect(publish).not.toHaveBeenCalled();
+  });
+});
+
+describe("channel configured", () => {
+  beforeEach(() => {
+    process.env.EVENTARC_CHANNEL = "projects/p/locations/l/channels/c";
+    setupEventChannel();
+  });
+
+  test("publishes the firestore-bigquery-export event type only", async () => {
+    await recordStartEvent({ a: 1 });
+    expect(publish).toHaveBeenCalledTimes(1);
+    expect(publish).toHaveBeenCalledWith(
+      expect.objectContaining({
+        type: "firebase.extensions.firestore-bigquery-export.v1.onStart",
+      })
+    );
+  });
+
+  test("error / success / completion map to their event types", async () => {
+    await recordErrorEvent(new Error("boom"));
+    await recordSuccessEvent({ subject: "doc1", data: { a: 1 } });
+    await recordCompletionEvent({ a: 1 });
+
+    const types = publish.mock.calls.map((c) => c[0].type);
+    expect(types).toEqual([
+      "firebase.extensions.firestore-bigquery-export.v1.onError",
+      "firebase.extensions.firestore-bigquery-export.v1.onSuccess",
+      "firebase.extensions.firestore-bigquery-export.v1.onCompletion",
+    ]);
+  });
+});

package/tests/export-config.test.ts

@@ -0,0 +1,114 @@
+/**
+ * Copyright 2026 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { describe, expect, test } from "vitest";
+import { resolveExportConfig, toTrackerConfig } from "../src/export-config";
+
+describe("resolveExportConfig", () => {
+  const minimal = {
+    collectionPath: "users",
+    datasetId: "analytics",
+    tableId: "users",
+    location: "us-central1",
+    projectId: "test-project",
+  };
+
+  test("applies defaults for omitted fields", () => {
+    const resolved = resolveExportConfig(minimal);
+
+    expect(resolved.projectId).toBe("test-project");
+    expect(resolved.datasetLocation).toBe("us");
+    expect(resolved.databaseId).toBe("(default)");
+    expect(resolved.viewType).toBe("view");
+    expect(resolved.wildcardIds).toBe(false);
+    expect(resolved.excludeOldData).toBe(false);
+    expect(resolved.clustering).toBeNull();
+    expect(resolved.logLevel).toBe("info");
+  });
+
+  test("keeps caller-supplied values", () => {
+    const resolved = resolveExportConfig({
+      ...minimal,
+      location: "europe-west2",
+      databaseId: "secondary",
+      wildcardIds: true,
+    });
+
+    expect(resolved.location).toBe("europe-west2");
+    expect(resolved.databaseId).toBe("secondary");
+    expect(resolved.wildcardIds).toBe(true);
+  });
+
+  test("carries required fields through", () => {
+    const resolved = resolveExportConfig(minimal);
+    expect(resolved.collectionPath).toBe("users");
+    expect(resolved.datasetId).toBe("analytics");
+    expect(resolved.tableId).toBe("users");
+  });
+});
+
+describe("toTrackerConfig", () => {
+  const base = {
+    collectionPath: "users",
+    datasetId: "analytics",
+    tableId: "users",
+    location: "us-central1",
+    projectId: "test-project",
+  };
+
+  test("derives materialized view flags from viewType", () => {
+    const plain = toTrackerConfig(resolveExportConfig(base));
+    expect(plain.useMaterializedView).toBe(false);
+    expect(plain.useIncrementalMaterializedView).toBe(false);
+
+    const incremental = toTrackerConfig(
+      resolveExportConfig({ ...base, viewType: "materialized_incremental" })
+    );
+    expect(incremental.useMaterializedView).toBe(true);
+    expect(incremental.useIncrementalMaterializedView).toBe(true);
+
+    const nonIncremental = toTrackerConfig(
+      resolveExportConfig({ ...base, viewType: "materialized_non_incremental" })
+    );
+    expect(nonIncremental.useMaterializedView).toBe(true);
+    expect(nonIncremental.useIncrementalMaterializedView).toBe(false);
+  });
+
+  test("always skips init (initialization is handled lazily)", () => {
+    const tracker = toTrackerConfig(resolveExportConfig(base));
+    expect(tracker.skipInit).toBe(true);
+  });
+
+  test("maps databaseId onto both firestore fields", () => {
+    const tracker = toTrackerConfig(
+      resolveExportConfig({ ...base, databaseId: "secondary" })
+    );
+    expect(tracker.databaseId).toBe("secondary");
+    expect(tracker.firestoreInstanceId).toBe("secondary");
+  });
+
+  test("uses projectId as the default BigQuery project", () => {
+    const tracker = toTrackerConfig(resolveExportConfig(base));
+    expect(tracker.bqProjectId).toBe("test-project");
+  });
+
+  test("keeps an explicitly configured BigQuery project", () => {
+    const tracker = toTrackerConfig(
+      resolveExportConfig({ ...base, bqProjectId: "analytics-project" })
+    );
+    expect(tracker.bqProjectId).toBe("analytics-project");
+  });
+});

package/tests/handlers.test.ts

@@ -0,0 +1,220 @@
+/**
+ * Copyright 2026 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { ChangeType } from "@firebaseextensions/firestore-bigquery-change-tracker";
+import { beforeEach, describe, expect, test, vi } from "vitest";
+import { resolveExportConfig } from "../src/export-config";
+import {
+  type DocumentWriteEvent,
+  type HandlerContext,
+  handleDocumentWrite,
+} from "../src/handlers";
+
+vi.mock("../src/events");
+vi.mock("../src/logs");
+
+import * as events from "../src/events";
+
+/** Fake Firestore snapshot with only the fields the handlers read. */
+function snap(exists: boolean, id: string, data: unknown = {}) {
+  return { exists, id, data: () => data };
+}
+
+/** Fake document-write event. */
+function writeEvent(
+  before: ReturnType<typeof snap>,
+  after: ReturnType<typeof snap>,
+  overrides: Partial<Record<string, unknown>> = {}
+): DocumentWriteEvent {
+  return {
+    data: { before, after },
+    id: "evt-1",
+    time: "2026-01-01T00:00:00Z",
+    document: "users/doc1",
+    params: { documentId: "doc1" },
+    ...overrides,
+  } as unknown as DocumentWriteEvent;
+}
+
+/** Builds a HandlerContext with spy tracker + config overrides. */
+function makeCtx(
+  configOverrides: Record<string, unknown> = {}
+): HandlerContext {
+  const tracker = {
+    record: vi.fn().mockResolvedValue(undefined),
+    serializeData: vi.fn((d: unknown) => d),
+  };
+  const config = {
+    ...resolveExportConfig({
+      collectionPath: "users",
+      datasetId: "ds",
+      tableId: "tbl",
+      projectId: "test-project",
+    }),
+    ...configOverrides,
+  };
+  return {
+    tracker: tracker as unknown as HandlerContext["tracker"],
+    config: config as HandlerContext["config"],
+    ensureInitialized: vi.fn().mockResolvedValue(undefined),
+  };
+}
+
+describe("handleDocumentWrite", () => {
+  beforeEach(() => vi.clearAllMocks());
+
+  test("returns early when the event has no data", async () => {
+    const ctx = makeCtx();
+    await handleDocumentWrite({ data: undefined } as DocumentWriteEvent, ctx);
+    expect(ctx.tracker.record).not.toHaveBeenCalled();
+  });
+
+  test("CREATE records the new data, no old data", async () => {
+    const ctx = makeCtx();
+    const event = writeEvent(snap(false, "doc1"), snap(true, "doc1", { a: 1 }));
+
+    await handleDocumentWrite(event, ctx);
+
+    expect(ctx.tracker.record).toHaveBeenCalledTimes(1);
+    const [[recorded]] = (ctx.tracker.record as ReturnType<typeof vi.fn>).mock
+      .calls;
+    expect(recorded[0].operation).toBe(ChangeType.CREATE);
+    expect(recorded[0].data).toEqual({ a: 1 });
+    expect(recorded[0].oldData).toBeUndefined();
+    expect(events.recordStartEvent).toHaveBeenCalledTimes(1);
+  });
+
+  test("UPDATE includes old data by default", async () => {
+    const ctx = makeCtx();
+    const event = writeEvent(
+      snap(true, "doc1", { a: 1 }),
+      snap(true, "doc1", { a: 2 })
+    );
+
+    await handleDocumentWrite(event, ctx);
+
+    const [[recorded]] = (ctx.tracker.record as ReturnType<typeof vi.fn>).mock
+      .calls;
+    expect(recorded[0].operation).toBe(ChangeType.UPDATE);
+    expect(recorded[0].data).toEqual({ a: 2 });
+    expect(recorded[0].oldData).toEqual({ a: 1 });
+  });
+
+  test("excludeOldData omits the previous state on update", async () => {
+    const ctx = makeCtx({ excludeOldData: true });
+    const event = writeEvent(
+      snap(true, "doc1", { a: 1 }),
+      snap(true, "doc1", { a: 2 })
+    );
+
+    await handleDocumentWrite(event, ctx);
+
+    const [[recorded]] = (ctx.tracker.record as ReturnType<typeof vi.fn>).mock
+      .calls;
+    expect(recorded[0].oldData).toBeUndefined();
+  });
+
+  test("DELETE records no new data", async () => {
+    const ctx = makeCtx();
+    const event = writeEvent(snap(true, "doc1", { a: 1 }), snap(false, "doc1"));
+
+    await handleDocumentWrite(event, ctx);
+
+    const [[recorded]] = (ctx.tracker.record as ReturnType<typeof vi.fn>).mock
+      .calls;
+    expect(recorded[0].operation).toBe(ChangeType.DELETE);
+    expect(recorded[0].data).toBeUndefined();
+  });
+
+  test("does not provision on the hot path", async () => {
+    const ctx = makeCtx();
+    await handleDocumentWrite(
+      writeEvent(snap(false, "doc1"), snap(true, "doc1", { a: 1 })),
+      ctx
+    );
+    expect(ctx.ensureInitialized).not.toHaveBeenCalled();
+  });
+
+  test("wildcardIds gates whether path params are recorded", async () => {
+    const withIds = makeCtx({ wildcardIds: true });
+    await handleDocumentWrite(
+      writeEvent(snap(false, "doc1"), snap(true, "doc1", { a: 1 })),
+      withIds
+    );
+    const [[recordedWith]] = (
+      withIds.tracker.record as ReturnType<typeof vi.fn>
+    ).mock.calls;
+    expect(recordedWith[0].pathParams).toEqual({ documentId: "doc1" });
+
+    const withoutIds = makeCtx({ wildcardIds: false });
+    await handleDocumentWrite(
+      writeEvent(snap(false, "doc1"), snap(true, "doc1", { a: 1 })),
+      withoutIds
+    );
+    const [[recordedWithout]] = (
+      withoutIds.tracker.record as ReturnType<typeof vi.fn>
+    ).mock.calls;
+    expect(recordedWithout[0].pathParams).toBeNull();
+  });
+
+  test("self-heals and retries the write when the inline write fails", async () => {
+    const ctx = makeCtx();
+    (ctx.tracker.record as ReturnType<typeof vi.fn>).mockRejectedValueOnce(
+      new Error("bq down")
+    );
+
+    await handleDocumentWrite(
+      writeEvent(snap(false, "doc1"), snap(true, "doc1", { a: 1 })),
+      ctx
+    );
+
+    expect(ctx.ensureInitialized).toHaveBeenCalledTimes(1);
+    expect(ctx.tracker.record).toHaveBeenCalledTimes(2);
+  });
+
+  test("rethrows when self-heal retry fails so runtime retry can replay", async () => {
+    const ctx = makeCtx();
+    (ctx.tracker.record as ReturnType<typeof vi.fn>)
+      .mockRejectedValueOnce(new Error("bq down"))
+      .mockRejectedValueOnce(new Error("still down"));
+
+    await expect(
+      handleDocumentWrite(
+        writeEvent(snap(false, "doc1"), snap(true, "doc1", { a: 1 })),
+        ctx
+      )
+    ).rejects.toThrow("still down");
+    expect(ctx.ensureInitialized).toHaveBeenCalledTimes(1);
+    expect(events.recordErrorEvent).toHaveBeenCalled();
+  });
+
+  test("rethrows when serialization fails", async () => {
+    const ctx = makeCtx();
+    (ctx.tracker.serializeData as ReturnType<typeof vi.fn>).mockImplementation(
+      () => {
+        throw new Error("bad data");
+      }
+    );
+
+    await expect(
+      handleDocumentWrite(
+        writeEvent(snap(false, "doc1"), snap(true, "doc1", { a: 1 })),
+        ctx
+      )
+    ).rejects.toThrow("bad data");
+    expect(ctx.tracker.record).not.toHaveBeenCalled();
+  });
+});

package/tests/init.test.ts

@@ -0,0 +1,76 @@
+/**
+ * Copyright 2026 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { FirestoreBigQueryEventHistoryTracker } from "@firebaseextensions/firestore-bigquery-change-tracker";
+import { describe, expect, test, vi } from "vitest";
+import { createEnsureInitialized } from "../src/init";
+
+function trackerWith(initialize: () => Promise<void>) {
+  return { initialize } as unknown as FirestoreBigQueryEventHistoryTracker;
+}
+
+describe("createEnsureInitialized", () => {
+  test("initializes the tracker once across sequential calls", async () => {
+    const initialize = vi.fn().mockResolvedValue(undefined);
+    const ensureInitialized = createEnsureInitialized(trackerWith(initialize));
+
+    await ensureInitialized();
+    await ensureInitialized();
+    await ensureInitialized();
+
+    expect(initialize).toHaveBeenCalledTimes(1);
+  });
+
+  test("concurrent calls share a single in-flight initialization", async () => {
+    let release!: () => void;
+    const gate = new Promise<void>((resolve) => {
+      release = resolve;
+    });
+    const initialize = vi.fn().mockImplementation(() => gate);
+    const ensureInitialized = createEnsureInitialized(trackerWith(initialize));
+
+    const first = ensureInitialized();
+    const second = ensureInitialized();
+    release();
+    await Promise.all([first, second]);
+
+    expect(initialize).toHaveBeenCalledTimes(1);
+  });
+
+  test("a failed initialization resets the guard so the next call retries", async () => {
+    const initialize = vi
+      .fn()
+      .mockRejectedValueOnce(new Error("transient"))
+      .mockResolvedValueOnce(undefined);
+    const ensureInitialized = createEnsureInitialized(trackerWith(initialize));
+
+    await expect(ensureInitialized()).rejects.toThrow("transient");
+    await expect(ensureInitialized()).resolves.toBeUndefined();
+
+    expect(initialize).toHaveBeenCalledTimes(2);
+  });
+
+  test("a success is cached — no retry after resolution", async () => {
+    const initialize = vi.fn().mockResolvedValue(undefined);
+    const ensureInitialized = createEnsureInitialized(trackerWith(initialize));
+
+    await ensureInitialized();
+    initialize.mockClear();
+    await ensureInitialized();
+
+    expect(initialize).not.toHaveBeenCalled();
+  });
+});