@firebase-function-kits/bigquery-firestore-export 0.0.2-rc.3 → 0.0.2-rc.5

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@firebase-function-kits/bigquery-firestore-export",
-  "version": "0.0.2-rc.3",
+  "version": "0.0.2-rc.5",
   "repository": {
     "type": "git",
     "url": "https://github.com/firebase/extensions.git",
@@ -35,15 +35,19 @@
     "@firebase/app": "^0.16.0",
     "@google-cloud/bigquery": "^8.3.1",
     "@google-cloud/bigquery-data-transfer": "^5.0.1",
-    "@google-cloud/pubsub": "^4.11.0",
-    "firebase-admin": "^14.1.0",
-    "firebase-functions": "^7.3.3-rc.0"
+    "@google-cloud/pubsub": "^4.11.0"
   },
   "devDependencies": {
+    "firebase-admin": "^14.4.0",
+    "firebase-functions": "^7.3.3-rc.3",
     "typescript": "^5.9.3",
     "vitest": "^4.1.10"
   },
   "overrides": {
     "uuid@9.0.1": "11.1.1"
+  },
+  "peerDependencies": {
+    "firebase-admin": "^14.4.0",
+    "firebase-functions": "^7.3.3-rc.3"
   }
 }

package/src/config.ts

@@ -14,12 +14,7 @@
  * limitations under the License.
  */
 
-import {
-  defineString,
-  expr,
-  projectID,
-  select,
-} from "firebase-functions/params";
+import { defineString, projectID, select } from "firebase-functions/params";
 import type {
   BigqueryFirestoreExportConfig,
   DeployTimeOptions,
@@ -27,10 +22,29 @@ import type {
 } from "./export-config";
 
 const LOG_LEVEL_OPTIONS = ["debug", "info", "warn", "error", "silent"] as const;
-const instanceId = defineString("INSTANCE_ID");
+
+// firebase-tools injects this for kit instances (set to the instance's key in
+// firebase.json) during discovery, in the emulator, and on deployed functions.
+// The FIREBASE_ prefix is reserved in .env files and the params machinery never
+// sees injected values, so it must be a plain env read, not a defineString.
+function instanceIdFromEnv(): string {
+  const instanceId = process.env.FIREBASE_KIT_INSTANCE_ID;
+  if (!instanceId) {
+    throw new Error(
+      "FIREBASE_KIT_INSTANCE_ID is not set. It is provided automatically to " +
+        "kit instances by firebase-tools >= 15.27.0; deploy or emulate this " +
+        "kit with a supported CLI version."
+    );
+  }
+  return instanceId;
+}
+
+// Resolved at import so the topic default is a concrete name at discovery. An
+// unsupported CLI fails the discovery pass here rather than freezing
+// "kit-undefined-processMessages" into the manifest.
+const instanceId = instanceIdFromEnv();
 
 const params = {
-  instanceId,
   bigqueryDatasetLocation: defineString("BIGQUERY_DATASET_LOCATION", {
     label: "BigQuery Dataset Location",
     description:
@@ -81,7 +95,7 @@ const params = {
     description:
       "Which Pub/Sub topic should receive BigQuery Data Transfer completion notifications? Leave the default unless you are migrating from the bigquery-firestore-export extension, whose topic is named ext-<instance id>-processMessages. Pointing this at the extension's topic keeps the existing scheduled query's notification settings untouched.",
 
-    default: expr`kit-${instanceId}-processMessages`,
+    default: `kit-${instanceId}-processMessages`,
     input: {
       text: {
         nonEmpty: true,
@@ -201,12 +215,10 @@ function normalizeLogLevel(value: string): LogLevel {
 
 /** Reads runtime values from Firebase deploy-time parameters. */
 export function configFromEnv(): BigqueryFirestoreExportConfig {
-  const resolvedInstanceId = params.instanceId.value();
-
   return {
     bigqueryDatasetLocation: params.bigqueryDatasetLocation.value(),
     projectId: projectID.value(),
-    instanceId: resolvedInstanceId,
+    instanceId: instanceIdFromEnv(),
     transferConfigName: optional(params.transferConfigName.value()),
     datasetId: params.datasetId.value(),
     tableName: params.tableName.value(),

package/src/dts.ts

@@ -194,6 +194,11 @@ export async function constructUpdateTransferConfigRequest(
     updatedFields.partitioning_field.stringValue = newPartitioningField;
   }
 
+  if (config.displayName !== transferConfig.displayName) {
+    updateMask.push("display_name");
+    updatedConfig.displayName = config.displayName;
+  }
+
   if (config.schedule !== transferConfig.schedule) {
     updateMask.push("schedule");
     updatedConfig.schedule = config.schedule;

package/src/helper.ts

@@ -178,10 +178,12 @@ export function convertUnsupportedDataTypes(
     return row as FirestoreRowValue;
   }
 
+  // A TIME is a time of day with no date, so no Timestamp can hold it without
+  // inventing one. BigQuery's own "HH:MM:SS[.ffffff]" string is kept instead.
+  if (row instanceof BigQueryTime) return row.value;
   if (
     row instanceof BigQueryTimestamp ||
     row instanceof BigQueryDate ||
-    row instanceof BigQueryTime ||
     row instanceof BigQueryDatetime
   ) {
     return Timestamp.fromDate(new Date(row.value));

package/src/index.ts

@@ -55,6 +55,10 @@ const REQUIRED_ROLES: ReadonlyArray<Role> = [
   "roles/run.invoker",
 ];
 const REQUIRED_APIS = [
+  {
+    api: "firestore.googleapis.com",
+    reason: "Writes BigQuery export run state to Cloud Firestore.",
+  },
   {
     api: "bigquery.googleapis.com",
     reason: "Runs scheduled queries and reads their destination tables.",

package/tests/config.test.ts

@@ -14,32 +14,59 @@
  * limitations under the License.
  */
 
-import { Expression } from "firebase-functions/params";
+import { declaredParams, Expression } from "firebase-functions/params";
 import { afterEach, describe, expect, test, vi } from "vitest";
-import { CONFIG_EXPRESSIONS, configFromEnv } from "../src/config";
+
+const INSTANCE_ID = "users-export";
+
+// The instance id is read when the module loads, so the environment has to be
+// in place before each import.
+async function importConfig(instanceId: string | undefined) {
+  vi.resetModules();
+  vi.stubEnv("FIREBASE_KIT_INSTANCE_ID", instanceId);
+  return import("../src/config");
+}
+
+function stubRuntimeEnv() {
+  vi.stubEnv("FIREBASE_CONFIG", JSON.stringify({ projectId: "test-project" }));
+  vi.stubEnv("BIGQUERY_DATASET_LOCATION", "EU");
+  vi.stubEnv("DATASET_ID", "analytics");
+  vi.stubEnv("TABLE_NAME", "users");
+  vi.stubEnv("QUERY_STRING", "SELECT * FROM source.users");
+  vi.stubEnv("DISPLAY_NAME", "Users export");
+  vi.stubEnv("SCHEDULE", "every 24 hours");
+  vi.stubEnv("COLLECTION_PATH", "transferConfigs");
+  vi.stubEnv("LOG_LEVEL", "info");
+}
 
 afterEach(() => {
   vi.unstubAllEnvs();
 });
 
 describe("CONFIG_EXPRESSIONS", () => {
-  test("binds the trigger to the Pub/Sub topic parameter", () => {
+  test("binds the trigger to the Pub/Sub topic parameter", async () => {
+    const { CONFIG_EXPRESSIONS } = await importConfig(INSTANCE_ID);
+
     expect(CONFIG_EXPRESSIONS.pubSubTopic).toBeInstanceOf(Expression);
     expect((CONFIG_EXPRESSIONS.pubSubTopic as Expression<string>).toCEL()).toBe(
       "{{ params.PUB_SUB_TOPIC }}"
     );
   });
 
-  test("defaults the topic parameter to the instance-namespaced kit topic", () => {
+  test("defaults the topic parameter to the instance-namespaced kit topic", async () => {
+    const { CONFIG_EXPRESSIONS } = await importConfig(INSTANCE_ID);
+
     const spec = (
       CONFIG_EXPRESSIONS.pubSubTopic as unknown as {
         toSpec: () => { default?: string };
       }
     ).toSpec();
-    expect(spec.default).toBe("kit-{{ params.INSTANCE_ID }}-processMessages");
+    expect(spec.default).toBe("kit-users-export-processMessages");
   });
 
-  test("accepts a topic ID but rejects a full resource name", () => {
+  test("accepts a topic ID but rejects a full resource name", async () => {
+    const { CONFIG_EXPRESSIONS } = await importConfig(INSTANCE_ID);
+
     const spec = (
       CONFIG_EXPRESSIONS.pubSubTopic as unknown as {
         toSpec: () => { input?: { text?: { validationRegex?: string } } };
@@ -60,22 +87,40 @@ describe("CONFIG_EXPRESSIONS", () => {
   });
 });
 
-describe("configFromEnv", () => {
-  test("reads runtime parameters and derives the same topic", () => {
-    vi.stubEnv(
-      "FIREBASE_CONFIG",
-      JSON.stringify({ projectId: "test-project" })
+describe("instance id", () => {
+  // The CLI injects FIREBASE_KIT_INSTANCE_ID as a reserved env var; declaring
+  // it (or INSTANCE_ID) as a param makes the CLI prompt for a value it cannot
+  // accept and abort loading the kit.
+  test("is not declared as a param", async () => {
+    await importConfig(INSTANCE_ID);
+
+    const declared = declaredParams.map((param) => param.name);
+    expect(declared).toContain("PUB_SUB_TOPIC");
+    expect(declared).not.toContain("INSTANCE_ID");
+    expect(declared).not.toContain("FIREBASE_KIT_INSTANCE_ID");
+  });
+
+  test("fails discovery when FIREBASE_KIT_INSTANCE_ID is missing", async () => {
+    await expect(importConfig(undefined)).rejects.toThrow(
+      /FIREBASE_KIT_INSTANCE_ID is not set/
     );
-    vi.stubEnv("INSTANCE_ID", "users-export");
-    vi.stubEnv("BIGQUERY_DATASET_LOCATION", "EU");
-    vi.stubEnv("DATASET_ID", "analytics");
-    vi.stubEnv("TABLE_NAME", "users");
-    vi.stubEnv("QUERY_STRING", "SELECT * FROM source.users");
-    vi.stubEnv("DISPLAY_NAME", "Users export");
-    vi.stubEnv("SCHEDULE", "every 24 hours");
-    vi.stubEnv("COLLECTION_PATH", "transferConfigs");
-    vi.stubEnv("LOG_LEVEL", "info");
+  });
+
+  test("throws at runtime when FIREBASE_KIT_INSTANCE_ID is missing", async () => {
+    const { configFromEnv } = await importConfig(INSTANCE_ID);
+    stubRuntimeEnv();
+    vi.stubEnv("FIREBASE_KIT_INSTANCE_ID", undefined);
 
+    expect(() => configFromEnv()).toThrow(
+      /FIREBASE_KIT_INSTANCE_ID is not set/
+    );
+  });
+});
+
+describe("configFromEnv", () => {
+  test("reads runtime parameters and derives the same topic", async () => {
+    const { configFromEnv } = await importConfig(INSTANCE_ID);
+    stubRuntimeEnv();
     vi.stubEnv("PUB_SUB_TOPIC", "kit-users-export-processMessages");
 
     expect(configFromEnv()).toMatchObject({
@@ -90,20 +135,9 @@ describe("configFromEnv", () => {
     });
   });
 
-  test("passes through a topic pointing at the extension's own topic", () => {
-    vi.stubEnv(
-      "FIREBASE_CONFIG",
-      JSON.stringify({ projectId: "test-project" })
-    );
-    vi.stubEnv("INSTANCE_ID", "users-export");
-    vi.stubEnv("BIGQUERY_DATASET_LOCATION", "EU");
-    vi.stubEnv("DATASET_ID", "analytics");
-    vi.stubEnv("TABLE_NAME", "users");
-    vi.stubEnv("QUERY_STRING", "SELECT * FROM source.users");
-    vi.stubEnv("DISPLAY_NAME", "Users export");
-    vi.stubEnv("SCHEDULE", "every 24 hours");
-    vi.stubEnv("COLLECTION_PATH", "transferConfigs");
-    vi.stubEnv("LOG_LEVEL", "info");
+  test("passes through a topic pointing at the extension's own topic", async () => {
+    const { configFromEnv } = await importConfig(INSTANCE_ID);
+    stubRuntimeEnv();
     vi.stubEnv("PUB_SUB_TOPIC", "ext-users-export-processMessages");
 
     expect(configFromEnv().pubSubTopic).toBe(

package/tests/dts.test.ts

@@ -85,6 +85,7 @@ describe("constructUpdateTransferConfigRequest", () => {
     const client = clientWithTransferConfig({
       name: "projects/p/locations/us/transferConfigs/c",
       destinationDatasetId: "analytics",
+      displayName: "Users export",
       schedule: "every 24 hours",
       notificationPubsubTopic:
         "projects/test-project/topics/kit-users-export-processMessages",
@@ -108,6 +109,63 @@ describe("constructUpdateTransferConfigRequest", () => {
     expect(request.updateMask?.paths).toEqual(["params"]);
   });
 
+  test("updates the display name when it changed", async () => {
+    const client = clientWithTransferConfig({
+      name: "projects/p/locations/us/transferConfigs/c",
+      destinationDatasetId: "analytics",
+      displayName: "Old export name",
+      schedule: "every 24 hours",
+      notificationPubsubTopic:
+        "projects/test-project/topics/kit-users-export-processMessages",
+      params: {
+        fields: {
+          query: { stringValue: config.queryString },
+          destination_table_name_template: {
+            stringValue: 'users_{run_time|"%H%M%S"}',
+          },
+          partitioning_field: { stringValue: "created_at" },
+        },
+      },
+    });
+
+    const request = await constructUpdateTransferConfigRequest(
+      client,
+      "projects/p/locations/us/transferConfigs/c",
+      config
+    );
+
+    expect(request.updateMask?.paths).toEqual(["display_name"]);
+    expect(request.transferConfig?.displayName).toBe("Users export");
+  });
+
+  test("leaves the display name out of the mask when unchanged", async () => {
+    const client = clientWithTransferConfig({
+      name: "projects/p/locations/us/transferConfigs/c",
+      destinationDatasetId: "analytics",
+      displayName: "Users export",
+      schedule: "every 12 hours",
+      notificationPubsubTopic:
+        "projects/test-project/topics/kit-users-export-processMessages",
+      params: {
+        fields: {
+          query: { stringValue: config.queryString },
+          destination_table_name_template: {
+            stringValue: 'users_{run_time|"%H%M%S"}',
+          },
+          partitioning_field: { stringValue: "created_at" },
+        },
+      },
+    });
+
+    const request = await constructUpdateTransferConfigRequest(
+      client,
+      "projects/p/locations/us/transferConfigs/c",
+      config
+    );
+
+    expect(request.updateMask?.paths).toEqual(["schedule"]);
+  });
+
   test("rejects clearing an existing partitioning field", async () => {
     const client = clientWithTransferConfig({
       name: "projects/p/locations/us/transferConfigs/c",

package/tests/helper-values.test.ts

@@ -110,10 +110,46 @@ describe("convertUnsupportedDataTypes", () => {
     );
   });
 
-  test("throws on a TIME value, which no Date can represent", () => {
-    expect(() =>
-      convertUnsupportedDataTypes({ time: new BigQueryTime("10:30:00") })
-    ).toThrow('Value for argument "seconds" is not a valid integer.');
+  // The strings here are the values a live query returns for
+  // TIME "10:30:00", TIME "10:30:00.123456" and TIME "00:00:00".
+  test("keeps a TIME value as the string BigQuery returned", () => {
+    expect(
+      convertUnsupportedDataTypes({
+        plain: new BigQueryTime("10:30:00"),
+        micros: new BigQueryTime("10:30:00.123456"),
+        midnight: new BigQueryTime("00:00:00"),
+      })
+    ).toEqual({
+      plain: "10:30:00",
+      micros: "10:30:00.123456",
+      midnight: "00:00:00",
+    });
+  });
+
+  test("keeps TIME values nested in arrays and structs", () => {
+    expect(
+      convertUnsupportedDataTypes({
+        times: [new BigQueryTime("01:02:03"), new BigQueryTime("04:05:06")],
+        outer: { inner: new BigQueryTime("07:08:09") },
+      })
+    ).toEqual({
+      times: ["01:02:03", "04:05:06"],
+      outer: { inner: "07:08:09" },
+    });
+  });
+
+  test("still converts the other temporal types alongside a TIME", () => {
+    const converted = convertUnsupportedDataTypes({
+      time: new BigQueryTime("10:30:00"),
+      timestamp: new BigQueryTimestamp("2023-01-15T10:30:00.000Z"),
+      date: new BigQueryDate("2023-01-15"),
+      datetime: new BigQueryDatetime("2023-01-15T10:30:00"),
+    });
+
+    expect(converted.time).toBe("10:30:00");
+    expect(converted.timestamp).toBeInstanceOf(Timestamp);
+    expect(converted.date).toBeInstanceOf(Timestamp);
+    expect(converted.datetime).toBeInstanceOf(Timestamp);
   });
 
   test("converts a plain Date to a Firestore Timestamp", () => {