@firebase-function-kits/bigquery-firestore-export 0.0.2-rc.0 → 0.0.2-rc.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@firebase-function-kits/bigquery-firestore-export",
-  "version": "0.0.2-rc.0",
+  "version": "0.0.2-rc.2",
   "repository": {
     "type": "git",
     "url": "https://github.com/firebase/extensions.git",
@@ -37,10 +37,7 @@
     "@google-cloud/bigquery-data-transfer": "^5.0.1",
     "@google-cloud/pubsub": "^4.11.0",
     "firebase-admin": "^14.1.0",
-    "firebase-functions": "7.3.2"
-  },
-  "devDependencies": {
-    "vitest": "^4.1.10"
+    "firebase-functions": "^7.3.2"
   },
   "overrides": {
     "uuid@9.0.1": "11.1.1"

package/src/config.ts

@@ -32,26 +32,159 @@ const instanceId = defineString("INSTANCE_ID");
 const params = {
   instanceId,
   bigqueryDatasetLocation: defineString("BIGQUERY_DATASET_LOCATION", {
+    label: "BigQuery Dataset Location",
+    description:
+      "What is the location of the BigQuery dataset referenced in the query?",
+
     default: "US",
+    input: select({
+      "Columbus, Ohio (us-east5)": "us-east5",
+      "Iowa (us-central1)": "us-central1",
+      "Las Vegas (us-west4)": "us-west4",
+      "Los Angeles (us-west2)": "us-west2",
+      "Montréal (northamerica-northeast1)": "northamerica-northeast1",
+      "Northern Virginia (us-east4)": "us-east4",
+      "Oregon (us-west1)": "us-west1",
+      "Salt Lake City (us-west3)": "us-west3",
+      "São Paulo (southamerica-east1)": "southamerica-east1",
+      "Santiago (southamerica-west1)": "southamerica-west1",
+      "South Carolina (us-east1)": "us-east1",
+      "Toronto (northamerica-northeast2)": "northamerica-northeast2",
+      "Delhi (asia-south2)": "asia-south2",
+      "Hong Kong (asia-east2)": "asia-east2",
+      "Jakarta (asia-southeast2)": "asia-southeast2",
+      "Melbourne (australia-southeast2)": "australia-southeast2",
+      "Mumbai (asia-south1)": "asia-south1",
+      "Osaka (asia-northeast2)": "asia-northeast2",
+      "Seoul (asia-northeast3)": "asia-northeast3",
+      "Singapore (asia-southeast1)": "asia-southeast1",
+      "Sydney (australia-southeast1)": "australia-southeast1",
+      "Taiwan (asia-east1)": "asia-east1",
+      "Tokyo (asia-northeast1)": "asia-northeast1",
+      "Belgium (europe-west1)": "europe-west1",
+      "Finland (europe-north1)": "europe-north1",
+      "Frankfurt (europe-west3)": "europe-west3",
+      "London (europe-west2)": "europe-west2",
+      "Madrid (europe-southwest1)": "europe-southwest1",
+      "Milan (europe-west8)": "europe-west8",
+      "Netherlands (europe-west4)": "europe-west4",
+      "Paris (europe-west9)": "europe-west9",
+      "Warsaw (europe-central2)": "europe-central2",
+      "Zürich (europe-west6)": "europe-west6",
+      "US Multi-Region (US)": "US",
+      "EU Mutli-Region (EU)": "EU",
+    }),
   }),
   transferConfigName: defineString("TRANSFER_CONFIG_NAME", { default: "" }),
-  datasetId: defineString("DATASET_ID"),
-  tableName: defineString("TABLE_NAME"),
-  queryString: defineString("QUERY_STRING"),
-  displayName: defineString("DISPLAY_NAME"),
-  partitioningField: defineString("PARTITIONING_FIELD", { default: "" }),
-  schedule: defineString("SCHEDULE"),
+  pubSubTopic: defineString("PUB_SUB_TOPIC", {
+    label: "Pub/Sub Topic",
+    description:
+      "Which Pub/Sub topic should receive BigQuery Data Transfer completion notifications? Leave the default unless you are migrating from the bigquery-firestore-export extension, whose topic is named ext-<instance id>-processMessages. Pointing this at the extension's topic keeps the existing scheduled query's notification settings untouched.",
+
+    default: expr`kit-${instanceId}-processMessages`,
+    input: {
+      text: {
+        nonEmpty: true,
+        example: "ext-my-instance-processMessages",
+        validationRegex: /^(?!goog)[a-zA-Z][a-zA-Z0-9\-_.~+%]{2,254}$/,
+        validationErrorMessage:
+          "Must be a Pub/Sub topic ID, not a full projects/<project>/topics/<topic> resource name. IDs are 3 to 255 characters, start with a letter, may contain letters, numbers and - _ . ~ + %, and cannot start with goog.",
+      },
+    },
+  }),
+  datasetId: defineString("DATASET_ID", {
+    label: "Dataset ID",
+    description:
+      "What's the BigQuery destination dataset you'd like to use? Each transfer run will write to a table in this destination dataset.",
+    input: {
+      text: {
+        nonEmpty: true,
+        example: "customer_data",
+      },
+    },
+  }),
+  tableName: defineString("TABLE_NAME", {
+    label: "Destination Table Name",
+    description:
+      "What's the destination table name prefix you'd like to use? Each transfer run will write to the table with this name, postfixed with the runtime.",
+    input: {
+      text: {
+        nonEmpty: true,
+        example: "transactions",
+      },
+    },
+  }),
+  queryString: defineString("QUERY_STRING", {
+    label: "Query String",
+    description: "What's the BQ query you'd like to execute?",
+    input: {
+      text: {
+        nonEmpty: true,
+
+        example: "SELECT * from <PROJECT_ID>.customer_data.transactions",
+      },
+    },
+  }),
+  displayName: defineString("DISPLAY_NAME", {
+    label: "Display Name",
+    description: "What display name would you like to use?",
+    input: {
+      text: {
+        nonEmpty: true,
+        example: "Daily Rollup - Customer Transactions",
+      },
+    },
+  }),
+  partitioningField: defineString("PARTITIONING_FIELD", {
+    label: "Partitioning Field",
+    description:
+      "What's the partitioning field on the destination table ID? Leave empty if not using a partitioning field.",
+    default: "",
+    input: { text: { example: "timestamp" } },
+  }),
+  schedule: defineString("SCHEDULE", {
+    label: "Schedule",
+    description:
+      "What's the execution schedule you'd like to use for this query?",
+    input: {
+      text: {
+        nonEmpty: true,
+        example: "every 15 minutes",
+      },
+    },
+  }),
   firestoreCollection: defineString("COLLECTION_PATH", {
+    label: "Firestore Collection",
+    description:
+      "What's the top-level Firestore Collection to store transfer configs, run metadata, and query output?",
+
     default: "transferConfigs",
+    input: {
+      text: {
+        example: "transferConfigs",
+
+        validationRegex: /^[^\/]+(\/[^\/]+\/[^\/]+)*$/,
+        validationErrorMessage: "Must be a valid Cloud Firestore Collection",
+      },
+    },
   }),
   logLevel: defineString("LOG_LEVEL", {
+    label: "Log Level",
+    description: "What's the log level you'd like to use for this extension?",
+
     default: "info",
-    input: select([...LOG_LEVEL_OPTIONS]),
+    input: select({
+      DEBUG: "debug",
+      INFO: "info",
+      WARN: "warn",
+      ERROR: "error",
+      SILENT: "silent",
+    }),
   }),
 };
 
 export const CONFIG_EXPRESSIONS: DeployTimeOptions = {
-  pubSubTopic: expr`kit-${instanceId}-processMessages`,
+  pubSubTopic: params.pubSubTopic,
 };
 
 function optional(value: string): string | undefined {
@@ -81,7 +214,7 @@ export function configFromEnv(): BigqueryFirestoreExportConfig {
     displayName: params.displayName.value(),
     partitioningField: optional(params.partitioningField.value()),
     schedule: params.schedule.value(),
-    pubSubTopic: `kit-${resolvedInstanceId}-processMessages`,
+    pubSubTopic: params.pubSubTopic.value(),
     firestoreCollection: params.firestoreCollection.value(),
     logLevel: normalizeLogLevel(params.logLevel.value()),
   };

package/src/dts.ts

@@ -61,14 +61,35 @@ function transferConfigFields(config: TransferConfig) {
   return fields;
 }
 
+/**
+ * Full resource name of the topic DTS publishes run notifications to.
+ *
+ * PUB_SUB_TOPIC is validated as a bare topic ID when prompted, but values from
+ * a dotenv file reach us unvalidated, and both the Pub/Sub client and the
+ * trigger accept a full resource name, so accept one here too.
+ */
+export function notificationTopicName(
+  config: ResolvedBigqueryFirestoreExportConfig
+): string {
+  return config.pubSubTopic.startsWith("projects/")
+    ? config.pubSubTopic
+    : `projects/${config.projectId}/topics/${config.pubSubTopic}`;
+}
+
 function stringField(value: string | undefined): { stringValue: string } {
   return { stringValue: value ?? "" };
 }
 
-/** Creates the protobuf-shaped request used for a scheduled query. */
+/**
+ * Creates the protobuf-shaped request used for a scheduled query.
+ *
+ * The query runs as whichever identity creates it, which is this function. To
+ * name a different account, `serviceAccountName` goes on the request rather
+ * than on the transfer config, and the caller needs `actAs` on the account it
+ * names, including its own.
+ */
 export function createTransferConfigRequest(
-  config: ResolvedBigqueryFirestoreExportConfig,
-  serviceAccountEmail?: string
+  config: ResolvedBigqueryFirestoreExportConfig
 ): bigqueryDataTransfer.protos.google.cloud.bigquery.datatransfer.v1.ICreateTransferConfigRequest {
   return {
     parent: `projects/${config.projectId}`,
@@ -87,10 +108,7 @@ export function createTransferConfigRequest(
         },
       },
       schedule: config.schedule,
-      notificationPubsubTopic: `projects/${config.projectId}/topics/${config.pubSubTopic}`,
-      ...(serviceAccountEmail
-        ? { serviceAccountName: serviceAccountEmail }
-        : {}),
+      notificationPubsubTopic: notificationTopicName(config),
     },
   };
 }
@@ -120,7 +138,7 @@ export async function createTransferConfig(
 ): Promise<TransferConfig> {
   logs.createTransferConfig();
   const [created] = await client.createTransferConfig(
-    createTransferConfigRequest(config, config.serviceAccount)
+    createTransferConfigRequest(config)
   );
   if (!created.name) {
     throw new Error("BigQuery API returned a transfer config without a name");
@@ -181,7 +199,7 @@ export async function constructUpdateTransferConfigRequest(
     updatedConfig.schedule = config.schedule;
   }
 
-  const expectedTopic = `projects/${config.projectId}/topics/${config.pubSubTopic}`;
+  const expectedTopic = notificationTopicName(config);
   if (expectedTopic !== transferConfig.notificationPubsubTopic) {
     updateMask.push("notification_pubsub_topic");
     updatedConfig.notificationPubsubTopic = expectedTopic;

package/src/export-config.ts

@@ -45,8 +45,6 @@ export interface BigqueryFirestoreExportConfig {
   pubSubTopic?: string;
   /** Root Firestore collection for configs and run output. */
   firestoreCollection?: string;
-  /** Runtime identity used when creating a DTS config. */
-  serviceAccount?: string;
   /** Log verbosity. Defaults to `info`. */
   logLevel?: LogLevel;
 }
@@ -65,7 +63,6 @@ export interface ResolvedBigqueryFirestoreExportConfig {
   schedule: string;
   pubSubTopic: string;
   firestoreCollection: string;
-  serviceAccount?: string;
   logLevel: LogLevel;
 }
 
@@ -117,7 +114,6 @@ export function resolveConfig(
       optional(config.pubSubTopic) ?? `kit-${instanceId}-processMessages`,
     firestoreCollection:
       optional(config.firestoreCollection) ?? "transferConfigs",
-    serviceAccount: optional(config.serviceAccount),
     logLevel,
   };
 }

package/src/handlers.ts

@@ -23,6 +23,8 @@ import {
   createTransferConfig,
   type DataTransferClient,
   getTransferConfig,
+  notificationTopicName,
+  PARTITIONING_FIELD_REMOVAL_ERROR_PREFIX,
   updateTransferConfig,
 } from "./dts";
 import type { ResolvedBigqueryFirestoreExportConfig } from "./export-config";
@@ -64,6 +66,13 @@ async function ensureNotificationTopic(ctx: HandlerContext): Promise<void> {
   }
 }
 
+function isPartitioningFieldRemovalError(err: unknown): err is Error {
+  return (
+    err instanceof Error &&
+    err.message.includes(PARTITIONING_FIELD_REMOVAL_ERROR_PREFIX)
+  );
+}
+
 async function storeTransferConfig(
   ctx: HandlerContext,
   transferConfig: Awaited<ReturnType<typeof getTransferConfig>>
@@ -105,8 +114,19 @@ export async function handleUpsertTransferConfig(
       ctx.config.transferConfigName
     );
     if (!linked) {
-      throw new Error(
-        `Transfer config not found: ${ctx.config.transferConfigName}`
+      // Only a redeploy with a corrected TRANSFER_CONFIG_NAME can resolve this,
+      // so retrying the task cannot help.
+      logs.linkedTransferConfigMissing(ctx.config.transferConfigName);
+      return;
+    }
+    // A linked config is adopted as-is, so a topic mismatch is the user's to
+    // resolve: rewriting it would repoint a config this deployment did not create.
+    const expectedTopic = notificationTopicName(ctx.config);
+    if (linked.notificationPubsubTopic !== expectedTopic) {
+      logs.linkedTopicMismatch(
+        ctx.config.transferConfigName,
+        linked.notificationPubsubTopic,
+        expectedTopic
       );
     }
     await storeTransferConfig(ctx, linked);
@@ -128,14 +148,21 @@ export async function handleUpsertTransferConfig(
   const transferConfigName = existing.docs[0].data().name;
   if (typeof transferConfigName !== "string" || !transferConfigName) {
     throw new Error(
-      `Existing transfer config document in ${ctx.config.firestoreCollection} is missing required 'name' field.`
+      `Existing transfer config document in ${ctx.config.firestoreCollection} is missing required 'name' field. Delete the document so a new scheduled query is created, then redeploy.`
     );
   }
 
-  const updated = await updateTransferConfig(
-    ctx.dataTransfer,
-    transferConfigName,
-    ctx.config
-  );
-  await storeTransferConfig(ctx, updated);
+  try {
+    const updated = await updateTransferConfig(
+      ctx.dataTransfer,
+      transferConfigName,
+      ctx.config
+    );
+    await storeTransferConfig(ctx, updated);
+  } catch (err) {
+    if (!isPartitioningFieldRemovalError(err)) throw err;
+    // The guard rejects the update while building the request, so nothing was
+    // sent and a retry hits the same guard.
+    logs.partitioningFieldRemovalAborted(err.message);
+  }
 }

package/src/helper.ts

@@ -227,6 +227,7 @@ export async function writeRunResultsToFirestore(
     message.json.destinationDatasetId,
     tableName
   );
+  logs.writeRunResultsToFirestore(runId);
   const collection = db.collection(
     `${config.firestoreCollection}/${transferConfigId}/runs/${runId}/output`
   );

package/src/index.ts

@@ -111,7 +111,7 @@ function getContext(): HandlerContext {
 export const processMessages = onMessagePublished<TransferRunPayload>(
   {
     topic: CONFIG_EXPRESSIONS.pubSubTopic,
-    retry: true,
+    retry: false,
   },
   (event) => handleMessagePublished(event, getContext())
 );

package/src/logs.ts

@@ -70,6 +70,10 @@ export function bigqueryQueryFailed(
   });
 }
 
+export function writeRunResultsToFirestore(runId: string): void {
+  logger.debug("Writing BigQuery results to Firestore", { runId });
+}
+
 export function runResultsWrittenToFirestore(
   runId: string,
   successCount: number,
@@ -140,6 +144,30 @@ export function partitioningFieldRemovalAttempted(
   });
 }
 
+export function linkedTransferConfigMissing(name: string): void {
+  logger.error(
+    "The scheduled query named by TRANSFER_CONFIG_NAME does not exist, so nothing was linked. Set it to a scheduled query that exists in this project and redeploy, or clear it to have this deployment create its own.",
+    { name }
+  );
+}
+
+// The reason carries the remediation, and an Error passed as structured data
+// serialises to {}, so it goes in the message.
+export function partitioningFieldRemovalAborted(reason: string): void {
+  logger.error(`Stopped without updating the scheduled query. ${reason}`);
+}
+
 export function topicCreated(name: string): void {
   logger.info("Created Pub/Sub topic for transfer notifications", { name });
 }
+
+export function linkedTopicMismatch(
+  name: string,
+  linkedTopic: string | null | undefined,
+  expectedTopic: string
+): void {
+  logger.warn(
+    "Linked transfer config notifies a different Pub/Sub topic, so its runs will not reach this kit. Set PUB_SUB_TOPIC to the linked topic, or point the transfer config at the configured one.",
+    { name, linkedTopic: linkedTopic ?? "", expectedTopic }
+  );
+}

package/tests/config.test.ts

@@ -23,12 +23,41 @@ afterEach(() => {
 });
 
 describe("CONFIG_EXPRESSIONS", () => {
-  test("namespaces the Pub/Sub topic with the required instance id", () => {
+  test("binds the trigger to the Pub/Sub topic parameter", () => {
     expect(CONFIG_EXPRESSIONS.pubSubTopic).toBeInstanceOf(Expression);
     expect((CONFIG_EXPRESSIONS.pubSubTopic as Expression<string>).toCEL()).toBe(
-      "kit-{{ params.INSTANCE_ID }}-processMessages"
+      "{{ params.PUB_SUB_TOPIC }}"
     );
   });
+
+  test("defaults the topic parameter to the instance-namespaced kit topic", () => {
+    const spec = (
+      CONFIG_EXPRESSIONS.pubSubTopic as unknown as {
+        toSpec: () => { default?: string };
+      }
+    ).toSpec();
+    expect(spec.default).toBe("kit-{{ params.INSTANCE_ID }}-processMessages");
+  });
+
+  test("accepts a topic ID but rejects a full resource name", () => {
+    const spec = (
+      CONFIG_EXPRESSIONS.pubSubTopic as unknown as {
+        toSpec: () => { input?: { text?: { validationRegex?: string } } };
+      }
+    ).toSpec();
+    const pattern = spec.input?.text?.validationRegex;
+    expect(pattern).toBeTypeOf("string");
+    const validate = (value: string) =>
+      new RegExp(pattern as string).test(value);
+
+    expect(validate("ext-users-export-processMessages")).toBe(true);
+    expect(validate("kit-users-export-processMessages")).toBe(true);
+    expect(
+      validate("projects/test-project/topics/ext-users-export-processMessages")
+    ).toBe(false);
+    expect(validate("")).toBe(false);
+    expect(validate("goog-reserved-prefix")).toBe(false);
+  });
 });
 
 describe("configFromEnv", () => {
@@ -47,6 +76,8 @@ describe("configFromEnv", () => {
     vi.stubEnv("COLLECTION_PATH", "transferConfigs");
     vi.stubEnv("LOG_LEVEL", "info");
 
+    vi.stubEnv("PUB_SUB_TOPIC", "kit-users-export-processMessages");
+
     expect(configFromEnv()).toMatchObject({
       projectId: "test-project",
       instanceId: "users-export",
@@ -58,4 +89,25 @@ describe("configFromEnv", () => {
       logLevel: "info",
     });
   });
+
+  test("passes through a topic pointing at the extension's own topic", () => {
+    vi.stubEnv(
+      "FIREBASE_CONFIG",
+      JSON.stringify({ projectId: "test-project" })
+    );
+    vi.stubEnv("INSTANCE_ID", "users-export");
+    vi.stubEnv("BIGQUERY_DATASET_LOCATION", "EU");
+    vi.stubEnv("DATASET_ID", "analytics");
+    vi.stubEnv("TABLE_NAME", "users");
+    vi.stubEnv("QUERY_STRING", "SELECT * FROM source.users");
+    vi.stubEnv("DISPLAY_NAME", "Users export");
+    vi.stubEnv("SCHEDULE", "every 24 hours");
+    vi.stubEnv("COLLECTION_PATH", "transferConfigs");
+    vi.stubEnv("LOG_LEVEL", "info");
+    vi.stubEnv("PUB_SUB_TOPIC", "ext-users-export-processMessages");
+
+    expect(configFromEnv().pubSubTopic).toBe(
+      "ext-users-export-processMessages"
+    );
+  });
 });