@firebase-function-kits/bigquery-firestore-export 0.0.2-rc.1 → 0.0.2-rc.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@firebase-function-kits/bigquery-firestore-export",
-  "version": "0.0.2-rc.1",
+  "version": "0.0.2-rc.2",
   "repository": {
     "type": "git",
     "url": "https://github.com/firebase/extensions.git",
@@ -37,10 +37,7 @@
     "@google-cloud/bigquery-data-transfer": "^5.0.1",
     "@google-cloud/pubsub": "^4.11.0",
     "firebase-admin": "^14.1.0",
-    "firebase-functions": "7.3.2"
-  },
-  "devDependencies": {
-    "vitest": "^4.1.10"
+    "firebase-functions": "^7.3.2"
   },
   "overrides": {
     "uuid@9.0.1": "11.1.1"

package/src/config.ts

@@ -76,6 +76,22 @@ const params = {
     }),
   }),
   transferConfigName: defineString("TRANSFER_CONFIG_NAME", { default: "" }),
+  pubSubTopic: defineString("PUB_SUB_TOPIC", {
+    label: "Pub/Sub Topic",
+    description:
+      "Which Pub/Sub topic should receive BigQuery Data Transfer completion notifications? Leave the default unless you are migrating from the bigquery-firestore-export extension, whose topic is named ext-<instance id>-processMessages. Pointing this at the extension's topic keeps the existing scheduled query's notification settings untouched.",
+
+    default: expr`kit-${instanceId}-processMessages`,
+    input: {
+      text: {
+        nonEmpty: true,
+        example: "ext-my-instance-processMessages",
+        validationRegex: /^(?!goog)[a-zA-Z][a-zA-Z0-9\-_.~+%]{2,254}$/,
+        validationErrorMessage:
+          "Must be a Pub/Sub topic ID, not a full projects/<project>/topics/<topic> resource name. IDs are 3 to 255 characters, start with a letter, may contain letters, numbers and - _ . ~ + %, and cannot start with goog.",
+      },
+    },
+  }),
   datasetId: defineString("DATASET_ID", {
     label: "Dataset ID",
     description:
@@ -168,7 +184,7 @@ const params = {
 };
 
 export const CONFIG_EXPRESSIONS: DeployTimeOptions = {
-  pubSubTopic: expr`kit-${instanceId}-processMessages`,
+  pubSubTopic: params.pubSubTopic,
 };
 
 function optional(value: string): string | undefined {
@@ -198,7 +214,7 @@ export function configFromEnv(): BigqueryFirestoreExportConfig {
     displayName: params.displayName.value(),
     partitioningField: optional(params.partitioningField.value()),
     schedule: params.schedule.value(),
-    pubSubTopic: `kit-${resolvedInstanceId}-processMessages`,
+    pubSubTopic: params.pubSubTopic.value(),
     firestoreCollection: params.firestoreCollection.value(),
     logLevel: normalizeLogLevel(params.logLevel.value()),
   };

package/src/dts.ts

@@ -61,6 +61,21 @@ function transferConfigFields(config: TransferConfig) {
   return fields;
 }
 
+/**
+ * Full resource name of the topic DTS publishes run notifications to.
+ *
+ * PUB_SUB_TOPIC is validated as a bare topic ID when prompted, but values from
+ * a dotenv file reach us unvalidated, and both the Pub/Sub client and the
+ * trigger accept a full resource name, so accept one here too.
+ */
+export function notificationTopicName(
+  config: ResolvedBigqueryFirestoreExportConfig
+): string {
+  return config.pubSubTopic.startsWith("projects/")
+    ? config.pubSubTopic
+    : `projects/${config.projectId}/topics/${config.pubSubTopic}`;
+}
+
 function stringField(value: string | undefined): { stringValue: string } {
   return { stringValue: value ?? "" };
 }
@@ -93,7 +108,7 @@ export function createTransferConfigRequest(
         },
       },
       schedule: config.schedule,
-      notificationPubsubTopic: `projects/${config.projectId}/topics/${config.pubSubTopic}`,
+      notificationPubsubTopic: notificationTopicName(config),
     },
   };
 }
@@ -184,7 +199,7 @@ export async function constructUpdateTransferConfigRequest(
     updatedConfig.schedule = config.schedule;
   }
 
-  const expectedTopic = `projects/${config.projectId}/topics/${config.pubSubTopic}`;
+  const expectedTopic = notificationTopicName(config);
   if (expectedTopic !== transferConfig.notificationPubsubTopic) {
     updateMask.push("notification_pubsub_topic");
     updatedConfig.notificationPubsubTopic = expectedTopic;

package/src/handlers.ts

@@ -23,6 +23,8 @@ import {
   createTransferConfig,
   type DataTransferClient,
   getTransferConfig,
+  notificationTopicName,
+  PARTITIONING_FIELD_REMOVAL_ERROR_PREFIX,
   updateTransferConfig,
 } from "./dts";
 import type { ResolvedBigqueryFirestoreExportConfig } from "./export-config";
@@ -64,6 +66,13 @@ async function ensureNotificationTopic(ctx: HandlerContext): Promise<void> {
   }
 }
 
+function isPartitioningFieldRemovalError(err: unknown): err is Error {
+  return (
+    err instanceof Error &&
+    err.message.includes(PARTITIONING_FIELD_REMOVAL_ERROR_PREFIX)
+  );
+}
+
 async function storeTransferConfig(
   ctx: HandlerContext,
   transferConfig: Awaited<ReturnType<typeof getTransferConfig>>
@@ -105,8 +114,19 @@ export async function handleUpsertTransferConfig(
       ctx.config.transferConfigName
     );
     if (!linked) {
-      throw new Error(
-        `Transfer config not found: ${ctx.config.transferConfigName}`
+      // Only a redeploy with a corrected TRANSFER_CONFIG_NAME can resolve this,
+      // so retrying the task cannot help.
+      logs.linkedTransferConfigMissing(ctx.config.transferConfigName);
+      return;
+    }
+    // A linked config is adopted as-is, so a topic mismatch is the user's to
+    // resolve: rewriting it would repoint a config this deployment did not create.
+    const expectedTopic = notificationTopicName(ctx.config);
+    if (linked.notificationPubsubTopic !== expectedTopic) {
+      logs.linkedTopicMismatch(
+        ctx.config.transferConfigName,
+        linked.notificationPubsubTopic,
+        expectedTopic
       );
     }
     await storeTransferConfig(ctx, linked);
@@ -128,14 +148,21 @@ export async function handleUpsertTransferConfig(
   const transferConfigName = existing.docs[0].data().name;
   if (typeof transferConfigName !== "string" || !transferConfigName) {
     throw new Error(
-      `Existing transfer config document in ${ctx.config.firestoreCollection} is missing required 'name' field.`
+      `Existing transfer config document in ${ctx.config.firestoreCollection} is missing required 'name' field. Delete the document so a new scheduled query is created, then redeploy.`
     );
   }
 
-  const updated = await updateTransferConfig(
-    ctx.dataTransfer,
-    transferConfigName,
-    ctx.config
-  );
-  await storeTransferConfig(ctx, updated);
+  try {
+    const updated = await updateTransferConfig(
+      ctx.dataTransfer,
+      transferConfigName,
+      ctx.config
+    );
+    await storeTransferConfig(ctx, updated);
+  } catch (err) {
+    if (!isPartitioningFieldRemovalError(err)) throw err;
+    // The guard rejects the update while building the request, so nothing was
+    // sent and a retry hits the same guard.
+    logs.partitioningFieldRemovalAborted(err.message);
+  }
 }

package/src/helper.ts

@@ -227,6 +227,7 @@ export async function writeRunResultsToFirestore(
     message.json.destinationDatasetId,
     tableName
   );
+  logs.writeRunResultsToFirestore(runId);
   const collection = db.collection(
     `${config.firestoreCollection}/${transferConfigId}/runs/${runId}/output`
   );

package/src/logs.ts

@@ -70,6 +70,10 @@ export function bigqueryQueryFailed(
   });
 }
 
+export function writeRunResultsToFirestore(runId: string): void {
+  logger.debug("Writing BigQuery results to Firestore", { runId });
+}
+
 export function runResultsWrittenToFirestore(
   runId: string,
   successCount: number,
@@ -140,6 +144,30 @@ export function partitioningFieldRemovalAttempted(
   });
 }
 
+export function linkedTransferConfigMissing(name: string): void {
+  logger.error(
+    "The scheduled query named by TRANSFER_CONFIG_NAME does not exist, so nothing was linked. Set it to a scheduled query that exists in this project and redeploy, or clear it to have this deployment create its own.",
+    { name }
+  );
+}
+
+// The reason carries the remediation, and an Error passed as structured data
+// serialises to {}, so it goes in the message.
+export function partitioningFieldRemovalAborted(reason: string): void {
+  logger.error(`Stopped without updating the scheduled query. ${reason}`);
+}
+
 export function topicCreated(name: string): void {
   logger.info("Created Pub/Sub topic for transfer notifications", { name });
 }
+
+export function linkedTopicMismatch(
+  name: string,
+  linkedTopic: string | null | undefined,
+  expectedTopic: string
+): void {
+  logger.warn(
+    "Linked transfer config notifies a different Pub/Sub topic, so its runs will not reach this kit. Set PUB_SUB_TOPIC to the linked topic, or point the transfer config at the configured one.",
+    { name, linkedTopic: linkedTopic ?? "", expectedTopic }
+  );
+}

package/tests/config.test.ts

@@ -23,12 +23,41 @@ afterEach(() => {
 });
 
 describe("CONFIG_EXPRESSIONS", () => {
-  test("namespaces the Pub/Sub topic with the required instance id", () => {
+  test("binds the trigger to the Pub/Sub topic parameter", () => {
     expect(CONFIG_EXPRESSIONS.pubSubTopic).toBeInstanceOf(Expression);
     expect((CONFIG_EXPRESSIONS.pubSubTopic as Expression<string>).toCEL()).toBe(
-      "kit-{{ params.INSTANCE_ID }}-processMessages"
+      "{{ params.PUB_SUB_TOPIC }}"
     );
   });
+
+  test("defaults the topic parameter to the instance-namespaced kit topic", () => {
+    const spec = (
+      CONFIG_EXPRESSIONS.pubSubTopic as unknown as {
+        toSpec: () => { default?: string };
+      }
+    ).toSpec();
+    expect(spec.default).toBe("kit-{{ params.INSTANCE_ID }}-processMessages");
+  });
+
+  test("accepts a topic ID but rejects a full resource name", () => {
+    const spec = (
+      CONFIG_EXPRESSIONS.pubSubTopic as unknown as {
+        toSpec: () => { input?: { text?: { validationRegex?: string } } };
+      }
+    ).toSpec();
+    const pattern = spec.input?.text?.validationRegex;
+    expect(pattern).toBeTypeOf("string");
+    const validate = (value: string) =>
+      new RegExp(pattern as string).test(value);
+
+    expect(validate("ext-users-export-processMessages")).toBe(true);
+    expect(validate("kit-users-export-processMessages")).toBe(true);
+    expect(
+      validate("projects/test-project/topics/ext-users-export-processMessages")
+    ).toBe(false);
+    expect(validate("")).toBe(false);
+    expect(validate("goog-reserved-prefix")).toBe(false);
+  });
 });
 
 describe("configFromEnv", () => {
@@ -47,6 +76,8 @@ describe("configFromEnv", () => {
     vi.stubEnv("COLLECTION_PATH", "transferConfigs");
     vi.stubEnv("LOG_LEVEL", "info");
 
+    vi.stubEnv("PUB_SUB_TOPIC", "kit-users-export-processMessages");
+
     expect(configFromEnv()).toMatchObject({
       projectId: "test-project",
       instanceId: "users-export",
@@ -58,4 +89,25 @@ describe("configFromEnv", () => {
       logLevel: "info",
     });
   });
+
+  test("passes through a topic pointing at the extension's own topic", () => {
+    vi.stubEnv(
+      "FIREBASE_CONFIG",
+      JSON.stringify({ projectId: "test-project" })
+    );
+    vi.stubEnv("INSTANCE_ID", "users-export");
+    vi.stubEnv("BIGQUERY_DATASET_LOCATION", "EU");
+    vi.stubEnv("DATASET_ID", "analytics");
+    vi.stubEnv("TABLE_NAME", "users");
+    vi.stubEnv("QUERY_STRING", "SELECT * FROM source.users");
+    vi.stubEnv("DISPLAY_NAME", "Users export");
+    vi.stubEnv("SCHEDULE", "every 24 hours");
+    vi.stubEnv("COLLECTION_PATH", "transferConfigs");
+    vi.stubEnv("LOG_LEVEL", "info");
+    vi.stubEnv("PUB_SUB_TOPIC", "ext-users-export-processMessages");
+
+    expect(configFromEnv().pubSubTopic).toBe(
+      "ext-users-export-processMessages"
+    );
+  });
 });