@firebase-function-kits/bigquery-firestore-export 0.0.2-rc.0 → 0.0.2-rc.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@firebase-function-kits/bigquery-firestore-export",
-  "version": "0.0.2-rc.0",
+  "version": "0.0.2-rc.1",
   "repository": {
     "type": "git",
     "url": "https://github.com/firebase/extensions.git",

package/src/config.ts

@@ -32,21 +32,138 @@ const instanceId = defineString("INSTANCE_ID");
 const params = {
   instanceId,
   bigqueryDatasetLocation: defineString("BIGQUERY_DATASET_LOCATION", {
+    label: "BigQuery Dataset Location",
+    description:
+      "What is the location of the BigQuery dataset referenced in the query?",
+
     default: "US",
+    input: select({
+      "Columbus, Ohio (us-east5)": "us-east5",
+      "Iowa (us-central1)": "us-central1",
+      "Las Vegas (us-west4)": "us-west4",
+      "Los Angeles (us-west2)": "us-west2",
+      "Montréal (northamerica-northeast1)": "northamerica-northeast1",
+      "Northern Virginia (us-east4)": "us-east4",
+      "Oregon (us-west1)": "us-west1",
+      "Salt Lake City (us-west3)": "us-west3",
+      "São Paulo (southamerica-east1)": "southamerica-east1",
+      "Santiago (southamerica-west1)": "southamerica-west1",
+      "South Carolina (us-east1)": "us-east1",
+      "Toronto (northamerica-northeast2)": "northamerica-northeast2",
+      "Delhi (asia-south2)": "asia-south2",
+      "Hong Kong (asia-east2)": "asia-east2",
+      "Jakarta (asia-southeast2)": "asia-southeast2",
+      "Melbourne (australia-southeast2)": "australia-southeast2",
+      "Mumbai (asia-south1)": "asia-south1",
+      "Osaka (asia-northeast2)": "asia-northeast2",
+      "Seoul (asia-northeast3)": "asia-northeast3",
+      "Singapore (asia-southeast1)": "asia-southeast1",
+      "Sydney (australia-southeast1)": "australia-southeast1",
+      "Taiwan (asia-east1)": "asia-east1",
+      "Tokyo (asia-northeast1)": "asia-northeast1",
+      "Belgium (europe-west1)": "europe-west1",
+      "Finland (europe-north1)": "europe-north1",
+      "Frankfurt (europe-west3)": "europe-west3",
+      "London (europe-west2)": "europe-west2",
+      "Madrid (europe-southwest1)": "europe-southwest1",
+      "Milan (europe-west8)": "europe-west8",
+      "Netherlands (europe-west4)": "europe-west4",
+      "Paris (europe-west9)": "europe-west9",
+      "Warsaw (europe-central2)": "europe-central2",
+      "Zürich (europe-west6)": "europe-west6",
+      "US Multi-Region (US)": "US",
+      "EU Mutli-Region (EU)": "EU",
+    }),
   }),
   transferConfigName: defineString("TRANSFER_CONFIG_NAME", { default: "" }),
-  datasetId: defineString("DATASET_ID"),
-  tableName: defineString("TABLE_NAME"),
-  queryString: defineString("QUERY_STRING"),
-  displayName: defineString("DISPLAY_NAME"),
-  partitioningField: defineString("PARTITIONING_FIELD", { default: "" }),
-  schedule: defineString("SCHEDULE"),
+  datasetId: defineString("DATASET_ID", {
+    label: "Dataset ID",
+    description:
+      "What's the BigQuery destination dataset you'd like to use? Each transfer run will write to a table in this destination dataset.",
+    input: {
+      text: {
+        nonEmpty: true,
+        example: "customer_data",
+      },
+    },
+  }),
+  tableName: defineString("TABLE_NAME", {
+    label: "Destination Table Name",
+    description:
+      "What's the destination table name prefix you'd like to use? Each transfer run will write to the table with this name, postfixed with the runtime.",
+    input: {
+      text: {
+        nonEmpty: true,
+        example: "transactions",
+      },
+    },
+  }),
+  queryString: defineString("QUERY_STRING", {
+    label: "Query String",
+    description: "What's the BQ query you'd like to execute?",
+    input: {
+      text: {
+        nonEmpty: true,
+
+        example: "SELECT * from <PROJECT_ID>.customer_data.transactions",
+      },
+    },
+  }),
+  displayName: defineString("DISPLAY_NAME", {
+    label: "Display Name",
+    description: "What display name would you like to use?",
+    input: {
+      text: {
+        nonEmpty: true,
+        example: "Daily Rollup - Customer Transactions",
+      },
+    },
+  }),
+  partitioningField: defineString("PARTITIONING_FIELD", {
+    label: "Partitioning Field",
+    description:
+      "What's the partitioning field on the destination table ID? Leave empty if not using a partitioning field.",
+    default: "",
+    input: { text: { example: "timestamp" } },
+  }),
+  schedule: defineString("SCHEDULE", {
+    label: "Schedule",
+    description:
+      "What's the execution schedule you'd like to use for this query?",
+    input: {
+      text: {
+        nonEmpty: true,
+        example: "every 15 minutes",
+      },
+    },
+  }),
   firestoreCollection: defineString("COLLECTION_PATH", {
+    label: "Firestore Collection",
+    description:
+      "What's the top-level Firestore Collection to store transfer configs, run metadata, and query output?",
+
     default: "transferConfigs",
+    input: {
+      text: {
+        example: "transferConfigs",
+
+        validationRegex: /^[^\/]+(\/[^\/]+\/[^\/]+)*$/,
+        validationErrorMessage: "Must be a valid Cloud Firestore Collection",
+      },
+    },
   }),
   logLevel: defineString("LOG_LEVEL", {
+    label: "Log Level",
+    description: "What's the log level you'd like to use for this extension?",
+
     default: "info",
-    input: select([...LOG_LEVEL_OPTIONS]),
+    input: select({
+      DEBUG: "debug",
+      INFO: "info",
+      WARN: "warn",
+      ERROR: "error",
+      SILENT: "silent",
+    }),
   }),
 };
 

package/src/dts.ts

@@ -65,10 +65,16 @@ function stringField(value: string | undefined): { stringValue: string } {
   return { stringValue: value ?? "" };
 }
 
-/** Creates the protobuf-shaped request used for a scheduled query. */
+/**
+ * Creates the protobuf-shaped request used for a scheduled query.
+ *
+ * The query runs as whichever identity creates it, which is this function. To
+ * name a different account, `serviceAccountName` goes on the request rather
+ * than on the transfer config, and the caller needs `actAs` on the account it
+ * names, including its own.
+ */
 export function createTransferConfigRequest(
-  config: ResolvedBigqueryFirestoreExportConfig,
-  serviceAccountEmail?: string
+  config: ResolvedBigqueryFirestoreExportConfig
 ): bigqueryDataTransfer.protos.google.cloud.bigquery.datatransfer.v1.ICreateTransferConfigRequest {
   return {
     parent: `projects/${config.projectId}`,
@@ -88,9 +94,6 @@ export function createTransferConfigRequest(
       },
       schedule: config.schedule,
       notificationPubsubTopic: `projects/${config.projectId}/topics/${config.pubSubTopic}`,
-      ...(serviceAccountEmail
-        ? { serviceAccountName: serviceAccountEmail }
-        : {}),
     },
   };
 }
@@ -120,7 +123,7 @@ export async function createTransferConfig(
 ): Promise<TransferConfig> {
   logs.createTransferConfig();
   const [created] = await client.createTransferConfig(
-    createTransferConfigRequest(config, config.serviceAccount)
+    createTransferConfigRequest(config)
   );
   if (!created.name) {
     throw new Error("BigQuery API returned a transfer config without a name");

package/src/export-config.ts

@@ -45,8 +45,6 @@ export interface BigqueryFirestoreExportConfig {
   pubSubTopic?: string;
   /** Root Firestore collection for configs and run output. */
   firestoreCollection?: string;
-  /** Runtime identity used when creating a DTS config. */
-  serviceAccount?: string;
   /** Log verbosity. Defaults to `info`. */
   logLevel?: LogLevel;
 }
@@ -65,7 +63,6 @@ export interface ResolvedBigqueryFirestoreExportConfig {
   schedule: string;
   pubSubTopic: string;
   firestoreCollection: string;
-  serviceAccount?: string;
   logLevel: LogLevel;
 }
 
@@ -117,7 +114,6 @@ export function resolveConfig(
       optional(config.pubSubTopic) ?? `kit-${instanceId}-processMessages`,
     firestoreCollection:
       optional(config.firestoreCollection) ?? "transferConfigs",
-    serviceAccount: optional(config.serviceAccount),
     logLevel,
   };
 }

package/src/index.ts

@@ -111,7 +111,7 @@ function getContext(): HandlerContext {
 export const processMessages = onMessagePublished<TransferRunPayload>(
   {
     topic: CONFIG_EXPRESSIONS.pubSubTopic,
-    retry: true,
+    retry: false,
   },
   (event) => handleMessagePublished(event, getContext())
 );

package/tests/dts.test.ts

@@ -43,10 +43,7 @@ function clientWithTransferConfig(transferConfig: object): DataTransferClient {
 
 describe("createTransferConfigRequest", () => {
   test("creates the scheduled-query request", () => {
-    const request = createTransferConfigRequest(
-      config,
-      "runtime@test-project.iam.gserviceaccount.com"
-    );
+    const request = createTransferConfigRequest(config);
 
     expect(request).toMatchObject({
       parent: "projects/test-project",
@@ -57,7 +54,6 @@ describe("createTransferConfigRequest", () => {
         schedule: "every 24 hours",
         notificationPubsubTopic:
           "projects/test-project/topics/kit-users-export-processMessages",
-        serviceAccountName: "runtime@test-project.iam.gserviceaccount.com",
       },
     });
     expect(

package/tests/export-config.test.ts

@@ -46,16 +46,12 @@ describe("resolveConfig", () => {
       ...minimal,
       transferConfigName: "  projects/p/locations/us/transferConfigs/c  ",
       partitioningField: "  created_at  ",
-      serviceAccount: "  runtime@test-project.iam.gserviceaccount.com  ",
     });
 
     expect(resolved.transferConfigName).toBe(
       "projects/p/locations/us/transferConfigs/c"
     );
     expect(resolved.partitioningField).toBe("created_at");
-    expect(resolved.serviceAccount).toBe(
-      "runtime@test-project.iam.gserviceaccount.com"
-    );
   });
 
   test.each(["instanceId", "datasetId", "tableName", "queryString"] as const)(