@firebase-function-kits/firestore-bigquery-export 0.0.1 → 0.0.2-rc.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. package/CHANGELOG.md +1 -0
  2. package/README.md +303 -0
  3. package/lib/config.d.ts +33 -0
  4. package/lib/config.d.ts.map +1 -0
  5. package/lib/config.js +569 -0
  6. package/lib/config.js.map +1 -0
  7. package/lib/events.d.ts +45 -0
  8. package/lib/events.d.ts.map +1 -0
  9. package/lib/events.js +154 -0
  10. package/lib/events.js.map +1 -0
  11. package/lib/export-config.d.ts +96 -0
  12. package/lib/export-config.d.ts.map +1 -0
  13. package/lib/export-config.js +77 -0
  14. package/lib/export-config.js.map +1 -0
  15. package/lib/handlers.d.ts +29 -0
  16. package/lib/handlers.d.ts.map +1 -0
  17. package/lib/handlers.js +165 -0
  18. package/lib/handlers.js.map +1 -0
  19. package/lib/index.d.ts +23 -0
  20. package/lib/index.d.ts.map +1 -0
  21. package/lib/index.js +179 -0
  22. package/lib/index.js.map +1 -0
  23. package/lib/init.d.ts +16 -0
  24. package/lib/init.d.ts.map +1 -0
  25. package/lib/init.js +44 -0
  26. package/lib/init.js.map +1 -0
  27. package/lib/lib.d.ts +20 -0
  28. package/lib/lib.d.ts.map +1 -0
  29. package/lib/lib.js +47 -0
  30. package/lib/lib.js.map +1 -0
  31. package/lib/logs.d.ts +39 -0
  32. package/lib/logs.d.ts.map +1 -0
  33. package/lib/logs.js +186 -0
  34. package/lib/logs.js.map +1 -0
  35. package/lib/util.d.ts +23 -0
  36. package/lib/util.d.ts.map +1 -0
  37. package/lib/util.js +66 -0
  38. package/lib/util.js.map +1 -0
  39. package/npm-shrinkwrap.json +7016 -0
  40. package/package.json +37 -3
  41. package/src/config.ts +696 -0
  42. package/src/events.ts +146 -0
  43. package/src/export-config.ts +205 -0
  44. package/src/handlers.ts +211 -0
  45. package/src/index.ts +174 -0
  46. package/src/init.ts +45 -0
  47. package/src/lib.ts +55 -0
  48. package/src/logs.ts +233 -0
  49. package/src/util.ts +64 -0
  50. package/tests/config.test.ts +182 -0
  51. package/tests/events.test.ts +84 -0
  52. package/tests/export-config.test.ts +114 -0
  53. package/tests/handlers.test.ts +220 -0
  54. package/tests/init.test.ts +76 -0
  55. package/tests/util.test.ts +102 -0
  56. package/tsconfig.json +18 -0
  57. package/tsconfig.tsbuildinfo +1 -0
package/src/config.ts ADDED
@@ -0,0 +1,696 @@
1
+ /*
2
+ * Copyright 2019 Google LLC
3
+ *
4
+ * Licensed under the Apache License, Version 2.0 (the "License");
5
+ * you may not use this file except in compliance with the License.
6
+ * You may obtain a copy of the License at
7
+ *
8
+ * https://www.apache.org/licenses/LICENSE-2.0
9
+ *
10
+ * Unless required by applicable law or agreed to in writing, software
11
+ * distributed under the License is distributed on an "AS IS" BASIS,
12
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ * See the License for the specific language governing permissions and
14
+ * limitations under the License.
15
+ */
16
+
17
+ import type {
18
+ ChangeTrackerConfig,
19
+ PartitioningFieldType,
20
+ TimePartitioningGranularity,
21
+ } from "@firebaseextensions/firestore-bigquery-change-tracker";
22
+ import { LogLevel } from "@firebaseextensions/firestore-bigquery-change-tracker";
23
+ import type { Expression } from "firebase-functions/params";
24
+ import {
25
+ defineBoolean,
26
+ defineString,
27
+ projectID,
28
+ select,
29
+ } from "firebase-functions/params";
30
+ import type { ExportConfig, ViewType } from "./export-config";
31
+
32
+ type TrackerLogLevel = "debug" | "info" | "warn" | "error" | "silent";
33
+ type ConfigExpression<T extends string | number | boolean> = Expression<T>;
34
+ const DECIMAL_RADIX = 10;
35
+ const DATASET_LOCATION_OPTIONS = [
36
+ "us-central1",
37
+ "us-west4",
38
+ "europe-central2",
39
+ "us-west2",
40
+ "northamerica-northeast1",
41
+ "us-east4",
42
+ "us-west1",
43
+ "us-west3",
44
+ "southamerica-east1",
45
+ "us-east1",
46
+ "europe-west1",
47
+ "europe-north1",
48
+ "europe-west3",
49
+ "europe-west2",
50
+ "europe-west4",
51
+ "europe-west6",
52
+ "asia-east1",
53
+ "asia-east2",
54
+ "asia-southeast2",
55
+ "asia-south1",
56
+ "asia-southeast1",
57
+ "asia-northeast2",
58
+ "asia-northeast3",
59
+ "australia-southeast1",
60
+ "asia-northeast1",
61
+ "us",
62
+ "eu",
63
+ "africa-south1",
64
+ "me-west1",
65
+ "me-central1",
66
+ "me-central2",
67
+ "europe-west12",
68
+ "europe-north2",
69
+ "europe-west9",
70
+ "europe-west8",
71
+ "europe-southwest1",
72
+ "europe-west10",
73
+ "australia-southeast2",
74
+ "asia-south2",
75
+ "northamerica-northeast2",
76
+ "southamerica-west1",
77
+ "northamerica-south1",
78
+ "us-south1",
79
+ "us-east5",
80
+ ] as const;
81
+ const DATABASE_REGION_OPTIONS = [
82
+ "eur3",
83
+ "nam5",
84
+ "nam7",
85
+ "us-central1",
86
+ "us-west1",
87
+ "us-west2",
88
+ "us-west3",
89
+ "us-west4",
90
+ "us-east1",
91
+ "us-east4",
92
+ "us-east5",
93
+ "us-south1",
94
+ "northamerica-northeast1",
95
+ "northamerica-northeast2",
96
+ "northamerica-south1",
97
+ "southamerica-east1",
98
+ "southamerica-west1",
99
+ "europe-west1",
100
+ "europe-west2",
101
+ "europe-west3",
102
+ "europe-west4",
103
+ "europe-west6",
104
+ "europe-west8",
105
+ "europe-west9",
106
+ "europe-west10",
107
+ "europe-west12",
108
+ "europe-southwest1",
109
+ "europe-north1",
110
+ "europe-north2",
111
+ "europe-central2",
112
+ "me-central1",
113
+ "me-central2",
114
+ "me-west1",
115
+ "asia-south1",
116
+ "asia-south2",
117
+ "asia-southeast1",
118
+ "asia-southeast2",
119
+ "asia-east1",
120
+ "asia-east2",
121
+ "asia-northeast1",
122
+ "asia-northeast2",
123
+ "asia-northeast3",
124
+ "australia-southeast1",
125
+ "australia-southeast2",
126
+ "africa-south1",
127
+ ] as const;
128
+ const TABLE_PARTITIONING_OPTIONS = [
129
+ "HOUR",
130
+ "DAY",
131
+ "MONTH",
132
+ "YEAR",
133
+ "NONE",
134
+ ] as const;
135
+ const TIME_PARTITIONING_FIELD_TYPE_OPTIONS = [
136
+ "TIMESTAMP",
137
+ "DATETIME",
138
+ "DATE",
139
+ "omit",
140
+ ] as const;
141
+ const VIEW_TYPE_OPTIONS = [
142
+ "view",
143
+ "materialized_incremental",
144
+ "materialized_non_incremental",
145
+ ] as const;
146
+ const LOG_LEVEL_OPTIONS = ["debug", "info", "warn", "error", "silent"] as const;
147
+ export interface ConfigExpressions {
148
+ collectionPath: ConfigExpression<string>;
149
+ datasetId: ConfigExpression<string>;
150
+ tableId: ConfigExpression<string>;
151
+ location: ConfigExpression<string>;
152
+ database: ConfigExpression<string>;
153
+ }
154
+
155
+ /**
156
+ * Deploy-time parameters. Set these via a `.env` / `.env.<project>` file or the
157
+ * interactive prompts shown by `firebase deploy`.
158
+ *
159
+ * @see https://firebase.google.com/docs/functions/config-env
160
+ */
161
+ const params = {
162
+ bigqueryProjectId: defineString("BIGQUERY_PROJECT_ID", {
163
+ label: "BigQuery Project ID",
164
+ description:
165
+ "Override the default project for BigQuery instance. This can allow updates to be directed to to a BigQuery instance on another GCP project.",
166
+
167
+ default: projectID,
168
+ }),
169
+ database: defineString("DATABASE", {
170
+ label: "Firestore Instance ID",
171
+ description:
172
+ 'The Firestore database to use. Use "(default)" for the default database. You can view your available Firestore databases at https://console.cloud.google.com/firestore/databases.',
173
+ default: "(default)",
174
+ input: { text: { example: "(default)" } },
175
+ }),
176
+ databaseRegion: defineString("DATABASE_REGION", {
177
+ label: "Firestore Instance Location",
178
+ description:
179
+ "Where is the Firestore database located? You can check your current database location at https://console.cloud.google.com/firestore/databases.",
180
+
181
+ input: select({
182
+ "Multi-region (Europe - Belgium and Netherlands)": "eur3",
183
+ "Multi-region (United States)": "nam5",
184
+ "Multi-region (Iowa, North Virginia, and Oklahoma)": "nam7",
185
+ "Iowa (us-central1)": "us-central1",
186
+ "Oregon (us-west1)": "us-west1",
187
+ "Los Angeles (us-west2)": "us-west2",
188
+ "Salt Lake City (us-west3)": "us-west3",
189
+ "Las Vegas (us-west4)": "us-west4",
190
+ "South Carolina (us-east1)": "us-east1",
191
+ "Northern Virginia (us-east4)": "us-east4",
192
+ "Columbus (us-east5)": "us-east5",
193
+ "Dallas (us-south1)": "us-south1",
194
+ "Montreal (northamerica-northeast1)": "northamerica-northeast1",
195
+ "Toronto (northamerica-northeast2)": "northamerica-northeast2",
196
+ "Queretaro (northamerica-south1)": "northamerica-south1",
197
+ "Sao Paulo (southamerica-east1)": "southamerica-east1",
198
+ "Santiago (southamerica-west1)": "southamerica-west1",
199
+ "Belgium (europe-west1)": "europe-west1",
200
+ "London (europe-west2)": "europe-west2",
201
+ "Frankfurt (europe-west3)": "europe-west3",
202
+ "Netherlands (europe-west4)": "europe-west4",
203
+ "Zurich (europe-west6)": "europe-west6",
204
+ "Milan (europe-west8)": "europe-west8",
205
+ "Paris (europe-west9)": "europe-west9",
206
+ "Berlin (europe-west10)": "europe-west10",
207
+ "Turin (europe-west12)": "europe-west12",
208
+ "Madrid (europe-southwest1)": "europe-southwest1",
209
+ "Finland (europe-north1)": "europe-north1",
210
+ "Stockholm (europe-north2)": "europe-north2",
211
+ "Warsaw (europe-central2)": "europe-central2",
212
+ "Doha (me-central1)": "me-central1",
213
+ "Dammam (me-central2)": "me-central2",
214
+ "Tel Aviv (me-west1)": "me-west1",
215
+ "Mumbai (asia-south1)": "asia-south1",
216
+ "Delhi (asia-south2)": "asia-south2",
217
+ "Singapore (asia-southeast1)": "asia-southeast1",
218
+ "Jakarta (asia-southeast2)": "asia-southeast2",
219
+ "Taiwan (asia-east1)": "asia-east1",
220
+ "Hong Kong (asia-east2)": "asia-east2",
221
+ "Tokyo (asia-northeast1)": "asia-northeast1",
222
+ "Osaka (asia-northeast2)": "asia-northeast2",
223
+ "Seoul (asia-northeast3)": "asia-northeast3",
224
+ "Sydney (australia-southeast1)": "australia-southeast1",
225
+ "Melbourne (australia-southeast2)": "australia-southeast2",
226
+ "Johannesburg (africa-south1)": "africa-south1",
227
+ }),
228
+ }),
229
+ collectionPath: defineString("COLLECTION_PATH", {
230
+ label: "Collection path",
231
+ description:
232
+ "What is the path of the collection that you would like to export? You may use `{wildcard}` notation to match a subcollection of all documents in a collection (for example: `chatrooms/{chatid}/posts`). Parent Firestore Document IDs from `{wildcards}` can be returned in `path_params` as a JSON formatted string.",
233
+
234
+ default: "posts",
235
+ input: {
236
+ text: {
237
+ example: "posts",
238
+
239
+ validationRegex: /^[^\/]+(\/[^\/]+\/[^\/]+)*$/,
240
+ validationErrorMessage:
241
+ 'Firestore collection paths must be an odd number of segments separated by slashes, e.g. "path/to/collection".',
242
+ },
243
+ },
244
+ }),
245
+ datasetId: defineString("DATASET_ID", {
246
+ label: "Dataset ID",
247
+ description:
248
+ "What ID would you like to use for your BigQuery dataset? This extension will create the dataset, if it doesn't already exist.",
249
+
250
+ default: "firestore_export",
251
+ input: {
252
+ text: {
253
+ example: "firestore_export",
254
+
255
+ validationRegex: /^[a-zA-Z0-9_]+$/,
256
+ validationErrorMessage:
257
+ "BigQuery dataset IDs must be alphanumeric (plus underscores) and must be no more than 1024 characters.",
258
+ },
259
+ },
260
+ }),
261
+ tableId: defineString("TABLE_ID", {
262
+ label: "Table ID",
263
+ description:
264
+ "What identifying prefix would you like to use for your table and view inside your BigQuery dataset? This extension will create the table and view, if they don't already exist.",
265
+
266
+ default: "posts",
267
+ input: {
268
+ text: {
269
+ example: "posts",
270
+
271
+ validationRegex: /^[a-zA-Z0-9_]+$/,
272
+ validationErrorMessage:
273
+ "BigQuery table IDs must be alphanumeric (plus underscores) and must be no more than 1024 characters.",
274
+ },
275
+ },
276
+ }),
277
+ datasetLocation: defineString("DATASET_LOCATION", {
278
+ label: "BigQuery Dataset location",
279
+ description:
280
+ "Where do you want to deploy the BigQuery dataset created for this extension? For help selecting a location, refer to the [location selection guide](https://cloud.google.com/bigquery/docs/locations).",
281
+
282
+ default: "us",
283
+ input: select({
284
+ "Iowa (us-central1)": "us-central1",
285
+ "Las Vegas (us-west4)": "us-west4",
286
+ "Warsaw (europe-central2)": "europe-central2",
287
+ "Los Angeles (us-west2)": "us-west2",
288
+ "Montreal (northamerica-northeast1)": "northamerica-northeast1",
289
+ "Northern Virginia (us-east4)": "us-east4",
290
+ "Oregon (us-west1)": "us-west1",
291
+ "Salt Lake City (us-west3)": "us-west3",
292
+ "Sao Paulo (southamerica-east1)": "southamerica-east1",
293
+ "South Carolina (us-east1)": "us-east1",
294
+ "Belgium (europe-west1)": "europe-west1",
295
+ "Finland (europe-north1)": "europe-north1",
296
+ "Frankfurt (europe-west3)": "europe-west3",
297
+ "London (europe-west2)": "europe-west2",
298
+ "Netherlands (europe-west4)": "europe-west4",
299
+ "Zurich (europe-west6)": "europe-west6",
300
+ "Taiwan (asia-east1)": "asia-east1",
301
+ "Hong Kong (asia-east2)": "asia-east2",
302
+ "Jakarta (asia-southeast2)": "asia-southeast2",
303
+ "Mumbai (asia-south1)": "asia-south1",
304
+ "Singapore (asia-southeast1)": "asia-southeast1",
305
+ "Osaka (asia-northeast2)": "asia-northeast2",
306
+ "Seoul (asia-northeast3)": "asia-northeast3",
307
+ "Sydney (australia-southeast1)": "australia-southeast1",
308
+ "Tokyo (asia-northeast1)": "asia-northeast1",
309
+ "United States (multi-regional)": "us",
310
+ "Europe (multi-regional)": "eu",
311
+ "Johannesburg (africa-south1)": "africa-south1",
312
+ "Tel Aviv (me-west1)": "me-west1",
313
+ "Doha (me-central1)": "me-central1",
314
+ "Dammam (me-central2)": "me-central2",
315
+ "Zürich (europe-west6)": "europe-west6",
316
+ "Turin (europe-west12)": "europe-west12",
317
+ "Stockholm (europe-north2)": "europe-north2",
318
+ "Paris (europe-west9)": "europe-west9",
319
+ "Milan (europe-west8)": "europe-west8",
320
+ "Madrid (europe-southwest1)": "europe-southwest1",
321
+ "Berlin (europe-west10)": "europe-west10",
322
+ "Melbourne (australia-southeast2)": "australia-southeast2",
323
+ "Delhi (asia-south2)": "asia-south2",
324
+ "Toronto (northamerica-northeast2)": "northamerica-northeast2",
325
+ "Santiago (southamerica-west1)": "southamerica-west1",
326
+ "Mexico (northamerica-south1)": "northamerica-south1",
327
+ "Dallas (us-south1)": "us-south1",
328
+ "Columbus, Ohio (us-east5)": "us-east5",
329
+ }),
330
+ }),
331
+ backupCollection: defineString("BACKUP_COLLECTION", {
332
+ label: "Backup Collection Name",
333
+ description:
334
+ "This (optional) parameter will allow you to specify a collection for which failed BigQuery updates will be written to.",
335
+ default: "",
336
+ }),
337
+ transformFunction: defineString("TRANSFORM_FUNCTION", {
338
+ label: "Transform function URL",
339
+ description:
340
+ "Specify a function URL to call that will transform the payload that will be written to BigQuery. See the pre-install documentation for more details.",
341
+ default: "",
342
+ input: {
343
+ text: {
344
+ example:
345
+ "https://us-west1-my-project-id.cloudfunctions.net/myTransformFunction",
346
+ },
347
+ },
348
+ }),
349
+ tablePartitioning: defineString("TABLE_PARTITIONING", {
350
+ label: "BigQuery SQL table Time Partitioning option type",
351
+ description:
352
+ "This parameter will allow you to partition the BigQuery table and BigQuery view created by the extension based on data ingestion time. You may select the granularity of partitioning based upon one of: HOUR, DAY, MONTH, YEAR. This will generate one partition per day, hour, month or year, respectively.",
353
+
354
+ default: "NONE",
355
+ input: select({
356
+ hour: "HOUR",
357
+ day: "DAY",
358
+ month: "MONTH",
359
+ year: "YEAR",
360
+ none: "NONE",
361
+ }),
362
+ }),
363
+ timePartitioningField: defineString("TIME_PARTITIONING_FIELD", {
364
+ label: "BigQuery Time Partitioning column name",
365
+ description:
366
+ "BigQuery table column/schema field name for TimePartitioning. You can choose schema available as `timestamp` OR a new custom defined column that will be assigned to the selected Firestore Document field below. Defaults to pseudo column _PARTITIONTIME if unspecified. Cannot be changed if Table is already partitioned.",
367
+
368
+ default: "",
369
+ }),
370
+ timePartitioningFieldType: defineString("TIME_PARTITIONING_FIELD_TYPE", {
371
+ label: "BigQuery SQL Time Partitioning table schema field(column) type",
372
+ description:
373
+ "Parameter for BigQuery SQL schema field type for the selected Time Partitioning Firestore Document field option. Cannot be changed if Table is already partitioned.",
374
+
375
+ default: "omit",
376
+ input: select([...TIME_PARTITIONING_FIELD_TYPE_OPTIONS]),
377
+ }),
378
+ timePartitioningFirestoreField: defineString(
379
+ "TIME_PARTITIONING_FIRESTORE_FIELD",
380
+ {
381
+ label:
382
+ "Firestore Document field name for BigQuery SQL Time Partitioning field option",
383
+ description:
384
+ "This parameter will allow you to partition the BigQuery table created by the extension based on the selected Firestore Document field. The Firestore Document field value must be a top-level TIMESTAMP, DATETIME, DATE field BigQuery string format or Firestore timestamp(will be converted to BigQuery TIMESTAMP). Cannot be changed if Table is already partitioned.\n example: `postDate`(Ensure that the Firestore-BigQuery export extension\ncreates the dataset and table before initiating any backfill scripts.\n This step is crucial for the partitioning to function correctly. It is\nessential for the script to insert data into an already partitioned table.)",
385
+ default: "",
386
+ }
387
+ ),
388
+ clustering: defineString("CLUSTERING", {
389
+ label: "BigQuery SQL table clustering",
390
+ description:
391
+ "This parameter allows you to set up clustering for the BigQuery table created by the extension. Specify up to 4 comma-separated fields (for example: `data,document_id,timestamp` - no whitespaces). The order of the specified columns determines the sort order of the data. \nNote: Cluster columns must be top-level, non-repeated columns of one of the following types: BIGNUMERIC, BOOL, DATE, DATETIME, GEOGRAPHY, INT64, NUMERIC, RANGE, STRING, TIMESTAMP. Clustering will not be added if a field with an invalid type is present in this parameter.\nAvailable schema extensions table fields for clustering include: `document_id, document_name, timestamp, event_id, operation, data`.",
392
+
393
+ default: "",
394
+ input: {
395
+ text: {
396
+ example: "data,document_id,timestamp",
397
+
398
+ // Extension regex, with an empty branch added: the param is optional.
399
+ validationRegex: /^(?:[^,\s]+(?:,[^,\s]+){0,3}|)$/,
400
+ validationErrorMessage:
401
+ "No whitespaces. Max 4 fields. e.g. `data,timestamp,event_id,operation`",
402
+ },
403
+ },
404
+ }),
405
+ wildcardIds: defineBoolean("WILDCARD_IDS", {
406
+ label: "Enable Wildcard Column field with Parent Firestore Document IDs",
407
+ description:
408
+ "If enabled, creates a column containing a JSON object of all wildcard ids from a documents path.",
409
+
410
+ default: false,
411
+ }),
412
+ useNewSnapshotQuerySyntax: defineBoolean("USE_NEW_SNAPSHOT_QUERY_SYNTAX", {
413
+ label: "Use new query syntax for snapshots",
414
+ description:
415
+ "If enabled, snapshots will be generated with the new query syntax, which should be more performant, and avoid potential resource limitations.",
416
+
417
+ default: false,
418
+ }),
419
+ excludeOldData: defineBoolean("EXCLUDE_OLD_DATA", {
420
+ label: "Exclude old data payloads",
421
+ description:
422
+ "If enabled, table rows will never contain old data (document snapshot before the Firestore onDocumentUpdate event: `change.before.data()`). The reduction in data should be more performant, and avoid potential resource limitations.",
423
+
424
+ default: false,
425
+ }),
426
+ viewType: defineString("VIEW_TYPE", {
427
+ label: "View Type",
428
+ description:
429
+ "Select the type of view to create in BigQuery. A regular view is a virtual table defined by a SQL query. A materialized view persists the results of a query for faster access, with either incremental or non-incremental updates. Please note that materialized views in this extension come with several important caveats and limitations - carefully review the pre-install documentation before selecting these options to ensure they are appropriate for your use case.",
430
+
431
+ default: "view",
432
+ input: select({
433
+ View: "view",
434
+ "Materialized View (Incremental)": "materialized_incremental",
435
+ "Materialized View (Non-incremental)": "materialized_non_incremental",
436
+ }),
437
+ }),
438
+ maxStaleness: defineString("MAX_STALENESS", {
439
+ label: "Maximum Staleness Duration",
440
+ description:
441
+ "For materialized views only: Specifies the maximum staleness acceptable for the materialized view. Should be specified as an INTERVAL value following BigQuery SQL syntax. This parameter will only take effect if View Type is set to a materialized view option.",
442
+ default: "",
443
+ input: { text: { example: 'INTERVAL "8:0:0" HOUR TO SECOND' } },
444
+ }),
445
+ refreshIntervalMinutes: defineString("REFRESH_INTERVAL_MINUTES", {
446
+ label: "Refresh Interval (Minutes)",
447
+ description:
448
+ "For materialized views only: Specifies how often the materialized view should be refreshed, in minutes. This parameter will only take effect if View Type is set to a materialized view option.",
449
+
450
+ default: "",
451
+ input: {
452
+ text: {
453
+ example: "60",
454
+
455
+ // Extension regex, with an empty branch added: the param is optional.
456
+ validationRegex: /^(?:[1-9][0-9]*|)$/,
457
+ validationErrorMessage: "Must be a positive integer",
458
+ },
459
+ },
460
+ }),
461
+ kmsKeyName: defineString("KMS_KEY_NAME", {
462
+ label: "Cloud KMS key name",
463
+ description:
464
+ "Instead of Google managing the key encryption keys that protect your data, you control and manage key encryption keys in Cloud KMS. If this parameter is set, the extension will specify the KMS key name when creating the BQ table. See the PREINSTALL.md for more details.",
465
+
466
+ default: "",
467
+ input: {
468
+ text: {
469
+ // Extension regex (unanchored, as upstream), with an empty branch
470
+ // added: the param is optional.
471
+ validationRegex:
472
+ /projects\/([^\/]+)\/locations\/([^\/]+)\/keyRings\/([^\/]+)\/cryptoKeys\/([^\/]+)|^$/,
473
+ validationErrorMessage:
474
+ "The key name must be of the format 'projects/PROJECT_NAME/locations/KEY_RING_LOCATION/keyRings/KEY_RING_ID/cryptoKeys/KEY_ID'.",
475
+ },
476
+ },
477
+ }),
478
+ logLevel: defineString("LOG_LEVEL", {
479
+ label: "Log level",
480
+ description:
481
+ "The log level for the extension. The log level controls the verbosity of the extension's logs. The available log levels are: debug, info, warn, and error. To reduce the volume of logs, use a log level of warn or error.",
482
+
483
+ default: "info",
484
+ input: select({
485
+ Debug: "debug",
486
+ Info: "info",
487
+ Warn: "warn",
488
+ Error: "error",
489
+ Silent: "silent",
490
+ }),
491
+ }),
492
+ };
493
+
494
+ export const CONFIG_EXPRESSIONS: ConfigExpressions = {
495
+ collectionPath: params.collectionPath,
496
+ datasetId: params.datasetId,
497
+ tableId: params.tableId,
498
+ location: params.databaseRegion,
499
+ database: params.database,
500
+ };
501
+
502
+ function timePartitioning(
503
+ type: string | undefined
504
+ ): TimePartitioningGranularity | null {
505
+ if (
506
+ type === "HOUR" ||
507
+ type === "DAY" ||
508
+ type === "MONTH" ||
509
+ type === "YEAR"
510
+ ) {
511
+ return type;
512
+ }
513
+
514
+ return null;
515
+ }
516
+
517
+ export function clustering(clusters: string | undefined) {
518
+ return clusters ? clusters.split(",").slice(0, 4) : null;
519
+ }
520
+
521
+ function normalizeOptionalPartitionValue(
522
+ value: string | undefined
523
+ ): string | undefined {
524
+ const normalized = value?.trim();
525
+
526
+ if (!normalized || normalized === "NONE" || normalized === "omit") {
527
+ return undefined;
528
+ }
529
+
530
+ return normalized;
531
+ }
532
+
533
+ function normalizePartitionFieldType(
534
+ value: string | undefined
535
+ ): PartitioningFieldType | undefined {
536
+ const normalized = normalizeOptionalPartitionValue(value);
537
+ if (
538
+ normalized === "TIMESTAMP" ||
539
+ normalized === "DATE" ||
540
+ normalized === "DATETIME"
541
+ ) {
542
+ return normalized;
543
+ }
544
+ return undefined;
545
+ }
546
+
547
+ export function buildPartitioningConfig(params: {
548
+ timePartitioning: TimePartitioningGranularity | null;
549
+ timePartitioningField: string | undefined;
550
+ timePartitioningFieldType: string | undefined;
551
+ timePartitioningFirestoreField: string | undefined;
552
+ }): ChangeTrackerConfig["partitioning"] {
553
+ const { timePartitioning } = params;
554
+ const rawFieldName = params.timePartitioningField?.trim();
555
+ const rawFieldType = params.timePartitioningFieldType?.trim();
556
+ const rawFirestoreField = params.timePartitioningFirestoreField?.trim();
557
+
558
+ const formatValue = (value: string | undefined): string =>
559
+ value && value.length > 0 ? `"${value}"` : "(empty)";
560
+
561
+ const throwInvalidPartitioningConfig = (detail: string): never => {
562
+ throw new Error(
563
+ [
564
+ "Invalid partitioning configuration for firestore-bigquery-export.",
565
+ detail,
566
+ `Received TABLE_PARTITIONING=${formatValue(
567
+ timePartitioning ?? undefined
568
+ )},`,
569
+ `TIME_PARTITIONING_FIELD=${formatValue(rawFieldName)},`,
570
+ `TIME_PARTITIONING_FIRESTORE_FIELD=${formatValue(rawFirestoreField)},`,
571
+ `TIME_PARTITIONING_FIELD_TYPE=${formatValue(rawFieldType)}.`,
572
+ "Valid combinations are:",
573
+ "1) Ingestion-time: TABLE_PARTITIONING set and all TIME_PARTITIONING_* values empty/NONE/omit.",
574
+ "2) Timestamp field: TABLE_PARTITIONING set, TIME_PARTITIONING_FIELD=timestamp, TIME_PARTITIONING_FIRESTORE_FIELD empty.",
575
+ "3) Custom field: TABLE_PARTITIONING set, and TIME_PARTITIONING_FIELD + TIME_PARTITIONING_FIRESTORE_FIELD + TIME_PARTITIONING_FIELD_TYPE all provided.",
576
+ ].join(" ")
577
+ );
578
+ };
579
+
580
+ const fieldName = normalizeOptionalPartitionValue(
581
+ params.timePartitioningField
582
+ );
583
+ const fieldType = normalizePartitionFieldType(
584
+ params.timePartitioningFieldType
585
+ );
586
+ const firestoreField = normalizeOptionalPartitionValue(
587
+ params.timePartitioningFirestoreField
588
+ );
589
+
590
+ if (!timePartitioning) {
591
+ if (fieldName || fieldType || firestoreField) {
592
+ return throwInvalidPartitioningConfig(
593
+ "Partition-specific fields cannot be provided when TABLE_PARTITIONING is NONE."
594
+ );
595
+ }
596
+ return { granularity: "NONE" };
597
+ }
598
+
599
+ if (!fieldName && !firestoreField) {
600
+ return { granularity: timePartitioning };
601
+ }
602
+
603
+ if (fieldName === "timestamp" && !firestoreField) {
604
+ return {
605
+ granularity: timePartitioning,
606
+ bigqueryColumnName: "timestamp",
607
+ ...(fieldType ? { bigqueryColumnType: fieldType } : {}),
608
+ };
609
+ }
610
+
611
+ if (fieldName && firestoreField && fieldType) {
612
+ return {
613
+ granularity: timePartitioning,
614
+ bigqueryColumnName: fieldName,
615
+ bigqueryColumnType: fieldType,
616
+ firestoreFieldName: firestoreField,
617
+ };
618
+ }
619
+
620
+ return throwInvalidPartitioningConfig(
621
+ "When TABLE_PARTITIONING is set, partitioning fields are either incomplete or invalid."
622
+ );
623
+ }
624
+
625
+ function normalizeLogLevel(level: string | undefined): TrackerLogLevel {
626
+ switch ((level || "").toLowerCase()) {
627
+ case "debug":
628
+ return "debug";
629
+ case "info":
630
+ return "info";
631
+ case "warn":
632
+ return "warn";
633
+ case "error":
634
+ return "error";
635
+ case "silent":
636
+ return "silent";
637
+ default:
638
+ return LogLevel.INFO;
639
+ }
640
+ }
641
+
642
+ function normalizePositiveInt(value: string): number | undefined {
643
+ const normalized = Number.parseInt(value, DECIMAL_RADIX);
644
+ return normalized > 0 ? normalized : undefined;
645
+ }
646
+
647
+ /** Coerce an empty-string param value to `undefined`. */
648
+ function optional(value: string): string | undefined {
649
+ return value.length > 0 ? value : undefined;
650
+ }
651
+
652
+ /**
653
+ * Resolves all deploy-time params into an {@link ExportConfig}.
654
+ *
655
+ * Param values are read when this is called. During the Firebase deploy-time
656
+ * discovery pass params return their declared defaults, so the default
657
+ * `TABLE_PARTITIONING=NONE` keeps {@link buildPartitioningConfig} from throwing.
658
+ * This is the bridge for the env-driven path: env params in, typed config out,
659
+ * which the main entry point wires into the exported functions.
660
+ *
661
+ * @returns The export configuration assembled from environment params.
662
+ */
663
+ export function configFromEnv(): ExportConfig {
664
+ const tablePartitioning = optional(params.tablePartitioning.value());
665
+
666
+ return {
667
+ collectionPath: params.collectionPath.value(),
668
+ datasetId: params.datasetId.value(),
669
+ tableId: params.tableId.value(),
670
+ location: params.databaseRegion.value(),
671
+ datasetLocation: optional(params.datasetLocation.value()),
672
+ bqProjectId: optional(params.bigqueryProjectId.value()),
673
+ projectId: projectID.value(),
674
+ databaseId: optional(params.database.value()) || "(default)",
675
+ wildcardIds: params.wildcardIds.value(),
676
+ excludeOldData: params.excludeOldData.value(),
677
+ useNewSnapshotQuerySyntax: params.useNewSnapshotQuerySyntax.value(),
678
+ viewType: (optional(params.viewType.value()) || "view") as ViewType,
679
+ partitioning: buildPartitioningConfig({
680
+ timePartitioning: timePartitioning(tablePartitioning),
681
+ timePartitioningField: params.timePartitioningField.value(),
682
+ timePartitioningFieldType: params.timePartitioningFieldType.value(),
683
+ timePartitioningFirestoreField:
684
+ params.timePartitioningFirestoreField.value(),
685
+ }),
686
+ clustering: clustering(optional(params.clustering.value())),
687
+ maxStaleness: optional(params.maxStaleness.value()),
688
+ refreshIntervalMinutes: normalizePositiveInt(
689
+ params.refreshIntervalMinutes.value()
690
+ ),
691
+ backupCollectionId: optional(params.backupCollection.value()),
692
+ transformFunction: optional(params.transformFunction.value()),
693
+ kmsKeyName: optional(params.kmsKeyName.value()),
694
+ logLevel: normalizeLogLevel(params.logLevel.value()),
695
+ };
696
+ }