@firebase-function-kits/firestore-bigquery-export 0.0.1 → 0.0.2-rc.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. package/CHANGELOG.md +1 -0
  2. package/README.md +234 -0
  3. package/lib/config.d.ts +33 -0
  4. package/lib/config.d.ts.map +1 -0
  5. package/lib/config.js +345 -0
  6. package/lib/config.js.map +1 -0
  7. package/lib/events.d.ts +45 -0
  8. package/lib/events.d.ts.map +1 -0
  9. package/lib/events.js +154 -0
  10. package/lib/events.js.map +1 -0
  11. package/lib/export-config.d.ts +96 -0
  12. package/lib/export-config.d.ts.map +1 -0
  13. package/lib/export-config.js +77 -0
  14. package/lib/export-config.js.map +1 -0
  15. package/lib/handlers.d.ts +29 -0
  16. package/lib/handlers.d.ts.map +1 -0
  17. package/lib/handlers.js +165 -0
  18. package/lib/handlers.js.map +1 -0
  19. package/lib/index.d.ts +23 -0
  20. package/lib/index.d.ts.map +1 -0
  21. package/lib/index.js +179 -0
  22. package/lib/index.js.map +1 -0
  23. package/lib/init.d.ts +16 -0
  24. package/lib/init.d.ts.map +1 -0
  25. package/lib/init.js +44 -0
  26. package/lib/init.js.map +1 -0
  27. package/lib/lib.d.ts +20 -0
  28. package/lib/lib.d.ts.map +1 -0
  29. package/lib/lib.js +47 -0
  30. package/lib/lib.js.map +1 -0
  31. package/lib/logs.d.ts +39 -0
  32. package/lib/logs.d.ts.map +1 -0
  33. package/lib/logs.js +186 -0
  34. package/lib/logs.js.map +1 -0
  35. package/lib/util.d.ts +23 -0
  36. package/lib/util.d.ts.map +1 -0
  37. package/lib/util.js +66 -0
  38. package/lib/util.js.map +1 -0
  39. package/package.json +33 -3
  40. package/src/config.ts +420 -0
  41. package/src/events.ts +146 -0
  42. package/src/export-config.ts +205 -0
  43. package/src/handlers.ts +211 -0
  44. package/src/index.ts +174 -0
  45. package/src/init.ts +45 -0
  46. package/src/lib.ts +55 -0
  47. package/src/logs.ts +233 -0
  48. package/src/util.ts +64 -0
  49. package/tests/config.test.ts +182 -0
  50. package/tests/events.test.ts +84 -0
  51. package/tests/export-config.test.ts +114 -0
  52. package/tests/handlers.test.ts +220 -0
  53. package/tests/init.test.ts +76 -0
  54. package/tests/util.test.ts +102 -0
  55. package/tsconfig.json +18 -0
  56. package/tsconfig.tsbuildinfo +1 -0
package/lib/util.js ADDED
@@ -0,0 +1,66 @@
1
+ "use strict";
2
+ /*
3
+ * Copyright 2019 Google LLC
4
+ *
5
+ * Licensed under the Apache License, Version 2.0 (the "License");
6
+ * you may not use this file except in compliance with the License.
7
+ * You may obtain a copy of the License at
8
+ *
9
+ * https://www.apache.org/licenses/LICENSE-2.0
10
+ *
11
+ * Unless required by applicable law or agreed to in writing, software
12
+ * distributed under the License is distributed on an "AS IS" BASIS,
13
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
+ * See the License for the specific language governing permissions and
15
+ * limitations under the License.
16
+ */
17
+ Object.defineProperty(exports, "__esModule", { value: true });
18
+ exports.resolveWildcardIds = void 0;
19
+ exports.getChangeType = getChangeType;
20
+ exports.getDocumentId = getDocumentId;
21
+ const firestore_bigquery_change_tracker_1 = require("@firebaseextensions/firestore-bigquery-change-tracker");
22
+ /**
23
+ * Get the change type (CREATE, UPDATE, DELETE) from the Firestore change.
24
+ * @param change Firestore document change object.
25
+ * @returns {ChangeType} The type of change.
26
+ */
27
+ function getChangeType(change) {
28
+ if (!change.after.exists) {
29
+ return firestore_bigquery_change_tracker_1.ChangeType.DELETE;
30
+ }
31
+ if (!change.before.exists) {
32
+ return firestore_bigquery_change_tracker_1.ChangeType.CREATE;
33
+ }
34
+ return firestore_bigquery_change_tracker_1.ChangeType.UPDATE;
35
+ }
36
+ /**
37
+ * Get the document ID from the Firestore change.
38
+ * @param change Firestore document change object.
39
+ * @returns {string} The document ID.
40
+ */
41
+ function getDocumentId(change) {
42
+ if (change.after.exists) {
43
+ return change.after.id;
44
+ }
45
+ return change.before.id;
46
+ }
47
+ /**
48
+ *
49
+ * @param template - eg, regions/{regionId}/countries
50
+ * @param text - eg, regions/asia/countries
51
+ *
52
+ * @return - eg, { regionId: "asia" }
53
+ */
54
+ const resolveWildcardIds = (template, text) => {
55
+ const textSegments = text.split("/");
56
+ return template
57
+ .split("/")
58
+ .reduce((previousValue, currentValue, currentIndex) => {
59
+ if (currentValue.startsWith("{") && currentValue.endsWith("}")) {
60
+ previousValue[currentValue.slice(1, -1)] = textSegments[currentIndex];
61
+ }
62
+ return previousValue;
63
+ }, {});
64
+ };
65
+ exports.resolveWildcardIds = resolveWildcardIds;
66
+ //# sourceMappingURL=util.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"util.js","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;GAcG;;;;;AAEH,6GAAmF;AAGnF;;;;GAIG;AACH,uBAA8B,MAAgC;IAC5D,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;QACzB,OAAO,8CAAU,CAAC,MAAM,CAAC;IAC3B,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QAC1B,OAAO,8CAAU,CAAC,MAAM,CAAC;IAC3B,CAAC;IACD,OAAO,8CAAU,CAAC,MAAM,CAAC;AAC3B,CAAC;AAED;;;;GAIG;AACH,uBAA8B,MAAgC;IAC5D,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;QACxB,OAAO,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;IACzB,CAAC;IACD,OAAO,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;AAC1B,CAAC;AAED;;;;;;GAMG;AACI,MAAM,kBAAkB,GAAG,CAAC,QAAgB,EAAE,IAAY,EAAE,EAAE;IACnE,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACrC,OAAO,QAAQ;SACZ,KAAK,CAAC,GAAG,CAAC;SACV,MAAM,CAAC,CAAC,aAAa,EAAE,YAAY,EAAE,YAAY,EAAE,EAAE;QACpD,IAAI,YAAY,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YAC/D,aAAa,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,YAAY,CAAC,YAAY,CAAC,CAAC;QACxE,CAAC;QACD,OAAO,aAAa,CAAC;IACvB,CAAC,EAAE,EAAE,CAAC,CAAC;AACX,CAAC,CAAC;AAVW,QAAA,kBAAkB,GAAlB,kBAAkB,CAU7B"}
package/package.json CHANGED
@@ -1,11 +1,41 @@
1
1
  {
2
2
  "name": "@firebase-function-kits/firestore-bigquery-export",
3
- "version": "0.0.1",
3
+ "version": "0.0.2-rc.0",
4
+ "repository": "firebase/extensions",
4
5
  "description": "Stream a Cloud Firestore collection to BigQuery as a deployable Firebase Function",
5
6
  "license": "Apache-2.0",
6
- "main": "index.js",
7
- "repository": "firebase/extensions",
7
+ "main": "lib/index.js",
8
+ "types": "lib/index.d.ts",
9
+ "exports": {
10
+ ".": {
11
+ "types": "./lib/index.d.ts",
12
+ "default": "./lib/index.js"
13
+ },
14
+ "./lib": {
15
+ "types": "./lib/lib.d.ts",
16
+ "default": "./lib/lib.js"
17
+ }
18
+ },
8
19
  "engines": {
9
20
  "node": ">=22"
21
+ },
22
+ "scripts": {
23
+ "build": "tsc -b",
24
+ "clean": "tsc -b --clean",
25
+ "test": "vitest run",
26
+ "deploy": "pnpm build && firebase deploy --only functions",
27
+ "serve": "firebase emulators:start --only functions"
28
+ },
29
+ "dependencies": {
30
+ "@firebaseextensions/firestore-bigquery-change-tracker": "^2.0.4",
31
+ "@google-cloud/bigquery": "^7.6.0",
32
+ "firebase-admin": "^14.1.0",
33
+ "firebase-functions": "7.3.2",
34
+ "generate-schema": "^2.6.0",
35
+ "lodash": "^4.17.14"
36
+ },
37
+ "devDependencies": {
38
+ "@types/lodash": "^4.17.0",
39
+ "vitest": "^3.2.4"
10
40
  }
11
41
  }
package/src/config.ts ADDED
@@ -0,0 +1,420 @@
1
+ /*
2
+ * Copyright 2019 Google LLC
3
+ *
4
+ * Licensed under the Apache License, Version 2.0 (the "License");
5
+ * you may not use this file except in compliance with the License.
6
+ * You may obtain a copy of the License at
7
+ *
8
+ * https://www.apache.org/licenses/LICENSE-2.0
9
+ *
10
+ * Unless required by applicable law or agreed to in writing, software
11
+ * distributed under the License is distributed on an "AS IS" BASIS,
12
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ * See the License for the specific language governing permissions and
14
+ * limitations under the License.
15
+ */
16
+
17
+ import type {
18
+ ChangeTrackerConfig,
19
+ PartitioningFieldType,
20
+ TimePartitioningGranularity,
21
+ } from "@firebaseextensions/firestore-bigquery-change-tracker";
22
+ import { LogLevel } from "@firebaseextensions/firestore-bigquery-change-tracker";
23
+ import type { Expression } from "firebase-functions/params";
24
+ import {
25
+ defineBoolean,
26
+ defineString,
27
+ projectID,
28
+ select,
29
+ } from "firebase-functions/params";
30
+ import type { ExportConfig, ViewType } from "./export-config";
31
+
32
+ type TrackerLogLevel = "debug" | "info" | "warn" | "error" | "silent";
33
+ type ConfigExpression<T extends string | number | boolean> = Expression<T>;
34
+ const DECIMAL_RADIX = 10;
35
+ const DATASET_LOCATION_OPTIONS = [
36
+ "us-central1",
37
+ "us-west4",
38
+ "europe-central2",
39
+ "us-west2",
40
+ "northamerica-northeast1",
41
+ "us-east4",
42
+ "us-west1",
43
+ "us-west3",
44
+ "southamerica-east1",
45
+ "us-east1",
46
+ "europe-west1",
47
+ "europe-north1",
48
+ "europe-west3",
49
+ "europe-west2",
50
+ "europe-west4",
51
+ "europe-west6",
52
+ "asia-east1",
53
+ "asia-east2",
54
+ "asia-southeast2",
55
+ "asia-south1",
56
+ "asia-southeast1",
57
+ "asia-northeast2",
58
+ "asia-northeast3",
59
+ "australia-southeast1",
60
+ "asia-northeast1",
61
+ "us",
62
+ "eu",
63
+ "africa-south1",
64
+ "me-west1",
65
+ "me-central1",
66
+ "me-central2",
67
+ "europe-west12",
68
+ "europe-north2",
69
+ "europe-west9",
70
+ "europe-west8",
71
+ "europe-southwest1",
72
+ "europe-west10",
73
+ "australia-southeast2",
74
+ "asia-south2",
75
+ "northamerica-northeast2",
76
+ "southamerica-west1",
77
+ "northamerica-south1",
78
+ "us-south1",
79
+ "us-east5",
80
+ ] as const;
81
+ const DATABASE_REGION_OPTIONS = [
82
+ "eur3",
83
+ "nam5",
84
+ "nam7",
85
+ "us-central1",
86
+ "us-west1",
87
+ "us-west2",
88
+ "us-west3",
89
+ "us-west4",
90
+ "us-east1",
91
+ "us-east4",
92
+ "us-east5",
93
+ "us-south1",
94
+ "northamerica-northeast1",
95
+ "northamerica-northeast2",
96
+ "northamerica-south1",
97
+ "southamerica-east1",
98
+ "southamerica-west1",
99
+ "europe-west1",
100
+ "europe-west2",
101
+ "europe-west3",
102
+ "europe-west4",
103
+ "europe-west6",
104
+ "europe-west8",
105
+ "europe-west9",
106
+ "europe-west10",
107
+ "europe-west12",
108
+ "europe-southwest1",
109
+ "europe-north1",
110
+ "europe-north2",
111
+ "europe-central2",
112
+ "me-central1",
113
+ "me-central2",
114
+ "me-west1",
115
+ "asia-south1",
116
+ "asia-south2",
117
+ "asia-southeast1",
118
+ "asia-southeast2",
119
+ "asia-east1",
120
+ "asia-east2",
121
+ "asia-northeast1",
122
+ "asia-northeast2",
123
+ "asia-northeast3",
124
+ "australia-southeast1",
125
+ "australia-southeast2",
126
+ "africa-south1",
127
+ ] as const;
128
+ const TABLE_PARTITIONING_OPTIONS = [
129
+ "HOUR",
130
+ "DAY",
131
+ "MONTH",
132
+ "YEAR",
133
+ "NONE",
134
+ ] as const;
135
+ const TIME_PARTITIONING_FIELD_TYPE_OPTIONS = [
136
+ "TIMESTAMP",
137
+ "DATETIME",
138
+ "DATE",
139
+ "omit",
140
+ ] as const;
141
+ const VIEW_TYPE_OPTIONS = [
142
+ "view",
143
+ "materialized_incremental",
144
+ "materialized_non_incremental",
145
+ ] as const;
146
+ const LOG_LEVEL_OPTIONS = ["debug", "info", "warn", "error", "silent"] as const;
147
+ export interface ConfigExpressions {
148
+ collectionPath: ConfigExpression<string>;
149
+ datasetId: ConfigExpression<string>;
150
+ tableId: ConfigExpression<string>;
151
+ location: ConfigExpression<string>;
152
+ database: ConfigExpression<string>;
153
+ }
154
+
155
+ /**
156
+ * Deploy-time parameters. Set these via a `.env` / `.env.<project>` file or the
157
+ * interactive prompts shown by `firebase deploy`.
158
+ *
159
+ * @see https://firebase.google.com/docs/functions/config-env
160
+ */
161
+ const params = {
162
+ bigqueryProjectId: defineString("BIGQUERY_PROJECT_ID", {
163
+ default: projectID,
164
+ }),
165
+ database: defineString("DATABASE", { default: "(default)" }),
166
+ databaseRegion: defineString("DATABASE_REGION", {
167
+ input: select([...DATABASE_REGION_OPTIONS]),
168
+ }),
169
+ collectionPath: defineString("COLLECTION_PATH", { default: "posts" }),
170
+ datasetId: defineString("DATASET_ID", { default: "firestore_export" }),
171
+ tableId: defineString("TABLE_ID", { default: "posts" }),
172
+ datasetLocation: defineString("DATASET_LOCATION", {
173
+ default: "us",
174
+ input: select([...DATASET_LOCATION_OPTIONS]),
175
+ }),
176
+ backupCollection: defineString("BACKUP_COLLECTION", { default: "" }),
177
+ transformFunction: defineString("TRANSFORM_FUNCTION", { default: "" }),
178
+ tablePartitioning: defineString("TABLE_PARTITIONING", {
179
+ default: "NONE",
180
+ input: select([...TABLE_PARTITIONING_OPTIONS]),
181
+ }),
182
+ timePartitioningField: defineString("TIME_PARTITIONING_FIELD", {
183
+ default: "",
184
+ }),
185
+ timePartitioningFieldType: defineString("TIME_PARTITIONING_FIELD_TYPE", {
186
+ default: "omit",
187
+ input: select([...TIME_PARTITIONING_FIELD_TYPE_OPTIONS]),
188
+ }),
189
+ timePartitioningFirestoreField: defineString(
190
+ "TIME_PARTITIONING_FIRESTORE_FIELD",
191
+ { default: "" }
192
+ ),
193
+ clustering: defineString("CLUSTERING", { default: "" }),
194
+ wildcardIds: defineBoolean("WILDCARD_IDS", {
195
+ default: false,
196
+ }),
197
+ useNewSnapshotQuerySyntax: defineBoolean("USE_NEW_SNAPSHOT_QUERY_SYNTAX", {
198
+ default: false,
199
+ }),
200
+ excludeOldData: defineBoolean("EXCLUDE_OLD_DATA", {
201
+ default: false,
202
+ }),
203
+ viewType: defineString("VIEW_TYPE", {
204
+ default: "view",
205
+ input: select([...VIEW_TYPE_OPTIONS]),
206
+ }),
207
+ maxStaleness: defineString("MAX_STALENESS", { default: "" }),
208
+ refreshIntervalMinutes: defineString("REFRESH_INTERVAL_MINUTES", {
209
+ default: "",
210
+ }),
211
+ kmsKeyName: defineString("KMS_KEY_NAME", { default: "" }),
212
+ logLevel: defineString("LOG_LEVEL", {
213
+ default: "info",
214
+ input: select([...LOG_LEVEL_OPTIONS]),
215
+ }),
216
+ };
217
+
218
+ export const CONFIG_EXPRESSIONS: ConfigExpressions = {
219
+ collectionPath: params.collectionPath,
220
+ datasetId: params.datasetId,
221
+ tableId: params.tableId,
222
+ location: params.databaseRegion,
223
+ database: params.database,
224
+ };
225
+
226
+ function timePartitioning(
227
+ type: string | undefined
228
+ ): TimePartitioningGranularity | null {
229
+ if (
230
+ type === "HOUR" ||
231
+ type === "DAY" ||
232
+ type === "MONTH" ||
233
+ type === "YEAR"
234
+ ) {
235
+ return type;
236
+ }
237
+
238
+ return null;
239
+ }
240
+
241
+ export function clustering(clusters: string | undefined) {
242
+ return clusters ? clusters.split(",").slice(0, 4) : null;
243
+ }
244
+
245
+ function normalizeOptionalPartitionValue(
246
+ value: string | undefined
247
+ ): string | undefined {
248
+ const normalized = value?.trim();
249
+
250
+ if (!normalized || normalized === "NONE" || normalized === "omit") {
251
+ return undefined;
252
+ }
253
+
254
+ return normalized;
255
+ }
256
+
257
+ function normalizePartitionFieldType(
258
+ value: string | undefined
259
+ ): PartitioningFieldType | undefined {
260
+ const normalized = normalizeOptionalPartitionValue(value);
261
+ if (
262
+ normalized === "TIMESTAMP" ||
263
+ normalized === "DATE" ||
264
+ normalized === "DATETIME"
265
+ ) {
266
+ return normalized;
267
+ }
268
+ return undefined;
269
+ }
270
+
271
+ export function buildPartitioningConfig(params: {
272
+ timePartitioning: TimePartitioningGranularity | null;
273
+ timePartitioningField: string | undefined;
274
+ timePartitioningFieldType: string | undefined;
275
+ timePartitioningFirestoreField: string | undefined;
276
+ }): ChangeTrackerConfig["partitioning"] {
277
+ const { timePartitioning } = params;
278
+ const rawFieldName = params.timePartitioningField?.trim();
279
+ const rawFieldType = params.timePartitioningFieldType?.trim();
280
+ const rawFirestoreField = params.timePartitioningFirestoreField?.trim();
281
+
282
+ const formatValue = (value: string | undefined): string =>
283
+ value && value.length > 0 ? `"${value}"` : "(empty)";
284
+
285
+ const throwInvalidPartitioningConfig = (detail: string): never => {
286
+ throw new Error(
287
+ [
288
+ "Invalid partitioning configuration for firestore-bigquery-export.",
289
+ detail,
290
+ `Received TABLE_PARTITIONING=${formatValue(
291
+ timePartitioning ?? undefined
292
+ )},`,
293
+ `TIME_PARTITIONING_FIELD=${formatValue(rawFieldName)},`,
294
+ `TIME_PARTITIONING_FIRESTORE_FIELD=${formatValue(rawFirestoreField)},`,
295
+ `TIME_PARTITIONING_FIELD_TYPE=${formatValue(rawFieldType)}.`,
296
+ "Valid combinations are:",
297
+ "1) Ingestion-time: TABLE_PARTITIONING set and all TIME_PARTITIONING_* values empty/NONE/omit.",
298
+ "2) Timestamp field: TABLE_PARTITIONING set, TIME_PARTITIONING_FIELD=timestamp, TIME_PARTITIONING_FIRESTORE_FIELD empty.",
299
+ "3) Custom field: TABLE_PARTITIONING set, and TIME_PARTITIONING_FIELD + TIME_PARTITIONING_FIRESTORE_FIELD + TIME_PARTITIONING_FIELD_TYPE all provided.",
300
+ ].join(" ")
301
+ );
302
+ };
303
+
304
+ const fieldName = normalizeOptionalPartitionValue(
305
+ params.timePartitioningField
306
+ );
307
+ const fieldType = normalizePartitionFieldType(
308
+ params.timePartitioningFieldType
309
+ );
310
+ const firestoreField = normalizeOptionalPartitionValue(
311
+ params.timePartitioningFirestoreField
312
+ );
313
+
314
+ if (!timePartitioning) {
315
+ if (fieldName || fieldType || firestoreField) {
316
+ return throwInvalidPartitioningConfig(
317
+ "Partition-specific fields cannot be provided when TABLE_PARTITIONING is NONE."
318
+ );
319
+ }
320
+ return { granularity: "NONE" };
321
+ }
322
+
323
+ if (!fieldName && !firestoreField) {
324
+ return { granularity: timePartitioning };
325
+ }
326
+
327
+ if (fieldName === "timestamp" && !firestoreField) {
328
+ return {
329
+ granularity: timePartitioning,
330
+ bigqueryColumnName: "timestamp",
331
+ ...(fieldType ? { bigqueryColumnType: fieldType } : {}),
332
+ };
333
+ }
334
+
335
+ if (fieldName && firestoreField && fieldType) {
336
+ return {
337
+ granularity: timePartitioning,
338
+ bigqueryColumnName: fieldName,
339
+ bigqueryColumnType: fieldType,
340
+ firestoreFieldName: firestoreField,
341
+ };
342
+ }
343
+
344
+ return throwInvalidPartitioningConfig(
345
+ "When TABLE_PARTITIONING is set, partitioning fields are either incomplete or invalid."
346
+ );
347
+ }
348
+
349
+ function normalizeLogLevel(level: string | undefined): TrackerLogLevel {
350
+ switch ((level || "").toLowerCase()) {
351
+ case "debug":
352
+ return "debug";
353
+ case "info":
354
+ return "info";
355
+ case "warn":
356
+ return "warn";
357
+ case "error":
358
+ return "error";
359
+ case "silent":
360
+ return "silent";
361
+ default:
362
+ return LogLevel.INFO;
363
+ }
364
+ }
365
+
366
+ function normalizePositiveInt(value: string): number | undefined {
367
+ const normalized = Number.parseInt(value, DECIMAL_RADIX);
368
+ return normalized > 0 ? normalized : undefined;
369
+ }
370
+
371
+ /** Coerce an empty-string param value to `undefined`. */
372
+ function optional(value: string): string | undefined {
373
+ return value.length > 0 ? value : undefined;
374
+ }
375
+
376
+ /**
377
+ * Resolves all deploy-time params into an {@link ExportConfig}.
378
+ *
379
+ * Param values are read when this is called. During the Firebase deploy-time
380
+ * discovery pass params return their declared defaults, so the default
381
+ * `TABLE_PARTITIONING=NONE` keeps {@link buildPartitioningConfig} from throwing.
382
+ * This is the bridge for the env-driven path: env params in, typed config out,
383
+ * which the main entry point wires into the exported functions.
384
+ *
385
+ * @returns The export configuration assembled from environment params.
386
+ */
387
+ export function configFromEnv(): ExportConfig {
388
+ const tablePartitioning = optional(params.tablePartitioning.value());
389
+
390
+ return {
391
+ collectionPath: params.collectionPath.value(),
392
+ datasetId: params.datasetId.value(),
393
+ tableId: params.tableId.value(),
394
+ location: params.databaseRegion.value(),
395
+ datasetLocation: optional(params.datasetLocation.value()),
396
+ bqProjectId: optional(params.bigqueryProjectId.value()),
397
+ projectId: projectID.value(),
398
+ databaseId: optional(params.database.value()) || "(default)",
399
+ wildcardIds: params.wildcardIds.value(),
400
+ excludeOldData: params.excludeOldData.value(),
401
+ useNewSnapshotQuerySyntax: params.useNewSnapshotQuerySyntax.value(),
402
+ viewType: (optional(params.viewType.value()) || "view") as ViewType,
403
+ partitioning: buildPartitioningConfig({
404
+ timePartitioning: timePartitioning(tablePartitioning),
405
+ timePartitioningField: params.timePartitioningField.value(),
406
+ timePartitioningFieldType: params.timePartitioningFieldType.value(),
407
+ timePartitioningFirestoreField:
408
+ params.timePartitioningFirestoreField.value(),
409
+ }),
410
+ clustering: clustering(optional(params.clustering.value())),
411
+ maxStaleness: optional(params.maxStaleness.value()),
412
+ refreshIntervalMinutes: normalizePositiveInt(
413
+ params.refreshIntervalMinutes.value()
414
+ ),
415
+ backupCollectionId: optional(params.backupCollection.value()),
416
+ transformFunction: optional(params.transformFunction.value()),
417
+ kmsKeyName: optional(params.kmsKeyName.value()),
418
+ logLevel: normalizeLogLevel(params.logLevel.value()),
419
+ };
420
+ }
package/src/events.ts ADDED
@@ -0,0 +1,146 @@
1
+ /*
2
+ * Copyright 2019 Google LLC
3
+ *
4
+ * Licensed under the Apache License, Version 2.0 (the "License");
5
+ * you may not use this file except in compliance with the License.
6
+ * You may obtain a copy of the License at
7
+ *
8
+ * https://www.apache.org/licenses/LICENSE-2.0
9
+ *
10
+ * Unless required by applicable law or agreed to in writing, software
11
+ * distributed under the License is distributed on an "AS IS" BASIS,
12
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ * See the License for the specific language governing permissions and
14
+ * limitations under the License.
15
+ */
16
+ import * as eventArc from "firebase-admin/eventarc";
17
+
18
+ const { getEventarc } = eventArc;
19
+
20
+ /**
21
+ * Builds the Eventarc event type for this extension.
22
+ *
23
+ * @param eventName The name of the event (e.g., "onStart", "onError", etc.)
24
+ * @returns The event type string.
25
+ */
26
+ const getEventTypes = (eventName: string) => [
27
+ `firebase.extensions.firestore-bigquery-export.v1.${eventName}`,
28
+ ];
29
+
30
+ let eventChannel: eventArc.Channel | undefined;
31
+
32
+ /**
33
+ * Sets up the Eventarc channel.
34
+ *
35
+ * This function retrieves the Eventarc channel based on the environment variables:
36
+ * - `EVENTARC_CHANNEL` specifies the channel to use for publishing events.
37
+ * - `EXT_SELECTED_EVENTS` defines the allowed event types.
38
+ *
39
+ * @function setupEventChannel
40
+ */
41
+ export const setupEventChannel = () => {
42
+ eventChannel = process.env.EVENTARC_CHANNEL
43
+ ? getEventarc().channel(process.env.EVENTARC_CHANNEL, {
44
+ allowedEventTypes: process.env.EXT_SELECTED_EVENTS,
45
+ })
46
+ : undefined;
47
+ };
48
+
49
+ /**
50
+ * Publishes a "start" event using both OLD and NEW event types.
51
+ *
52
+ * @param data The payload to send with the event. Can be a string or an object.
53
+ * @returns A Promise resolving when both events are published.
54
+ */
55
+ export const recordStartEvent = async (data: string | object) => {
56
+ if (!eventChannel) return Promise.resolve();
57
+
58
+ const eventTypes = getEventTypes("onStart");
59
+
60
+ // Publish events for both OLD and NEW event types
61
+ return Promise.all(
62
+ eventTypes.map((type) =>
63
+ eventChannel.publish({
64
+ type,
65
+ data,
66
+ })
67
+ )
68
+ );
69
+ };
70
+
71
+ /**
72
+ * Publishes an "error" event using both OLD and NEW event types.
73
+ *
74
+ * @param err The Error object containing the error message.
75
+ * @param subject (Optional) Subject identifier related to the error event.
76
+ * @returns A Promise resolving when both events are published.
77
+ */
78
+ export const recordErrorEvent = async (err: Error, subject?: string) => {
79
+ if (!eventChannel) return Promise.resolve();
80
+
81
+ const eventTypes = getEventTypes("onError");
82
+
83
+ // Publish events for both OLD and NEW event types
84
+ return Promise.all(
85
+ eventTypes.map((type) =>
86
+ eventChannel.publish({
87
+ type,
88
+ data: { message: err.message },
89
+ subject,
90
+ })
91
+ )
92
+ );
93
+ };
94
+
95
+ /**
96
+ * Publishes a "success" event using both OLD and NEW event types.
97
+ *
98
+ * @param params An object containing the subject and the event data.
99
+ * @param params.subject A string representing the subject of the event.
100
+ * @param params.data The payload to send with the event.
101
+ * @returns A Promise resolving when both events are published.
102
+ */
103
+ export const recordSuccessEvent = async ({
104
+ subject,
105
+ data,
106
+ }: {
107
+ subject: string;
108
+ data: string | object;
109
+ }) => {
110
+ if (!eventChannel) return Promise.resolve();
111
+
112
+ const eventTypes = getEventTypes("onSuccess");
113
+
114
+ // Publish events for both OLD and NEW event types
115
+ return Promise.all(
116
+ eventTypes.map((type) =>
117
+ eventChannel.publish({
118
+ type,
119
+ subject,
120
+ data,
121
+ })
122
+ )
123
+ );
124
+ };
125
+
126
+ /**
127
+ * Publishes a "completion" event using both OLD and NEW event types.
128
+ *
129
+ * @param data The payload to send with the event. Can be a string or an object.
130
+ * @returns A Promise resolving when both events are published.
131
+ */
132
+ export const recordCompletionEvent = async (data: string | object) => {
133
+ if (!eventChannel) return Promise.resolve();
134
+
135
+ const eventTypes = getEventTypes("onCompletion");
136
+
137
+ // Publish events for both OLD and NEW event types
138
+ return Promise.all(
139
+ eventTypes.map((type) =>
140
+ eventChannel.publish({
141
+ type,
142
+ data,
143
+ })
144
+ )
145
+ );
146
+ };