@firebase-function-kits/firestore-bigquery-export 0.0.1 → 0.0.2-rc.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +1 -0
- package/README.md +234 -0
- package/lib/config.d.ts +33 -0
- package/lib/config.d.ts.map +1 -0
- package/lib/config.js +345 -0
- package/lib/config.js.map +1 -0
- package/lib/events.d.ts +45 -0
- package/lib/events.d.ts.map +1 -0
- package/lib/events.js +154 -0
- package/lib/events.js.map +1 -0
- package/lib/export-config.d.ts +96 -0
- package/lib/export-config.d.ts.map +1 -0
- package/lib/export-config.js +77 -0
- package/lib/export-config.js.map +1 -0
- package/lib/handlers.d.ts +29 -0
- package/lib/handlers.d.ts.map +1 -0
- package/lib/handlers.js +165 -0
- package/lib/handlers.js.map +1 -0
- package/lib/index.d.ts +23 -0
- package/lib/index.d.ts.map +1 -0
- package/lib/index.js +179 -0
- package/lib/index.js.map +1 -0
- package/lib/init.d.ts +16 -0
- package/lib/init.d.ts.map +1 -0
- package/lib/init.js +44 -0
- package/lib/init.js.map +1 -0
- package/lib/lib.d.ts +20 -0
- package/lib/lib.d.ts.map +1 -0
- package/lib/lib.js +47 -0
- package/lib/lib.js.map +1 -0
- package/lib/logs.d.ts +39 -0
- package/lib/logs.d.ts.map +1 -0
- package/lib/logs.js +186 -0
- package/lib/logs.js.map +1 -0
- package/lib/util.d.ts +23 -0
- package/lib/util.d.ts.map +1 -0
- package/lib/util.js +66 -0
- package/lib/util.js.map +1 -0
- package/package.json +33 -3
- package/src/config.ts +420 -0
- package/src/events.ts +146 -0
- package/src/export-config.ts +205 -0
- package/src/handlers.ts +211 -0
- package/src/index.ts +174 -0
- package/src/init.ts +45 -0
- package/src/lib.ts +55 -0
- package/src/logs.ts +233 -0
- package/src/util.ts +64 -0
- package/tests/config.test.ts +182 -0
- package/tests/events.test.ts +84 -0
- package/tests/export-config.test.ts +114 -0
- package/tests/handlers.test.ts +220 -0
- package/tests/init.test.ts +76 -0
- package/tests/util.test.ts +102 -0
- package/tsconfig.json +18 -0
- package/tsconfig.tsbuildinfo +1 -0
package/lib/util.js
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/*
|
|
3
|
+
* Copyright 2019 Google LLC
|
|
4
|
+
*
|
|
5
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
6
|
+
* you may not use this file except in compliance with the License.
|
|
7
|
+
* You may obtain a copy of the License at
|
|
8
|
+
*
|
|
9
|
+
* https://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
*
|
|
11
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
12
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
13
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
14
|
+
* See the License for the specific language governing permissions and
|
|
15
|
+
* limitations under the License.
|
|
16
|
+
*/
|
|
17
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
18
|
+
exports.resolveWildcardIds = void 0;
|
|
19
|
+
exports.getChangeType = getChangeType;
|
|
20
|
+
exports.getDocumentId = getDocumentId;
|
|
21
|
+
const firestore_bigquery_change_tracker_1 = require("@firebaseextensions/firestore-bigquery-change-tracker");
|
|
22
|
+
/**
|
|
23
|
+
* Get the change type (CREATE, UPDATE, DELETE) from the Firestore change.
|
|
24
|
+
* @param change Firestore document change object.
|
|
25
|
+
* @returns {ChangeType} The type of change.
|
|
26
|
+
*/
|
|
27
|
+
function getChangeType(change) {
|
|
28
|
+
if (!change.after.exists) {
|
|
29
|
+
return firestore_bigquery_change_tracker_1.ChangeType.DELETE;
|
|
30
|
+
}
|
|
31
|
+
if (!change.before.exists) {
|
|
32
|
+
return firestore_bigquery_change_tracker_1.ChangeType.CREATE;
|
|
33
|
+
}
|
|
34
|
+
return firestore_bigquery_change_tracker_1.ChangeType.UPDATE;
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Get the document ID from the Firestore change.
|
|
38
|
+
* @param change Firestore document change object.
|
|
39
|
+
* @returns {string} The document ID.
|
|
40
|
+
*/
|
|
41
|
+
function getDocumentId(change) {
|
|
42
|
+
if (change.after.exists) {
|
|
43
|
+
return change.after.id;
|
|
44
|
+
}
|
|
45
|
+
return change.before.id;
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
*
|
|
49
|
+
* @param template - eg, regions/{regionId}/countries
|
|
50
|
+
* @param text - eg, regions/asia/countries
|
|
51
|
+
*
|
|
52
|
+
* @return - eg, { regionId: "asia" }
|
|
53
|
+
*/
|
|
54
|
+
const resolveWildcardIds = (template, text) => {
|
|
55
|
+
const textSegments = text.split("/");
|
|
56
|
+
return template
|
|
57
|
+
.split("/")
|
|
58
|
+
.reduce((previousValue, currentValue, currentIndex) => {
|
|
59
|
+
if (currentValue.startsWith("{") && currentValue.endsWith("}")) {
|
|
60
|
+
previousValue[currentValue.slice(1, -1)] = textSegments[currentIndex];
|
|
61
|
+
}
|
|
62
|
+
return previousValue;
|
|
63
|
+
}, {});
|
|
64
|
+
};
|
|
65
|
+
exports.resolveWildcardIds = resolveWildcardIds;
|
|
66
|
+
//# sourceMappingURL=util.js.map
|
package/lib/util.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"util.js","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;GAcG;;;;;AAEH,6GAAmF;AAGnF;;;;GAIG;AACH,uBAA8B,MAAgC;IAC5D,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;QACzB,OAAO,8CAAU,CAAC,MAAM,CAAC;IAC3B,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QAC1B,OAAO,8CAAU,CAAC,MAAM,CAAC;IAC3B,CAAC;IACD,OAAO,8CAAU,CAAC,MAAM,CAAC;AAC3B,CAAC;AAED;;;;GAIG;AACH,uBAA8B,MAAgC;IAC5D,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;QACxB,OAAO,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;IACzB,CAAC;IACD,OAAO,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;AAC1B,CAAC;AAED;;;;;;GAMG;AACI,MAAM,kBAAkB,GAAG,CAAC,QAAgB,EAAE,IAAY,EAAE,EAAE;IACnE,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACrC,OAAO,QAAQ;SACZ,KAAK,CAAC,GAAG,CAAC;SACV,MAAM,CAAC,CAAC,aAAa,EAAE,YAAY,EAAE,YAAY,EAAE,EAAE;QACpD,IAAI,YAAY,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YAC/D,aAAa,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,YAAY,CAAC,YAAY,CAAC,CAAC;QACxE,CAAC;QACD,OAAO,aAAa,CAAC;IACvB,CAAC,EAAE,EAAE,CAAC,CAAC;AACX,CAAC,CAAC;AAVW,QAAA,kBAAkB,GAAlB,kBAAkB,CAU7B"}
|
package/package.json
CHANGED
|
@@ -1,11 +1,41 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@firebase-function-kits/firestore-bigquery-export",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.2-rc.0",
|
|
4
|
+
"repository": "firebase/extensions",
|
|
4
5
|
"description": "Stream a Cloud Firestore collection to BigQuery as a deployable Firebase Function",
|
|
5
6
|
"license": "Apache-2.0",
|
|
6
|
-
"main": "index.js",
|
|
7
|
-
"
|
|
7
|
+
"main": "lib/index.js",
|
|
8
|
+
"types": "lib/index.d.ts",
|
|
9
|
+
"exports": {
|
|
10
|
+
".": {
|
|
11
|
+
"types": "./lib/index.d.ts",
|
|
12
|
+
"default": "./lib/index.js"
|
|
13
|
+
},
|
|
14
|
+
"./lib": {
|
|
15
|
+
"types": "./lib/lib.d.ts",
|
|
16
|
+
"default": "./lib/lib.js"
|
|
17
|
+
}
|
|
18
|
+
},
|
|
8
19
|
"engines": {
|
|
9
20
|
"node": ">=22"
|
|
21
|
+
},
|
|
22
|
+
"scripts": {
|
|
23
|
+
"build": "tsc -b",
|
|
24
|
+
"clean": "tsc -b --clean",
|
|
25
|
+
"test": "vitest run",
|
|
26
|
+
"deploy": "pnpm build && firebase deploy --only functions",
|
|
27
|
+
"serve": "firebase emulators:start --only functions"
|
|
28
|
+
},
|
|
29
|
+
"dependencies": {
|
|
30
|
+
"@firebaseextensions/firestore-bigquery-change-tracker": "^2.0.4",
|
|
31
|
+
"@google-cloud/bigquery": "^7.6.0",
|
|
32
|
+
"firebase-admin": "^14.1.0",
|
|
33
|
+
"firebase-functions": "7.3.2",
|
|
34
|
+
"generate-schema": "^2.6.0",
|
|
35
|
+
"lodash": "^4.17.14"
|
|
36
|
+
},
|
|
37
|
+
"devDependencies": {
|
|
38
|
+
"@types/lodash": "^4.17.0",
|
|
39
|
+
"vitest": "^3.2.4"
|
|
10
40
|
}
|
|
11
41
|
}
|
package/src/config.ts
ADDED
|
@@ -0,0 +1,420 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Copyright 2019 Google LLC
|
|
3
|
+
*
|
|
4
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
* you may not use this file except in compliance with the License.
|
|
6
|
+
* You may obtain a copy of the License at
|
|
7
|
+
*
|
|
8
|
+
* https://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
*
|
|
10
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
* See the License for the specific language governing permissions and
|
|
14
|
+
* limitations under the License.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import type {
|
|
18
|
+
ChangeTrackerConfig,
|
|
19
|
+
PartitioningFieldType,
|
|
20
|
+
TimePartitioningGranularity,
|
|
21
|
+
} from "@firebaseextensions/firestore-bigquery-change-tracker";
|
|
22
|
+
import { LogLevel } from "@firebaseextensions/firestore-bigquery-change-tracker";
|
|
23
|
+
import type { Expression } from "firebase-functions/params";
|
|
24
|
+
import {
|
|
25
|
+
defineBoolean,
|
|
26
|
+
defineString,
|
|
27
|
+
projectID,
|
|
28
|
+
select,
|
|
29
|
+
} from "firebase-functions/params";
|
|
30
|
+
import type { ExportConfig, ViewType } from "./export-config";
|
|
31
|
+
|
|
32
|
+
type TrackerLogLevel = "debug" | "info" | "warn" | "error" | "silent";
|
|
33
|
+
type ConfigExpression<T extends string | number | boolean> = Expression<T>;
|
|
34
|
+
const DECIMAL_RADIX = 10;
|
|
35
|
+
const DATASET_LOCATION_OPTIONS = [
|
|
36
|
+
"us-central1",
|
|
37
|
+
"us-west4",
|
|
38
|
+
"europe-central2",
|
|
39
|
+
"us-west2",
|
|
40
|
+
"northamerica-northeast1",
|
|
41
|
+
"us-east4",
|
|
42
|
+
"us-west1",
|
|
43
|
+
"us-west3",
|
|
44
|
+
"southamerica-east1",
|
|
45
|
+
"us-east1",
|
|
46
|
+
"europe-west1",
|
|
47
|
+
"europe-north1",
|
|
48
|
+
"europe-west3",
|
|
49
|
+
"europe-west2",
|
|
50
|
+
"europe-west4",
|
|
51
|
+
"europe-west6",
|
|
52
|
+
"asia-east1",
|
|
53
|
+
"asia-east2",
|
|
54
|
+
"asia-southeast2",
|
|
55
|
+
"asia-south1",
|
|
56
|
+
"asia-southeast1",
|
|
57
|
+
"asia-northeast2",
|
|
58
|
+
"asia-northeast3",
|
|
59
|
+
"australia-southeast1",
|
|
60
|
+
"asia-northeast1",
|
|
61
|
+
"us",
|
|
62
|
+
"eu",
|
|
63
|
+
"africa-south1",
|
|
64
|
+
"me-west1",
|
|
65
|
+
"me-central1",
|
|
66
|
+
"me-central2",
|
|
67
|
+
"europe-west12",
|
|
68
|
+
"europe-north2",
|
|
69
|
+
"europe-west9",
|
|
70
|
+
"europe-west8",
|
|
71
|
+
"europe-southwest1",
|
|
72
|
+
"europe-west10",
|
|
73
|
+
"australia-southeast2",
|
|
74
|
+
"asia-south2",
|
|
75
|
+
"northamerica-northeast2",
|
|
76
|
+
"southamerica-west1",
|
|
77
|
+
"northamerica-south1",
|
|
78
|
+
"us-south1",
|
|
79
|
+
"us-east5",
|
|
80
|
+
] as const;
|
|
81
|
+
const DATABASE_REGION_OPTIONS = [
|
|
82
|
+
"eur3",
|
|
83
|
+
"nam5",
|
|
84
|
+
"nam7",
|
|
85
|
+
"us-central1",
|
|
86
|
+
"us-west1",
|
|
87
|
+
"us-west2",
|
|
88
|
+
"us-west3",
|
|
89
|
+
"us-west4",
|
|
90
|
+
"us-east1",
|
|
91
|
+
"us-east4",
|
|
92
|
+
"us-east5",
|
|
93
|
+
"us-south1",
|
|
94
|
+
"northamerica-northeast1",
|
|
95
|
+
"northamerica-northeast2",
|
|
96
|
+
"northamerica-south1",
|
|
97
|
+
"southamerica-east1",
|
|
98
|
+
"southamerica-west1",
|
|
99
|
+
"europe-west1",
|
|
100
|
+
"europe-west2",
|
|
101
|
+
"europe-west3",
|
|
102
|
+
"europe-west4",
|
|
103
|
+
"europe-west6",
|
|
104
|
+
"europe-west8",
|
|
105
|
+
"europe-west9",
|
|
106
|
+
"europe-west10",
|
|
107
|
+
"europe-west12",
|
|
108
|
+
"europe-southwest1",
|
|
109
|
+
"europe-north1",
|
|
110
|
+
"europe-north2",
|
|
111
|
+
"europe-central2",
|
|
112
|
+
"me-central1",
|
|
113
|
+
"me-central2",
|
|
114
|
+
"me-west1",
|
|
115
|
+
"asia-south1",
|
|
116
|
+
"asia-south2",
|
|
117
|
+
"asia-southeast1",
|
|
118
|
+
"asia-southeast2",
|
|
119
|
+
"asia-east1",
|
|
120
|
+
"asia-east2",
|
|
121
|
+
"asia-northeast1",
|
|
122
|
+
"asia-northeast2",
|
|
123
|
+
"asia-northeast3",
|
|
124
|
+
"australia-southeast1",
|
|
125
|
+
"australia-southeast2",
|
|
126
|
+
"africa-south1",
|
|
127
|
+
] as const;
|
|
128
|
+
const TABLE_PARTITIONING_OPTIONS = [
|
|
129
|
+
"HOUR",
|
|
130
|
+
"DAY",
|
|
131
|
+
"MONTH",
|
|
132
|
+
"YEAR",
|
|
133
|
+
"NONE",
|
|
134
|
+
] as const;
|
|
135
|
+
const TIME_PARTITIONING_FIELD_TYPE_OPTIONS = [
|
|
136
|
+
"TIMESTAMP",
|
|
137
|
+
"DATETIME",
|
|
138
|
+
"DATE",
|
|
139
|
+
"omit",
|
|
140
|
+
] as const;
|
|
141
|
+
const VIEW_TYPE_OPTIONS = [
|
|
142
|
+
"view",
|
|
143
|
+
"materialized_incremental",
|
|
144
|
+
"materialized_non_incremental",
|
|
145
|
+
] as const;
|
|
146
|
+
const LOG_LEVEL_OPTIONS = ["debug", "info", "warn", "error", "silent"] as const;
|
|
147
|
+
export interface ConfigExpressions {
|
|
148
|
+
collectionPath: ConfigExpression<string>;
|
|
149
|
+
datasetId: ConfigExpression<string>;
|
|
150
|
+
tableId: ConfigExpression<string>;
|
|
151
|
+
location: ConfigExpression<string>;
|
|
152
|
+
database: ConfigExpression<string>;
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/**
|
|
156
|
+
* Deploy-time parameters. Set these via a `.env` / `.env.<project>` file or the
|
|
157
|
+
* interactive prompts shown by `firebase deploy`.
|
|
158
|
+
*
|
|
159
|
+
* @see https://firebase.google.com/docs/functions/config-env
|
|
160
|
+
*/
|
|
161
|
+
const params = {
|
|
162
|
+
bigqueryProjectId: defineString("BIGQUERY_PROJECT_ID", {
|
|
163
|
+
default: projectID,
|
|
164
|
+
}),
|
|
165
|
+
database: defineString("DATABASE", { default: "(default)" }),
|
|
166
|
+
databaseRegion: defineString("DATABASE_REGION", {
|
|
167
|
+
input: select([...DATABASE_REGION_OPTIONS]),
|
|
168
|
+
}),
|
|
169
|
+
collectionPath: defineString("COLLECTION_PATH", { default: "posts" }),
|
|
170
|
+
datasetId: defineString("DATASET_ID", { default: "firestore_export" }),
|
|
171
|
+
tableId: defineString("TABLE_ID", { default: "posts" }),
|
|
172
|
+
datasetLocation: defineString("DATASET_LOCATION", {
|
|
173
|
+
default: "us",
|
|
174
|
+
input: select([...DATASET_LOCATION_OPTIONS]),
|
|
175
|
+
}),
|
|
176
|
+
backupCollection: defineString("BACKUP_COLLECTION", { default: "" }),
|
|
177
|
+
transformFunction: defineString("TRANSFORM_FUNCTION", { default: "" }),
|
|
178
|
+
tablePartitioning: defineString("TABLE_PARTITIONING", {
|
|
179
|
+
default: "NONE",
|
|
180
|
+
input: select([...TABLE_PARTITIONING_OPTIONS]),
|
|
181
|
+
}),
|
|
182
|
+
timePartitioningField: defineString("TIME_PARTITIONING_FIELD", {
|
|
183
|
+
default: "",
|
|
184
|
+
}),
|
|
185
|
+
timePartitioningFieldType: defineString("TIME_PARTITIONING_FIELD_TYPE", {
|
|
186
|
+
default: "omit",
|
|
187
|
+
input: select([...TIME_PARTITIONING_FIELD_TYPE_OPTIONS]),
|
|
188
|
+
}),
|
|
189
|
+
timePartitioningFirestoreField: defineString(
|
|
190
|
+
"TIME_PARTITIONING_FIRESTORE_FIELD",
|
|
191
|
+
{ default: "" }
|
|
192
|
+
),
|
|
193
|
+
clustering: defineString("CLUSTERING", { default: "" }),
|
|
194
|
+
wildcardIds: defineBoolean("WILDCARD_IDS", {
|
|
195
|
+
default: false,
|
|
196
|
+
}),
|
|
197
|
+
useNewSnapshotQuerySyntax: defineBoolean("USE_NEW_SNAPSHOT_QUERY_SYNTAX", {
|
|
198
|
+
default: false,
|
|
199
|
+
}),
|
|
200
|
+
excludeOldData: defineBoolean("EXCLUDE_OLD_DATA", {
|
|
201
|
+
default: false,
|
|
202
|
+
}),
|
|
203
|
+
viewType: defineString("VIEW_TYPE", {
|
|
204
|
+
default: "view",
|
|
205
|
+
input: select([...VIEW_TYPE_OPTIONS]),
|
|
206
|
+
}),
|
|
207
|
+
maxStaleness: defineString("MAX_STALENESS", { default: "" }),
|
|
208
|
+
refreshIntervalMinutes: defineString("REFRESH_INTERVAL_MINUTES", {
|
|
209
|
+
default: "",
|
|
210
|
+
}),
|
|
211
|
+
kmsKeyName: defineString("KMS_KEY_NAME", { default: "" }),
|
|
212
|
+
logLevel: defineString("LOG_LEVEL", {
|
|
213
|
+
default: "info",
|
|
214
|
+
input: select([...LOG_LEVEL_OPTIONS]),
|
|
215
|
+
}),
|
|
216
|
+
};
|
|
217
|
+
|
|
218
|
+
export const CONFIG_EXPRESSIONS: ConfigExpressions = {
|
|
219
|
+
collectionPath: params.collectionPath,
|
|
220
|
+
datasetId: params.datasetId,
|
|
221
|
+
tableId: params.tableId,
|
|
222
|
+
location: params.databaseRegion,
|
|
223
|
+
database: params.database,
|
|
224
|
+
};
|
|
225
|
+
|
|
226
|
+
function timePartitioning(
|
|
227
|
+
type: string | undefined
|
|
228
|
+
): TimePartitioningGranularity | null {
|
|
229
|
+
if (
|
|
230
|
+
type === "HOUR" ||
|
|
231
|
+
type === "DAY" ||
|
|
232
|
+
type === "MONTH" ||
|
|
233
|
+
type === "YEAR"
|
|
234
|
+
) {
|
|
235
|
+
return type;
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
return null;
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
export function clustering(clusters: string | undefined) {
|
|
242
|
+
return clusters ? clusters.split(",").slice(0, 4) : null;
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
function normalizeOptionalPartitionValue(
|
|
246
|
+
value: string | undefined
|
|
247
|
+
): string | undefined {
|
|
248
|
+
const normalized = value?.trim();
|
|
249
|
+
|
|
250
|
+
if (!normalized || normalized === "NONE" || normalized === "omit") {
|
|
251
|
+
return undefined;
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
return normalized;
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
function normalizePartitionFieldType(
|
|
258
|
+
value: string | undefined
|
|
259
|
+
): PartitioningFieldType | undefined {
|
|
260
|
+
const normalized = normalizeOptionalPartitionValue(value);
|
|
261
|
+
if (
|
|
262
|
+
normalized === "TIMESTAMP" ||
|
|
263
|
+
normalized === "DATE" ||
|
|
264
|
+
normalized === "DATETIME"
|
|
265
|
+
) {
|
|
266
|
+
return normalized;
|
|
267
|
+
}
|
|
268
|
+
return undefined;
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
export function buildPartitioningConfig(params: {
|
|
272
|
+
timePartitioning: TimePartitioningGranularity | null;
|
|
273
|
+
timePartitioningField: string | undefined;
|
|
274
|
+
timePartitioningFieldType: string | undefined;
|
|
275
|
+
timePartitioningFirestoreField: string | undefined;
|
|
276
|
+
}): ChangeTrackerConfig["partitioning"] {
|
|
277
|
+
const { timePartitioning } = params;
|
|
278
|
+
const rawFieldName = params.timePartitioningField?.trim();
|
|
279
|
+
const rawFieldType = params.timePartitioningFieldType?.trim();
|
|
280
|
+
const rawFirestoreField = params.timePartitioningFirestoreField?.trim();
|
|
281
|
+
|
|
282
|
+
const formatValue = (value: string | undefined): string =>
|
|
283
|
+
value && value.length > 0 ? `"${value}"` : "(empty)";
|
|
284
|
+
|
|
285
|
+
const throwInvalidPartitioningConfig = (detail: string): never => {
|
|
286
|
+
throw new Error(
|
|
287
|
+
[
|
|
288
|
+
"Invalid partitioning configuration for firestore-bigquery-export.",
|
|
289
|
+
detail,
|
|
290
|
+
`Received TABLE_PARTITIONING=${formatValue(
|
|
291
|
+
timePartitioning ?? undefined
|
|
292
|
+
)},`,
|
|
293
|
+
`TIME_PARTITIONING_FIELD=${formatValue(rawFieldName)},`,
|
|
294
|
+
`TIME_PARTITIONING_FIRESTORE_FIELD=${formatValue(rawFirestoreField)},`,
|
|
295
|
+
`TIME_PARTITIONING_FIELD_TYPE=${formatValue(rawFieldType)}.`,
|
|
296
|
+
"Valid combinations are:",
|
|
297
|
+
"1) Ingestion-time: TABLE_PARTITIONING set and all TIME_PARTITIONING_* values empty/NONE/omit.",
|
|
298
|
+
"2) Timestamp field: TABLE_PARTITIONING set, TIME_PARTITIONING_FIELD=timestamp, TIME_PARTITIONING_FIRESTORE_FIELD empty.",
|
|
299
|
+
"3) Custom field: TABLE_PARTITIONING set, and TIME_PARTITIONING_FIELD + TIME_PARTITIONING_FIRESTORE_FIELD + TIME_PARTITIONING_FIELD_TYPE all provided.",
|
|
300
|
+
].join(" ")
|
|
301
|
+
);
|
|
302
|
+
};
|
|
303
|
+
|
|
304
|
+
const fieldName = normalizeOptionalPartitionValue(
|
|
305
|
+
params.timePartitioningField
|
|
306
|
+
);
|
|
307
|
+
const fieldType = normalizePartitionFieldType(
|
|
308
|
+
params.timePartitioningFieldType
|
|
309
|
+
);
|
|
310
|
+
const firestoreField = normalizeOptionalPartitionValue(
|
|
311
|
+
params.timePartitioningFirestoreField
|
|
312
|
+
);
|
|
313
|
+
|
|
314
|
+
if (!timePartitioning) {
|
|
315
|
+
if (fieldName || fieldType || firestoreField) {
|
|
316
|
+
return throwInvalidPartitioningConfig(
|
|
317
|
+
"Partition-specific fields cannot be provided when TABLE_PARTITIONING is NONE."
|
|
318
|
+
);
|
|
319
|
+
}
|
|
320
|
+
return { granularity: "NONE" };
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
if (!fieldName && !firestoreField) {
|
|
324
|
+
return { granularity: timePartitioning };
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
if (fieldName === "timestamp" && !firestoreField) {
|
|
328
|
+
return {
|
|
329
|
+
granularity: timePartitioning,
|
|
330
|
+
bigqueryColumnName: "timestamp",
|
|
331
|
+
...(fieldType ? { bigqueryColumnType: fieldType } : {}),
|
|
332
|
+
};
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
if (fieldName && firestoreField && fieldType) {
|
|
336
|
+
return {
|
|
337
|
+
granularity: timePartitioning,
|
|
338
|
+
bigqueryColumnName: fieldName,
|
|
339
|
+
bigqueryColumnType: fieldType,
|
|
340
|
+
firestoreFieldName: firestoreField,
|
|
341
|
+
};
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
return throwInvalidPartitioningConfig(
|
|
345
|
+
"When TABLE_PARTITIONING is set, partitioning fields are either incomplete or invalid."
|
|
346
|
+
);
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
function normalizeLogLevel(level: string | undefined): TrackerLogLevel {
|
|
350
|
+
switch ((level || "").toLowerCase()) {
|
|
351
|
+
case "debug":
|
|
352
|
+
return "debug";
|
|
353
|
+
case "info":
|
|
354
|
+
return "info";
|
|
355
|
+
case "warn":
|
|
356
|
+
return "warn";
|
|
357
|
+
case "error":
|
|
358
|
+
return "error";
|
|
359
|
+
case "silent":
|
|
360
|
+
return "silent";
|
|
361
|
+
default:
|
|
362
|
+
return LogLevel.INFO;
|
|
363
|
+
}
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
function normalizePositiveInt(value: string): number | undefined {
|
|
367
|
+
const normalized = Number.parseInt(value, DECIMAL_RADIX);
|
|
368
|
+
return normalized > 0 ? normalized : undefined;
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
/** Coerce an empty-string param value to `undefined`. */
|
|
372
|
+
function optional(value: string): string | undefined {
|
|
373
|
+
return value.length > 0 ? value : undefined;
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
/**
|
|
377
|
+
* Resolves all deploy-time params into an {@link ExportConfig}.
|
|
378
|
+
*
|
|
379
|
+
* Param values are read when this is called. During the Firebase deploy-time
|
|
380
|
+
* discovery pass params return their declared defaults, so the default
|
|
381
|
+
* `TABLE_PARTITIONING=NONE` keeps {@link buildPartitioningConfig} from throwing.
|
|
382
|
+
* This is the bridge for the env-driven path: env params in, typed config out,
|
|
383
|
+
* which the main entry point wires into the exported functions.
|
|
384
|
+
*
|
|
385
|
+
* @returns The export configuration assembled from environment params.
|
|
386
|
+
*/
|
|
387
|
+
export function configFromEnv(): ExportConfig {
|
|
388
|
+
const tablePartitioning = optional(params.tablePartitioning.value());
|
|
389
|
+
|
|
390
|
+
return {
|
|
391
|
+
collectionPath: params.collectionPath.value(),
|
|
392
|
+
datasetId: params.datasetId.value(),
|
|
393
|
+
tableId: params.tableId.value(),
|
|
394
|
+
location: params.databaseRegion.value(),
|
|
395
|
+
datasetLocation: optional(params.datasetLocation.value()),
|
|
396
|
+
bqProjectId: optional(params.bigqueryProjectId.value()),
|
|
397
|
+
projectId: projectID.value(),
|
|
398
|
+
databaseId: optional(params.database.value()) || "(default)",
|
|
399
|
+
wildcardIds: params.wildcardIds.value(),
|
|
400
|
+
excludeOldData: params.excludeOldData.value(),
|
|
401
|
+
useNewSnapshotQuerySyntax: params.useNewSnapshotQuerySyntax.value(),
|
|
402
|
+
viewType: (optional(params.viewType.value()) || "view") as ViewType,
|
|
403
|
+
partitioning: buildPartitioningConfig({
|
|
404
|
+
timePartitioning: timePartitioning(tablePartitioning),
|
|
405
|
+
timePartitioningField: params.timePartitioningField.value(),
|
|
406
|
+
timePartitioningFieldType: params.timePartitioningFieldType.value(),
|
|
407
|
+
timePartitioningFirestoreField:
|
|
408
|
+
params.timePartitioningFirestoreField.value(),
|
|
409
|
+
}),
|
|
410
|
+
clustering: clustering(optional(params.clustering.value())),
|
|
411
|
+
maxStaleness: optional(params.maxStaleness.value()),
|
|
412
|
+
refreshIntervalMinutes: normalizePositiveInt(
|
|
413
|
+
params.refreshIntervalMinutes.value()
|
|
414
|
+
),
|
|
415
|
+
backupCollectionId: optional(params.backupCollection.value()),
|
|
416
|
+
transformFunction: optional(params.transformFunction.value()),
|
|
417
|
+
kmsKeyName: optional(params.kmsKeyName.value()),
|
|
418
|
+
logLevel: normalizeLogLevel(params.logLevel.value()),
|
|
419
|
+
};
|
|
420
|
+
}
|
package/src/events.ts
ADDED
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Copyright 2019 Google LLC
|
|
3
|
+
*
|
|
4
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
* you may not use this file except in compliance with the License.
|
|
6
|
+
* You may obtain a copy of the License at
|
|
7
|
+
*
|
|
8
|
+
* https://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
*
|
|
10
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
* See the License for the specific language governing permissions and
|
|
14
|
+
* limitations under the License.
|
|
15
|
+
*/
|
|
16
|
+
import * as eventArc from "firebase-admin/eventarc";
|
|
17
|
+
|
|
18
|
+
const { getEventarc } = eventArc;
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* Builds the Eventarc event type for this extension.
|
|
22
|
+
*
|
|
23
|
+
* @param eventName The name of the event (e.g., "onStart", "onError", etc.)
|
|
24
|
+
* @returns The event type string.
|
|
25
|
+
*/
|
|
26
|
+
const getEventTypes = (eventName: string) => [
|
|
27
|
+
`firebase.extensions.firestore-bigquery-export.v1.${eventName}`,
|
|
28
|
+
];
|
|
29
|
+
|
|
30
|
+
let eventChannel: eventArc.Channel | undefined;
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Sets up the Eventarc channel.
|
|
34
|
+
*
|
|
35
|
+
* This function retrieves the Eventarc channel based on the environment variables:
|
|
36
|
+
* - `EVENTARC_CHANNEL` specifies the channel to use for publishing events.
|
|
37
|
+
* - `EXT_SELECTED_EVENTS` defines the allowed event types.
|
|
38
|
+
*
|
|
39
|
+
* @function setupEventChannel
|
|
40
|
+
*/
|
|
41
|
+
export const setupEventChannel = () => {
|
|
42
|
+
eventChannel = process.env.EVENTARC_CHANNEL
|
|
43
|
+
? getEventarc().channel(process.env.EVENTARC_CHANNEL, {
|
|
44
|
+
allowedEventTypes: process.env.EXT_SELECTED_EVENTS,
|
|
45
|
+
})
|
|
46
|
+
: undefined;
|
|
47
|
+
};
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Publishes a "start" event using both OLD and NEW event types.
|
|
51
|
+
*
|
|
52
|
+
* @param data The payload to send with the event. Can be a string or an object.
|
|
53
|
+
* @returns A Promise resolving when both events are published.
|
|
54
|
+
*/
|
|
55
|
+
export const recordStartEvent = async (data: string | object) => {
|
|
56
|
+
if (!eventChannel) return Promise.resolve();
|
|
57
|
+
|
|
58
|
+
const eventTypes = getEventTypes("onStart");
|
|
59
|
+
|
|
60
|
+
// Publish events for both OLD and NEW event types
|
|
61
|
+
return Promise.all(
|
|
62
|
+
eventTypes.map((type) =>
|
|
63
|
+
eventChannel.publish({
|
|
64
|
+
type,
|
|
65
|
+
data,
|
|
66
|
+
})
|
|
67
|
+
)
|
|
68
|
+
);
|
|
69
|
+
};
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* Publishes an "error" event using both OLD and NEW event types.
|
|
73
|
+
*
|
|
74
|
+
* @param err The Error object containing the error message.
|
|
75
|
+
* @param subject (Optional) Subject identifier related to the error event.
|
|
76
|
+
* @returns A Promise resolving when both events are published.
|
|
77
|
+
*/
|
|
78
|
+
export const recordErrorEvent = async (err: Error, subject?: string) => {
|
|
79
|
+
if (!eventChannel) return Promise.resolve();
|
|
80
|
+
|
|
81
|
+
const eventTypes = getEventTypes("onError");
|
|
82
|
+
|
|
83
|
+
// Publish events for both OLD and NEW event types
|
|
84
|
+
return Promise.all(
|
|
85
|
+
eventTypes.map((type) =>
|
|
86
|
+
eventChannel.publish({
|
|
87
|
+
type,
|
|
88
|
+
data: { message: err.message },
|
|
89
|
+
subject,
|
|
90
|
+
})
|
|
91
|
+
)
|
|
92
|
+
);
|
|
93
|
+
};
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* Publishes a "success" event using both OLD and NEW event types.
|
|
97
|
+
*
|
|
98
|
+
* @param params An object containing the subject and the event data.
|
|
99
|
+
* @param params.subject A string representing the subject of the event.
|
|
100
|
+
* @param params.data The payload to send with the event.
|
|
101
|
+
* @returns A Promise resolving when both events are published.
|
|
102
|
+
*/
|
|
103
|
+
export const recordSuccessEvent = async ({
|
|
104
|
+
subject,
|
|
105
|
+
data,
|
|
106
|
+
}: {
|
|
107
|
+
subject: string;
|
|
108
|
+
data: string | object;
|
|
109
|
+
}) => {
|
|
110
|
+
if (!eventChannel) return Promise.resolve();
|
|
111
|
+
|
|
112
|
+
const eventTypes = getEventTypes("onSuccess");
|
|
113
|
+
|
|
114
|
+
// Publish events for both OLD and NEW event types
|
|
115
|
+
return Promise.all(
|
|
116
|
+
eventTypes.map((type) =>
|
|
117
|
+
eventChannel.publish({
|
|
118
|
+
type,
|
|
119
|
+
subject,
|
|
120
|
+
data,
|
|
121
|
+
})
|
|
122
|
+
)
|
|
123
|
+
);
|
|
124
|
+
};
|
|
125
|
+
|
|
126
|
+
/**
|
|
127
|
+
* Publishes a "completion" event using both OLD and NEW event types.
|
|
128
|
+
*
|
|
129
|
+
* @param data The payload to send with the event. Can be a string or an object.
|
|
130
|
+
* @returns A Promise resolving when both events are published.
|
|
131
|
+
*/
|
|
132
|
+
export const recordCompletionEvent = async (data: string | object) => {
|
|
133
|
+
if (!eventChannel) return Promise.resolve();
|
|
134
|
+
|
|
135
|
+
const eventTypes = getEventTypes("onCompletion");
|
|
136
|
+
|
|
137
|
+
// Publish events for both OLD and NEW event types
|
|
138
|
+
return Promise.all(
|
|
139
|
+
eventTypes.map((type) =>
|
|
140
|
+
eventChannel.publish({
|
|
141
|
+
type,
|
|
142
|
+
data,
|
|
143
|
+
})
|
|
144
|
+
)
|
|
145
|
+
);
|
|
146
|
+
};
|