@firebase-function-kits/bigquery-firestore-export 0.0.2-rc.0 → 0.0.2-rc.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +1 -1
- package/README.md +138 -4
- package/lib/config.d.ts.map +1 -1
- package/lib/config.js +128 -9
- package/lib/config.js.map +1 -1
- package/lib/dts.d.ts +17 -2
- package/lib/dts.d.ts.map +1 -1
- package/lib/dts.js +25 -8
- package/lib/dts.js.map +1 -1
- package/lib/export-config.d.ts +0 -3
- package/lib/export-config.d.ts.map +1 -1
- package/lib/export-config.js +0 -1
- package/lib/export-config.js.map +1 -1
- package/lib/handlers.d.ts.map +1 -1
- package/lib/handlers.js +26 -4
- package/lib/handlers.js.map +1 -1
- package/lib/helper.d.ts.map +1 -1
- package/lib/helper.js +1 -0
- package/lib/helper.js.map +1 -1
- package/lib/index.js +1 -1
- package/lib/index.js.map +1 -1
- package/lib/logs.d.ts +4 -0
- package/lib/logs.d.ts.map +1 -1
- package/lib/logs.js +18 -0
- package/lib/logs.js.map +1 -1
- package/lib/metadata.d.ts.map +1 -1
- package/npm-shrinkwrap.json +3841 -0
- package/package.json +2 -5
- package/src/config.ts +142 -9
- package/src/dts.ts +27 -9
- package/src/export-config.ts +0 -4
- package/src/handlers.ts +36 -9
- package/src/helper.ts +1 -0
- package/src/index.ts +1 -1
- package/src/logs.ts +28 -0
- package/tests/config.test.ts +54 -2
- package/tests/dts-transfer-config.test.ts +400 -0
- package/tests/dts.test.ts +18 -5
- package/tests/export-config.test.ts +0 -4
- package/tests/handlers.test.ts +113 -5
- package/tests/helper-values.test.ts +168 -0
- package/tests/helper.test.ts +65 -3
- package/tests/notification-topic.test.ts +151 -0
- package/tests/run-results.test.ts +373 -0
- package/tsconfig.tsbuildinfo +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
- Initial release
|
|
1
|
+
- Initial release of kit, see README for differences between the legacy extension and this kit
|
package/README.md
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# @firebase/bigquery-firestore-export
|
|
1
|
+
# @firebase-function-kits/bigquery-firestore-export
|
|
2
2
|
|
|
3
3
|
Schedule a BigQuery query and write each run's rows and metadata to Firestore.
|
|
4
4
|
This is the Export BigQuery to Firestore Firebase Extension as an npm package
|
|
@@ -12,7 +12,7 @@ there is no hosted version, so you deploy them yourself.
|
|
|
12
12
|
## Install
|
|
13
13
|
|
|
14
14
|
```sh
|
|
15
|
-
npm install @firebase/bigquery-firestore-export
|
|
15
|
+
npm install @firebase-function-kits/bigquery-firestore-export
|
|
16
16
|
```
|
|
17
17
|
|
|
18
18
|
## Required IAM
|
|
@@ -43,7 +43,7 @@ Export both functions from your functions codebase entry:
|
|
|
43
43
|
export {
|
|
44
44
|
processMessages,
|
|
45
45
|
upsertTransferConfig,
|
|
46
|
-
} from "@firebase/bigquery-firestore-export";
|
|
46
|
+
} from "@firebase-function-kits/bigquery-firestore-export";
|
|
47
47
|
```
|
|
48
48
|
|
|
49
49
|
and configure them with a `.env` (or `.env.<projectId>`):
|
|
@@ -114,6 +114,7 @@ loads them at deploy time and prompts for required values that are missing.
|
|
|
114
114
|
| `displayName` | `DISPLAY_NAME` | yes | — | Human-readable scheduled-query name |
|
|
115
115
|
| `partitioningField` | `PARTITIONING_FIELD` | no | (empty) | Destination-table partitioning field |
|
|
116
116
|
| `schedule` | `SCHEDULE` | yes | — | DTS schedule, such as `every 24 hours` |
|
|
117
|
+
| `pubSubTopic` | `PUB_SUB_TOPIC` | no | `kit-<INSTANCE_ID>-processMessages` | Pub/Sub topic ID, not a full resource name, receiving DTS completion notifications |
|
|
117
118
|
| `firestoreCollection` | `COLLECTION_PATH` | no | `transferConfigs` | Root Firestore collection for configs and output |
|
|
118
119
|
| `logLevel` | `LOG_LEVEL` | no | `info` | `debug`, `info`, `warn`, `error`, or `silent` |
|
|
119
120
|
|
|
@@ -173,9 +174,142 @@ The run document stores DTS metadata and row counts. Its `output` subcollection
|
|
|
173
174
|
contains converted query rows. The `latest` document is updated transactionally
|
|
174
175
|
so an older completion message cannot replace a newer run.
|
|
175
176
|
|
|
177
|
+
## Differences from the Export BigQuery to Firestore extension
|
|
178
|
+
|
|
179
|
+
This kit is version 0.2.2 of the extension repackaged as an npm package. It is a
|
|
180
|
+
close port: the same two functions, the same BigQuery Data Transfer scheduled
|
|
181
|
+
query, the same `transferConfigs/{configId}/runs/{runId}` and `runs/latest`
|
|
182
|
+
documents, the same `WRITE_TRUNCATE` destination table naming and the same
|
|
183
|
+
row-by-row copy into Firestore. Every setting keeps its extension environment
|
|
184
|
+
variable name and default, so a `.env` copied from your installed instance needs
|
|
185
|
+
no value changes. What changes is the instance id, the Pub/Sub topic, the identity
|
|
186
|
+
the scheduled query runs as, and how repeated BigQuery columns land in Firestore.
|
|
187
|
+
|
|
188
|
+
### You set `INSTANCE_ID` yourself, and the Pub/Sub topic is renamed
|
|
189
|
+
|
|
190
|
+
The extension derived an instance id at install and used it to name its
|
|
191
|
+
notification topic (`ext-<instance id>-processMessages`) and to tag its transfer
|
|
192
|
+
config document with `extInstanceId`. Here `INSTANCE_ID` is a setting you
|
|
193
|
+
provide, and it must match this instance's key in the `instances` map in
|
|
194
|
+
`firebase.json`.
|
|
195
|
+
|
|
196
|
+
The topic defaults to `kit-<INSTANCE_ID>-processMessages`, and the kit creates it
|
|
197
|
+
on first run if it does not already exist. Set `INSTANCE_ID` to your installed
|
|
198
|
+
instance's id if you want the kit to adopt the scheduled query that instance
|
|
199
|
+
created, because the lookup is by `extInstanceId` on the documents in
|
|
200
|
+
`COLLECTION_PATH`. With a different id the kit finds nothing, creates a second
|
|
201
|
+
scheduled query, and you end up with two writing into the same collection.
|
|
202
|
+
|
|
203
|
+
An adopted transfer config still notifies the extension's `ext-` topic, and the
|
|
204
|
+
kit reconciles `notification_pubsub_topic` to whatever `PUB_SUB_TOPIC` names. So
|
|
205
|
+
there are two ways to migrate:
|
|
206
|
+
|
|
207
|
+
- Keep the extension's topic. Set
|
|
208
|
+
`PUB_SUB_TOPIC=ext-<instance id>-processMessages`, and the transfer config is
|
|
209
|
+
left untouched: anything else subscribed to that topic, including an extension
|
|
210
|
+
instance still installed, keeps receiving run notifications.
|
|
211
|
+
- Take the default. The first deploy repoints the transfer config at
|
|
212
|
+
`kit-<INSTANCE_ID>-processMessages`, which stops notifications reaching the
|
|
213
|
+
extension and any other subscriber on the old topic. Do this once the extension
|
|
214
|
+
is uninstalled, and the old topic can then be deleted.
|
|
215
|
+
|
|
216
|
+
`TRANSFER_CONFIG_NAME` is the exception. A linked config is adopted as-is and is
|
|
217
|
+
never repointed, so if it notifies a topic other than `PUB_SUB_TOPIC` the kit
|
|
218
|
+
logs a warning and its runs never reach the kit's `processMessages` function.
|
|
219
|
+
|
|
220
|
+
### Repeated BigQuery columns are now written as arrays
|
|
221
|
+
|
|
222
|
+
A repeated (`ARRAY`) column used to arrive in Firestore as a map keyed by
|
|
223
|
+
position, `{ "0": ..., "1": ... }`, because the conversion treated every
|
|
224
|
+
non-scalar value as an object. The kit writes a real Firestore array instead.
|
|
225
|
+
Anything reading those fields by numeric string key needs updating, and rows
|
|
226
|
+
written before and after the change are not the same shape. Scalars, timestamps,
|
|
227
|
+
dates, times, datetimes, bytes and geography values convert exactly as before.
|
|
228
|
+
|
|
229
|
+
### The scheduled query runs as a different service account
|
|
230
|
+
|
|
231
|
+
The extension created the transfer config with
|
|
232
|
+
`serviceAccountName: ext-<instance id>@<project>.iam.gserviceaccount.com`, so the
|
|
233
|
+
query ran as the extension's own service account. The kit creates it without a
|
|
234
|
+
service account name, so BigQuery Data Transfer runs it as the identity that
|
|
235
|
+
created it, which is your function's runtime service account (the default compute
|
|
236
|
+
service account unless you have set one).
|
|
237
|
+
|
|
238
|
+
That account needs to be able to read whatever `QUERY_STRING` touches and write
|
|
239
|
+
to `DATASET_ID`. `roles/bigquery.admin` on the function covers this for datasets
|
|
240
|
+
in the same project; a cross-project query needs the grant made explicitly. This
|
|
241
|
+
was not exercised against a live deploy.
|
|
242
|
+
|
|
243
|
+
Note also that a service account cannot be changed on an existing transfer config,
|
|
244
|
+
so a scheduled query originally created by an installed extension instance keeps
|
|
245
|
+
running as the extension's service account even after the kit adopts it. Delete
|
|
246
|
+
and recreate the scheduled query if you want it moved.
|
|
247
|
+
|
|
248
|
+
### The setup step runs on every deploy
|
|
249
|
+
|
|
250
|
+
The install, update and configure hooks are replaced by an `upsertTransferConfig`
|
|
251
|
+
task that the CLI runs after your first deploy and after every redeploy. It does
|
|
252
|
+
the same work: create the scheduled query if this instance has none, otherwise
|
|
253
|
+
reconcile the existing one against your current `QUERY_STRING`, `DATASET_ID`,
|
|
254
|
+
`TABLE_NAME`, `PARTITIONING_FIELD` and `SCHEDULE`.
|
|
255
|
+
|
|
256
|
+
Two consequences of it now being an ordinary task rather than a lifecycle event.
|
|
257
|
+
There is no install UI to report progress into, so failures show up in the task's
|
|
258
|
+
function logs, and the task retries up to five times with a 30 second minimum
|
|
259
|
+
backoff. And `DISPLAY_NAME` is no longer immutable, but changing it does not
|
|
260
|
+
rename an existing scheduled query, because display name is not part of the
|
|
261
|
+
update; it only applies to a config the kit creates.
|
|
262
|
+
|
|
263
|
+
Removing `PARTITIONING_FIELD` once it has been set still fails, with the same
|
|
264
|
+
explanation, because the BigQuery Data Transfer API cannot clear it.
|
|
265
|
+
|
|
266
|
+
### You can link an existing scheduled query
|
|
267
|
+
|
|
268
|
+
`TRANSFER_CONFIG_NAME` is a new setting. Point it at the full resource name of a
|
|
269
|
+
scheduled query you already have
|
|
270
|
+
(`projects/<project>/locations/<location>/transferConfigs/<id>`) and the kit
|
|
271
|
+
records that config in Firestore and consumes its notifications instead of
|
|
272
|
+
creating one of its own. The extension carried the code for this but no setting to
|
|
273
|
+
reach it. Leave it empty for the create-or-reconcile behaviour described above.
|
|
274
|
+
|
|
275
|
+
### Region, and no location setting
|
|
276
|
+
|
|
277
|
+
`LOCATION` is gone. Both functions deploy to your codebase's default region
|
|
278
|
+
(`us-central1` unless you have changed it) rather than the immutable location you
|
|
279
|
+
picked at install. `BIGQUERY_DATASET_LOCATION` is unchanged and still tells the
|
|
280
|
+
result query where your dataset lives.
|
|
281
|
+
|
|
282
|
+
### Failed notifications are retried
|
|
283
|
+
|
|
284
|
+
`processMessages` is a 2nd gen Pub/Sub function with retries enabled, where the
|
|
285
|
+
extension's 1st gen trigger did not retry. A run whose results fail to copy, for
|
|
286
|
+
example because BigQuery or Firestore is briefly unavailable, is now retried
|
|
287
|
+
rather than dropped. A notification that keeps failing, such as one for a transfer
|
|
288
|
+
config not tagged with this `INSTANCE_ID`, is also retried until Pub/Sub gives up.
|
|
289
|
+
|
|
290
|
+
Both functions' service accounts need `roles/eventarc.eventReceiver` and
|
|
291
|
+
`roles/run.invoker` on top of the three roles the extension asked for, and the
|
|
292
|
+
Pub/Sub API is now requested explicitly. The Firebase CLI handles all of this.
|
|
293
|
+
|
|
294
|
+
### Unchanged
|
|
295
|
+
|
|
296
|
+
- `COLLECTION_PATH` still defaults to `transferConfigs`, and the document layout
|
|
297
|
+
under it is identical: the transfer config document keyed by config id, a `runs`
|
|
298
|
+
subcollection keyed by run id holding `runMetadata`, `totalRowCount` and
|
|
299
|
+
`failedRowCount`, a `latest` document, and an `output` collection of rows per
|
|
300
|
+
run.
|
|
301
|
+
- The destination table is still `TABLE_NAME_{run_time|"%H%M%S"}` with
|
|
302
|
+
`WRITE_TRUNCATE`, and results are still read with `SELECT *` in
|
|
303
|
+
`BIGQUERY_DATASET_LOCATION`.
|
|
304
|
+
- Runs that do not succeed still write a run document with zeroed counts and still
|
|
305
|
+
update `latest`, and `latest` is still only moved forward by a newer run.
|
|
306
|
+
- Rows are still written one document at a time in chunks of 10,000, with
|
|
307
|
+
per-row failures logged and counted rather than aborting the run.
|
|
308
|
+
- `LOG_LEVEL` still accepts `debug`, `info`, `warn`, `error` and `silent`.
|
|
309
|
+
|
|
176
310
|
## API surface
|
|
177
311
|
|
|
178
|
-
- **Main entry** (`@firebase/bigquery-firestore-export`): exports
|
|
312
|
+
- **Main entry** (`@firebase-function-kits/bigquery-firestore-export`): exports
|
|
179
313
|
`processMessages` and `upsertTransferConfig`, registers first-deploy and
|
|
180
314
|
redeploy lifecycle hooks, and resolves runtime dependencies lazily. Use this
|
|
181
315
|
entry from Firebase deploy/emulator/runtime.
|
package/lib/config.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAsBA,OAAO,KAAK,EACV,6BAA6B,EAC7B,iBAAiB,EAElB,MAAM,iBAAiB,CAAC;
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAsBA,OAAO,KAAK,EACV,6BAA6B,EAC7B,iBAAiB,EAElB,MAAM,iBAAiB,CAAC;AA+JzB,eAAO,MAAM,kBAAkB,EAAE,iBAEhC,CAAC;AAcF,iEAAiE;AACjE,wBAAgB,aAAa,IAAI,6BAA6B,CAkB7D"}
|
package/lib/config.js
CHANGED
|
@@ -23,25 +23,144 @@ const instanceId = (0, params_1.defineString)("INSTANCE_ID");
|
|
|
23
23
|
const params = {
|
|
24
24
|
instanceId,
|
|
25
25
|
bigqueryDatasetLocation: (0, params_1.defineString)("BIGQUERY_DATASET_LOCATION", {
|
|
26
|
+
label: "BigQuery Dataset Location",
|
|
27
|
+
description: "What is the location of the BigQuery dataset referenced in the query?",
|
|
26
28
|
default: "US",
|
|
29
|
+
input: (0, params_1.select)({
|
|
30
|
+
"Columbus, Ohio (us-east5)": "us-east5",
|
|
31
|
+
"Iowa (us-central1)": "us-central1",
|
|
32
|
+
"Las Vegas (us-west4)": "us-west4",
|
|
33
|
+
"Los Angeles (us-west2)": "us-west2",
|
|
34
|
+
"Montréal (northamerica-northeast1)": "northamerica-northeast1",
|
|
35
|
+
"Northern Virginia (us-east4)": "us-east4",
|
|
36
|
+
"Oregon (us-west1)": "us-west1",
|
|
37
|
+
"Salt Lake City (us-west3)": "us-west3",
|
|
38
|
+
"São Paulo (southamerica-east1)": "southamerica-east1",
|
|
39
|
+
"Santiago (southamerica-west1)": "southamerica-west1",
|
|
40
|
+
"South Carolina (us-east1)": "us-east1",
|
|
41
|
+
"Toronto (northamerica-northeast2)": "northamerica-northeast2",
|
|
42
|
+
"Delhi (asia-south2)": "asia-south2",
|
|
43
|
+
"Hong Kong (asia-east2)": "asia-east2",
|
|
44
|
+
"Jakarta (asia-southeast2)": "asia-southeast2",
|
|
45
|
+
"Melbourne (australia-southeast2)": "australia-southeast2",
|
|
46
|
+
"Mumbai (asia-south1)": "asia-south1",
|
|
47
|
+
"Osaka (asia-northeast2)": "asia-northeast2",
|
|
48
|
+
"Seoul (asia-northeast3)": "asia-northeast3",
|
|
49
|
+
"Singapore (asia-southeast1)": "asia-southeast1",
|
|
50
|
+
"Sydney (australia-southeast1)": "australia-southeast1",
|
|
51
|
+
"Taiwan (asia-east1)": "asia-east1",
|
|
52
|
+
"Tokyo (asia-northeast1)": "asia-northeast1",
|
|
53
|
+
"Belgium (europe-west1)": "europe-west1",
|
|
54
|
+
"Finland (europe-north1)": "europe-north1",
|
|
55
|
+
"Frankfurt (europe-west3)": "europe-west3",
|
|
56
|
+
"London (europe-west2)": "europe-west2",
|
|
57
|
+
"Madrid (europe-southwest1)": "europe-southwest1",
|
|
58
|
+
"Milan (europe-west8)": "europe-west8",
|
|
59
|
+
"Netherlands (europe-west4)": "europe-west4",
|
|
60
|
+
"Paris (europe-west9)": "europe-west9",
|
|
61
|
+
"Warsaw (europe-central2)": "europe-central2",
|
|
62
|
+
"Zürich (europe-west6)": "europe-west6",
|
|
63
|
+
"US Multi-Region (US)": "US",
|
|
64
|
+
"EU Mutli-Region (EU)": "EU",
|
|
65
|
+
}),
|
|
27
66
|
}),
|
|
28
67
|
transferConfigName: (0, params_1.defineString)("TRANSFER_CONFIG_NAME", { default: "" }),
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
68
|
+
pubSubTopic: (0, params_1.defineString)("PUB_SUB_TOPIC", {
|
|
69
|
+
label: "Pub/Sub Topic",
|
|
70
|
+
description: "Which Pub/Sub topic should receive BigQuery Data Transfer completion notifications? Leave the default unless you are migrating from the bigquery-firestore-export extension, whose topic is named ext-<instance id>-processMessages. Pointing this at the extension's topic keeps the existing scheduled query's notification settings untouched.",
|
|
71
|
+
default: (0, params_1.expr) `kit-${instanceId}-processMessages`,
|
|
72
|
+
input: {
|
|
73
|
+
text: {
|
|
74
|
+
nonEmpty: true,
|
|
75
|
+
example: "ext-my-instance-processMessages",
|
|
76
|
+
validationRegex: /^(?!goog)[a-zA-Z][a-zA-Z0-9\-_.~+%]{2,254}$/,
|
|
77
|
+
validationErrorMessage: "Must be a Pub/Sub topic ID, not a full projects/<project>/topics/<topic> resource name. IDs are 3 to 255 characters, start with a letter, may contain letters, numbers and - _ . ~ + %, and cannot start with goog.",
|
|
78
|
+
},
|
|
79
|
+
},
|
|
80
|
+
}),
|
|
81
|
+
datasetId: (0, params_1.defineString)("DATASET_ID", {
|
|
82
|
+
label: "Dataset ID",
|
|
83
|
+
description: "What's the BigQuery destination dataset you'd like to use? Each transfer run will write to a table in this destination dataset.",
|
|
84
|
+
input: {
|
|
85
|
+
text: {
|
|
86
|
+
nonEmpty: true,
|
|
87
|
+
example: "customer_data",
|
|
88
|
+
},
|
|
89
|
+
},
|
|
90
|
+
}),
|
|
91
|
+
tableName: (0, params_1.defineString)("TABLE_NAME", {
|
|
92
|
+
label: "Destination Table Name",
|
|
93
|
+
description: "What's the destination table name prefix you'd like to use? Each transfer run will write to the table with this name, postfixed with the runtime.",
|
|
94
|
+
input: {
|
|
95
|
+
text: {
|
|
96
|
+
nonEmpty: true,
|
|
97
|
+
example: "transactions",
|
|
98
|
+
},
|
|
99
|
+
},
|
|
100
|
+
}),
|
|
101
|
+
queryString: (0, params_1.defineString)("QUERY_STRING", {
|
|
102
|
+
label: "Query String",
|
|
103
|
+
description: "What's the BQ query you'd like to execute?",
|
|
104
|
+
input: {
|
|
105
|
+
text: {
|
|
106
|
+
nonEmpty: true,
|
|
107
|
+
example: "SELECT * from <PROJECT_ID>.customer_data.transactions",
|
|
108
|
+
},
|
|
109
|
+
},
|
|
110
|
+
}),
|
|
111
|
+
displayName: (0, params_1.defineString)("DISPLAY_NAME", {
|
|
112
|
+
label: "Display Name",
|
|
113
|
+
description: "What display name would you like to use?",
|
|
114
|
+
input: {
|
|
115
|
+
text: {
|
|
116
|
+
nonEmpty: true,
|
|
117
|
+
example: "Daily Rollup - Customer Transactions",
|
|
118
|
+
},
|
|
119
|
+
},
|
|
120
|
+
}),
|
|
121
|
+
partitioningField: (0, params_1.defineString)("PARTITIONING_FIELD", {
|
|
122
|
+
label: "Partitioning Field",
|
|
123
|
+
description: "What's the partitioning field on the destination table ID? Leave empty if not using a partitioning field.",
|
|
124
|
+
default: "",
|
|
125
|
+
input: { text: { example: "timestamp" } },
|
|
126
|
+
}),
|
|
127
|
+
schedule: (0, params_1.defineString)("SCHEDULE", {
|
|
128
|
+
label: "Schedule",
|
|
129
|
+
description: "What's the execution schedule you'd like to use for this query?",
|
|
130
|
+
input: {
|
|
131
|
+
text: {
|
|
132
|
+
nonEmpty: true,
|
|
133
|
+
example: "every 15 minutes",
|
|
134
|
+
},
|
|
135
|
+
},
|
|
136
|
+
}),
|
|
35
137
|
firestoreCollection: (0, params_1.defineString)("COLLECTION_PATH", {
|
|
138
|
+
label: "Firestore Collection",
|
|
139
|
+
description: "What's the top-level Firestore Collection to store transfer configs, run metadata, and query output?",
|
|
36
140
|
default: "transferConfigs",
|
|
141
|
+
input: {
|
|
142
|
+
text: {
|
|
143
|
+
example: "transferConfigs",
|
|
144
|
+
validationRegex: /^[^\/]+(\/[^\/]+\/[^\/]+)*$/,
|
|
145
|
+
validationErrorMessage: "Must be a valid Cloud Firestore Collection",
|
|
146
|
+
},
|
|
147
|
+
},
|
|
37
148
|
}),
|
|
38
149
|
logLevel: (0, params_1.defineString)("LOG_LEVEL", {
|
|
150
|
+
label: "Log Level",
|
|
151
|
+
description: "What's the log level you'd like to use for this extension?",
|
|
39
152
|
default: "info",
|
|
40
|
-
input: (0, params_1.select)(
|
|
153
|
+
input: (0, params_1.select)({
|
|
154
|
+
DEBUG: "debug",
|
|
155
|
+
INFO: "info",
|
|
156
|
+
WARN: "warn",
|
|
157
|
+
ERROR: "error",
|
|
158
|
+
SILENT: "silent",
|
|
159
|
+
}),
|
|
41
160
|
}),
|
|
42
161
|
};
|
|
43
162
|
exports.CONFIG_EXPRESSIONS = {
|
|
44
|
-
pubSubTopic:
|
|
163
|
+
pubSubTopic: params.pubSubTopic,
|
|
45
164
|
};
|
|
46
165
|
function optional(value) {
|
|
47
166
|
const normalized = value.trim();
|
|
@@ -67,7 +186,7 @@ function configFromEnv() {
|
|
|
67
186
|
displayName: params.displayName.value(),
|
|
68
187
|
partitioningField: optional(params.partitioningField.value()),
|
|
69
188
|
schedule: params.schedule.value(),
|
|
70
|
-
pubSubTopic:
|
|
189
|
+
pubSubTopic: params.pubSubTopic.value(),
|
|
71
190
|
firestoreCollection: params.firestoreCollection.value(),
|
|
72
191
|
logLevel: normalizeLogLevel(params.logLevel.value()),
|
|
73
192
|
};
|
package/lib/config.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;GAcG
|
|
1
|
+
{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;GAcG;;;AA4LH,sCAkBC;AA5MD,sDAKmC;AAOnC,MAAM,iBAAiB,GAAG,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAU,CAAC;AAChF,MAAM,UAAU,GAAG,IAAA,qBAAY,EAAC,aAAa,CAAC,CAAC;AAE/C,MAAM,MAAM,GAAG;IACb,UAAU;IACV,uBAAuB,EAAE,IAAA,qBAAY,EAAC,2BAA2B,EAAE;QACjE,KAAK,EAAE,2BAA2B;QAClC,WAAW,EACT,uEAAuE;QAEzE,OAAO,EAAE,IAAI;QACb,KAAK,EAAE,IAAA,eAAM,EAAC;YACZ,2BAA2B,EAAE,UAAU;YACvC,oBAAoB,EAAE,aAAa;YACnC,sBAAsB,EAAE,UAAU;YAClC,wBAAwB,EAAE,UAAU;YACpC,oCAAoC,EAAE,yBAAyB;YAC/D,8BAA8B,EAAE,UAAU;YAC1C,mBAAmB,EAAE,UAAU;YAC/B,2BAA2B,EAAE,UAAU;YACvC,gCAAgC,EAAE,oBAAoB;YACtD,+BAA+B,EAAE,oBAAoB;YACrD,2BAA2B,EAAE,UAAU;YACvC,mCAAmC,EAAE,yBAAyB;YAC9D,qBAAqB,EAAE,aAAa;YACpC,wBAAwB,EAAE,YAAY;YACtC,2BAA2B,EAAE,iBAAiB;YAC9C,kCAAkC,EAAE,sBAAsB;YAC1D,sBAAsB,EAAE,aAAa;YACrC,yBAAyB,EAAE,iBAAiB;YAC5C,yBAAyB,EAAE,iBAAiB;YAC5C,6BAA6B,EAAE,iBAAiB;YAChD,+BAA+B,EAAE,sBAAsB;YACvD,qBAAqB,EAAE,YAAY;YACnC,yBAAyB,EAAE,iBAAiB;YAC5C,wBAAwB,EAAE,cAAc;YACxC,yBAAyB,EAAE,eAAe;YAC1C,0BAA0B,EAAE,cAAc;YAC1C,uBAAuB,EAAE,cAAc;YACvC,4BAA4B,EAAE,mBAAmB;YACjD,sBAAsB,EAAE,cAAc;YACtC,4BAA4B,EAAE,cAAc;YAC5C,sBAAsB,EAAE,cAAc;YACtC,0BAA0B,EAAE,iBAAiB;YAC7C,uBAAuB,EAAE,cAAc;YACvC,sBAAsB,EAAE,IAAI;YAC5B,sBAAsB,EAAE,IAAI;SAC7B,CAAC;KACH,CAAC;IACF,kBAAkB,EAAE,IAAA,qBAAY,EAAC,sBAAsB,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;IACzE,WAAW,EAAE,IAAA,qBAAY,EAAC,eAAe,EAAE;QACzC,KAAK,EAAE,eAAe;QACtB,WAAW,EACT,mVAAmV;QAErV,OAAO,EAAE,IAAA,aAAI,EAAA,OAAO,UAAU,kBAAkB;QAChD,KAAK,EAAE;YACL,IAAI,EAAE;gBACJ,QAAQ,EAAE,IAAI;gBACd,OAAO,EAAE,iCAAiC;gBAC1C,eAAe,EAAE,6CAA6C;gBAC9D,sBAAsB,EACpB,qNAAqN;aACxN;SACF;KACF,CAAC;IACF,SAAS,EAAE,IAAA,qBAAY,EAAC,YAAY,EAAE;QACpC,KAAK,EAAE,YAAY;QACnB,WAAW,EACT,iIAAiI;QACnI,KAAK,EAAE;YACL,IAAI,EAAE;gBACJ,QAAQ,EAAE,IAAI;gBACd,OAAO,EAAE,eAAe;aACzB;SACF;KACF,CAAC;IACF,SAAS,EAAE,IAAA,qBAAY,EAAC,YAAY,EAAE;QACpC,KAAK,EAAE,wBAAwB;QAC/B,WAAW,EACT,mJAAmJ;QACrJ,KAAK,EAAE;YACL,IAAI,EAAE;gBACJ,QAAQ,EAAE,IAAI;gBACd,OAAO,EAAE,cAAc;aACxB;SACF;KACF,CAAC;IACF,WAAW,EAAE,IAAA,qBAAY,EAAC,cAAc,EAAE;QACxC,KAAK,EAAE,cAAc;QACrB,WAAW,EAAE,4CAA4C;QACzD,KAAK,EAAE;YACL,IAAI,EAAE;gBACJ,QAAQ,EAAE,IAAI;gBAEd,OAAO,EAAE,uDAAuD;aACjE;SACF;KACF,CAAC;IACF,WAAW,EAAE,IAAA,qBAAY,EAAC,cAAc,EAAE;QACxC,KAAK,EAAE,cAAc;QACrB,WAAW,EAAE,0CAA0C;QACvD,KAAK,EAAE;YACL,IAAI,EAAE;gBACJ,QAAQ,EAAE,IAAI;gBACd,OAAO,EAAE,sCAAsC;aAChD;SACF;KACF,CAAC;IACF,iBAAiB,EAAE,IAAA,qBAAY,EAAC,oBAAoB,EAAE;QACpD,KAAK,EAAE,oBAAoB;QAC3B,WAAW,EACT,2GAA2G;QAC7G,OAAO,EAAE,EAAE;QACX,KAAK,EAAE,EAAE,IAAI,EAAE,EAAE,OAAO,EAAE,WAAW,EAAE,EAAE;KAC1C,CAAC;IACF,QAAQ,EAAE,IAAA,qBAAY,EAAC,UAAU,EAAE;QACjC,KAAK,EAAE,UAAU;QACjB,WAAW,EACT,iEAAiE;QACnE,KAAK,EAAE;YACL,IAAI,EAAE;gBACJ,QAAQ,EAAE,IAAI;gBACd,OAAO,EAAE,kBAAkB;aAC5B;SACF;KACF,CAAC;IACF,mBAAmB,EAAE,IAAA,qBAAY,EAAC,iBAAiB,EAAE;QACnD,KAAK,EAAE,sBAAsB;QAC7B,WAAW,EACT,sGAAsG;QAExG,OAAO,EAAE,iBAAiB;QAC1B,KAAK,EAAE;YACL,IAAI,EAAE;gBACJ,OAAO,EAAE,iBAAiB;gBAE1B,eAAe,EAAE,6BAA6B;gBAC9C,sBAAsB,EAAE,4CAA4C;aACrE;SACF;KACF,CAAC;IACF,QAAQ,EAAE,IAAA,qBAAY,EAAC,WAAW,EAAE;QAClC,KAAK,EAAE,WAAW;QAClB,WAAW,EAAE,4DAA4D;QAEzE,OAAO,EAAE,MAAM;QACf,KAAK,EAAE,IAAA,eAAM,EAAC;YACZ,KAAK,EAAE,OAAO;YACd,IAAI,EAAE,MAAM;YACZ,IAAI,EAAE,MAAM;YACZ,KAAK,EAAE,OAAO;YACd,MAAM,EAAE,QAAQ;SACjB,CAAC;KACH,CAAC;CACH,CAAC;AAEW,QAAA,kBAAkB,GAAsB;IACnD,WAAW,EAAE,MAAM,CAAC,WAAW;CAChC,CAAC;AAEF,SAAS,QAAQ,CAAC,KAAa;IAC7B,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;IAChC,OAAO,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC;AACxD,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAa;IACtC,MAAM,UAAU,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC;IACvC,OAAO,iBAAiB,CAAC,QAAQ,CAAC,UAAsB,CAAC;QACvD,CAAC,CAAE,UAAuB;QAC1B,CAAC,CAAC,MAAM,CAAC;AACb,CAAC;AAED,iEAAiE;AACjE,SAAgB,aAAa;IAC3B,MAAM,kBAAkB,GAAG,MAAM,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;IAErD,OAAO;QACL,uBAAuB,EAAE,MAAM,CAAC,uBAAuB,CAAC,KAAK,EAAE;QAC/D,SAAS,EAAE,kBAAS,CAAC,KAAK,EAAE;QAC5B,UAAU,EAAE,kBAAkB;QAC9B,kBAAkB,EAAE,QAAQ,CAAC,MAAM,CAAC,kBAAkB,CAAC,KAAK,EAAE,CAAC;QAC/D,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC,KAAK,EAAE;QACnC,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC,KAAK,EAAE;QACnC,WAAW,EAAE,MAAM,CAAC,WAAW,CAAC,KAAK,EAAE;QACvC,WAAW,EAAE,MAAM,CAAC,WAAW,CAAC,KAAK,EAAE;QACvC,iBAAiB,EAAE,QAAQ,CAAC,MAAM,CAAC,iBAAiB,CAAC,KAAK,EAAE,CAAC;QAC7D,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE;QACjC,WAAW,EAAE,MAAM,CAAC,WAAW,CAAC,KAAK,EAAE;QACvC,mBAAmB,EAAE,MAAM,CAAC,mBAAmB,CAAC,KAAK,EAAE;QACvD,QAAQ,EAAE,iBAAiB,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;KACrD,CAAC;AACJ,CAAC"}
|
package/lib/dts.d.ts
CHANGED
|
@@ -4,8 +4,23 @@ export type DataTransferClient = bigqueryDataTransfer.v1.DataTransferServiceClie
|
|
|
4
4
|
export type TransferConfig = bigqueryDataTransfer.protos.google.cloud.bigquery.datatransfer.v1.ITransferConfig;
|
|
5
5
|
export declare const PARTITIONING_FIELD_REMOVAL_ERROR_PREFIX = "Cannot remove partitioning_field from an existing transfer config";
|
|
6
6
|
export declare const PARTITIONING_FIELD_REMOVAL_ERROR = "Cannot remove partitioning_field from an existing transfer config. The BigQuery Data Transfer API does not support clearing this parameter once it has been set. To change partitioning, create a new transfer config with the desired setting.";
|
|
7
|
-
/**
|
|
8
|
-
|
|
7
|
+
/**
|
|
8
|
+
* Full resource name of the topic DTS publishes run notifications to.
|
|
9
|
+
*
|
|
10
|
+
* PUB_SUB_TOPIC is validated as a bare topic ID when prompted, but values from
|
|
11
|
+
* a dotenv file reach us unvalidated, and both the Pub/Sub client and the
|
|
12
|
+
* trigger accept a full resource name, so accept one here too.
|
|
13
|
+
*/
|
|
14
|
+
export declare function notificationTopicName(config: ResolvedBigqueryFirestoreExportConfig): string;
|
|
15
|
+
/**
|
|
16
|
+
* Creates the protobuf-shaped request used for a scheduled query.
|
|
17
|
+
*
|
|
18
|
+
* The query runs as whichever identity creates it, which is this function. To
|
|
19
|
+
* name a different account, `serviceAccountName` goes on the request rather
|
|
20
|
+
* than on the transfer config, and the caller needs `actAs` on the account it
|
|
21
|
+
* names, including its own.
|
|
22
|
+
*/
|
|
23
|
+
export declare function createTransferConfigRequest(config: ResolvedBigqueryFirestoreExportConfig): bigqueryDataTransfer.protos.google.cloud.bigquery.datatransfer.v1.ICreateTransferConfigRequest;
|
|
9
24
|
export declare function getTransferConfig(client: DataTransferClient, transferConfigName: string): Promise<TransferConfig | null>;
|
|
10
25
|
export declare function createTransferConfig(client: DataTransferClient, config: ResolvedBigqueryFirestoreExportConfig): Promise<TransferConfig>;
|
|
11
26
|
/** Builds a minimal update mask while retaining unsupported immutable fields. */
|
package/lib/dts.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dts.d.ts","sourceRoot":"","sources":["../src/dts.ts"],"names":[],"mappings":"AAgBA,OAAO,KAAK,oBAAoB,MAAM,sCAAsC,CAAC;AAC7E,OAAO,KAAK,EAAE,qCAAqC,EAAE,MAAM,iBAAiB,CAAC;AAG7E,MAAM,MAAM,kBAAkB,GAC5B,oBAAoB,CAAC,EAAE,CAAC,yBAAyB,CAAC;AACpD,MAAM,MAAM,cAAc,GACxB,oBAAoB,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC,eAAe,CAAC;AAMpF,eAAO,MAAM,uCAAuC,sEACxB,CAAC;AAC7B,eAAO,MAAM,gCAAgC,oPAAgN,CAAC;
|
|
1
|
+
{"version":3,"file":"dts.d.ts","sourceRoot":"","sources":["../src/dts.ts"],"names":[],"mappings":"AAgBA,OAAO,KAAK,oBAAoB,MAAM,sCAAsC,CAAC;AAC7E,OAAO,KAAK,EAAE,qCAAqC,EAAE,MAAM,iBAAiB,CAAC;AAG7E,MAAM,MAAM,kBAAkB,GAC5B,oBAAoB,CAAC,EAAE,CAAC,yBAAyB,CAAC;AACpD,MAAM,MAAM,cAAc,GACxB,oBAAoB,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC,eAAe,CAAC;AAMpF,eAAO,MAAM,uCAAuC,sEACxB,CAAC;AAC7B,eAAO,MAAM,gCAAgC,oPAAgN,CAAC;AAgC9P;;;;;;GAMG;AACH,wBAAgB,qBAAqB,CACnC,MAAM,EAAE,qCAAqC,GAC5C,MAAM,CAIR;AAMD;;;;;;;GAOG;AACH,wBAAgB,2BAA2B,CACzC,MAAM,EAAE,qCAAqC,GAC5C,oBAAoB,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC,4BAA4B,CAqBhG;AAED,wBAAsB,iBAAiB,CACrC,MAAM,EAAE,kBAAkB,EAC1B,kBAAkB,EAAE,MAAM,GACzB,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC,CAchC;AAED,wBAAsB,oBAAoB,CACxC,MAAM,EAAE,kBAAkB,EAC1B,MAAM,EAAE,qCAAqC,GAC5C,OAAO,CAAC,cAAc,CAAC,CAUzB;AAED,iFAAiF;AACjF,wBAAsB,oCAAoC,CACxD,MAAM,EAAE,kBAAkB,EAC1B,kBAAkB,EAAE,MAAM,EAC1B,MAAM,EAAE,qCAAqC,GAC5C,OAAO,CAAC,oBAAoB,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC,4BAA4B,CAAC,CAyDzG;AAED,wBAAsB,oBAAoB,CACxC,MAAM,EAAE,kBAAkB,EAC1B,kBAAkB,EAAE,MAAM,EAC1B,MAAM,EAAE,qCAAqC,GAC5C,OAAO,CAAC,cAAc,CAAC,CAczB"}
|
package/lib/dts.js
CHANGED
|
@@ -49,6 +49,7 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
49
49
|
})();
|
|
50
50
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
51
51
|
exports.PARTITIONING_FIELD_REMOVAL_ERROR = exports.PARTITIONING_FIELD_REMOVAL_ERROR_PREFIX = void 0;
|
|
52
|
+
exports.notificationTopicName = notificationTopicName;
|
|
52
53
|
exports.createTransferConfigRequest = createTransferConfigRequest;
|
|
53
54
|
exports.getTransferConfig = getTransferConfig;
|
|
54
55
|
exports.createTransferConfig = createTransferConfig;
|
|
@@ -79,11 +80,30 @@ function transferConfigFields(config) {
|
|
|
79
80
|
}
|
|
80
81
|
return fields;
|
|
81
82
|
}
|
|
83
|
+
/**
|
|
84
|
+
* Full resource name of the topic DTS publishes run notifications to.
|
|
85
|
+
*
|
|
86
|
+
* PUB_SUB_TOPIC is validated as a bare topic ID when prompted, but values from
|
|
87
|
+
* a dotenv file reach us unvalidated, and both the Pub/Sub client and the
|
|
88
|
+
* trigger accept a full resource name, so accept one here too.
|
|
89
|
+
*/
|
|
90
|
+
function notificationTopicName(config) {
|
|
91
|
+
return config.pubSubTopic.startsWith("projects/")
|
|
92
|
+
? config.pubSubTopic
|
|
93
|
+
: `projects/${config.projectId}/topics/${config.pubSubTopic}`;
|
|
94
|
+
}
|
|
82
95
|
function stringField(value) {
|
|
83
96
|
return { stringValue: value ?? "" };
|
|
84
97
|
}
|
|
85
|
-
/**
|
|
86
|
-
|
|
98
|
+
/**
|
|
99
|
+
* Creates the protobuf-shaped request used for a scheduled query.
|
|
100
|
+
*
|
|
101
|
+
* The query runs as whichever identity creates it, which is this function. To
|
|
102
|
+
* name a different account, `serviceAccountName` goes on the request rather
|
|
103
|
+
* than on the transfer config, and the caller needs `actAs` on the account it
|
|
104
|
+
* names, including its own.
|
|
105
|
+
*/
|
|
106
|
+
function createTransferConfigRequest(config) {
|
|
87
107
|
return {
|
|
88
108
|
parent: `projects/${config.projectId}`,
|
|
89
109
|
transferConfig: {
|
|
@@ -99,10 +119,7 @@ function createTransferConfigRequest(config, serviceAccountEmail) {
|
|
|
99
119
|
},
|
|
100
120
|
},
|
|
101
121
|
schedule: config.schedule,
|
|
102
|
-
notificationPubsubTopic:
|
|
103
|
-
...(serviceAccountEmail
|
|
104
|
-
? { serviceAccountName: serviceAccountEmail }
|
|
105
|
-
: {}),
|
|
122
|
+
notificationPubsubTopic: notificationTopicName(config),
|
|
106
123
|
},
|
|
107
124
|
};
|
|
108
125
|
}
|
|
@@ -124,7 +141,7 @@ async function getTransferConfig(client, transferConfigName) {
|
|
|
124
141
|
}
|
|
125
142
|
async function createTransferConfig(client, config) {
|
|
126
143
|
logs.createTransferConfig();
|
|
127
|
-
const [created] = await client.createTransferConfig(createTransferConfigRequest(config
|
|
144
|
+
const [created] = await client.createTransferConfig(createTransferConfigRequest(config));
|
|
128
145
|
if (!created.name) {
|
|
129
146
|
throw new Error("BigQuery API returned a transfer config without a name");
|
|
130
147
|
}
|
|
@@ -168,7 +185,7 @@ async function constructUpdateTransferConfigRequest(client, transferConfigName,
|
|
|
168
185
|
updateMask.push("schedule");
|
|
169
186
|
updatedConfig.schedule = config.schedule;
|
|
170
187
|
}
|
|
171
|
-
const expectedTopic =
|
|
188
|
+
const expectedTopic = notificationTopicName(config);
|
|
172
189
|
if (expectedTopic !== transferConfig.notificationPubsubTopic) {
|
|
173
190
|
updateMask.push("notification_pubsub_topic");
|
|
174
191
|
updatedConfig.notificationPubsubTopic = expectedTopic;
|
package/lib/dts.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dts.js","sourceRoot":"","sources":["../src/dts.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;GAcG
|
|
1
|
+
{"version":3,"file":"dts.js","sourceRoot":"","sources":["../src/dts.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;GAcG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAwDH,sDAMC;AAcD,kEAuBC;AAED,8CAiBC;AAED,oDAaC;AAGD,oFA6DC;AAED,oDAkBC;AAvND,2FAA6E;AAE7E,6CAA+B;AAO/B,MAAM,cAAc,GAAG,CAAC,CAAC;AACzB,MAAM,0BAA0B,GAC9B,mEAAmE,CAAC;AAEzD,QAAA,uCAAuC,GAClD,0BAA0B,CAAC;AAChB,QAAA,gCAAgC,GAAG,GAAG,0BAA0B,gLAAgL,CAAC;AAE9P,SAAS,eAAe,CAAC,GAAY;IACnC,OAAO,CACL,OAAO,GAAG,KAAK,QAAQ;QACvB,GAAG,KAAK,IAAI;QACZ,MAAM,IAAI,GAAG;QACb,GAAG,CAAC,IAAI,KAAK,cAAc,CAC5B,CAAC;AACJ,CAAC;AAED,SAAS,oBAAoB,CAAC,MAAsB;IAClD,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC;IACrC,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,IAAI,KAAK,CACb,8DAA8D,CAC/D,CAAC;IACJ,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QAClB,MAAM,IAAI,KAAK,CACb,oEAAoE,CACrE,CAAC;IACJ,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,+BAA+B,EAAE,CAAC;QAC5C,MAAM,IAAI,KAAK,CACb,8FAA8F,CAC/F,CAAC;IACJ,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;;GAMG;AACH,SAAgB,qBAAqB,CACnC,MAA6C;IAE7C,OAAO,MAAM,CAAC,WAAW,CAAC,UAAU,CAAC,WAAW,CAAC;QAC/C,CAAC,CAAC,MAAM,CAAC,WAAW;QACpB,CAAC,CAAC,YAAY,MAAM,CAAC,SAAS,WAAW,MAAM,CAAC,WAAW,EAAE,CAAC;AAClE,CAAC;AAED,SAAS,WAAW,CAAC,KAAyB;IAC5C,OAAO,EAAE,WAAW,EAAE,KAAK,IAAI,EAAE,EAAE,CAAC;AACtC,CAAC;AAED;;;;;;;GAOG;AACH,SAAgB,2BAA2B,CACzC,MAA6C;IAE7C,OAAO;QACL,MAAM,EAAE,YAAY,MAAM,CAAC,SAAS,EAAE;QACtC,cAAc,EAAE;YACd,oBAAoB,EAAE,MAAM,CAAC,SAAS;YACtC,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,YAAY,EAAE,iBAAiB;YAC/B,MAAM,EAAE;gBACN,MAAM,EAAE;oBACN,KAAK,EAAE,WAAW,CAAC,MAAM,CAAC,WAAW,CAAC;oBACtC,+BAA+B,EAAE,WAAW,CAC1C,GAAG,MAAM,CAAC,SAAS,sBAAsB,CAC1C;oBACD,iBAAiB,EAAE,WAAW,CAAC,gBAAgB,CAAC;oBAChD,kBAAkB,EAAE,WAAW,CAAC,MAAM,CAAC,iBAAiB,CAAC;iBAC1D;aACF;YACD,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,uBAAuB,EAAE,qBAAqB,CAAC,MAAM,CAAC;SACvD;KACF,CAAC;AACJ,CAAC;AAEM,KAAK,UAAU,iBAAiB,CACrC,MAA0B,EAC1B,kBAA0B;IAE1B,IAAI,CAAC;QACH,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,MAAM,CAAC,iBAAiB,CAAC;YAC9C,IAAI,EAAE,kBAAkB;SACzB,CAAC,CAAC;QACH,OAAO,MAAM,CAAC;IAChB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,eAAe,CAAC,GAAG,CAAC,EAAE,CAAC;YACzB,IAAI,CAAC,sBAAsB,CAAC,kBAAkB,CAAC,CAAC;YAChD,OAAO,IAAI,CAAC;QACd,CAAC;QACD,IAAI,CAAC,uBAAuB,CAAC,kBAAkB,EAAE,GAAG,CAAC,CAAC;QACtD,MAAM,GAAG,CAAC;IACZ,CAAC;AACH,CAAC;AAEM,KAAK,UAAU,oBAAoB,CACxC,MAA0B,EAC1B,MAA6C;IAE7C,IAAI,CAAC,oBAAoB,EAAE,CAAC;IAC5B,MAAM,CAAC,OAAO,CAAC,GAAG,MAAM,MAAM,CAAC,oBAAoB,CACjD,2BAA2B,CAAC,MAAM,CAAC,CACpC,CAAC;IACF,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAClB,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC;IAC5E,CAAC;IACD,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IACzC,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,iFAAiF;AAC1E,KAAK,UAAU,oCAAoC,CACxD,MAA0B,EAC1B,kBAA0B,EAC1B,MAA6C;IAE7C,MAAM,cAAc,GAAG,MAAM,iBAAiB,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAC;IAC3E,IAAI,CAAC,cAAc;QAAE,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAElE,MAAM,MAAM,GAAG,oBAAoB,CAAC,cAAc,CAAC,CAAC;IACpD,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAC9B,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,CACb,CAAC;IACpB,MAAM,aAAa,GAAG,oBAAoB,CAAC,aAAa,CAAC,CAAC;IAC1D,MAAM,UAAU,GAAa,EAAE,CAAC;IAEhC,IAAI,MAAM,CAAC,SAAS,KAAK,cAAc,CAAC,oBAAoB,EAAE,CAAC;QAC7D,UAAU,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;QAC1C,aAAa,CAAC,oBAAoB,GAAG,MAAM,CAAC,SAAS,CAAC;IACxD,CAAC;IACD,IAAI,MAAM,CAAC,WAAW,KAAK,MAAM,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC;QACpD,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC1B,aAAa,CAAC,KAAK,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;IACvD,CAAC;IAED,MAAM,aAAa,GAAG,GAAG,MAAM,CAAC,SAAS,sBAAsB,CAAC;IAChE,IAAI,aAAa,KAAK,MAAM,CAAC,+BAA+B,CAAC,WAAW,EAAE,CAAC;QACzE,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC1B,aAAa,CAAC,+BAA+B,CAAC,WAAW,GAAG,aAAa,CAAC;IAC5E,CAAC;IAED,MAAM,yBAAyB,GAC7B,MAAM,CAAC,kBAAkB,EAAE,WAAW,IAAI,EAAE,CAAC;IAC/C,MAAM,oBAAoB,GAAG,MAAM,CAAC,iBAAiB,IAAI,EAAE,CAAC;IAC5D,IAAI,oBAAoB,KAAK,yBAAyB,EAAE,CAAC;QACvD,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAC1B,IAAI,CAAC,iCAAiC,CACpC,kBAAkB,EAClB,yBAAyB,CAC1B,CAAC;YACF,MAAM,IAAI,KAAK,CAAC,wCAAgC,CAAC,CAAC;QACpD,CAAC;QACD,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC1B,aAAa,CAAC,kBAAkB,KAAK,EAAE,CAAC;QACxC,aAAa,CAAC,kBAAkB,CAAC,WAAW,GAAG,oBAAoB,CAAC;IACtE,CAAC;IAED,IAAI,MAAM,CAAC,QAAQ,KAAK,cAAc,CAAC,QAAQ,EAAE,CAAC;QAChD,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAC5B,aAAa,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;IAC3C,CAAC;IAED,MAAM,aAAa,GAAG,qBAAqB,CAAC,MAAM,CAAC,CAAC;IACpD,IAAI,aAAa,KAAK,cAAc,CAAC,uBAAuB,EAAE,CAAC;QAC7D,UAAU,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;QAC7C,aAAa,CAAC,uBAAuB,GAAG,aAAa,CAAC;IACxD,CAAC;IAED,OAAO;QACL,cAAc,EAAE,aAAa;QAC7B,UAAU,EAAE,EAAE,KAAK,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC,EAAE;KAChD,CAAC;AACJ,CAAC;AAEM,KAAK,UAAU,oBAAoB,CACxC,MAA0B,EAC1B,kBAA0B,EAC1B,MAA6C;IAE7C,MAAM,OAAO,GAAG,MAAM,oCAAoC,CACxD,MAAM,EACN,kBAAkB,EAClB,MAAM,CACP,CAAC;IACF,IAAI,CAAC,oBAAoB,CAAC,kBAAkB,CAAC,CAAC;IAC9C,MAAM,SAAS,GACb,oBAAoB,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC,2BAA2B,CAAC,UAAU,CACtG,OAAO,CACR,CAAC;IACJ,MAAM,CAAC,OAAO,CAAC,GAAG,MAAM,MAAM,CAAC,oBAAoB,CAAC,SAAS,CAAC,CAAC;IAC/D,IAAI,CAAC,qBAAqB,CAAC,kBAAkB,CAAC,CAAC;IAC/C,OAAO,OAAO,CAAC;AACjB,CAAC"}
|
package/lib/export-config.d.ts
CHANGED
|
@@ -27,8 +27,6 @@ export interface BigqueryFirestoreExportConfig {
|
|
|
27
27
|
pubSubTopic?: string;
|
|
28
28
|
/** Root Firestore collection for configs and run output. */
|
|
29
29
|
firestoreCollection?: string;
|
|
30
|
-
/** Runtime identity used when creating a DTS config. */
|
|
31
|
-
serviceAccount?: string;
|
|
32
30
|
/** Log verbosity. Defaults to `info`. */
|
|
33
31
|
logLevel?: LogLevel;
|
|
34
32
|
}
|
|
@@ -46,7 +44,6 @@ export interface ResolvedBigqueryFirestoreExportConfig {
|
|
|
46
44
|
schedule: string;
|
|
47
45
|
pubSubTopic: string;
|
|
48
46
|
firestoreCollection: string;
|
|
49
|
-
serviceAccount?: string;
|
|
50
47
|
logLevel: LogLevel;
|
|
51
48
|
}
|
|
52
49
|
/** Deploy-time values used to construct the v2 triggers. */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"export-config.d.ts","sourceRoot":"","sources":["../src/export-config.ts"],"names":[],"mappings":"AAgBA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,2BAA2B,CAAC;AAE5D,sDAAsD;AACtD,MAAM,MAAM,QAAQ,GAAG,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,QAAQ,CAAC;AAEtE,oEAAoE;AACpE,MAAM,WAAW,6BAA6B;IAC5C,sEAAsE;IACtE,uBAAuB,EAAE,MAAM,CAAC;IAChC,+BAA+B;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,qEAAqE;IACrE,UAAU,EAAE,MAAM,CAAC;IACnB,6DAA6D;IAC7D,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,uCAAuC;IACvC,SAAS,EAAE,MAAM,CAAC;IAClB,6CAA6C;IAC7C,SAAS,EAAE,MAAM,CAAC;IAClB,4BAA4B;IAC5B,WAAW,EAAE,MAAM,CAAC;IACpB,+CAA+C;IAC/C,WAAW,EAAE,MAAM,CAAC;IACpB,qDAAqD;IACrD,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,uEAAuE;IACvE,QAAQ,EAAE,MAAM,CAAC;IACjB,4DAA4D;IAC5D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,4DAA4D;IAC5D,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,
|
|
1
|
+
{"version":3,"file":"export-config.d.ts","sourceRoot":"","sources":["../src/export-config.ts"],"names":[],"mappings":"AAgBA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,2BAA2B,CAAC;AAE5D,sDAAsD;AACtD,MAAM,MAAM,QAAQ,GAAG,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,QAAQ,CAAC;AAEtE,oEAAoE;AACpE,MAAM,WAAW,6BAA6B;IAC5C,sEAAsE;IACtE,uBAAuB,EAAE,MAAM,CAAC;IAChC,+BAA+B;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,qEAAqE;IACrE,UAAU,EAAE,MAAM,CAAC;IACnB,6DAA6D;IAC7D,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,uCAAuC;IACvC,SAAS,EAAE,MAAM,CAAC;IAClB,6CAA6C;IAC7C,SAAS,EAAE,MAAM,CAAC;IAClB,4BAA4B;IAC5B,WAAW,EAAE,MAAM,CAAC;IACpB,+CAA+C;IAC/C,WAAW,EAAE,MAAM,CAAC;IACpB,qDAAqD;IACrD,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,uEAAuE;IACvE,QAAQ,EAAE,MAAM,CAAC;IACjB,4DAA4D;IAC5D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,4DAA4D;IAC5D,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,yCAAyC;IACzC,QAAQ,CAAC,EAAE,QAAQ,CAAC;CACrB;AAED,qEAAqE;AACrE,MAAM,WAAW,qCAAqC;IACpD,uBAAuB,EAAE,MAAM,CAAC;IAChC,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,QAAQ,EAAE,QAAQ,CAAC;CACpB;AAED,4DAA4D;AAC5D,MAAM,WAAW,iBAAiB;IAChC,WAAW,EAAE,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;CAC1C;AAgBD,4EAA4E;AAC5E,wBAAgB,aAAa,CAC3B,MAAM,EAAE,6BAA6B,GACpC,qCAAqC,CA4BvC"}
|
package/lib/export-config.js
CHANGED
|
@@ -49,7 +49,6 @@ function resolveConfig(config) {
|
|
|
49
49
|
schedule: required(config.schedule, "schedule"),
|
|
50
50
|
pubSubTopic: optional(config.pubSubTopic) ?? `kit-${instanceId}-processMessages`,
|
|
51
51
|
firestoreCollection: optional(config.firestoreCollection) ?? "transferConfigs",
|
|
52
|
-
serviceAccount: optional(config.serviceAccount),
|
|
53
52
|
logLevel,
|
|
54
53
|
};
|
|
55
54
|
}
|
package/lib/export-config.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"export-config.js","sourceRoot":"","sources":["../src/export-config.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;GAcG
|
|
1
|
+
{"version":3,"file":"export-config.js","sourceRoot":"","sources":["../src/export-config.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;GAcG;;AA0EH,sCA8BC;AA7CD,SAAS,QAAQ,CAAC,KAAa,EAAE,KAAa;IAC5C,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;IAChC,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,8BAA8B,CAAC,CAAC;IAC1D,CAAC;IACD,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,SAAS,QAAQ,CAAC,KAAyB;IACzC,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAC1C,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;IAChC,OAAO,UAAU,IAAI,SAAS,CAAC;AACjC,CAAC;AAED,4EAA4E;AAC5E,SAAgB,aAAa,CAC3B,MAAqC;IAErC,MAAM,UAAU,GAAG,QAAQ,CAAC,MAAM,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC;IAC7D,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC;IAE3C,IAAI,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QACrE,MAAM,IAAI,KAAK,CAAC,yBAAyB,QAAQ,EAAE,CAAC,CAAC;IACvD,CAAC;IAED,OAAO;QACL,uBAAuB,EAAE,QAAQ,CAC/B,MAAM,CAAC,uBAAuB,EAC9B,yBAAyB,CAC1B;QACD,SAAS,EAAE,QAAQ,CAAC,MAAM,CAAC,SAAS,EAAE,WAAW,CAAC;QAClD,UAAU;QACV,kBAAkB,EAAE,QAAQ,CAAC,MAAM,CAAC,kBAAkB,CAAC;QACvD,SAAS,EAAE,QAAQ,CAAC,MAAM,CAAC,SAAS,EAAE,WAAW,CAAC;QAClD,SAAS,EAAE,QAAQ,CAAC,MAAM,CAAC,SAAS,EAAE,WAAW,CAAC;QAClD,WAAW,EAAE,QAAQ,CAAC,MAAM,CAAC,WAAW,EAAE,aAAa,CAAC;QACxD,WAAW,EAAE,QAAQ,CAAC,MAAM,CAAC,WAAW,EAAE,aAAa,CAAC;QACxD,iBAAiB,EAAE,QAAQ,CAAC,MAAM,CAAC,iBAAiB,CAAC;QACrD,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,QAAQ,EAAE,UAAU,CAAC;QAC/C,WAAW,EACT,QAAQ,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI,OAAO,UAAU,kBAAkB;QACrE,mBAAmB,EACjB,QAAQ,CAAC,MAAM,CAAC,mBAAmB,CAAC,IAAI,iBAAiB;QAC3D,QAAQ;KACT,CAAC;AACJ,CAAC"}
|
package/lib/handlers.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"handlers.d.ts","sourceRoot":"","sources":["../src/handlers.ts"],"names":[],"mappings":"AAgBA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AACvD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AAC1D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,8BAA8B,CAAC;AACzE,OAAO,EAEL,KAAK,kBAAkB,
|
|
1
|
+
{"version":3,"file":"handlers.d.ts","sourceRoot":"","sources":["../src/handlers.ts"],"names":[],"mappings":"AAgBA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AACvD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AAC1D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,8BAA8B,CAAC;AACzE,OAAO,EAEL,KAAK,kBAAkB,EAKxB,MAAM,OAAO,CAAC;AACf,OAAO,KAAK,EAAE,qCAAqC,EAAE,MAAM,iBAAiB,CAAC;AAG7E,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,SAAS,CAAC;AAElD,MAAM,MAAM,gBAAgB,GAAG,UAAU,CACvC,oBAAoB,CAAC,kBAAkB,CAAC,CACzC,CAAC;AAEF,yEAAyE;AACzE,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,SAAS,CAAC;IACd,QAAQ,EAAE,QAAQ,CAAC;IACnB,YAAY,EAAE,kBAAkB,CAAC;IACjC,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,qCAAqC,CAAC;CAC/C;AA4CD,sEAAsE;AACtE,wBAAsB,sBAAsB,CAC1C,KAAK,EAAE,gBAAgB,EACvB,GAAG,EAAE,cAAc,GAClB,OAAO,CAAC,IAAI,CAAC,CASf;AAED,4EAA4E;AAC5E,wBAAsB,0BAA0B,CAC9C,GAAG,EAAE,cAAc,GAClB,OAAO,CAAC,IAAI,CAAC,CA4Df"}
|
package/lib/handlers.js
CHANGED
|
@@ -72,6 +72,10 @@ async function ensureNotificationTopic(ctx) {
|
|
|
72
72
|
}
|
|
73
73
|
}
|
|
74
74
|
}
|
|
75
|
+
function isPartitioningFieldRemovalError(err) {
|
|
76
|
+
return (err instanceof Error &&
|
|
77
|
+
err.message.includes(dts_1.PARTITIONING_FIELD_REMOVAL_ERROR_PREFIX));
|
|
78
|
+
}
|
|
75
79
|
async function storeTransferConfig(ctx, transferConfig) {
|
|
76
80
|
if (!transferConfig?.name) {
|
|
77
81
|
throw new Error("BigQuery transfer config is missing its resource name");
|
|
@@ -100,7 +104,16 @@ async function handleUpsertTransferConfig(ctx) {
|
|
|
100
104
|
if (ctx.config.transferConfigName) {
|
|
101
105
|
const linked = await (0, dts_1.getTransferConfig)(ctx.dataTransfer, ctx.config.transferConfigName);
|
|
102
106
|
if (!linked) {
|
|
103
|
-
|
|
107
|
+
// Only a redeploy with a corrected TRANSFER_CONFIG_NAME can resolve this,
|
|
108
|
+
// so retrying the task cannot help.
|
|
109
|
+
logs.linkedTransferConfigMissing(ctx.config.transferConfigName);
|
|
110
|
+
return;
|
|
111
|
+
}
|
|
112
|
+
// A linked config is adopted as-is, so a topic mismatch is the user's to
|
|
113
|
+
// resolve: rewriting it would repoint a config this deployment did not create.
|
|
114
|
+
const expectedTopic = (0, dts_1.notificationTopicName)(ctx.config);
|
|
115
|
+
if (linked.notificationPubsubTopic !== expectedTopic) {
|
|
116
|
+
logs.linkedTopicMismatch(ctx.config.transferConfigName, linked.notificationPubsubTopic, expectedTopic);
|
|
104
117
|
}
|
|
105
118
|
await storeTransferConfig(ctx, linked);
|
|
106
119
|
return;
|
|
@@ -117,9 +130,18 @@ async function handleUpsertTransferConfig(ctx) {
|
|
|
117
130
|
}
|
|
118
131
|
const transferConfigName = existing.docs[0].data().name;
|
|
119
132
|
if (typeof transferConfigName !== "string" || !transferConfigName) {
|
|
120
|
-
throw new Error(`Existing transfer config document in ${ctx.config.firestoreCollection} is missing required 'name' field.`);
|
|
133
|
+
throw new Error(`Existing transfer config document in ${ctx.config.firestoreCollection} is missing required 'name' field. Delete the document so a new scheduled query is created, then redeploy.`);
|
|
134
|
+
}
|
|
135
|
+
try {
|
|
136
|
+
const updated = await (0, dts_1.updateTransferConfig)(ctx.dataTransfer, transferConfigName, ctx.config);
|
|
137
|
+
await storeTransferConfig(ctx, updated);
|
|
138
|
+
}
|
|
139
|
+
catch (err) {
|
|
140
|
+
if (!isPartitioningFieldRemovalError(err))
|
|
141
|
+
throw err;
|
|
142
|
+
// The guard rejects the update while building the request, so nothing was
|
|
143
|
+
// sent and a retry hits the same guard.
|
|
144
|
+
logs.partitioningFieldRemovalAborted(err.message);
|
|
121
145
|
}
|
|
122
|
-
const updated = await (0, dts_1.updateTransferConfig)(ctx.dataTransfer, transferConfigName, ctx.config);
|
|
123
|
-
await storeTransferConfig(ctx, updated);
|
|
124
146
|
}
|
|
125
147
|
//# sourceMappingURL=handlers.js.map
|
package/lib/handlers.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"handlers.js","sourceRoot":"","sources":["../src/handlers.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;GAcG
|
|
1
|
+
{"version":3,"file":"handlers.js","sourceRoot":"","sources":["../src/handlers.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;GAcG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA4EH,wDAYC;AAGD,gEA8DC;AAlJD,+BAOe;AAEf,qCAA6E;AAC7E,6CAA+B;AAgB/B,KAAK,UAAU,uBAAuB,CAAC,GAAmB;IACxD,MAAM,KAAK,GAAG,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;IACvD,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,MAAM,EAAE,CAAC;IACtC,IAAI,MAAM;QAAE,OAAO;IAEnB,IAAI,CAAC;QACH,MAAM,GAAG,CAAC,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;QACrD,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;IAC5C,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,6EAA6E;QAC7E,IACE,OAAO,GAAG,KAAK,QAAQ;YACvB,GAAG,KAAK,IAAI;YACZ,CAAC,CAAC,MAAM,IAAI,GAAG,CAAC;YAChB,GAAG,CAAC,IAAI,KAAK,CAAC,EACd,CAAC;YACD,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,+BAA+B,CAAC,GAAY;IACnD,OAAO,CACL,GAAG,YAAY,KAAK;QACpB,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,6CAAuC,CAAC,CAC9D,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,mBAAmB,CAChC,GAAmB,EACnB,cAA6D;IAE7D,IAAI,CAAC,cAAc,EAAE,IAAI,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAC;IAC3E,CAAC;IACD,MAAM,EAAE,gBAAgB,EAAE,GAAG,IAAA,gCAAuB,EAAC,cAAc,CAAC,IAAI,CAAC,CAAC;IAC1E,MAAM,GAAG,CAAC,EAAE;SACT,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,mBAAmB,CAAC;SAC1C,GAAG,CAAC,gBAAgB,CAAC;SACrB,GAAG,CAAC,EAAE,aAAa,EAAE,GAAG,CAAC,MAAM,CAAC,UAAU,EAAE,GAAG,cAAc,EAAE,CAAC,CAAC;AACtE,CAAC;AAED,sEAAsE;AAC/D,KAAK,UAAU,sBAAsB,CAC1C,KAAuB,EACvB,GAAmB;IAEnB,IAAI,CAAC,KAAK,EAAE,CAAC;IACb,IAAI,CAAC;QACH,MAAM,IAAA,iCAAwB,EAAC,GAAG,EAAE,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;QACvE,IAAI,CAAC,QAAQ,EAAE,CAAC;IAClB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAChB,MAAM,GAAG,CAAC;IACZ,CAAC;AACH,CAAC;AAED,4EAA4E;AACrE,KAAK,UAAU,0BAA0B,CAC9C,GAAmB;IAEnB,MAAM,uBAAuB,CAAC,GAAG,CAAC,CAAC;IAEnC,IAAI,GAAG,CAAC,MAAM,CAAC,kBAAkB,EAAE,CAAC;QAClC,MAAM,MAAM,GAAG,MAAM,IAAA,uBAAiB,EACpC,GAAG,CAAC,YAAY,EAChB,GAAG,CAAC,MAAM,CAAC,kBAAkB,CAC9B,CAAC;QACF,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,0EAA0E;YAC1E,oCAAoC;YACpC,IAAI,CAAC,2BAA2B,CAAC,GAAG,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC;YAChE,OAAO;QACT,CAAC;QACD,yEAAyE;QACzE,+EAA+E;QAC/E,MAAM,aAAa,GAAG,IAAA,2BAAqB,EAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACxD,IAAI,MAAM,CAAC,uBAAuB,KAAK,aAAa,EAAE,CAAC;YACrD,IAAI,CAAC,mBAAmB,CACtB,GAAG,CAAC,MAAM,CAAC,kBAAkB,EAC7B,MAAM,CAAC,uBAAuB,EAC9B,aAAa,CACd,CAAC;QACJ,CAAC;QACD,MAAM,mBAAmB,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;QACvC,OAAO;IACT,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,GAAG,CAAC,EAAE;SAC1B,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,mBAAmB,CAAC;SAC1C,KAAK,CAAC,eAAe,EAAE,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC;SACnD,KAAK,CAAC,CAAC,CAAC;SACR,GAAG,EAAE,CAAC;IAET,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;QACnB,MAAM,OAAO,GAAG,MAAM,IAAA,0BAAoB,EAAC,GAAG,CAAC,YAAY,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;QACzE,MAAM,mBAAmB,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QACxC,OAAO;IACT,CAAC;IAED,MAAM,kBAAkB,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC;IACxD,IAAI,OAAO,kBAAkB,KAAK,QAAQ,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAClE,MAAM,IAAI,KAAK,CACb,wCAAwC,GAAG,CAAC,MAAM,CAAC,mBAAmB,4GAA4G,CACnL,CAAC;IACJ,CAAC;IAED,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,IAAA,0BAAoB,EACxC,GAAG,CAAC,YAAY,EAChB,kBAAkB,EAClB,GAAG,CAAC,MAAM,CACX,CAAC;QACF,MAAM,mBAAmB,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAC1C,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,+BAA+B,CAAC,GAAG,CAAC;YAAE,MAAM,GAAG,CAAC;QACrD,0EAA0E;QAC1E,wCAAwC;QACxC,IAAI,CAAC,+BAA+B,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IACpD,CAAC;AACH,CAAC"}
|