@firebase-function-kits/bigquery-firestore-export 0.0.2-rc.1 → 0.0.2-rc.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -9
- package/lib/config.d.ts.map +1 -1
- package/lib/config.js +15 -2
- package/lib/config.js.map +1 -1
- package/lib/dts.d.ts +8 -0
- package/lib/dts.d.ts.map +1 -1
- package/lib/dts.js +15 -2
- package/lib/dts.js.map +1 -1
- package/lib/export-config.js.map +1 -1
- package/lib/handlers.d.ts.map +1 -1
- package/lib/handlers.js +26 -4
- package/lib/handlers.js.map +1 -1
- package/lib/helper.d.ts.map +1 -1
- package/lib/helper.js +1 -0
- package/lib/helper.js.map +1 -1
- package/lib/index.js.map +1 -1
- package/lib/logs.d.ts +4 -0
- package/lib/logs.d.ts.map +1 -1
- package/lib/logs.js +18 -0
- package/lib/logs.js.map +1 -1
- package/lib/metadata.d.ts.map +1 -1
- package/npm-shrinkwrap.json +21 -6
- package/package.json +3 -2
- package/src/config.ts +18 -2
- package/src/dts.ts +17 -2
- package/src/handlers.ts +36 -9
- package/src/helper.ts +1 -0
- package/src/logs.ts +28 -0
- package/tests/config.test.ts +54 -2
- package/tests/dts-transfer-config.test.ts +400 -0
- package/tests/dts.test.ts +17 -0
- package/tests/handlers.test.ts +113 -5
- package/tests/helper-values.test.ts +168 -0
- package/tests/helper.test.ts +65 -3
- package/tests/notification-topic.test.ts +151 -0
- package/tests/run-results.test.ts +373 -0
- package/tsconfig.tsbuildinfo +1 -1
package/README.md
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# @firebase/bigquery-firestore-export
|
|
1
|
+
# @firebase-function-kits/bigquery-firestore-export
|
|
2
2
|
|
|
3
3
|
Schedule a BigQuery query and write each run's rows and metadata to Firestore.
|
|
4
4
|
This is the Export BigQuery to Firestore Firebase Extension as an npm package
|
|
@@ -12,7 +12,7 @@ there is no hosted version, so you deploy them yourself.
|
|
|
12
12
|
## Install
|
|
13
13
|
|
|
14
14
|
```sh
|
|
15
|
-
npm install @firebase/bigquery-firestore-export
|
|
15
|
+
npm install @firebase-function-kits/bigquery-firestore-export
|
|
16
16
|
```
|
|
17
17
|
|
|
18
18
|
## Required IAM
|
|
@@ -43,7 +43,7 @@ Export both functions from your functions codebase entry:
|
|
|
43
43
|
export {
|
|
44
44
|
processMessages,
|
|
45
45
|
upsertTransferConfig,
|
|
46
|
-
} from "@firebase/bigquery-firestore-export";
|
|
46
|
+
} from "@firebase-function-kits/bigquery-firestore-export";
|
|
47
47
|
```
|
|
48
48
|
|
|
49
49
|
and configure them with a `.env` (or `.env.<projectId>`):
|
|
@@ -114,6 +114,7 @@ loads them at deploy time and prompts for required values that are missing.
|
|
|
114
114
|
| `displayName` | `DISPLAY_NAME` | yes | — | Human-readable scheduled-query name |
|
|
115
115
|
| `partitioningField` | `PARTITIONING_FIELD` | no | (empty) | Destination-table partitioning field |
|
|
116
116
|
| `schedule` | `SCHEDULE` | yes | — | DTS schedule, such as `every 24 hours` |
|
|
117
|
+
| `pubSubTopic` | `PUB_SUB_TOPIC` | no | `kit-<INSTANCE_ID>-processMessages` | Pub/Sub topic ID, not a full resource name, receiving DTS completion notifications |
|
|
117
118
|
| `firestoreCollection` | `COLLECTION_PATH` | no | `transferConfigs` | Root Firestore collection for configs and output |
|
|
118
119
|
| `logLevel` | `LOG_LEVEL` | no | `info` | `debug`, `info`, `warn`, `error`, or `silent` |
|
|
119
120
|
|
|
@@ -192,16 +193,29 @@ config document with `extInstanceId`. Here `INSTANCE_ID` is a setting you
|
|
|
192
193
|
provide, and it must match this instance's key in the `instances` map in
|
|
193
194
|
`firebase.json`.
|
|
194
195
|
|
|
195
|
-
The topic
|
|
196
|
-
first run if it does not already exist. Set `INSTANCE_ID` to your installed
|
|
196
|
+
The topic defaults to `kit-<INSTANCE_ID>-processMessages`, and the kit creates it
|
|
197
|
+
on first run if it does not already exist. Set `INSTANCE_ID` to your installed
|
|
197
198
|
instance's id if you want the kit to adopt the scheduled query that instance
|
|
198
199
|
created, because the lookup is by `extInstanceId` on the documents in
|
|
199
200
|
`COLLECTION_PATH`. With a different id the kit finds nothing, creates a second
|
|
200
201
|
scheduled query, and you end up with two writing into the same collection.
|
|
201
202
|
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
203
|
+
An adopted transfer config still notifies the extension's `ext-` topic, and the
|
|
204
|
+
kit reconciles `notification_pubsub_topic` to whatever `PUB_SUB_TOPIC` names. So
|
|
205
|
+
there are two ways to migrate:
|
|
206
|
+
|
|
207
|
+
- Keep the extension's topic. Set
|
|
208
|
+
`PUB_SUB_TOPIC=ext-<instance id>-processMessages`, and the transfer config is
|
|
209
|
+
left untouched: anything else subscribed to that topic, including an extension
|
|
210
|
+
instance still installed, keeps receiving run notifications.
|
|
211
|
+
- Take the default. The first deploy repoints the transfer config at
|
|
212
|
+
`kit-<INSTANCE_ID>-processMessages`, which stops notifications reaching the
|
|
213
|
+
extension and any other subscriber on the old topic. Do this once the extension
|
|
214
|
+
is uninstalled, and the old topic can then be deleted.
|
|
215
|
+
|
|
216
|
+
`TRANSFER_CONFIG_NAME` is the exception. A linked config is adopted as-is and is
|
|
217
|
+
never repointed, so if it notifies a topic other than `PUB_SUB_TOPIC` the kit
|
|
218
|
+
logs a warning and its runs never reach the kit's `processMessages` function.
|
|
205
219
|
|
|
206
220
|
### Repeated BigQuery columns are now written as arrays
|
|
207
221
|
|
|
@@ -295,7 +309,7 @@ Pub/Sub API is now requested explicitly. The Firebase CLI handles all of this.
|
|
|
295
309
|
|
|
296
310
|
## API surface
|
|
297
311
|
|
|
298
|
-
- **Main entry** (`@firebase/bigquery-firestore-export`): exports
|
|
312
|
+
- **Main entry** (`@firebase-function-kits/bigquery-firestore-export`): exports
|
|
299
313
|
`processMessages` and `upsertTransferConfig`, registers first-deploy and
|
|
300
314
|
redeploy lifecycle hooks, and resolves runtime dependencies lazily. Use this
|
|
301
315
|
entry from Firebase deploy/emulator/runtime.
|
package/lib/config.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAsBA,OAAO,KAAK,EACV,6BAA6B,EAC7B,iBAAiB,EAElB,MAAM,iBAAiB,CAAC;AA+
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAsBA,OAAO,KAAK,EACV,6BAA6B,EAC7B,iBAAiB,EAElB,MAAM,iBAAiB,CAAC;AA+JzB,eAAO,MAAM,kBAAkB,EAAE,iBAEhC,CAAC;AAcF,iEAAiE;AACjE,wBAAgB,aAAa,IAAI,6BAA6B,CAkB7D"}
|
package/lib/config.js
CHANGED
|
@@ -65,6 +65,19 @@ const params = {
|
|
|
65
65
|
}),
|
|
66
66
|
}),
|
|
67
67
|
transferConfigName: (0, params_1.defineString)("TRANSFER_CONFIG_NAME", { default: "" }),
|
|
68
|
+
pubSubTopic: (0, params_1.defineString)("PUB_SUB_TOPIC", {
|
|
69
|
+
label: "Pub/Sub Topic",
|
|
70
|
+
description: "Which Pub/Sub topic should receive BigQuery Data Transfer completion notifications? Leave the default unless you are migrating from the bigquery-firestore-export extension, whose topic is named ext-<instance id>-processMessages. Pointing this at the extension's topic keeps the existing scheduled query's notification settings untouched.",
|
|
71
|
+
default: (0, params_1.expr) `kit-${instanceId}-processMessages`,
|
|
72
|
+
input: {
|
|
73
|
+
text: {
|
|
74
|
+
nonEmpty: true,
|
|
75
|
+
example: "ext-my-instance-processMessages",
|
|
76
|
+
validationRegex: /^(?!goog)[a-zA-Z][a-zA-Z0-9\-_.~+%]{2,254}$/,
|
|
77
|
+
validationErrorMessage: "Must be a Pub/Sub topic ID, not a full projects/<project>/topics/<topic> resource name. IDs are 3 to 255 characters, start with a letter, may contain letters, numbers and - _ . ~ + %, and cannot start with goog.",
|
|
78
|
+
},
|
|
79
|
+
},
|
|
80
|
+
}),
|
|
68
81
|
datasetId: (0, params_1.defineString)("DATASET_ID", {
|
|
69
82
|
label: "Dataset ID",
|
|
70
83
|
description: "What's the BigQuery destination dataset you'd like to use? Each transfer run will write to a table in this destination dataset.",
|
|
@@ -147,7 +160,7 @@ const params = {
|
|
|
147
160
|
}),
|
|
148
161
|
};
|
|
149
162
|
exports.CONFIG_EXPRESSIONS = {
|
|
150
|
-
pubSubTopic:
|
|
163
|
+
pubSubTopic: params.pubSubTopic,
|
|
151
164
|
};
|
|
152
165
|
function optional(value) {
|
|
153
166
|
const normalized = value.trim();
|
|
@@ -173,7 +186,7 @@ function configFromEnv() {
|
|
|
173
186
|
displayName: params.displayName.value(),
|
|
174
187
|
partitioningField: optional(params.partitioningField.value()),
|
|
175
188
|
schedule: params.schedule.value(),
|
|
176
|
-
pubSubTopic:
|
|
189
|
+
pubSubTopic: params.pubSubTopic.value(),
|
|
177
190
|
firestoreCollection: params.firestoreCollection.value(),
|
|
178
191
|
logLevel: normalizeLogLevel(params.logLevel.value()),
|
|
179
192
|
};
|
package/lib/config.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;GAcG
|
|
1
|
+
{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;GAcG;;;AA4LH,sCAkBC;AA5MD,sDAKmC;AAOnC,MAAM,iBAAiB,GAAG,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAU,CAAC;AAChF,MAAM,UAAU,GAAG,IAAA,qBAAY,EAAC,aAAa,CAAC,CAAC;AAE/C,MAAM,MAAM,GAAG;IACb,UAAU;IACV,uBAAuB,EAAE,IAAA,qBAAY,EAAC,2BAA2B,EAAE;QACjE,KAAK,EAAE,2BAA2B;QAClC,WAAW,EACT,uEAAuE;QAEzE,OAAO,EAAE,IAAI;QACb,KAAK,EAAE,IAAA,eAAM,EAAC;YACZ,2BAA2B,EAAE,UAAU;YACvC,oBAAoB,EAAE,aAAa;YACnC,sBAAsB,EAAE,UAAU;YAClC,wBAAwB,EAAE,UAAU;YACpC,oCAAoC,EAAE,yBAAyB;YAC/D,8BAA8B,EAAE,UAAU;YAC1C,mBAAmB,EAAE,UAAU;YAC/B,2BAA2B,EAAE,UAAU;YACvC,gCAAgC,EAAE,oBAAoB;YACtD,+BAA+B,EAAE,oBAAoB;YACrD,2BAA2B,EAAE,UAAU;YACvC,mCAAmC,EAAE,yBAAyB;YAC9D,qBAAqB,EAAE,aAAa;YACpC,wBAAwB,EAAE,YAAY;YACtC,2BAA2B,EAAE,iBAAiB;YAC9C,kCAAkC,EAAE,sBAAsB;YAC1D,sBAAsB,EAAE,aAAa;YACrC,yBAAyB,EAAE,iBAAiB;YAC5C,yBAAyB,EAAE,iBAAiB;YAC5C,6BAA6B,EAAE,iBAAiB;YAChD,+BAA+B,EAAE,sBAAsB;YACvD,qBAAqB,EAAE,YAAY;YACnC,yBAAyB,EAAE,iBAAiB;YAC5C,wBAAwB,EAAE,cAAc;YACxC,yBAAyB,EAAE,eAAe;YAC1C,0BAA0B,EAAE,cAAc;YAC1C,uBAAuB,EAAE,cAAc;YACvC,4BAA4B,EAAE,mBAAmB;YACjD,sBAAsB,EAAE,cAAc;YACtC,4BAA4B,EAAE,cAAc;YAC5C,sBAAsB,EAAE,cAAc;YACtC,0BAA0B,EAAE,iBAAiB;YAC7C,uBAAuB,EAAE,cAAc;YACvC,sBAAsB,EAAE,IAAI;YAC5B,sBAAsB,EAAE,IAAI;SAC7B,CAAC;KACH,CAAC;IACF,kBAAkB,EAAE,IAAA,qBAAY,EAAC,sBAAsB,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;IACzE,WAAW,EAAE,IAAA,qBAAY,EAAC,eAAe,EAAE;QACzC,KAAK,EAAE,eAAe;QACtB,WAAW,EACT,mVAAmV;QAErV,OAAO,EAAE,IAAA,aAAI,EAAA,OAAO,UAAU,kBAAkB;QAChD,KAAK,EAAE;YACL,IAAI,EAAE;gBACJ,QAAQ,EAAE,IAAI;gBACd,OAAO,EAAE,iCAAiC;gBAC1C,eAAe,EAAE,6CAA6C;gBAC9D,sBAAsB,EACpB,qNAAqN;aACxN;SACF;KACF,CAAC;IACF,SAAS,EAAE,IAAA,qBAAY,EAAC,YAAY,EAAE;QACpC,KAAK,EAAE,YAAY;QACnB,WAAW,EACT,iIAAiI;QACnI,KAAK,EAAE;YACL,IAAI,EAAE;gBACJ,QAAQ,EAAE,IAAI;gBACd,OAAO,EAAE,eAAe;aACzB;SACF;KACF,CAAC;IACF,SAAS,EAAE,IAAA,qBAAY,EAAC,YAAY,EAAE;QACpC,KAAK,EAAE,wBAAwB;QAC/B,WAAW,EACT,mJAAmJ;QACrJ,KAAK,EAAE;YACL,IAAI,EAAE;gBACJ,QAAQ,EAAE,IAAI;gBACd,OAAO,EAAE,cAAc;aACxB;SACF;KACF,CAAC;IACF,WAAW,EAAE,IAAA,qBAAY,EAAC,cAAc,EAAE;QACxC,KAAK,EAAE,cAAc;QACrB,WAAW,EAAE,4CAA4C;QACzD,KAAK,EAAE;YACL,IAAI,EAAE;gBACJ,QAAQ,EAAE,IAAI;gBAEd,OAAO,EAAE,uDAAuD;aACjE;SACF;KACF,CAAC;IACF,WAAW,EAAE,IAAA,qBAAY,EAAC,cAAc,EAAE;QACxC,KAAK,EAAE,cAAc;QACrB,WAAW,EAAE,0CAA0C;QACvD,KAAK,EAAE;YACL,IAAI,EAAE;gBACJ,QAAQ,EAAE,IAAI;gBACd,OAAO,EAAE,sCAAsC;aAChD;SACF;KACF,CAAC;IACF,iBAAiB,EAAE,IAAA,qBAAY,EAAC,oBAAoB,EAAE;QACpD,KAAK,EAAE,oBAAoB;QAC3B,WAAW,EACT,2GAA2G;QAC7G,OAAO,EAAE,EAAE;QACX,KAAK,EAAE,EAAE,IAAI,EAAE,EAAE,OAAO,EAAE,WAAW,EAAE,EAAE;KAC1C,CAAC;IACF,QAAQ,EAAE,IAAA,qBAAY,EAAC,UAAU,EAAE;QACjC,KAAK,EAAE,UAAU;QACjB,WAAW,EACT,iEAAiE;QACnE,KAAK,EAAE;YACL,IAAI,EAAE;gBACJ,QAAQ,EAAE,IAAI;gBACd,OAAO,EAAE,kBAAkB;aAC5B;SACF;KACF,CAAC;IACF,mBAAmB,EAAE,IAAA,qBAAY,EAAC,iBAAiB,EAAE;QACnD,KAAK,EAAE,sBAAsB;QAC7B,WAAW,EACT,sGAAsG;QAExG,OAAO,EAAE,iBAAiB;QAC1B,KAAK,EAAE;YACL,IAAI,EAAE;gBACJ,OAAO,EAAE,iBAAiB;gBAE1B,eAAe,EAAE,6BAA6B;gBAC9C,sBAAsB,EAAE,4CAA4C;aACrE;SACF;KACF,CAAC;IACF,QAAQ,EAAE,IAAA,qBAAY,EAAC,WAAW,EAAE;QAClC,KAAK,EAAE,WAAW;QAClB,WAAW,EAAE,4DAA4D;QAEzE,OAAO,EAAE,MAAM;QACf,KAAK,EAAE,IAAA,eAAM,EAAC;YACZ,KAAK,EAAE,OAAO;YACd,IAAI,EAAE,MAAM;YACZ,IAAI,EAAE,MAAM;YACZ,KAAK,EAAE,OAAO;YACd,MAAM,EAAE,QAAQ;SACjB,CAAC;KACH,CAAC;CACH,CAAC;AAEW,QAAA,kBAAkB,GAAsB;IACnD,WAAW,EAAE,MAAM,CAAC,WAAW;CAChC,CAAC;AAEF,SAAS,QAAQ,CAAC,KAAa;IAC7B,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;IAChC,OAAO,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC;AACxD,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAa;IACtC,MAAM,UAAU,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC;IACvC,OAAO,iBAAiB,CAAC,QAAQ,CAAC,UAAsB,CAAC;QACvD,CAAC,CAAE,UAAuB;QAC1B,CAAC,CAAC,MAAM,CAAC;AACb,CAAC;AAED,iEAAiE;AACjE,SAAgB,aAAa;IAC3B,MAAM,kBAAkB,GAAG,MAAM,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;IAErD,OAAO;QACL,uBAAuB,EAAE,MAAM,CAAC,uBAAuB,CAAC,KAAK,EAAE;QAC/D,SAAS,EAAE,kBAAS,CAAC,KAAK,EAAE;QAC5B,UAAU,EAAE,kBAAkB;QAC9B,kBAAkB,EAAE,QAAQ,CAAC,MAAM,CAAC,kBAAkB,CAAC,KAAK,EAAE,CAAC;QAC/D,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC,KAAK,EAAE;QACnC,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC,KAAK,EAAE;QACnC,WAAW,EAAE,MAAM,CAAC,WAAW,CAAC,KAAK,EAAE;QACvC,WAAW,EAAE,MAAM,CAAC,WAAW,CAAC,KAAK,EAAE;QACvC,iBAAiB,EAAE,QAAQ,CAAC,MAAM,CAAC,iBAAiB,CAAC,KAAK,EAAE,CAAC;QAC7D,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE;QACjC,WAAW,EAAE,MAAM,CAAC,WAAW,CAAC,KAAK,EAAE;QACvC,mBAAmB,EAAE,MAAM,CAAC,mBAAmB,CAAC,KAAK,EAAE;QACvD,QAAQ,EAAE,iBAAiB,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;KACrD,CAAC;AACJ,CAAC"}
|
package/lib/dts.d.ts
CHANGED
|
@@ -4,6 +4,14 @@ export type DataTransferClient = bigqueryDataTransfer.v1.DataTransferServiceClie
|
|
|
4
4
|
export type TransferConfig = bigqueryDataTransfer.protos.google.cloud.bigquery.datatransfer.v1.ITransferConfig;
|
|
5
5
|
export declare const PARTITIONING_FIELD_REMOVAL_ERROR_PREFIX = "Cannot remove partitioning_field from an existing transfer config";
|
|
6
6
|
export declare const PARTITIONING_FIELD_REMOVAL_ERROR = "Cannot remove partitioning_field from an existing transfer config. The BigQuery Data Transfer API does not support clearing this parameter once it has been set. To change partitioning, create a new transfer config with the desired setting.";
|
|
7
|
+
/**
|
|
8
|
+
* Full resource name of the topic DTS publishes run notifications to.
|
|
9
|
+
*
|
|
10
|
+
* PUB_SUB_TOPIC is validated as a bare topic ID when prompted, but values from
|
|
11
|
+
* a dotenv file reach us unvalidated, and both the Pub/Sub client and the
|
|
12
|
+
* trigger accept a full resource name, so accept one here too.
|
|
13
|
+
*/
|
|
14
|
+
export declare function notificationTopicName(config: ResolvedBigqueryFirestoreExportConfig): string;
|
|
7
15
|
/**
|
|
8
16
|
* Creates the protobuf-shaped request used for a scheduled query.
|
|
9
17
|
*
|
package/lib/dts.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dts.d.ts","sourceRoot":"","sources":["../src/dts.ts"],"names":[],"mappings":"AAgBA,OAAO,KAAK,oBAAoB,MAAM,sCAAsC,CAAC;AAC7E,OAAO,KAAK,EAAE,qCAAqC,EAAE,MAAM,iBAAiB,CAAC;AAG7E,MAAM,MAAM,kBAAkB,GAC5B,oBAAoB,CAAC,EAAE,CAAC,yBAAyB,CAAC;AACpD,MAAM,MAAM,cAAc,GACxB,oBAAoB,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC,eAAe,CAAC;AAMpF,eAAO,MAAM,uCAAuC,sEACxB,CAAC;AAC7B,eAAO,MAAM,gCAAgC,oPAAgN,CAAC;
|
|
1
|
+
{"version":3,"file":"dts.d.ts","sourceRoot":"","sources":["../src/dts.ts"],"names":[],"mappings":"AAgBA,OAAO,KAAK,oBAAoB,MAAM,sCAAsC,CAAC;AAC7E,OAAO,KAAK,EAAE,qCAAqC,EAAE,MAAM,iBAAiB,CAAC;AAG7E,MAAM,MAAM,kBAAkB,GAC5B,oBAAoB,CAAC,EAAE,CAAC,yBAAyB,CAAC;AACpD,MAAM,MAAM,cAAc,GACxB,oBAAoB,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC,eAAe,CAAC;AAMpF,eAAO,MAAM,uCAAuC,sEACxB,CAAC;AAC7B,eAAO,MAAM,gCAAgC,oPAAgN,CAAC;AAgC9P;;;;;;GAMG;AACH,wBAAgB,qBAAqB,CACnC,MAAM,EAAE,qCAAqC,GAC5C,MAAM,CAIR;AAMD;;;;;;;GAOG;AACH,wBAAgB,2BAA2B,CACzC,MAAM,EAAE,qCAAqC,GAC5C,oBAAoB,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC,4BAA4B,CAqBhG;AAED,wBAAsB,iBAAiB,CACrC,MAAM,EAAE,kBAAkB,EAC1B,kBAAkB,EAAE,MAAM,GACzB,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC,CAchC;AAED,wBAAsB,oBAAoB,CACxC,MAAM,EAAE,kBAAkB,EAC1B,MAAM,EAAE,qCAAqC,GAC5C,OAAO,CAAC,cAAc,CAAC,CAUzB;AAED,iFAAiF;AACjF,wBAAsB,oCAAoC,CACxD,MAAM,EAAE,kBAAkB,EAC1B,kBAAkB,EAAE,MAAM,EAC1B,MAAM,EAAE,qCAAqC,GAC5C,OAAO,CAAC,oBAAoB,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC,4BAA4B,CAAC,CAyDzG;AAED,wBAAsB,oBAAoB,CACxC,MAAM,EAAE,kBAAkB,EAC1B,kBAAkB,EAAE,MAAM,EAC1B,MAAM,EAAE,qCAAqC,GAC5C,OAAO,CAAC,cAAc,CAAC,CAczB"}
|
package/lib/dts.js
CHANGED
|
@@ -49,6 +49,7 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
49
49
|
})();
|
|
50
50
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
51
51
|
exports.PARTITIONING_FIELD_REMOVAL_ERROR = exports.PARTITIONING_FIELD_REMOVAL_ERROR_PREFIX = void 0;
|
|
52
|
+
exports.notificationTopicName = notificationTopicName;
|
|
52
53
|
exports.createTransferConfigRequest = createTransferConfigRequest;
|
|
53
54
|
exports.getTransferConfig = getTransferConfig;
|
|
54
55
|
exports.createTransferConfig = createTransferConfig;
|
|
@@ -79,6 +80,18 @@ function transferConfigFields(config) {
|
|
|
79
80
|
}
|
|
80
81
|
return fields;
|
|
81
82
|
}
|
|
83
|
+
/**
|
|
84
|
+
* Full resource name of the topic DTS publishes run notifications to.
|
|
85
|
+
*
|
|
86
|
+
* PUB_SUB_TOPIC is validated as a bare topic ID when prompted, but values from
|
|
87
|
+
* a dotenv file reach us unvalidated, and both the Pub/Sub client and the
|
|
88
|
+
* trigger accept a full resource name, so accept one here too.
|
|
89
|
+
*/
|
|
90
|
+
function notificationTopicName(config) {
|
|
91
|
+
return config.pubSubTopic.startsWith("projects/")
|
|
92
|
+
? config.pubSubTopic
|
|
93
|
+
: `projects/${config.projectId}/topics/${config.pubSubTopic}`;
|
|
94
|
+
}
|
|
82
95
|
function stringField(value) {
|
|
83
96
|
return { stringValue: value ?? "" };
|
|
84
97
|
}
|
|
@@ -106,7 +119,7 @@ function createTransferConfigRequest(config) {
|
|
|
106
119
|
},
|
|
107
120
|
},
|
|
108
121
|
schedule: config.schedule,
|
|
109
|
-
notificationPubsubTopic:
|
|
122
|
+
notificationPubsubTopic: notificationTopicName(config),
|
|
110
123
|
},
|
|
111
124
|
};
|
|
112
125
|
}
|
|
@@ -172,7 +185,7 @@ async function constructUpdateTransferConfigRequest(client, transferConfigName,
|
|
|
172
185
|
updateMask.push("schedule");
|
|
173
186
|
updatedConfig.schedule = config.schedule;
|
|
174
187
|
}
|
|
175
|
-
const expectedTopic =
|
|
188
|
+
const expectedTopic = notificationTopicName(config);
|
|
176
189
|
if (expectedTopic !== transferConfig.notificationPubsubTopic) {
|
|
177
190
|
updateMask.push("notification_pubsub_topic");
|
|
178
191
|
updatedConfig.notificationPubsubTopic = expectedTopic;
|
package/lib/dts.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dts.js","sourceRoot":"","sources":["../src/dts.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;GAcG
|
|
1
|
+
{"version":3,"file":"dts.js","sourceRoot":"","sources":["../src/dts.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;GAcG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAwDH,sDAMC;AAcD,kEAuBC;AAED,8CAiBC;AAED,oDAaC;AAGD,oFA6DC;AAED,oDAkBC;AAvND,2FAA6E;AAE7E,6CAA+B;AAO/B,MAAM,cAAc,GAAG,CAAC,CAAC;AACzB,MAAM,0BAA0B,GAC9B,mEAAmE,CAAC;AAEzD,QAAA,uCAAuC,GAClD,0BAA0B,CAAC;AAChB,QAAA,gCAAgC,GAAG,GAAG,0BAA0B,gLAAgL,CAAC;AAE9P,SAAS,eAAe,CAAC,GAAY;IACnC,OAAO,CACL,OAAO,GAAG,KAAK,QAAQ;QACvB,GAAG,KAAK,IAAI;QACZ,MAAM,IAAI,GAAG;QACb,GAAG,CAAC,IAAI,KAAK,cAAc,CAC5B,CAAC;AACJ,CAAC;AAED,SAAS,oBAAoB,CAAC,MAAsB;IAClD,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC;IACrC,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,IAAI,KAAK,CACb,8DAA8D,CAC/D,CAAC;IACJ,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QAClB,MAAM,IAAI,KAAK,CACb,oEAAoE,CACrE,CAAC;IACJ,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,+BAA+B,EAAE,CAAC;QAC5C,MAAM,IAAI,KAAK,CACb,8FAA8F,CAC/F,CAAC;IACJ,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;;GAMG;AACH,SAAgB,qBAAqB,CACnC,MAA6C;IAE7C,OAAO,MAAM,CAAC,WAAW,CAAC,UAAU,CAAC,WAAW,CAAC;QAC/C,CAAC,CAAC,MAAM,CAAC,WAAW;QACpB,CAAC,CAAC,YAAY,MAAM,CAAC,SAAS,WAAW,MAAM,CAAC,WAAW,EAAE,CAAC;AAClE,CAAC;AAED,SAAS,WAAW,CAAC,KAAyB;IAC5C,OAAO,EAAE,WAAW,EAAE,KAAK,IAAI,EAAE,EAAE,CAAC;AACtC,CAAC;AAED;;;;;;;GAOG;AACH,SAAgB,2BAA2B,CACzC,MAA6C;IAE7C,OAAO;QACL,MAAM,EAAE,YAAY,MAAM,CAAC,SAAS,EAAE;QACtC,cAAc,EAAE;YACd,oBAAoB,EAAE,MAAM,CAAC,SAAS;YACtC,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,YAAY,EAAE,iBAAiB;YAC/B,MAAM,EAAE;gBACN,MAAM,EAAE;oBACN,KAAK,EAAE,WAAW,CAAC,MAAM,CAAC,WAAW,CAAC;oBACtC,+BAA+B,EAAE,WAAW,CAC1C,GAAG,MAAM,CAAC,SAAS,sBAAsB,CAC1C;oBACD,iBAAiB,EAAE,WAAW,CAAC,gBAAgB,CAAC;oBAChD,kBAAkB,EAAE,WAAW,CAAC,MAAM,CAAC,iBAAiB,CAAC;iBAC1D;aACF;YACD,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,uBAAuB,EAAE,qBAAqB,CAAC,MAAM,CAAC;SACvD;KACF,CAAC;AACJ,CAAC;AAEM,KAAK,UAAU,iBAAiB,CACrC,MAA0B,EAC1B,kBAA0B;IAE1B,IAAI,CAAC;QACH,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,MAAM,CAAC,iBAAiB,CAAC;YAC9C,IAAI,EAAE,kBAAkB;SACzB,CAAC,CAAC;QACH,OAAO,MAAM,CAAC;IAChB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,eAAe,CAAC,GAAG,CAAC,EAAE,CAAC;YACzB,IAAI,CAAC,sBAAsB,CAAC,kBAAkB,CAAC,CAAC;YAChD,OAAO,IAAI,CAAC;QACd,CAAC;QACD,IAAI,CAAC,uBAAuB,CAAC,kBAAkB,EAAE,GAAG,CAAC,CAAC;QACtD,MAAM,GAAG,CAAC;IACZ,CAAC;AACH,CAAC;AAEM,KAAK,UAAU,oBAAoB,CACxC,MAA0B,EAC1B,MAA6C;IAE7C,IAAI,CAAC,oBAAoB,EAAE,CAAC;IAC5B,MAAM,CAAC,OAAO,CAAC,GAAG,MAAM,MAAM,CAAC,oBAAoB,CACjD,2BAA2B,CAAC,MAAM,CAAC,CACpC,CAAC;IACF,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAClB,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC;IAC5E,CAAC;IACD,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IACzC,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,iFAAiF;AAC1E,KAAK,UAAU,oCAAoC,CACxD,MAA0B,EAC1B,kBAA0B,EAC1B,MAA6C;IAE7C,MAAM,cAAc,GAAG,MAAM,iBAAiB,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAC;IAC3E,IAAI,CAAC,cAAc;QAAE,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAElE,MAAM,MAAM,GAAG,oBAAoB,CAAC,cAAc,CAAC,CAAC;IACpD,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAC9B,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,CACb,CAAC;IACpB,MAAM,aAAa,GAAG,oBAAoB,CAAC,aAAa,CAAC,CAAC;IAC1D,MAAM,UAAU,GAAa,EAAE,CAAC;IAEhC,IAAI,MAAM,CAAC,SAAS,KAAK,cAAc,CAAC,oBAAoB,EAAE,CAAC;QAC7D,UAAU,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;QAC1C,aAAa,CAAC,oBAAoB,GAAG,MAAM,CAAC,SAAS,CAAC;IACxD,CAAC;IACD,IAAI,MAAM,CAAC,WAAW,KAAK,MAAM,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC;QACpD,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC1B,aAAa,CAAC,KAAK,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;IACvD,CAAC;IAED,MAAM,aAAa,GAAG,GAAG,MAAM,CAAC,SAAS,sBAAsB,CAAC;IAChE,IAAI,aAAa,KAAK,MAAM,CAAC,+BAA+B,CAAC,WAAW,EAAE,CAAC;QACzE,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC1B,aAAa,CAAC,+BAA+B,CAAC,WAAW,GAAG,aAAa,CAAC;IAC5E,CAAC;IAED,MAAM,yBAAyB,GAC7B,MAAM,CAAC,kBAAkB,EAAE,WAAW,IAAI,EAAE,CAAC;IAC/C,MAAM,oBAAoB,GAAG,MAAM,CAAC,iBAAiB,IAAI,EAAE,CAAC;IAC5D,IAAI,oBAAoB,KAAK,yBAAyB,EAAE,CAAC;QACvD,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAC1B,IAAI,CAAC,iCAAiC,CACpC,kBAAkB,EAClB,yBAAyB,CAC1B,CAAC;YACF,MAAM,IAAI,KAAK,CAAC,wCAAgC,CAAC,CAAC;QACpD,CAAC;QACD,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC1B,aAAa,CAAC,kBAAkB,KAAK,EAAE,CAAC;QACxC,aAAa,CAAC,kBAAkB,CAAC,WAAW,GAAG,oBAAoB,CAAC;IACtE,CAAC;IAED,IAAI,MAAM,CAAC,QAAQ,KAAK,cAAc,CAAC,QAAQ,EAAE,CAAC;QAChD,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAC5B,aAAa,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;IAC3C,CAAC;IAED,MAAM,aAAa,GAAG,qBAAqB,CAAC,MAAM,CAAC,CAAC;IACpD,IAAI,aAAa,KAAK,cAAc,CAAC,uBAAuB,EAAE,CAAC;QAC7D,UAAU,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;QAC7C,aAAa,CAAC,uBAAuB,GAAG,aAAa,CAAC;IACxD,CAAC;IAED,OAAO;QACL,cAAc,EAAE,aAAa;QAC7B,UAAU,EAAE,EAAE,KAAK,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC,EAAE;KAChD,CAAC;AACJ,CAAC;AAEM,KAAK,UAAU,oBAAoB,CACxC,MAA0B,EAC1B,kBAA0B,EAC1B,MAA6C;IAE7C,MAAM,OAAO,GAAG,MAAM,oCAAoC,CACxD,MAAM,EACN,kBAAkB,EAClB,MAAM,CACP,CAAC;IACF,IAAI,CAAC,oBAAoB,CAAC,kBAAkB,CAAC,CAAC;IAC9C,MAAM,SAAS,GACb,oBAAoB,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC,2BAA2B,CAAC,UAAU,CACtG,OAAO,CACR,CAAC;IACJ,MAAM,CAAC,OAAO,CAAC,GAAG,MAAM,MAAM,CAAC,oBAAoB,CAAC,SAAS,CAAC,CAAC;IAC/D,IAAI,CAAC,qBAAqB,CAAC,kBAAkB,CAAC,CAAC;IAC/C,OAAO,OAAO,CAAC;AACjB,CAAC"}
|
package/lib/export-config.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"export-config.js","sourceRoot":"","sources":["../src/export-config.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;GAcG
|
|
1
|
+
{"version":3,"file":"export-config.js","sourceRoot":"","sources":["../src/export-config.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;GAcG;;AA0EH,sCA8BC;AA7CD,SAAS,QAAQ,CAAC,KAAa,EAAE,KAAa;IAC5C,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;IAChC,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,8BAA8B,CAAC,CAAC;IAC1D,CAAC;IACD,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,SAAS,QAAQ,CAAC,KAAyB;IACzC,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAC1C,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;IAChC,OAAO,UAAU,IAAI,SAAS,CAAC;AACjC,CAAC;AAED,4EAA4E;AAC5E,SAAgB,aAAa,CAC3B,MAAqC;IAErC,MAAM,UAAU,GAAG,QAAQ,CAAC,MAAM,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC;IAC7D,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC;IAE3C,IAAI,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QACrE,MAAM,IAAI,KAAK,CAAC,yBAAyB,QAAQ,EAAE,CAAC,CAAC;IACvD,CAAC;IAED,OAAO;QACL,uBAAuB,EAAE,QAAQ,CAC/B,MAAM,CAAC,uBAAuB,EAC9B,yBAAyB,CAC1B;QACD,SAAS,EAAE,QAAQ,CAAC,MAAM,CAAC,SAAS,EAAE,WAAW,CAAC;QAClD,UAAU;QACV,kBAAkB,EAAE,QAAQ,CAAC,MAAM,CAAC,kBAAkB,CAAC;QACvD,SAAS,EAAE,QAAQ,CAAC,MAAM,CAAC,SAAS,EAAE,WAAW,CAAC;QAClD,SAAS,EAAE,QAAQ,CAAC,MAAM,CAAC,SAAS,EAAE,WAAW,CAAC;QAClD,WAAW,EAAE,QAAQ,CAAC,MAAM,CAAC,WAAW,EAAE,aAAa,CAAC;QACxD,WAAW,EAAE,QAAQ,CAAC,MAAM,CAAC,WAAW,EAAE,aAAa,CAAC;QACxD,iBAAiB,EAAE,QAAQ,CAAC,MAAM,CAAC,iBAAiB,CAAC;QACrD,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,QAAQ,EAAE,UAAU,CAAC;QAC/C,WAAW,EACT,QAAQ,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI,OAAO,UAAU,kBAAkB;QACrE,mBAAmB,EACjB,QAAQ,CAAC,MAAM,CAAC,mBAAmB,CAAC,IAAI,iBAAiB;QAC3D,QAAQ;KACT,CAAC;AACJ,CAAC"}
|
package/lib/handlers.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"handlers.d.ts","sourceRoot":"","sources":["../src/handlers.ts"],"names":[],"mappings":"AAgBA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AACvD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AAC1D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,8BAA8B,CAAC;AACzE,OAAO,EAEL,KAAK,kBAAkB,
|
|
1
|
+
{"version":3,"file":"handlers.d.ts","sourceRoot":"","sources":["../src/handlers.ts"],"names":[],"mappings":"AAgBA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AACvD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AAC1D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,8BAA8B,CAAC;AACzE,OAAO,EAEL,KAAK,kBAAkB,EAKxB,MAAM,OAAO,CAAC;AACf,OAAO,KAAK,EAAE,qCAAqC,EAAE,MAAM,iBAAiB,CAAC;AAG7E,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,SAAS,CAAC;AAElD,MAAM,MAAM,gBAAgB,GAAG,UAAU,CACvC,oBAAoB,CAAC,kBAAkB,CAAC,CACzC,CAAC;AAEF,yEAAyE;AACzE,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,SAAS,CAAC;IACd,QAAQ,EAAE,QAAQ,CAAC;IACnB,YAAY,EAAE,kBAAkB,CAAC;IACjC,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,qCAAqC,CAAC;CAC/C;AA4CD,sEAAsE;AACtE,wBAAsB,sBAAsB,CAC1C,KAAK,EAAE,gBAAgB,EACvB,GAAG,EAAE,cAAc,GAClB,OAAO,CAAC,IAAI,CAAC,CASf;AAED,4EAA4E;AAC5E,wBAAsB,0BAA0B,CAC9C,GAAG,EAAE,cAAc,GAClB,OAAO,CAAC,IAAI,CAAC,CA4Df"}
|
package/lib/handlers.js
CHANGED
|
@@ -72,6 +72,10 @@ async function ensureNotificationTopic(ctx) {
|
|
|
72
72
|
}
|
|
73
73
|
}
|
|
74
74
|
}
|
|
75
|
+
function isPartitioningFieldRemovalError(err) {
|
|
76
|
+
return (err instanceof Error &&
|
|
77
|
+
err.message.includes(dts_1.PARTITIONING_FIELD_REMOVAL_ERROR_PREFIX));
|
|
78
|
+
}
|
|
75
79
|
async function storeTransferConfig(ctx, transferConfig) {
|
|
76
80
|
if (!transferConfig?.name) {
|
|
77
81
|
throw new Error("BigQuery transfer config is missing its resource name");
|
|
@@ -100,7 +104,16 @@ async function handleUpsertTransferConfig(ctx) {
|
|
|
100
104
|
if (ctx.config.transferConfigName) {
|
|
101
105
|
const linked = await (0, dts_1.getTransferConfig)(ctx.dataTransfer, ctx.config.transferConfigName);
|
|
102
106
|
if (!linked) {
|
|
103
|
-
|
|
107
|
+
// Only a redeploy with a corrected TRANSFER_CONFIG_NAME can resolve this,
|
|
108
|
+
// so retrying the task cannot help.
|
|
109
|
+
logs.linkedTransferConfigMissing(ctx.config.transferConfigName);
|
|
110
|
+
return;
|
|
111
|
+
}
|
|
112
|
+
// A linked config is adopted as-is, so a topic mismatch is the user's to
|
|
113
|
+
// resolve: rewriting it would repoint a config this deployment did not create.
|
|
114
|
+
const expectedTopic = (0, dts_1.notificationTopicName)(ctx.config);
|
|
115
|
+
if (linked.notificationPubsubTopic !== expectedTopic) {
|
|
116
|
+
logs.linkedTopicMismatch(ctx.config.transferConfigName, linked.notificationPubsubTopic, expectedTopic);
|
|
104
117
|
}
|
|
105
118
|
await storeTransferConfig(ctx, linked);
|
|
106
119
|
return;
|
|
@@ -117,9 +130,18 @@ async function handleUpsertTransferConfig(ctx) {
|
|
|
117
130
|
}
|
|
118
131
|
const transferConfigName = existing.docs[0].data().name;
|
|
119
132
|
if (typeof transferConfigName !== "string" || !transferConfigName) {
|
|
120
|
-
throw new Error(`Existing transfer config document in ${ctx.config.firestoreCollection} is missing required 'name' field.`);
|
|
133
|
+
throw new Error(`Existing transfer config document in ${ctx.config.firestoreCollection} is missing required 'name' field. Delete the document so a new scheduled query is created, then redeploy.`);
|
|
134
|
+
}
|
|
135
|
+
try {
|
|
136
|
+
const updated = await (0, dts_1.updateTransferConfig)(ctx.dataTransfer, transferConfigName, ctx.config);
|
|
137
|
+
await storeTransferConfig(ctx, updated);
|
|
138
|
+
}
|
|
139
|
+
catch (err) {
|
|
140
|
+
if (!isPartitioningFieldRemovalError(err))
|
|
141
|
+
throw err;
|
|
142
|
+
// The guard rejects the update while building the request, so nothing was
|
|
143
|
+
// sent and a retry hits the same guard.
|
|
144
|
+
logs.partitioningFieldRemovalAborted(err.message);
|
|
121
145
|
}
|
|
122
|
-
const updated = await (0, dts_1.updateTransferConfig)(ctx.dataTransfer, transferConfigName, ctx.config);
|
|
123
|
-
await storeTransferConfig(ctx, updated);
|
|
124
146
|
}
|
|
125
147
|
//# sourceMappingURL=handlers.js.map
|
package/lib/handlers.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"handlers.js","sourceRoot":"","sources":["../src/handlers.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;GAcG
|
|
1
|
+
{"version":3,"file":"handlers.js","sourceRoot":"","sources":["../src/handlers.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;GAcG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA4EH,wDAYC;AAGD,gEA8DC;AAlJD,+BAOe;AAEf,qCAA6E;AAC7E,6CAA+B;AAgB/B,KAAK,UAAU,uBAAuB,CAAC,GAAmB;IACxD,MAAM,KAAK,GAAG,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;IACvD,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,MAAM,EAAE,CAAC;IACtC,IAAI,MAAM;QAAE,OAAO;IAEnB,IAAI,CAAC;QACH,MAAM,GAAG,CAAC,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;QACrD,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;IAC5C,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,6EAA6E;QAC7E,IACE,OAAO,GAAG,KAAK,QAAQ;YACvB,GAAG,KAAK,IAAI;YACZ,CAAC,CAAC,MAAM,IAAI,GAAG,CAAC;YAChB,GAAG,CAAC,IAAI,KAAK,CAAC,EACd,CAAC;YACD,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,+BAA+B,CAAC,GAAY;IACnD,OAAO,CACL,GAAG,YAAY,KAAK;QACpB,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,6CAAuC,CAAC,CAC9D,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,mBAAmB,CAChC,GAAmB,EACnB,cAA6D;IAE7D,IAAI,CAAC,cAAc,EAAE,IAAI,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAC;IAC3E,CAAC;IACD,MAAM,EAAE,gBAAgB,EAAE,GAAG,IAAA,gCAAuB,EAAC,cAAc,CAAC,IAAI,CAAC,CAAC;IAC1E,MAAM,GAAG,CAAC,EAAE;SACT,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,mBAAmB,CAAC;SAC1C,GAAG,CAAC,gBAAgB,CAAC;SACrB,GAAG,CAAC,EAAE,aAAa,EAAE,GAAG,CAAC,MAAM,CAAC,UAAU,EAAE,GAAG,cAAc,EAAE,CAAC,CAAC;AACtE,CAAC;AAED,sEAAsE;AAC/D,KAAK,UAAU,sBAAsB,CAC1C,KAAuB,EACvB,GAAmB;IAEnB,IAAI,CAAC,KAAK,EAAE,CAAC;IACb,IAAI,CAAC;QACH,MAAM,IAAA,iCAAwB,EAAC,GAAG,EAAE,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;QACvE,IAAI,CAAC,QAAQ,EAAE,CAAC;IAClB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAChB,MAAM,GAAG,CAAC;IACZ,CAAC;AACH,CAAC;AAED,4EAA4E;AACrE,KAAK,UAAU,0BAA0B,CAC9C,GAAmB;IAEnB,MAAM,uBAAuB,CAAC,GAAG,CAAC,CAAC;IAEnC,IAAI,GAAG,CAAC,MAAM,CAAC,kBAAkB,EAAE,CAAC;QAClC,MAAM,MAAM,GAAG,MAAM,IAAA,uBAAiB,EACpC,GAAG,CAAC,YAAY,EAChB,GAAG,CAAC,MAAM,CAAC,kBAAkB,CAC9B,CAAC;QACF,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,0EAA0E;YAC1E,oCAAoC;YACpC,IAAI,CAAC,2BAA2B,CAAC,GAAG,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC;YAChE,OAAO;QACT,CAAC;QACD,yEAAyE;QACzE,+EAA+E;QAC/E,MAAM,aAAa,GAAG,IAAA,2BAAqB,EAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACxD,IAAI,MAAM,CAAC,uBAAuB,KAAK,aAAa,EAAE,CAAC;YACrD,IAAI,CAAC,mBAAmB,CACtB,GAAG,CAAC,MAAM,CAAC,kBAAkB,EAC7B,MAAM,CAAC,uBAAuB,EAC9B,aAAa,CACd,CAAC;QACJ,CAAC;QACD,MAAM,mBAAmB,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;QACvC,OAAO;IACT,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,GAAG,CAAC,EAAE;SAC1B,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,mBAAmB,CAAC;SAC1C,KAAK,CAAC,eAAe,EAAE,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC;SACnD,KAAK,CAAC,CAAC,CAAC;SACR,GAAG,EAAE,CAAC;IAET,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;QACnB,MAAM,OAAO,GAAG,MAAM,IAAA,0BAAoB,EAAC,GAAG,CAAC,YAAY,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;QACzE,MAAM,mBAAmB,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QACxC,OAAO;IACT,CAAC;IAED,MAAM,kBAAkB,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC;IACxD,IAAI,OAAO,kBAAkB,KAAK,QAAQ,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAClE,MAAM,IAAI,KAAK,CACb,wCAAwC,GAAG,CAAC,MAAM,CAAC,mBAAmB,4GAA4G,CACnL,CAAC;IACJ,CAAC;IAED,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,IAAA,0BAAoB,EACxC,GAAG,CAAC,YAAY,EAChB,kBAAkB,EAClB,GAAG,CAAC,MAAM,CACX,CAAC;QACF,MAAM,mBAAmB,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAC1C,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,+BAA+B,CAAC,GAAG,CAAC;YAAE,MAAM,GAAG,CAAC;QACrD,0EAA0E;QAC1E,wCAAwC;QACxC,IAAI,CAAC,+BAA+B,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IACpD,CAAC;AACH,CAAC"}
|
package/lib/helper.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"helper.d.ts","sourceRoot":"","sources":["../src/helper.ts"],"names":[],"mappings":"AAgBA,OAAO,EACL,KAAK,QAAQ,EAMd,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAGL,KAAK,SAAS,EAEf,MAAM,0BAA0B,CAAC;AAClC,OAAO,KAAK,EAAE,qCAAqC,EAAE,MAAM,iBAAiB,CAAC;AAE7E,OAAO,KAAK,EACV,WAAW,EACX,gBAAgB,EAChB,YAAY,EACZ,iBAAiB,EACjB,kBAAkB,EACnB,MAAM,SAAS,CAAC;AAEjB,MAAM,WAAW,qBAAqB;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,gBAAgB,EAAE,MAAM,CAAC;IACzB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,wBAAwB;IACvC,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,oBAAoB;IACnC,EAAE,EAAE,SAAS,CAAC;IACd,QAAQ,EAAE,QAAQ,CAAC;IACnB,MAAM,EAAE,qCAAqC,CAAC;CAC/C;AAQD,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,MAAM,GAAG,qBAAqB,CAcxE;AAED,wBAAgB,uBAAuB,CACrC,IAAI,EAAE,MAAM,GACX,wBAAwB,CAa1B;AAED,wBAAsB,uBAAuB,CAC3C,EAAE,EAAE,SAAS,EACb,MAAM,EAAE,qCAAqC,EAC7C,gBAAgB,EAAE,MAAM,EACxB,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,kBAAkB,EAC3B,SAAS,EAAE;IAAE,cAAc,EAAE,MAAM,CAAC;IAAC,aAAa,EAAE,MAAM,CAAA;CAAE,GAC3D,OAAO,CAAC,IAAI,CAAC,CAiCf;AAED,wBAAsB,kBAAkB,CACtC,QAAQ,EAAE,QAAQ,EAClB,MAAM,EAAE,qCAAqC,EAC7C,gBAAgB,EAAE,MAAM,EACxB,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,WAAW,EAAE,CAAC,CAgBxB;AAED,wBAAgB,2BAA2B,CAAC,GAAG,EAAE,IAAI,GAAG,IAAI,CAAC;AAC7D,wBAAgB,2BAA2B,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAC;AACjE,wBAAgB,2BAA2B,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAC;AACjE,wBAAgB,2BAA2B,CAAC,GAAG,EAAE,OAAO,GAAG,OAAO,CAAC;AACnE,wBAAgB,2BAA2B,CAAC,GAAG,EAAE,WAAW,GAAG,YAAY,CAAC;AAC5E,wBAAgB,2BAA2B,CACzC,GAAG,EAAE,gBAAgB,GACpB,iBAAiB,CAAC;AA+BrB,wBAAsB,0BAA0B,CAC9C,GAAG,EAAE,oBAAoB,EACzB,OAAO,EAAE,kBAAkB,GAC1B,OAAO,CAAC,IAAI,CAAC,
|
|
1
|
+
{"version":3,"file":"helper.d.ts","sourceRoot":"","sources":["../src/helper.ts"],"names":[],"mappings":"AAgBA,OAAO,EACL,KAAK,QAAQ,EAMd,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAGL,KAAK,SAAS,EAEf,MAAM,0BAA0B,CAAC;AAClC,OAAO,KAAK,EAAE,qCAAqC,EAAE,MAAM,iBAAiB,CAAC;AAE7E,OAAO,KAAK,EACV,WAAW,EACX,gBAAgB,EAChB,YAAY,EACZ,iBAAiB,EACjB,kBAAkB,EACnB,MAAM,SAAS,CAAC;AAEjB,MAAM,WAAW,qBAAqB;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,gBAAgB,EAAE,MAAM,CAAC;IACzB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,wBAAwB;IACvC,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,oBAAoB;IACnC,EAAE,EAAE,SAAS,CAAC;IACd,QAAQ,EAAE,QAAQ,CAAC;IACnB,MAAM,EAAE,qCAAqC,CAAC;CAC/C;AAQD,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,MAAM,GAAG,qBAAqB,CAcxE;AAED,wBAAgB,uBAAuB,CACrC,IAAI,EAAE,MAAM,GACX,wBAAwB,CAa1B;AAED,wBAAsB,uBAAuB,CAC3C,EAAE,EAAE,SAAS,EACb,MAAM,EAAE,qCAAqC,EAC7C,gBAAgB,EAAE,MAAM,EACxB,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,kBAAkB,EAC3B,SAAS,EAAE;IAAE,cAAc,EAAE,MAAM,CAAC;IAAC,aAAa,EAAE,MAAM,CAAA;CAAE,GAC3D,OAAO,CAAC,IAAI,CAAC,CAiCf;AAED,wBAAsB,kBAAkB,CACtC,QAAQ,EAAE,QAAQ,EAClB,MAAM,EAAE,qCAAqC,EAC7C,gBAAgB,EAAE,MAAM,EACxB,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,WAAW,EAAE,CAAC,CAgBxB;AAED,wBAAgB,2BAA2B,CAAC,GAAG,EAAE,IAAI,GAAG,IAAI,CAAC;AAC7D,wBAAgB,2BAA2B,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAC;AACjE,wBAAgB,2BAA2B,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAC;AACjE,wBAAgB,2BAA2B,CAAC,GAAG,EAAE,OAAO,GAAG,OAAO,CAAC;AACnE,wBAAgB,2BAA2B,CAAC,GAAG,EAAE,WAAW,GAAG,YAAY,CAAC;AAC5E,wBAAgB,2BAA2B,CACzC,GAAG,EAAE,gBAAgB,GACpB,iBAAiB,CAAC;AA+BrB,wBAAsB,0BAA0B,CAC9C,GAAG,EAAE,oBAAoB,EACzB,OAAO,EAAE,kBAAkB,GAC1B,OAAO,CAAC,IAAI,CAAC,CAgEf;AAED,wBAAsB,oCAAoC,CACxD,EAAE,EAAE,SAAS,EACb,MAAM,EAAE,qCAAqC,EAC7C,gBAAgB,EAAE,MAAM,GACvB,OAAO,CAAC,OAAO,CAAC,CAMlB;AAED,wBAAsB,wBAAwB,CAC5C,GAAG,EAAE,oBAAoB,EACzB,OAAO,EAAE,kBAAkB,GAC1B,OAAO,CAAC,IAAI,CAAC,CAiCf"}
|
package/lib/helper.js
CHANGED
|
@@ -165,6 +165,7 @@ async function writeRunResultsToFirestore(ctx, message) {
|
|
|
165
165
|
.join("");
|
|
166
166
|
const tableName = message.json.params.destination_table_name_template.replace('{run_time|"%H%M%S"}', runTimeSuffix);
|
|
167
167
|
const rows = await getBigqueryResults(bigquery, config, transferConfigId, runId, message.json.destinationDatasetId, tableName);
|
|
168
|
+
logs.writeRunResultsToFirestore(runId);
|
|
168
169
|
const collection = db.collection(`${config.firestoreCollection}/${transferConfigId}/runs/${runId}/output`);
|
|
169
170
|
let succeededRowCount = 0;
|
|
170
171
|
for (let i = 0; i < rows.length; i += FIRESTORE_WRITE_CHUNK_SIZE) {
|
package/lib/helper.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"helper.js","sourceRoot":"","sources":["../src/helper.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;GAcG
|
|
1
|
+
{"version":3,"file":"helper.js","sourceRoot":"","sources":["../src/helper.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;GAcG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAmDH,oDAcC;AAED,0DAeC;AAED,0DAwCC;AAED,gDAuBC;AAUD,kEA4BC;AAED,gEAmEC;AAED,oFAUC;AAED,4DAoCC;AAhTD,qDAOgC;AAChC,wDAKkC;AAElC,6CAA+B;AA4B/B,MAAM,uBAAuB,GAC3B,kFAAkF,CAAC;AACrF,MAAM,0BAA0B,GAC9B,mEAAmE,CAAC;AACtE,MAAM,0BAA0B,GAAG,MAAM,CAAC;AAE1C,SAAgB,oBAAoB,CAAC,IAAY;IAC/C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAC;IAClD,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CACb,sCAAsC,IAAI,uGAAuG,CAClJ,CAAC;IACJ,CAAC;IAED,OAAO;QACL,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC;QACnB,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC;QAClB,gBAAgB,EAAE,KAAK,CAAC,CAAC,CAAC;QAC1B,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC;KAChB,CAAC;AACJ,CAAC;AAED,SAAgB,uBAAuB,CACrC,IAAY;IAEZ,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC;IACrD,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CACb,yCAAyC,IAAI,0FAA0F,CACxI,CAAC;IACJ,CAAC;IAED,OAAO;QACL,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC;QACnB,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC;QAClB,gBAAgB,EAAE,KAAK,CAAC,CAAC,CAAC;KAC3B,CAAC;AACJ,CAAC;AAEM,KAAK,UAAU,uBAAuB,CAC3C,EAAa,EACb,MAA6C,EAC7C,gBAAwB,EACxB,KAAa,EACb,OAA2B,EAC3B,SAA4D;IAE5D,MAAM,SAAS,GAAG,EAAE;SACjB,UAAU,CAAC,GAAG,MAAM,CAAC,mBAAmB,IAAI,gBAAgB,OAAO,CAAC;SACpE,GAAG,CAAC,QAAQ,CAAC,CAAC;IACjB,MAAM,OAAO,GAAG,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC/C,MAAM,SAAS,GAAG;QAChB,WAAW,EAAE,OAAO,CAAC,IAAI;QACzB,WAAW,EAAE,KAAK;QAClB,GAAG,SAAS;KACb,CAAC;IAEF,MAAM,EAAE,CAAC,cAAc,CAAC,KAAK,EAAE,WAAW,EAAE,EAAE;QAC5C,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAChD,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC;QACjC,MAAM,eAAe,GAAG,UAAU,EAAE,WAAW,EAAE,OAAO,CAAC;QACzD,MAAM,aAAa,GAAG,UAAU,EAAE,WAAW,CAAC;QAC9C,MAAM,YAAY,GAChB,CAAC,UAAU;YACX,CAAC,eAAe;YAChB,IAAI,IAAI,CAAC,eAAe,CAAC,GAAG,OAAO;YACnC,aAAa,KAAK,KAAK,CAAC;QAE1B,IAAI,YAAY,EAAE,CAAC;YACjB,WAAW,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;YACtC,OAAO;QACT,CAAC;QAED,IAAI,CAAC,sBAAsB,CACzB,gBAAgB,EAChB,KAAK,EACL,0BAA0B,eAAe,OAAO,OAAO,CAAC,IAAI,CAAC,OAAO,GAAG,CACxE,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAEM,KAAK,UAAU,kBAAkB,CACtC,QAAkB,EAClB,MAA6C,EAC7C,gBAAwB,EACxB,KAAa,EACb,SAAiB,EACjB,SAAiB;IAEjB,MAAM,KAAK,GAAG,mBAAmB,MAAM,CAAC,SAAS,IAAI,SAAS,IAAI,SAAS,IAAI,CAAC;IAEhF,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,QAAQ,CAAC,cAAc,CAAC;YAC1C,KAAK;YACL,QAAQ,EAAE,MAAM,CAAC,uBAAuB;SACzC,CAAC,CAAC;QACH,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAChC,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,GAAG,CAAC,eAAe,EAAE,CAAC;QAC3C,IAAI,CAAC,uBAAuB,CAAC,gBAAgB,EAAE,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QACnE,OAAO,IAAqB,CAAC;IAC/B,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC;QAClE,MAAM,GAAG,CAAC;IACZ,CAAC;AACH,CAAC;AAUD,SAAgB,2BAA2B,CACzC,GAAqB;IAErB,IAAI,GAAG,KAAK,IAAI,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QAC5C,OAAO,GAAwB,CAAC;IAClC,CAAC;IAED,IACE,GAAG,YAAY,4BAAiB;QAChC,GAAG,YAAY,uBAAY;QAC3B,GAAG,YAAY,uBAAY;QAC3B,GAAG,YAAY,2BAAgB,EAC/B,CAAC;QACD,OAAO,qBAAS,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;IACjD,CAAC;IACD,IAAI,GAAG,YAAY,IAAI;QAAE,OAAO,qBAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;IACxD,IAAI,GAAG,YAAY,MAAM;QAAE,OAAO,IAAI,UAAU,CAAC,GAAG,CAAC,CAAC;IACtD,IAAI,GAAG,YAAY,oBAAS;QAAE,OAAO,GAAG,CAAC,KAAK,CAAC;IAC/C,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QACvB,OAAO,GAAG,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,2BAA2B,CAAC,KAAK,CAAC,CAAC,CAAC;IAChE,CAAC;IAED,OAAO,MAAM,CAAC,WAAW,CACvB,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC;QACxC,GAAG;QACH,2BAA2B,CAAC,KAAK,CAAC;KACnC,CAAC,CACa,CAAC;AACpB,CAAC;AAEM,KAAK,UAAU,0BAA0B,CAC9C,GAAyB,EACzB,OAA2B;IAE3B,MAAM,EAAE,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,GAAG,CAAC;IACrC,MAAM,EAAE,gBAAgB,EAAE,KAAK,EAAE,GAAG,oBAAoB,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC5E,MAAM,OAAO,GAAG,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC/C,MAAM,aAAa,GAAG;QACpB,OAAO,CAAC,WAAW,EAAE;QACrB,OAAO,CAAC,aAAa,EAAE;QACvB,OAAO,CAAC,aAAa,EAAE;KACxB;SACE,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;SAC5C,IAAI,CAAC,EAAE,CAAC,CAAC;IACZ,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,+BAA+B,CAAC,OAAO,CAC3E,qBAAqB,EACrB,aAAa,CACd,CAAC;IACF,MAAM,IAAI,GAAG,MAAM,kBAAkB,CACnC,QAAQ,EACR,MAAM,EACN,gBAAgB,EAChB,KAAK,EACL,OAAO,CAAC,IAAI,CAAC,oBAAoB,EACjC,SAAS,CACV,CAAC;IACF,IAAI,CAAC,0BAA0B,CAAC,KAAK,CAAC,CAAC;IACvC,MAAM,UAAU,GAAG,EAAE,CAAC,UAAU,CAC9B,GAAG,MAAM,CAAC,mBAAmB,IAAI,gBAAgB,SAAS,KAAK,SAAS,CACzE,CAAC;IACF,IAAI,iBAAiB,GAAG,CAAC,CAAC;IAE1B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,0BAA0B,EAAE,CAAC;QACjE,MAAM,MAAM,GAAoD,EAAE,CAAC;QACnE,KACE,IAAI,CAAC,GAAG,CAAC,EACT,CAAC,GAAG,CAAC,GAAG,0BAA0B,IAAI,CAAC,GAAG,IAAI,CAAC,MAAM,EACrD,CAAC,EAAE,EACH,CAAC;YACD,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,2BAA2B,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACpE,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;QACjD,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW;gBAAE,iBAAiB,EAAE,CAAC;;gBAClD,IAAI,CAAC,uBAAuB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACnD,CAAC;IACH,CAAC;IAED,MAAM,SAAS,GAAG;QAChB,cAAc,EAAE,IAAI,CAAC,MAAM,GAAG,iBAAiB;QAC/C,aAAa,EAAE,IAAI,CAAC,MAAM;KAC3B,CAAC;IACF,IAAI,CAAC,4BAA4B,CAAC,KAAK,EAAE,iBAAiB,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IAEzE,MAAM,EAAE;SACL,UAAU,CAAC,GAAG,MAAM,CAAC,mBAAmB,IAAI,gBAAgB,OAAO,CAAC;SACpE,GAAG,CAAC,KAAK,CAAC;SACV,GAAG,CAAC,EAAE,WAAW,EAAE,OAAO,CAAC,IAAI,EAAE,GAAG,SAAS,EAAE,CAAC,CAAC;IACpD,MAAM,uBAAuB,CAC3B,EAAE,EACF,MAAM,EACN,gBAAgB,EAChB,KAAK,EACL,OAAO,EACP,SAAS,CACV,CAAC;AACJ,CAAC;AAEM,KAAK,UAAU,oCAAoC,CACxD,EAAa,EACb,MAA6C,EAC7C,gBAAwB;IAExB,MAAM,OAAO,GAAG,MAAM,EAAE;SACrB,UAAU,CAAC,MAAM,CAAC,mBAAmB,CAAC;SACtC,KAAK,CAAC,eAAe,EAAE,IAAI,EAAE,MAAM,CAAC,UAAU,CAAC;SAC/C,GAAG,EAAE,CAAC;IACT,OAAO,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,gBAAgB,CAAC,CAAC;AACjE,CAAC;AAEM,KAAK,UAAU,wBAAwB,CAC5C,GAAyB,EACzB,OAA2B;IAE3B,MAAM,EAAE,gBAAgB,EAAE,KAAK,EAAE,GAAG,oBAAoB,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC5E,MAAM,UAAU,GAAG,MAAM,oCAAoC,CAC3D,GAAG,CAAC,EAAE,EACN,GAAG,CAAC,MAAM,EACV,gBAAgB,CACjB,CAAC;IAEF,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,MAAM,IAAI,KAAK,CACb,4DAA4D,gBAAgB,gDAAgD,GAAG,CAAC,MAAM,CAAC,UAAU,IAAI,CACtJ,CAAC;IACJ,CAAC;IAED,IAAI,OAAO,CAAC,IAAI,CAAC,KAAK,KAAK,WAAW,EAAE,CAAC;QACvC,MAAM,0BAA0B,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QAC/C,OAAO;IACT,CAAC;IAED,IAAI,CAAC,qBAAqB,CAAC,gBAAgB,EAAE,KAAK,EAAE,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACxE,MAAM,SAAS,GAAG,EAAE,cAAc,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,CAAC;IAC1D,MAAM,GAAG,CAAC,EAAE;SACT,UAAU,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,mBAAmB,IAAI,gBAAgB,OAAO,CAAC;SACxE,GAAG,CAAC,KAAK,CAAC;SACV,GAAG,CAAC,EAAE,WAAW,EAAE,OAAO,CAAC,IAAI,EAAE,GAAG,SAAS,EAAE,CAAC,CAAC;IACpD,MAAM,uBAAuB,CAC3B,GAAG,CAAC,EAAE,EACN,GAAG,CAAC,MAAM,EACV,gBAAgB,EAChB,KAAK,EACL,OAAO,EACP,SAAS,CACV,CAAC;AACJ,CAAC"}
|
package/lib/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;GAcG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEH;;;;GAIG;AAEH,qDAAkD;AAClD,iFAAkF;AAClF,iDAA8C;AAC9C,4CAA4D;AAC5D,wDAAwD;AACxD,yDAAkE;AAClE,uDAA+D;AAE/D,8CAAkE;AAClE,+DAGyC;AACzC,qCAA6D;AAE7D,mDAAgD;AAChD,yCAIoB;AACpB,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;GAcG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEH;;;;GAIG;AAEH,qDAAkD;AAClD,iFAAkF;AAClF,iDAA8C;AAC9C,4CAA4D;AAC5D,wDAAwD;AACxD,yDAAkE;AAClE,uDAA+D;AAE/D,8CAAkE;AAClE,+DAGyC;AACzC,qCAA6D;AAE7D,mDAAgD;AAChD,yCAIoB;AACpB,6CAA+B;AAG/B,wCAAsB;AAEtB,MAAM,eAAe,GAAG,sBAAsB,CAAC;AAC/C,MAAM,cAAc,GAAwB;IAC1C,sBAAsB;IACtB,sBAAsB;IACtB,oBAAoB;IACpB,8BAA8B;IAC9B,mBAAmB;CACpB,CAAC;AACF,MAAM,aAAa,GAAG;IACpB;QACE,GAAG,EAAE,yBAAyB;QAC9B,MAAM,EAAE,4DAA4D;KACrE;IACD;QACE,GAAG,EAAE,qCAAqC;QAC1C,MAAM,EAAE,6DAA6D;KACtE;IACD;QACE,GAAG,EAAE,uBAAuB;QAC5B,MAAM,EAAE,sDAAsD;KAC/D;CACO,CAAC;AAEX,KAAK,MAAM,IAAI,IAAI,cAAc,EAAE,CAAC;IAClC,IAAA,iBAAY,EAAC,IAAI,CAAC,CAAC;AACrB,CAAC;AAED,KAAK,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,aAAa,EAAE,CAAC;IAC5C,IAAA,gBAAW,EAAC,GAAG,EAAE,MAAM,CAAC,CAAC;AAC3B,CAAC;AAED,IAAA,4BAAgB,EAAC;IACf,IAAI,EAAE,EAAE,QAAQ,EAAE,eAAe,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,EAAE;CACxD,CAAC,CAAC;AACH,IAAA,yBAAa,EAAC;IACZ,IAAI,EAAE,EAAE,QAAQ,EAAE,eAAe,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,EAAE;CACxD,CAAC,CAAC;AAEH,IAAI,GAA+B,CAAC;AAEpC,SAAS,UAAU;IACjB,IAAI,GAAG;QAAE,OAAO,GAAG,CAAC;IAEpB,MAAM,MAAM,GAA0C,IAAA,6BAAa,EACjE,IAAA,sBAAa,GAAE,CAChB,CAAC;IACF,IAAI,IAAA,aAAO,GAAE,CAAC,MAAM,KAAK,CAAC;QAAE,IAAA,mBAAa,EAAC,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC;IAC3E,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAElB,GAAG,GAAG;QACJ,EAAE,EAAE,IAAA,wBAAY,GAAE;QAClB,QAAQ,EAAE,IAAI,mBAAQ,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,CAAC;QACvD,YAAY,EAAE,IAAI,2BAAoB,CAAC,yBAAyB,CAAC;YAC/D,SAAS,EAAE,MAAM,CAAC,SAAS;SAC5B,CAAC;QACF,MAAM,EAAE,IAAI,eAAM,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,CAAC;QACnD,MAAM;KACP,CAAC;IACF,OAAO,GAAG,CAAC;AACb,CAAC;AAED,gEAAgE;AACnD,QAAA,eAAe,GAAG,IAAA,2BAAkB,EAC/C;IACE,KAAK,EAAE,2BAAkB,CAAC,WAAW;IACrC,KAAK,EAAE,KAAK;CACb,EACD,CAAC,KAAK,EAAE,EAAE,CAAC,IAAA,iCAAsB,EAAC,KAAK,EAAE,UAAU,EAAE,CAAC,CACvD,CAAC;AAEF,uEAAuE;AAC1D,QAAA,oBAAoB,GAAG,IAAA,wBAAgB,EAClD;IACE,MAAM,EAAE,MAAM;IACd,WAAW,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE;CACvD,EACD,GAAG,EAAE,CAAC,IAAA,qCAA0B,EAAC,UAAU,EAAE,CAAC,CAC/C,CAAC"}
|
package/lib/logs.d.ts
CHANGED
|
@@ -7,6 +7,7 @@ export declare function errorWritingToFirestore(err: unknown): void;
|
|
|
7
7
|
export declare function bigqueryJobStarted(jobId: string | undefined): void;
|
|
8
8
|
export declare function bigqueryResultsRowCount(transferConfigId: string, runId: string, count: number): void;
|
|
9
9
|
export declare function bigqueryQueryFailed(transferConfigId: string, runId: string, tableName: string, err: unknown): void;
|
|
10
|
+
export declare function writeRunResultsToFirestore(runId: string): void;
|
|
10
11
|
export declare function runResultsWrittenToFirestore(runId: string, successCount: number, totalCount: number): void;
|
|
11
12
|
export declare function latestDocUpdateSkipped(transferConfigId: string, runId: string, reason: string): void;
|
|
12
13
|
export declare function handlingNonSuccessRun(transferConfigId: string, runId: string, state: string): void;
|
|
@@ -17,5 +18,8 @@ export declare function transferConfigUpdated(name: string): void;
|
|
|
17
18
|
export declare function transferConfigNotFound(name: string): void;
|
|
18
19
|
export declare function getTransferConfigFailed(name: string, err: unknown): void;
|
|
19
20
|
export declare function partitioningFieldRemovalAttempted(name: string, existingField: string): void;
|
|
21
|
+
export declare function linkedTransferConfigMissing(name: string): void;
|
|
22
|
+
export declare function partitioningFieldRemovalAborted(reason: string): void;
|
|
20
23
|
export declare function topicCreated(name: string): void;
|
|
24
|
+
export declare function linkedTopicMismatch(name: string, linkedTopic: string | null | undefined, expectedTopic: string): void;
|
|
21
25
|
//# sourceMappingURL=logs.d.ts.map
|
package/lib/logs.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"logs.d.ts","sourceRoot":"","sources":["../src/logs.ts"],"names":[],"mappings":"AAiBA,OAAO,KAAK,EAAE,qCAAqC,EAAE,MAAM,iBAAiB,CAAC;AAE7E,wBAAgB,IAAI,CAAC,MAAM,EAAE,qCAAqC,GAAG,IAAI,CAKxE;AAED,wBAAgB,KAAK,IAAI,IAAI,CAE5B;AAED,wBAAgB,QAAQ,IAAI,IAAI,CAE/B;AAED,wBAAgB,KAAK,CAAC,GAAG,EAAE,OAAO,GAAG,IAAI,CAExC;AAED,wBAAgB,uBAAuB,CAAC,GAAG,EAAE,OAAO,GAAG,IAAI,CAE1D;AAED,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI,CAElE;AAED,wBAAgB,uBAAuB,CACrC,gBAAgB,EAAE,MAAM,EACxB,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,GACZ,IAAI,CAMN;AAED,wBAAgB,mBAAmB,CACjC,gBAAgB,EAAE,MAAM,EACxB,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,MAAM,EACjB,GAAG,EAAE,OAAO,GACX,IAAI,CAON;AAED,wBAAgB,4BAA4B,CAC1C,KAAK,EAAE,MAAM,EACb,YAAY,EAAE,MAAM,EACpB,UAAU,EAAE,MAAM,GACjB,IAAI,CAMN;AAED,wBAAgB,sBAAsB,CACpC,gBAAgB,EAAE,MAAM,EACxB,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,GACb,IAAI,CAMN;AAED,wBAAgB,qBAAqB,CACnC,gBAAgB,EAAE,MAAM,EACxB,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,GACZ,IAAI,CAMN;AAED,wBAAgB,oBAAoB,IAAI,IAAI,CAE3C;AAED,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAExD;AAED,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAEvD;AAED,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAExD;AAED,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAEzD;AAED,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,GAAG,IAAI,CAExE;AAED,wBAAgB,iCAAiC,CAC/C,IAAI,EAAE,MAAM,EACZ,aAAa,EAAE,MAAM,GACpB,IAAI,CAKN;AAED,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAE/C"}
|
|
1
|
+
{"version":3,"file":"logs.d.ts","sourceRoot":"","sources":["../src/logs.ts"],"names":[],"mappings":"AAiBA,OAAO,KAAK,EAAE,qCAAqC,EAAE,MAAM,iBAAiB,CAAC;AAE7E,wBAAgB,IAAI,CAAC,MAAM,EAAE,qCAAqC,GAAG,IAAI,CAKxE;AAED,wBAAgB,KAAK,IAAI,IAAI,CAE5B;AAED,wBAAgB,QAAQ,IAAI,IAAI,CAE/B;AAED,wBAAgB,KAAK,CAAC,GAAG,EAAE,OAAO,GAAG,IAAI,CAExC;AAED,wBAAgB,uBAAuB,CAAC,GAAG,EAAE,OAAO,GAAG,IAAI,CAE1D;AAED,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI,CAElE;AAED,wBAAgB,uBAAuB,CACrC,gBAAgB,EAAE,MAAM,EACxB,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,GACZ,IAAI,CAMN;AAED,wBAAgB,mBAAmB,CACjC,gBAAgB,EAAE,MAAM,EACxB,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,MAAM,EACjB,GAAG,EAAE,OAAO,GACX,IAAI,CAON;AAED,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAE9D;AAED,wBAAgB,4BAA4B,CAC1C,KAAK,EAAE,MAAM,EACb,YAAY,EAAE,MAAM,EACpB,UAAU,EAAE,MAAM,GACjB,IAAI,CAMN;AAED,wBAAgB,sBAAsB,CACpC,gBAAgB,EAAE,MAAM,EACxB,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,GACb,IAAI,CAMN;AAED,wBAAgB,qBAAqB,CACnC,gBAAgB,EAAE,MAAM,EACxB,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,GACZ,IAAI,CAMN;AAED,wBAAgB,oBAAoB,IAAI,IAAI,CAE3C;AAED,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAExD;AAED,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAEvD;AAED,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAExD;AAED,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAEzD;AAED,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,GAAG,IAAI,CAExE;AAED,wBAAgB,iCAAiC,CAC/C,IAAI,EAAE,MAAM,EACZ,aAAa,EAAE,MAAM,GACpB,IAAI,CAKN;AAED,wBAAgB,2BAA2B,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAK9D;AAID,wBAAgB,+BAA+B,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAEpE;AAED,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAE/C;AAED,wBAAgB,mBAAmB,CACjC,IAAI,EAAE,MAAM,EACZ,WAAW,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,EACtC,aAAa,EAAE,MAAM,GACpB,IAAI,CAKN"}
|
package/lib/logs.js
CHANGED
|
@@ -23,6 +23,7 @@ exports.errorWritingToFirestore = errorWritingToFirestore;
|
|
|
23
23
|
exports.bigqueryJobStarted = bigqueryJobStarted;
|
|
24
24
|
exports.bigqueryResultsRowCount = bigqueryResultsRowCount;
|
|
25
25
|
exports.bigqueryQueryFailed = bigqueryQueryFailed;
|
|
26
|
+
exports.writeRunResultsToFirestore = writeRunResultsToFirestore;
|
|
26
27
|
exports.runResultsWrittenToFirestore = runResultsWrittenToFirestore;
|
|
27
28
|
exports.latestDocUpdateSkipped = latestDocUpdateSkipped;
|
|
28
29
|
exports.handlingNonSuccessRun = handlingNonSuccessRun;
|
|
@@ -33,7 +34,10 @@ exports.transferConfigUpdated = transferConfigUpdated;
|
|
|
33
34
|
exports.transferConfigNotFound = transferConfigNotFound;
|
|
34
35
|
exports.getTransferConfigFailed = getTransferConfigFailed;
|
|
35
36
|
exports.partitioningFieldRemovalAttempted = partitioningFieldRemovalAttempted;
|
|
37
|
+
exports.linkedTransferConfigMissing = linkedTransferConfigMissing;
|
|
38
|
+
exports.partitioningFieldRemovalAborted = partitioningFieldRemovalAborted;
|
|
36
39
|
exports.topicCreated = topicCreated;
|
|
40
|
+
exports.linkedTopicMismatch = linkedTopicMismatch;
|
|
37
41
|
const firebase_functions_1 = require("firebase-functions");
|
|
38
42
|
function init(config) {
|
|
39
43
|
firebase_functions_1.logger.info("Initializing BigQuery Firestore export", {
|
|
@@ -71,6 +75,9 @@ function bigqueryQueryFailed(transferConfigId, runId, tableName, err) {
|
|
|
71
75
|
err,
|
|
72
76
|
});
|
|
73
77
|
}
|
|
78
|
+
function writeRunResultsToFirestore(runId) {
|
|
79
|
+
firebase_functions_1.logger.debug("Writing BigQuery results to Firestore", { runId });
|
|
80
|
+
}
|
|
74
81
|
function runResultsWrittenToFirestore(runId, successCount, totalCount) {
|
|
75
82
|
firebase_functions_1.logger.info("Finished writing BigQuery results to Firestore", {
|
|
76
83
|
runId,
|
|
@@ -116,7 +123,18 @@ function partitioningFieldRemovalAttempted(name, existingField) {
|
|
|
116
123
|
existingField,
|
|
117
124
|
});
|
|
118
125
|
}
|
|
126
|
+
function linkedTransferConfigMissing(name) {
|
|
127
|
+
firebase_functions_1.logger.error("The scheduled query named by TRANSFER_CONFIG_NAME does not exist, so nothing was linked. Set it to a scheduled query that exists in this project and redeploy, or clear it to have this deployment create its own.", { name });
|
|
128
|
+
}
|
|
129
|
+
// The reason carries the remediation, and an Error passed as structured data
|
|
130
|
+
// serialises to {}, so it goes in the message.
|
|
131
|
+
function partitioningFieldRemovalAborted(reason) {
|
|
132
|
+
firebase_functions_1.logger.error(`Stopped without updating the scheduled query. ${reason}`);
|
|
133
|
+
}
|
|
119
134
|
function topicCreated(name) {
|
|
120
135
|
firebase_functions_1.logger.info("Created Pub/Sub topic for transfer notifications", { name });
|
|
121
136
|
}
|
|
137
|
+
function linkedTopicMismatch(name, linkedTopic, expectedTopic) {
|
|
138
|
+
firebase_functions_1.logger.warn("Linked transfer config notifies a different Pub/Sub topic, so its runs will not reach this kit. Set PUB_SUB_TOPIC to the linked topic, or point the transfer config at the configured one.", { name, linkedTopic: linkedTopic ?? "", expectedTopic });
|
|
139
|
+
}
|
|
122
140
|
//# sourceMappingURL=logs.js.map
|
package/lib/logs.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"logs.js","sourceRoot":"","sources":["../src/logs.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;GAcG
|
|
1
|
+
{"version":3,"file":"logs.js","sourceRoot":"","sources":["../src/logs.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;GAcG;;AAKH,oBAKC;AAED,sBAEC;AAED,4BAEC;AAED,sBAEC;AAED,0DAEC;AAED,gDAEC;AAED,0DAUC;AAED,kDAYC;AAED,gEAEC;AAED,oEAUC;AAED,wDAUC;AAED,sDAUC;AAED,oDAEC;AAED,sDAEC;AAED,oDAEC;AAED,sDAEC;AAED,wDAEC;AAED,0DAEC;AAED,8EAQC;AAED,kEAKC;AAID,0EAEC;AAED,oCAEC;AAED,kDASC;AA5JD,2DAA4C;AAG5C,SAAgB,IAAI,CAAC,MAA6C;IAChE,2BAAM,CAAC,IAAI,CAAC,wCAAwC,EAAE;QACpD,GAAG,MAAM;QACT,WAAW,EAAE,WAAW;KACzB,CAAC,CAAC;AACL,CAAC;AAED,SAAgB,KAAK;IACnB,2BAAM,CAAC,IAAI,CAAC,6CAA6C,CAAC,CAAC;AAC7D,CAAC;AAED,SAAgB,QAAQ;IACtB,2BAAM,CAAC,IAAI,CAAC,+CAA+C,CAAC,CAAC;AAC/D,CAAC;AAED,SAAgB,KAAK,CAAC,GAAY;IAChC,2BAAM,CAAC,KAAK,CAAC,kDAAkD,EAAE,GAAG,CAAC,CAAC;AACxE,CAAC;AAED,SAAgB,uBAAuB,CAAC,GAAY;IAClD,2BAAM,CAAC,KAAK,CAAC,2CAA2C,EAAE,GAAG,CAAC,CAAC;AACjE,CAAC;AAED,SAAgB,kBAAkB,CAAC,KAAyB;IAC1D,2BAAM,CAAC,IAAI,CAAC,gBAAgB,KAAK,IAAI,WAAW,UAAU,CAAC,CAAC;AAC9D,CAAC;AAED,SAAgB,uBAAuB,CACrC,gBAAwB,EACxB,KAAa,EACb,KAAa;IAEb,2BAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE;QAC7C,gBAAgB;QAChB,KAAK;QACL,KAAK;KACN,CAAC,CAAC;AACL,CAAC;AAED,SAAgB,mBAAmB,CACjC,gBAAwB,EACxB,KAAa,EACb,SAAiB,EACjB,GAAY;IAEZ,2BAAM,CAAC,KAAK,CAAC,8BAA8B,EAAE;QAC3C,gBAAgB;QAChB,KAAK;QACL,SAAS;QACT,GAAG;KACJ,CAAC,CAAC;AACL,CAAC;AAED,SAAgB,0BAA0B,CAAC,KAAa;IACtD,2BAAM,CAAC,KAAK,CAAC,uCAAuC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;AACnE,CAAC;AAED,SAAgB,4BAA4B,CAC1C,KAAa,EACb,YAAoB,EACpB,UAAkB;IAElB,2BAAM,CAAC,IAAI,CAAC,gDAAgD,EAAE;QAC5D,KAAK;QACL,YAAY;QACZ,UAAU;KACX,CAAC,CAAC;AACL,CAAC;AAED,SAAgB,sBAAsB,CACpC,gBAAwB,EACxB,KAAa,EACb,MAAc;IAEd,2BAAM,CAAC,KAAK,CAAC,+CAA+C,EAAE;QAC5D,gBAAgB;QAChB,KAAK;QACL,MAAM;KACP,CAAC,CAAC;AACL,CAAC;AAED,SAAgB,qBAAqB,CACnC,gBAAwB,EACxB,KAAa,EACb,KAAa;IAEb,2BAAM,CAAC,IAAI,CAAC,gDAAgD,EAAE;QAC5D,gBAAgB;QAChB,KAAK;QACL,KAAK;KACN,CAAC,CAAC;AACL,CAAC;AAED,SAAgB,oBAAoB;IAClC,2BAAM,CAAC,IAAI,CAAC,wCAAwC,CAAC,CAAC;AACxD,CAAC;AAED,SAAgB,qBAAqB,CAAC,IAAY;IAChD,2BAAM,CAAC,IAAI,CAAC,uCAAuC,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;AACjE,CAAC;AAED,SAAgB,oBAAoB,CAAC,IAAY;IAC/C,2BAAM,CAAC,IAAI,CAAC,wCAAwC,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;AAClE,CAAC;AAED,SAAgB,qBAAqB,CAAC,IAAY;IAChD,2BAAM,CAAC,IAAI,CAAC,uCAAuC,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;AACjE,CAAC;AAED,SAAgB,sBAAsB,CAAC,IAAY;IACjD,2BAAM,CAAC,KAAK,CAAC,yCAAyC,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;AACpE,CAAC;AAED,SAAgB,uBAAuB,CAAC,IAAY,EAAE,GAAY;IAChE,2BAAM,CAAC,KAAK,CAAC,8CAA8C,EAAE,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC;AAC9E,CAAC;AAED,SAAgB,iCAAiC,CAC/C,IAAY,EACZ,aAAqB;IAErB,2BAAM,CAAC,IAAI,CAAC,kDAAkD,EAAE;QAC9D,IAAI;QACJ,aAAa;KACd,CAAC,CAAC;AACL,CAAC;AAED,SAAgB,2BAA2B,CAAC,IAAY;IACtD,2BAAM,CAAC,KAAK,CACV,oNAAoN,EACpN,EAAE,IAAI,EAAE,CACT,CAAC;AACJ,CAAC;AAED,6EAA6E;AAC7E,+CAA+C;AAC/C,SAAgB,+BAA+B,CAAC,MAAc;IAC5D,2BAAM,CAAC,KAAK,CAAC,iDAAiD,MAAM,EAAE,CAAC,CAAC;AAC1E,CAAC;AAED,SAAgB,YAAY,CAAC,IAAY;IACvC,2BAAM,CAAC,IAAI,CAAC,kDAAkD,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;AAC5E,CAAC;AAED,SAAgB,mBAAmB,CACjC,IAAY,EACZ,WAAsC,EACtC,aAAqB;IAErB,2BAAM,CAAC,IAAI,CACT,4LAA4L,EAC5L,EAAE,IAAI,EAAE,WAAW,EAAE,WAAW,IAAI,EAAE,EAAE,aAAa,EAAE,CACxD,CAAC;AACJ,CAAC"}
|
package/lib/metadata.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"metadata.d.ts","sourceRoot":"","sources":["../src/metadata.ts"],"names":[],"mappings":"AAgBA,eAAO,MAAM,QAAQ
|
|
1
|
+
{"version":3,"file":"metadata.d.ts","sourceRoot":"","sources":["../src/metadata.ts"],"names":[],"mappings":"AAgBA,eAAO,MAAM,QAAQ;;;;CAcX,CAAC"}
|
package/npm-shrinkwrap.json
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@firebase-function-kits/bigquery-firestore-export",
|
|
3
|
-
"version": "0.0.2-rc.
|
|
3
|
+
"version": "0.0.2-rc.3",
|
|
4
4
|
"lockfileVersion": 3,
|
|
5
5
|
"requires": true,
|
|
6
6
|
"packages": {
|
|
7
7
|
"": {
|
|
8
8
|
"name": "@firebase-function-kits/bigquery-firestore-export",
|
|
9
|
-
"version": "0.0.2-rc.
|
|
9
|
+
"version": "0.0.2-rc.3",
|
|
10
10
|
"license": "Apache-2.0",
|
|
11
11
|
"dependencies": {
|
|
12
12
|
"@firebase/app": "^0.16.0",
|
|
@@ -14,9 +14,10 @@
|
|
|
14
14
|
"@google-cloud/bigquery-data-transfer": "^5.0.1",
|
|
15
15
|
"@google-cloud/pubsub": "^4.11.0",
|
|
16
16
|
"firebase-admin": "^14.1.0",
|
|
17
|
-
"firebase-functions": "7.3.
|
|
17
|
+
"firebase-functions": "^7.3.3-rc.0"
|
|
18
18
|
},
|
|
19
19
|
"devDependencies": {
|
|
20
|
+
"typescript": "^5.9.3",
|
|
20
21
|
"vitest": "^4.1.10"
|
|
21
22
|
},
|
|
22
23
|
"engines": {
|
|
@@ -2317,9 +2318,9 @@
|
|
|
2317
2318
|
}
|
|
2318
2319
|
},
|
|
2319
2320
|
"node_modules/firebase-functions": {
|
|
2320
|
-
"version": "7.3.
|
|
2321
|
-
"resolved": "https://registry.npmjs.org/firebase-functions/-/firebase-functions-7.3.
|
|
2322
|
-
"integrity": "sha512-
|
|
2321
|
+
"version": "7.3.3-rc.0",
|
|
2322
|
+
"resolved": "https://registry.npmjs.org/firebase-functions/-/firebase-functions-7.3.3-rc.0.tgz",
|
|
2323
|
+
"integrity": "sha512-yMVax5196sPp150ySihdXrhWtepeQ0yRmSBl3GoIogzy6DFoEi5CQbvMuFjTPz65MacRrNiJ55HS3oIlHUO/wg==",
|
|
2323
2324
|
"license": "MIT",
|
|
2324
2325
|
"dependencies": {
|
|
2325
2326
|
"@types/cors": "^2.8.5",
|
|
@@ -4551,6 +4552,20 @@
|
|
|
4551
4552
|
"url": "https://opencollective.com/express"
|
|
4552
4553
|
}
|
|
4553
4554
|
},
|
|
4555
|
+
"node_modules/typescript": {
|
|
4556
|
+
"version": "5.9.3",
|
|
4557
|
+
"resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz",
|
|
4558
|
+
"integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==",
|
|
4559
|
+
"dev": true,
|
|
4560
|
+
"license": "Apache-2.0",
|
|
4561
|
+
"bin": {
|
|
4562
|
+
"tsc": "bin/tsc",
|
|
4563
|
+
"tsserver": "bin/tsserver"
|
|
4564
|
+
},
|
|
4565
|
+
"engines": {
|
|
4566
|
+
"node": ">=14.17"
|
|
4567
|
+
}
|
|
4568
|
+
},
|
|
4554
4569
|
"node_modules/undici-types": {
|
|
4555
4570
|
"version": "8.3.0",
|
|
4556
4571
|
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.3.0.tgz",
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@firebase-function-kits/bigquery-firestore-export",
|
|
3
|
-
"version": "0.0.2-rc.
|
|
3
|
+
"version": "0.0.2-rc.3",
|
|
4
4
|
"repository": {
|
|
5
5
|
"type": "git",
|
|
6
6
|
"url": "https://github.com/firebase/extensions.git",
|
|
@@ -37,9 +37,10 @@
|
|
|
37
37
|
"@google-cloud/bigquery-data-transfer": "^5.0.1",
|
|
38
38
|
"@google-cloud/pubsub": "^4.11.0",
|
|
39
39
|
"firebase-admin": "^14.1.0",
|
|
40
|
-
"firebase-functions": "7.3.
|
|
40
|
+
"firebase-functions": "^7.3.3-rc.0"
|
|
41
41
|
},
|
|
42
42
|
"devDependencies": {
|
|
43
|
+
"typescript": "^5.9.3",
|
|
43
44
|
"vitest": "^4.1.10"
|
|
44
45
|
},
|
|
45
46
|
"overrides": {
|
package/src/config.ts
CHANGED
|
@@ -76,6 +76,22 @@ const params = {
|
|
|
76
76
|
}),
|
|
77
77
|
}),
|
|
78
78
|
transferConfigName: defineString("TRANSFER_CONFIG_NAME", { default: "" }),
|
|
79
|
+
pubSubTopic: defineString("PUB_SUB_TOPIC", {
|
|
80
|
+
label: "Pub/Sub Topic",
|
|
81
|
+
description:
|
|
82
|
+
"Which Pub/Sub topic should receive BigQuery Data Transfer completion notifications? Leave the default unless you are migrating from the bigquery-firestore-export extension, whose topic is named ext-<instance id>-processMessages. Pointing this at the extension's topic keeps the existing scheduled query's notification settings untouched.",
|
|
83
|
+
|
|
84
|
+
default: expr`kit-${instanceId}-processMessages`,
|
|
85
|
+
input: {
|
|
86
|
+
text: {
|
|
87
|
+
nonEmpty: true,
|
|
88
|
+
example: "ext-my-instance-processMessages",
|
|
89
|
+
validationRegex: /^(?!goog)[a-zA-Z][a-zA-Z0-9\-_.~+%]{2,254}$/,
|
|
90
|
+
validationErrorMessage:
|
|
91
|
+
"Must be a Pub/Sub topic ID, not a full projects/<project>/topics/<topic> resource name. IDs are 3 to 255 characters, start with a letter, may contain letters, numbers and - _ . ~ + %, and cannot start with goog.",
|
|
92
|
+
},
|
|
93
|
+
},
|
|
94
|
+
}),
|
|
79
95
|
datasetId: defineString("DATASET_ID", {
|
|
80
96
|
label: "Dataset ID",
|
|
81
97
|
description:
|
|
@@ -168,7 +184,7 @@ const params = {
|
|
|
168
184
|
};
|
|
169
185
|
|
|
170
186
|
export const CONFIG_EXPRESSIONS: DeployTimeOptions = {
|
|
171
|
-
pubSubTopic:
|
|
187
|
+
pubSubTopic: params.pubSubTopic,
|
|
172
188
|
};
|
|
173
189
|
|
|
174
190
|
function optional(value: string): string | undefined {
|
|
@@ -198,7 +214,7 @@ export function configFromEnv(): BigqueryFirestoreExportConfig {
|
|
|
198
214
|
displayName: params.displayName.value(),
|
|
199
215
|
partitioningField: optional(params.partitioningField.value()),
|
|
200
216
|
schedule: params.schedule.value(),
|
|
201
|
-
pubSubTopic:
|
|
217
|
+
pubSubTopic: params.pubSubTopic.value(),
|
|
202
218
|
firestoreCollection: params.firestoreCollection.value(),
|
|
203
219
|
logLevel: normalizeLogLevel(params.logLevel.value()),
|
|
204
220
|
};
|