@firebase-function-kits/bigquery-firestore-export 0.0.1 → 0.0.2-rc.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +1 -0
- package/README.md +188 -0
- package/lib/config.d.ts +5 -0
- package/lib/config.d.ts.map +1 -0
- package/lib/config.js +75 -0
- package/lib/config.js.map +1 -0
- package/lib/dts.d.ts +14 -0
- package/lib/dts.d.ts.map +1 -0
- package/lib/dts.js +189 -0
- package/lib/dts.js.map +1 -0
- package/lib/export-config.d.ts +58 -0
- package/lib/export-config.d.ts.map +1 -0
- package/lib/export-config.js +56 -0
- package/lib/export-config.js.map +1 -0
- package/lib/handlers.d.ts +22 -0
- package/lib/handlers.d.ts.map +1 -0
- package/lib/handlers.js +125 -0
- package/lib/handlers.js.map +1 -0
- package/lib/helper.d.ts +37 -0
- package/lib/helper.d.ts.map +1 -0
- package/lib/helper.js +219 -0
- package/lib/helper.js.map +1 -0
- package/lib/index.d.ts +7 -0
- package/lib/index.d.ts.map +1 -0
- package/lib/index.js +137 -0
- package/lib/index.js.map +1 -0
- package/lib/lib.d.ts +11 -0
- package/lib/lib.d.ts.map +1 -0
- package/lib/lib.js +47 -0
- package/lib/lib.js.map +1 -0
- package/lib/logs.d.ts +21 -0
- package/lib/logs.d.ts.map +1 -0
- package/lib/logs.js +122 -0
- package/lib/logs.js.map +1 -0
- package/lib/metadata.d.ts +6 -0
- package/lib/metadata.d.ts.map +1 -0
- package/lib/metadata.js +34 -0
- package/lib/metadata.js.map +1 -0
- package/lib/types.d.ts +44 -0
- package/lib/types.d.ts.map +1 -0
- package/lib/types.js +18 -0
- package/lib/types.js.map +1 -0
- package/package.json +40 -4
- package/src/config.ts +88 -0
- package/src/dts.ts +214 -0
- package/src/export-config.ts +123 -0
- package/src/handlers.ts +141 -0
- package/src/helper.ts +320 -0
- package/src/index.ts +126 -0
- package/src/lib.ts +69 -0
- package/src/logs.ts +145 -0
- package/src/metadata.ts +31 -0
- package/src/types.ts +100 -0
- package/tests/config.test.ts +61 -0
- package/tests/dts.test.ts +124 -0
- package/tests/export-config.test.ts +78 -0
- package/tests/handlers.test.ts +170 -0
- package/tests/helper.test.ts +74 -0
- package/tests/lib.test.ts +43 -0
- package/tsconfig.json +18 -0
- package/tsconfig.tsbuildinfo +1 -0
- package/index.js +0 -0
package/CHANGELOG.md
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
- Initial release
|
package/README.md
ADDED
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
# @firebase/bigquery-firestore-export
|
|
2
|
+
|
|
3
|
+
Schedule a BigQuery query and write each run's rows and metadata to Firestore.
|
|
4
|
+
This is the Export BigQuery to Firestore Firebase Extension as an npm package
|
|
5
|
+
you add to your own Firebase Functions codebase and deploy.
|
|
6
|
+
|
|
7
|
+
The kit creates or reconciles a BigQuery Data Transfer scheduled query, listens
|
|
8
|
+
for its Pub/Sub completion notifications, reads the destination table, and
|
|
9
|
+
writes the rows to Firestore. The functions run in your own Firebase project;
|
|
10
|
+
there is no hosted version, so you deploy them yourself.
|
|
11
|
+
|
|
12
|
+
## Install
|
|
13
|
+
|
|
14
|
+
```sh
|
|
15
|
+
npm install @firebase/bigquery-firestore-export
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
## Required IAM
|
|
19
|
+
|
|
20
|
+
Deploy needs these Google Cloud roles and APIs for the function's service
|
|
21
|
+
account. Firebase CLI 15.23.0 or later creates that account, grants the roles
|
|
22
|
+
below, enables the listed APIs, and attaches the account to every function in
|
|
23
|
+
this kit. Do not set a custom runtime service account for this codebase—it
|
|
24
|
+
conflicts with that automatic setup.
|
|
25
|
+
|
|
26
|
+
| Role / API | Why |
|
|
27
|
+
| ------------------------------------- | ------------------------------------------------------------------- |
|
|
28
|
+
| `roles/datastore.user` | store transfer configs, run metadata, and query output in Firestore |
|
|
29
|
+
| `roles/bigquery.admin` | create scheduled queries and read their destination tables |
|
|
30
|
+
| `roles/pubsub.admin` | create the transfer-notification topic |
|
|
31
|
+
| `roles/eventarc.eventReceiver` | receive Gen2 Pub/Sub trigger events |
|
|
32
|
+
| `roles/run.invoker` | allow Eventarc/Tasks to invoke the Gen2 Cloud Run services |
|
|
33
|
+
| `bigquery.googleapis.com` | run queries and read destination tables |
|
|
34
|
+
| `bigquerydatatransfer.googleapis.com` | create and reconcile scheduled-query transfer configs |
|
|
35
|
+
| `pubsub.googleapis.com` | deliver transfer-completion notifications |
|
|
36
|
+
|
|
37
|
+
## Usage
|
|
38
|
+
|
|
39
|
+
Export both functions from your functions codebase entry:
|
|
40
|
+
|
|
41
|
+
```ts
|
|
42
|
+
// functions/src/index.ts
|
|
43
|
+
export {
|
|
44
|
+
processMessages,
|
|
45
|
+
upsertTransferConfig,
|
|
46
|
+
} from "@firebase/bigquery-firestore-export";
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
and configure them with a `.env` (or `.env.<projectId>`):
|
|
50
|
+
|
|
51
|
+
```sh
|
|
52
|
+
INSTANCE_ID=analytics-export
|
|
53
|
+
BIGQUERY_DATASET_LOCATION=US
|
|
54
|
+
DATASET_ID=analytics
|
|
55
|
+
TABLE_NAME=users
|
|
56
|
+
QUERY_STRING=SELECT * FROM `my-project.source.users`
|
|
57
|
+
DISPLAY_NAME=Export users to Firestore
|
|
58
|
+
SCHEDULE=every 24 hours
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
- `processMessages` consumes BigQuery Data Transfer completion messages.
|
|
62
|
+
- `upsertTransferConfig` is the idempotent lifecycle task that creates, links,
|
|
63
|
+
or updates the scheduled query and its notification topic.
|
|
64
|
+
|
|
65
|
+
Importing the package without exporting its functions deploys nothing—the CLI
|
|
66
|
+
only deploys what your entry file exports.
|
|
67
|
+
|
|
68
|
+
## Deploy
|
|
69
|
+
|
|
70
|
+
The package's `firebase.json` declares a `kit` stanza (Firebase CLI 15.25.1 or
|
|
71
|
+
later, behind the `kits` experiment):
|
|
72
|
+
|
|
73
|
+
```json
|
|
74
|
+
{
|
|
75
|
+
"functions": [
|
|
76
|
+
{
|
|
77
|
+
"source": ".",
|
|
78
|
+
"kit": "bigquery-firestore-export",
|
|
79
|
+
"instances": {
|
|
80
|
+
"default": "."
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
]
|
|
84
|
+
}
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
`instances` maps each instance id to the directory (relative to
|
|
88
|
+
`firebase.json`) holding that instance's `.env`. The CLI prefixes every
|
|
89
|
+
function and task queue name with `kit-<instance id>-`, so the functions above
|
|
90
|
+
deploy as `kit-default-processMessages` and
|
|
91
|
+
`kit-default-upsertTransferConfig`.
|
|
92
|
+
|
|
93
|
+
```sh
|
|
94
|
+
firebase experiments:enable kits
|
|
95
|
+
firebase deploy --only functions
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
Deploy a single instance with `firebase deploy --only functions:<instance id>`.
|
|
99
|
+
|
|
100
|
+
## Configuration
|
|
101
|
+
|
|
102
|
+
Set these values in a `.env` (or `.env.<projectId>`) file. The Firebase CLI
|
|
103
|
+
loads them at deploy time and prompts for required values that are missing.
|
|
104
|
+
`PROJECT_ID` is supplied by the Firebase CLI.
|
|
105
|
+
|
|
106
|
+
| Field | Env var | Required | Default | Description |
|
|
107
|
+
| ------------------------- | --------------------------- | -------- | ----------------- | ------------------------------------------------------------ |
|
|
108
|
+
| `instanceId` | `INSTANCE_ID` | yes | — | Must match this instance's key in the `instances` map |
|
|
109
|
+
| `bigqueryDatasetLocation` | `BIGQUERY_DATASET_LOCATION` | no | `US` | BigQuery destination dataset location |
|
|
110
|
+
| `transferConfigName` | `TRANSFER_CONFIG_NAME` | no | (empty) | Existing DTS config resource to link instead of creating one |
|
|
111
|
+
| `datasetId` | `DATASET_ID` | yes | — | BigQuery destination dataset id |
|
|
112
|
+
| `tableName` | `TABLE_NAME` | yes | — | Prefix for per-run destination tables |
|
|
113
|
+
| `queryString` | `QUERY_STRING` | yes | — | Scheduled Standard SQL query |
|
|
114
|
+
| `displayName` | `DISPLAY_NAME` | yes | — | Human-readable scheduled-query name |
|
|
115
|
+
| `partitioningField` | `PARTITIONING_FIELD` | no | (empty) | Destination-table partitioning field |
|
|
116
|
+
| `schedule` | `SCHEDULE` | yes | — | DTS schedule, such as `every 24 hours` |
|
|
117
|
+
| `firestoreCollection` | `COLLECTION_PATH` | no | `transferConfigs` | Root Firestore collection for configs and output |
|
|
118
|
+
| `logLevel` | `LOG_LEVEL` | no | `info` | `debug`, `info`, `warn`, `error`, or `silent` |
|
|
119
|
+
|
|
120
|
+
## Multiple instances
|
|
121
|
+
|
|
122
|
+
To run several reverse-sync instances, add one entry per instance to the
|
|
123
|
+
`instances` map, each pointing at its own config directory with its own `.env`:
|
|
124
|
+
|
|
125
|
+
```json
|
|
126
|
+
{
|
|
127
|
+
"functions": [
|
|
128
|
+
{
|
|
129
|
+
"source": ".",
|
|
130
|
+
"kit": "bigquery-firestore-export",
|
|
131
|
+
"instances": {
|
|
132
|
+
"users": "instances/users",
|
|
133
|
+
"orders": "instances/orders"
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
]
|
|
137
|
+
}
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
Instance ids must be unique across all kit stanzas in the project, and every
|
|
141
|
+
instance's function names are namespaced by its `kit-<instance id>-` prefix, so
|
|
142
|
+
the instances cannot collide. Set `INSTANCE_ID` in each config directory to the
|
|
143
|
+
same value as that directory's key in the `instances` map; it also namespaces
|
|
144
|
+
the Pub/Sub notification topic and associates the deployment with its transfer
|
|
145
|
+
config.
|
|
146
|
+
|
|
147
|
+
## Provisioning
|
|
148
|
+
|
|
149
|
+
The kit wires `upsertTransferConfig` to both `afterFirstDeploy` and
|
|
150
|
+
`afterRedeploy`. The lifecycle task creates the Pub/Sub topic and scheduled
|
|
151
|
+
query on first deploy, then reconciles supported query, table, schedule,
|
|
152
|
+
dataset, partitioning, and topic changes on later deploys. It retries transient
|
|
153
|
+
failures up to five times with at least 30 seconds of backoff.
|
|
154
|
+
|
|
155
|
+
Set `TRANSFER_CONFIG_NAME` to link an existing scheduled-query config without
|
|
156
|
+
changing it. Otherwise the kit stores `extInstanceId` on the Firestore config
|
|
157
|
+
document and uses that value to find the config on later deploys. BigQuery DTS
|
|
158
|
+
does not support clearing a partitioning field once set; create a new transfer
|
|
159
|
+
config to remove partitioning.
|
|
160
|
+
|
|
161
|
+
## Firestore layout
|
|
162
|
+
|
|
163
|
+
For a transfer config `{configId}` and run `{runId}`, the kit writes:
|
|
164
|
+
|
|
165
|
+
```text
|
|
166
|
+
transferConfigs/{configId}
|
|
167
|
+
transferConfigs/{configId}/runs/{runId}
|
|
168
|
+
transferConfigs/{configId}/runs/{runId}/output/{rowId}
|
|
169
|
+
transferConfigs/{configId}/runs/latest
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
The run document stores DTS metadata and row counts. Its `output` subcollection
|
|
173
|
+
contains converted query rows. The `latest` document is updated transactionally
|
|
174
|
+
so an older completion message cannot replace a newer run.
|
|
175
|
+
|
|
176
|
+
## API surface
|
|
177
|
+
|
|
178
|
+
- **Main entry** (`@firebase/bigquery-firestore-export`): exports
|
|
179
|
+
`processMessages` and `upsertTransferConfig`, registers first-deploy and
|
|
180
|
+
redeploy lifecycle hooks, and resolves runtime dependencies lazily. Use this
|
|
181
|
+
entry from Firebase deploy/emulator/runtime.
|
|
182
|
+
- **Library entry** (`./lib`): side-effect-free config helpers, DTS helpers,
|
|
183
|
+
injectable handlers, BigQuery-to-Firestore conversion helpers, and public
|
|
184
|
+
message/config types for consumers that own trigger registration.
|
|
185
|
+
|
|
186
|
+
## License
|
|
187
|
+
|
|
188
|
+
Apache-2.0
|
package/lib/config.d.ts
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import type { BigqueryFirestoreExportConfig, DeployTimeOptions } from "./export-config";
|
|
2
|
+
export declare const CONFIG_EXPRESSIONS: DeployTimeOptions;
|
|
3
|
+
/** Reads runtime values from Firebase deploy-time parameters. */
|
|
4
|
+
export declare function configFromEnv(): BigqueryFirestoreExportConfig;
|
|
5
|
+
//# sourceMappingURL=config.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAsBA,OAAO,KAAK,EACV,6BAA6B,EAC7B,iBAAiB,EAElB,MAAM,iBAAiB,CAAC;AA0BzB,eAAO,MAAM,kBAAkB,EAAE,iBAEhC,CAAC;AAcF,iEAAiE;AACjE,wBAAgB,aAAa,IAAI,6BAA6B,CAkB7D"}
|
package/lib/config.js
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/*
|
|
3
|
+
* Copyright 2026 Google LLC
|
|
4
|
+
*
|
|
5
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
6
|
+
* you may not use this file except in compliance with the License.
|
|
7
|
+
* You may obtain a copy of the License at
|
|
8
|
+
*
|
|
9
|
+
* https://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
*
|
|
11
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
12
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
13
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
14
|
+
* See the License for the specific language governing permissions and
|
|
15
|
+
* limitations under the License.
|
|
16
|
+
*/
|
|
17
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
18
|
+
exports.CONFIG_EXPRESSIONS = void 0;
|
|
19
|
+
exports.configFromEnv = configFromEnv;
|
|
20
|
+
const params_1 = require("firebase-functions/params");
|
|
21
|
+
const LOG_LEVEL_OPTIONS = ["debug", "info", "warn", "error", "silent"];
|
|
22
|
+
const instanceId = (0, params_1.defineString)("INSTANCE_ID");
|
|
23
|
+
const params = {
|
|
24
|
+
instanceId,
|
|
25
|
+
bigqueryDatasetLocation: (0, params_1.defineString)("BIGQUERY_DATASET_LOCATION", {
|
|
26
|
+
default: "US",
|
|
27
|
+
}),
|
|
28
|
+
transferConfigName: (0, params_1.defineString)("TRANSFER_CONFIG_NAME", { default: "" }),
|
|
29
|
+
datasetId: (0, params_1.defineString)("DATASET_ID"),
|
|
30
|
+
tableName: (0, params_1.defineString)("TABLE_NAME"),
|
|
31
|
+
queryString: (0, params_1.defineString)("QUERY_STRING"),
|
|
32
|
+
displayName: (0, params_1.defineString)("DISPLAY_NAME"),
|
|
33
|
+
partitioningField: (0, params_1.defineString)("PARTITIONING_FIELD", { default: "" }),
|
|
34
|
+
schedule: (0, params_1.defineString)("SCHEDULE"),
|
|
35
|
+
firestoreCollection: (0, params_1.defineString)("COLLECTION_PATH", {
|
|
36
|
+
default: "transferConfigs",
|
|
37
|
+
}),
|
|
38
|
+
logLevel: (0, params_1.defineString)("LOG_LEVEL", {
|
|
39
|
+
default: "info",
|
|
40
|
+
input: (0, params_1.select)([...LOG_LEVEL_OPTIONS]),
|
|
41
|
+
}),
|
|
42
|
+
};
|
|
43
|
+
exports.CONFIG_EXPRESSIONS = {
|
|
44
|
+
pubSubTopic: (0, params_1.expr) `kit-${instanceId}-processMessages`,
|
|
45
|
+
};
|
|
46
|
+
function optional(value) {
|
|
47
|
+
const normalized = value.trim();
|
|
48
|
+
return normalized.length > 0 ? normalized : undefined;
|
|
49
|
+
}
|
|
50
|
+
function normalizeLogLevel(value) {
|
|
51
|
+
const normalized = value.toLowerCase();
|
|
52
|
+
return LOG_LEVEL_OPTIONS.includes(normalized)
|
|
53
|
+
? normalized
|
|
54
|
+
: "info";
|
|
55
|
+
}
|
|
56
|
+
/** Reads runtime values from Firebase deploy-time parameters. */
|
|
57
|
+
function configFromEnv() {
|
|
58
|
+
const resolvedInstanceId = params.instanceId.value();
|
|
59
|
+
return {
|
|
60
|
+
bigqueryDatasetLocation: params.bigqueryDatasetLocation.value(),
|
|
61
|
+
projectId: params_1.projectID.value(),
|
|
62
|
+
instanceId: resolvedInstanceId,
|
|
63
|
+
transferConfigName: optional(params.transferConfigName.value()),
|
|
64
|
+
datasetId: params.datasetId.value(),
|
|
65
|
+
tableName: params.tableName.value(),
|
|
66
|
+
queryString: params.queryString.value(),
|
|
67
|
+
displayName: params.displayName.value(),
|
|
68
|
+
partitioningField: optional(params.partitioningField.value()),
|
|
69
|
+
schedule: params.schedule.value(),
|
|
70
|
+
pubSubTopic: `kit-${resolvedInstanceId}-processMessages`,
|
|
71
|
+
firestoreCollection: params.firestoreCollection.value(),
|
|
72
|
+
logLevel: normalizeLogLevel(params.logLevel.value()),
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
//# sourceMappingURL=config.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;GAcG;;;;AAEH,sDAKmC;AAOnC,MAAM,iBAAiB,GAAG,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAU,CAAC;AAChF,MAAM,UAAU,GAAG,IAAA,qBAAY,EAAC,aAAa,CAAC,CAAC;AAE/C,MAAM,MAAM,GAAG;IACb,UAAU;IACV,uBAAuB,EAAE,IAAA,qBAAY,EAAC,2BAA2B,EAAE;QACjE,OAAO,EAAE,IAAI;KACd,CAAC;IACF,kBAAkB,EAAE,IAAA,qBAAY,EAAC,sBAAsB,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;IACzE,SAAS,EAAE,IAAA,qBAAY,EAAC,YAAY,CAAC;IACrC,SAAS,EAAE,IAAA,qBAAY,EAAC,YAAY,CAAC;IACrC,WAAW,EAAE,IAAA,qBAAY,EAAC,cAAc,CAAC;IACzC,WAAW,EAAE,IAAA,qBAAY,EAAC,cAAc,CAAC;IACzC,iBAAiB,EAAE,IAAA,qBAAY,EAAC,oBAAoB,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;IACtE,QAAQ,EAAE,IAAA,qBAAY,EAAC,UAAU,CAAC;IAClC,mBAAmB,EAAE,IAAA,qBAAY,EAAC,iBAAiB,EAAE;QACnD,OAAO,EAAE,iBAAiB;KAC3B,CAAC;IACF,QAAQ,EAAE,IAAA,qBAAY,EAAC,WAAW,EAAE;QAClC,OAAO,EAAE,MAAM;QACf,KAAK,EAAE,IAAA,eAAM,EAAC,CAAC,GAAG,iBAAiB,CAAC,CAAC;KACtC,CAAC;CACH,CAAC;AAEW,QAAA,kBAAkB,GAAsB;IACnD,WAAW,EAAE,IAAA,aAAI,EAAA,OAAO,UAAU,kBAAkB;CACrD,CAAC;AAEF,SAAS,QAAQ,CAAC,KAAa;IAC7B,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;IAChC,OAAO,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC;AACxD,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAa;IACtC,MAAM,UAAU,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC;IACvC,OAAO,iBAAiB,CAAC,QAAQ,CAAC,UAAsB,CAAC;QACvD,CAAC,CAAE,UAAuB;QAC1B,CAAC,CAAC,MAAM,CAAC;AACb,CAAC;AAED,iEAAiE;AACjE;IACE,MAAM,kBAAkB,GAAG,MAAM,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;IAErD,OAAO;QACL,uBAAuB,EAAE,MAAM,CAAC,uBAAuB,CAAC,KAAK,EAAE;QAC/D,SAAS,EAAE,kBAAS,CAAC,KAAK,EAAE;QAC5B,UAAU,EAAE,kBAAkB;QAC9B,kBAAkB,EAAE,QAAQ,CAAC,MAAM,CAAC,kBAAkB,CAAC,KAAK,EAAE,CAAC;QAC/D,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC,KAAK,EAAE;QACnC,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC,KAAK,EAAE;QACnC,WAAW,EAAE,MAAM,CAAC,WAAW,CAAC,KAAK,EAAE;QACvC,WAAW,EAAE,MAAM,CAAC,WAAW,CAAC,KAAK,EAAE;QACvC,iBAAiB,EAAE,QAAQ,CAAC,MAAM,CAAC,iBAAiB,CAAC,KAAK,EAAE,CAAC;QAC7D,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE;QACjC,WAAW,EAAE,OAAO,kBAAkB,kBAAkB;QACxD,mBAAmB,EAAE,MAAM,CAAC,mBAAmB,CAAC,KAAK,EAAE;QACvD,QAAQ,EAAE,iBAAiB,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;KACrD,CAAC;AACJ,CAAC"}
|
package/lib/dts.d.ts
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import * as bigqueryDataTransfer from "@google-cloud/bigquery-data-transfer";
|
|
2
|
+
import type { ResolvedBigqueryFirestoreExportConfig } from "./export-config";
|
|
3
|
+
export type DataTransferClient = bigqueryDataTransfer.v1.DataTransferServiceClient;
|
|
4
|
+
export type TransferConfig = bigqueryDataTransfer.protos.google.cloud.bigquery.datatransfer.v1.ITransferConfig;
|
|
5
|
+
export declare const PARTITIONING_FIELD_REMOVAL_ERROR_PREFIX = "Cannot remove partitioning_field from an existing transfer config";
|
|
6
|
+
export declare const PARTITIONING_FIELD_REMOVAL_ERROR = "Cannot remove partitioning_field from an existing transfer config. The BigQuery Data Transfer API does not support clearing this parameter once it has been set. To change partitioning, create a new transfer config with the desired setting.";
|
|
7
|
+
/** Creates the protobuf-shaped request used for a scheduled query. */
|
|
8
|
+
export declare function createTransferConfigRequest(config: ResolvedBigqueryFirestoreExportConfig, serviceAccountEmail?: string): bigqueryDataTransfer.protos.google.cloud.bigquery.datatransfer.v1.ICreateTransferConfigRequest;
|
|
9
|
+
export declare function getTransferConfig(client: DataTransferClient, transferConfigName: string): Promise<TransferConfig | null>;
|
|
10
|
+
export declare function createTransferConfig(client: DataTransferClient, config: ResolvedBigqueryFirestoreExportConfig): Promise<TransferConfig>;
|
|
11
|
+
/** Builds a minimal update mask while retaining unsupported immutable fields. */
|
|
12
|
+
export declare function constructUpdateTransferConfigRequest(client: DataTransferClient, transferConfigName: string, config: ResolvedBigqueryFirestoreExportConfig): Promise<bigqueryDataTransfer.protos.google.cloud.bigquery.datatransfer.v1.IUpdateTransferConfigRequest>;
|
|
13
|
+
export declare function updateTransferConfig(client: DataTransferClient, transferConfigName: string, config: ResolvedBigqueryFirestoreExportConfig): Promise<TransferConfig>;
|
|
14
|
+
//# sourceMappingURL=dts.d.ts.map
|
package/lib/dts.d.ts.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"dts.d.ts","sourceRoot":"","sources":["../src/dts.ts"],"names":[],"mappings":"AAgBA,OAAO,KAAK,oBAAoB,MAAM,sCAAsC,CAAC;AAC7E,OAAO,KAAK,EAAE,qCAAqC,EAAE,MAAM,iBAAiB,CAAC;AAG7E,MAAM,MAAM,kBAAkB,GAC5B,oBAAoB,CAAC,EAAE,CAAC,yBAAyB,CAAC;AACpD,MAAM,MAAM,cAAc,GACxB,oBAAoB,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC,eAAe,CAAC;AAMpF,eAAO,MAAM,uCAAuC,sEACxB,CAAC;AAC7B,eAAO,MAAM,gCAAgC,oPAAgN,CAAC;AAoC9P,sEAAsE;AACtE,wBAAgB,2BAA2B,CACzC,MAAM,EAAE,qCAAqC,EAC7C,mBAAmB,CAAC,EAAE,MAAM,GAC3B,oBAAoB,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC,4BAA4B,CAwBhG;AAED,wBAAsB,iBAAiB,CACrC,MAAM,EAAE,kBAAkB,EAC1B,kBAAkB,EAAE,MAAM,GACzB,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC,CAchC;AAED,wBAAsB,oBAAoB,CACxC,MAAM,EAAE,kBAAkB,EAC1B,MAAM,EAAE,qCAAqC,GAC5C,OAAO,CAAC,cAAc,CAAC,CAUzB;AAED,iFAAiF;AACjF,wBAAsB,oCAAoC,CACxD,MAAM,EAAE,kBAAkB,EAC1B,kBAAkB,EAAE,MAAM,EAC1B,MAAM,EAAE,qCAAqC,GAC5C,OAAO,CAAC,oBAAoB,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC,4BAA4B,CAAC,CAyDzG;AAED,wBAAsB,oBAAoB,CACxC,MAAM,EAAE,kBAAkB,EAC1B,kBAAkB,EAAE,MAAM,EAC1B,MAAM,EAAE,qCAAqC,GAC5C,OAAO,CAAC,cAAc,CAAC,CAczB"}
|
package/lib/dts.js
ADDED
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/*
|
|
3
|
+
* Copyright 2025 Google LLC
|
|
4
|
+
*
|
|
5
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
6
|
+
* you may not use this file except in compliance with the License.
|
|
7
|
+
* You may obtain a copy of the License at
|
|
8
|
+
*
|
|
9
|
+
* https://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
*
|
|
11
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
12
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
13
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
14
|
+
* See the License for the specific language governing permissions and
|
|
15
|
+
* limitations under the License.
|
|
16
|
+
*/
|
|
17
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
18
|
+
if (k2 === undefined) k2 = k;
|
|
19
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
20
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
21
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
22
|
+
}
|
|
23
|
+
Object.defineProperty(o, k2, desc);
|
|
24
|
+
}) : (function(o, m, k, k2) {
|
|
25
|
+
if (k2 === undefined) k2 = k;
|
|
26
|
+
o[k2] = m[k];
|
|
27
|
+
}));
|
|
28
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
29
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
30
|
+
}) : function(o, v) {
|
|
31
|
+
o["default"] = v;
|
|
32
|
+
});
|
|
33
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
34
|
+
var ownKeys = function(o) {
|
|
35
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
36
|
+
var ar = [];
|
|
37
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
38
|
+
return ar;
|
|
39
|
+
};
|
|
40
|
+
return ownKeys(o);
|
|
41
|
+
};
|
|
42
|
+
return function (mod) {
|
|
43
|
+
if (mod && mod.__esModule) return mod;
|
|
44
|
+
var result = {};
|
|
45
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
46
|
+
__setModuleDefault(result, mod);
|
|
47
|
+
return result;
|
|
48
|
+
};
|
|
49
|
+
})();
|
|
50
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
51
|
+
exports.PARTITIONING_FIELD_REMOVAL_ERROR = exports.PARTITIONING_FIELD_REMOVAL_ERROR_PREFIX = void 0;
|
|
52
|
+
exports.createTransferConfigRequest = createTransferConfigRequest;
|
|
53
|
+
exports.getTransferConfig = getTransferConfig;
|
|
54
|
+
exports.createTransferConfig = createTransferConfig;
|
|
55
|
+
exports.constructUpdateTransferConfigRequest = constructUpdateTransferConfigRequest;
|
|
56
|
+
exports.updateTransferConfig = updateTransferConfig;
|
|
57
|
+
const bigqueryDataTransfer = __importStar(require("@google-cloud/bigquery-data-transfer"));
|
|
58
|
+
const logs = __importStar(require("./logs"));
|
|
59
|
+
const GRPC_NOT_FOUND = 5;
|
|
60
|
+
const PACKAGE_PARTITIONING_ERROR = "Cannot remove partitioning_field from an existing transfer config";
|
|
61
|
+
exports.PARTITIONING_FIELD_REMOVAL_ERROR_PREFIX = PACKAGE_PARTITIONING_ERROR;
|
|
62
|
+
exports.PARTITIONING_FIELD_REMOVAL_ERROR = `${PACKAGE_PARTITIONING_ERROR}. The BigQuery Data Transfer API does not support clearing this parameter once it has been set. To change partitioning, create a new transfer config with the desired setting.`;
|
|
63
|
+
function isNotFoundError(err) {
|
|
64
|
+
return (typeof err === "object" &&
|
|
65
|
+
err !== null &&
|
|
66
|
+
"code" in err &&
|
|
67
|
+
err.code === GRPC_NOT_FOUND);
|
|
68
|
+
}
|
|
69
|
+
function transferConfigFields(config) {
|
|
70
|
+
const fields = config.params?.fields;
|
|
71
|
+
if (!fields) {
|
|
72
|
+
throw new Error("Transfer config has invalid structure: missing params.fields");
|
|
73
|
+
}
|
|
74
|
+
if (!fields.query) {
|
|
75
|
+
throw new Error("Transfer config has invalid structure: missing params.fields.query");
|
|
76
|
+
}
|
|
77
|
+
if (!fields.destination_table_name_template) {
|
|
78
|
+
throw new Error("Transfer config has invalid structure: missing params.fields.destination_table_name_template");
|
|
79
|
+
}
|
|
80
|
+
return fields;
|
|
81
|
+
}
|
|
82
|
+
function stringField(value) {
|
|
83
|
+
return { stringValue: value ?? "" };
|
|
84
|
+
}
|
|
85
|
+
/** Creates the protobuf-shaped request used for a scheduled query. */
|
|
86
|
+
function createTransferConfigRequest(config, serviceAccountEmail) {
|
|
87
|
+
return {
|
|
88
|
+
parent: `projects/${config.projectId}`,
|
|
89
|
+
transferConfig: {
|
|
90
|
+
destinationDatasetId: config.datasetId,
|
|
91
|
+
displayName: config.displayName,
|
|
92
|
+
dataSourceId: "scheduled_query",
|
|
93
|
+
params: {
|
|
94
|
+
fields: {
|
|
95
|
+
query: stringField(config.queryString),
|
|
96
|
+
destination_table_name_template: stringField(`${config.tableName}_{run_time|"%H%M%S"}`),
|
|
97
|
+
write_disposition: stringField("WRITE_TRUNCATE"),
|
|
98
|
+
partitioning_field: stringField(config.partitioningField),
|
|
99
|
+
},
|
|
100
|
+
},
|
|
101
|
+
schedule: config.schedule,
|
|
102
|
+
notificationPubsubTopic: `projects/${config.projectId}/topics/${config.pubSubTopic}`,
|
|
103
|
+
...(serviceAccountEmail
|
|
104
|
+
? { serviceAccountName: serviceAccountEmail }
|
|
105
|
+
: {}),
|
|
106
|
+
},
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
async function getTransferConfig(client, transferConfigName) {
|
|
110
|
+
try {
|
|
111
|
+
const [config] = await client.getTransferConfig({
|
|
112
|
+
name: transferConfigName,
|
|
113
|
+
});
|
|
114
|
+
return config;
|
|
115
|
+
}
|
|
116
|
+
catch (err) {
|
|
117
|
+
if (isNotFoundError(err)) {
|
|
118
|
+
logs.transferConfigNotFound(transferConfigName);
|
|
119
|
+
return null;
|
|
120
|
+
}
|
|
121
|
+
logs.getTransferConfigFailed(transferConfigName, err);
|
|
122
|
+
throw err;
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
async function createTransferConfig(client, config) {
|
|
126
|
+
logs.createTransferConfig();
|
|
127
|
+
const [created] = await client.createTransferConfig(createTransferConfigRequest(config, config.serviceAccount));
|
|
128
|
+
if (!created.name) {
|
|
129
|
+
throw new Error("BigQuery API returned a transfer config without a name");
|
|
130
|
+
}
|
|
131
|
+
logs.transferConfigCreated(created.name);
|
|
132
|
+
return created;
|
|
133
|
+
}
|
|
134
|
+
/** Builds a minimal update mask while retaining unsupported immutable fields. */
|
|
135
|
+
async function constructUpdateTransferConfigRequest(client, transferConfigName, config) {
|
|
136
|
+
const transferConfig = await getTransferConfig(client, transferConfigName);
|
|
137
|
+
if (!transferConfig)
|
|
138
|
+
throw new Error("Transfer config not found");
|
|
139
|
+
const fields = transferConfigFields(transferConfig);
|
|
140
|
+
const updatedConfig = JSON.parse(JSON.stringify(transferConfig));
|
|
141
|
+
const updatedFields = transferConfigFields(updatedConfig);
|
|
142
|
+
const updateMask = [];
|
|
143
|
+
if (config.datasetId !== transferConfig.destinationDatasetId) {
|
|
144
|
+
updateMask.push("destination_dataset_id");
|
|
145
|
+
updatedConfig.destinationDatasetId = config.datasetId;
|
|
146
|
+
}
|
|
147
|
+
if (config.queryString !== fields.query.stringValue) {
|
|
148
|
+
updateMask.push("params");
|
|
149
|
+
updatedFields.query.stringValue = config.queryString;
|
|
150
|
+
}
|
|
151
|
+
const tableTemplate = `${config.tableName}_{run_time|"%H%M%S"}`;
|
|
152
|
+
if (tableTemplate !== fields.destination_table_name_template.stringValue) {
|
|
153
|
+
updateMask.push("params");
|
|
154
|
+
updatedFields.destination_table_name_template.stringValue = tableTemplate;
|
|
155
|
+
}
|
|
156
|
+
const existingPartitioningField = fields.partitioning_field?.stringValue ?? "";
|
|
157
|
+
const newPartitioningField = config.partitioningField ?? "";
|
|
158
|
+
if (newPartitioningField !== existingPartitioningField) {
|
|
159
|
+
if (!newPartitioningField) {
|
|
160
|
+
logs.partitioningFieldRemovalAttempted(transferConfigName, existingPartitioningField);
|
|
161
|
+
throw new Error(exports.PARTITIONING_FIELD_REMOVAL_ERROR);
|
|
162
|
+
}
|
|
163
|
+
updateMask.push("params");
|
|
164
|
+
updatedFields.partitioning_field ??= {};
|
|
165
|
+
updatedFields.partitioning_field.stringValue = newPartitioningField;
|
|
166
|
+
}
|
|
167
|
+
if (config.schedule !== transferConfig.schedule) {
|
|
168
|
+
updateMask.push("schedule");
|
|
169
|
+
updatedConfig.schedule = config.schedule;
|
|
170
|
+
}
|
|
171
|
+
const expectedTopic = `projects/${config.projectId}/topics/${config.pubSubTopic}`;
|
|
172
|
+
if (expectedTopic !== transferConfig.notificationPubsubTopic) {
|
|
173
|
+
updateMask.push("notification_pubsub_topic");
|
|
174
|
+
updatedConfig.notificationPubsubTopic = expectedTopic;
|
|
175
|
+
}
|
|
176
|
+
return {
|
|
177
|
+
transferConfig: updatedConfig,
|
|
178
|
+
updateMask: { paths: [...new Set(updateMask)] },
|
|
179
|
+
};
|
|
180
|
+
}
|
|
181
|
+
async function updateTransferConfig(client, transferConfigName, config) {
|
|
182
|
+
const request = await constructUpdateTransferConfigRequest(client, transferConfigName, config);
|
|
183
|
+
logs.updateTransferConfig(transferConfigName);
|
|
184
|
+
const converted = bigqueryDataTransfer.protos.google.cloud.bigquery.datatransfer.v1.UpdateTransferConfigRequest.fromObject(request);
|
|
185
|
+
const [updated] = await client.updateTransferConfig(converted);
|
|
186
|
+
logs.transferConfigUpdated(transferConfigName);
|
|
187
|
+
return updated;
|
|
188
|
+
}
|
|
189
|
+
//# sourceMappingURL=dts.js.map
|
package/lib/dts.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"dts.js","sourceRoot":"","sources":["../src/dts.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;GAcG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEH,MAAY,oBAAoB,iEAA6C;AAE7E,MAAY,IAAI,mCAAe;AAO/B,MAAM,cAAc,GAAG,CAAC,CAAC;AACzB,MAAM,0BAA0B,GAC9B,mEAAmE,CAAC;AAEzD,QAAA,uCAAuC,GAClD,0BAA0B,CAAC;AAChB,QAAA,gCAAgC,GAAG,GAAG,0BAA0B,gLAAgL,CAAC;AAE9P,SAAS,eAAe,CAAC,GAAY;IACnC,OAAO,CACL,OAAO,GAAG,KAAK,QAAQ;QACvB,GAAG,KAAK,IAAI;QACZ,MAAM,IAAI,GAAG;QACb,GAAG,CAAC,IAAI,KAAK,cAAc,CAC5B,CAAC;AACJ,CAAC;AAED,SAAS,oBAAoB,CAAC,MAAsB;IAClD,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC;IACrC,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,IAAI,KAAK,CACb,8DAA8D,CAC/D,CAAC;IACJ,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QAClB,MAAM,IAAI,KAAK,CACb,oEAAoE,CACrE,CAAC;IACJ,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,+BAA+B,EAAE,CAAC;QAC5C,MAAM,IAAI,KAAK,CACb,8FAA8F,CAC/F,CAAC;IACJ,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,WAAW,CAAC,KAAyB;IAC5C,OAAO,EAAE,WAAW,EAAE,KAAK,IAAI,EAAE,EAAE,CAAC;AACtC,CAAC;AAED,sEAAsE;AACtE,qCACE,MAA6C,EAC7C,mBAA4B;IAE5B,OAAO;QACL,MAAM,EAAE,YAAY,MAAM,CAAC,SAAS,EAAE;QACtC,cAAc,EAAE;YACd,oBAAoB,EAAE,MAAM,CAAC,SAAS;YACtC,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,YAAY,EAAE,iBAAiB;YAC/B,MAAM,EAAE;gBACN,MAAM,EAAE;oBACN,KAAK,EAAE,WAAW,CAAC,MAAM,CAAC,WAAW,CAAC;oBACtC,+BAA+B,EAAE,WAAW,CAC1C,GAAG,MAAM,CAAC,SAAS,sBAAsB,CAC1C;oBACD,iBAAiB,EAAE,WAAW,CAAC,gBAAgB,CAAC;oBAChD,kBAAkB,EAAE,WAAW,CAAC,MAAM,CAAC,iBAAiB,CAAC;iBAC1D;aACF;YACD,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,uBAAuB,EAAE,YAAY,MAAM,CAAC,SAAS,WAAW,MAAM,CAAC,WAAW,EAAE;YACpF,GAAG,CAAC,mBAAmB;gBACrB,CAAC,CAAC,EAAE,kBAAkB,EAAE,mBAAmB,EAAE;gBAC7C,CAAC,CAAC,EAAE,CAAC;SACR;KACF,CAAC;AACJ,CAAC;AAEM,KAAK,4BACV,MAA0B,EAC1B,kBAA0B;IAE1B,IAAI,CAAC;QACH,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,MAAM,CAAC,iBAAiB,CAAC;YAC9C,IAAI,EAAE,kBAAkB;SACzB,CAAC,CAAC;QACH,OAAO,MAAM,CAAC;IAChB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,eAAe,CAAC,GAAG,CAAC,EAAE,CAAC;YACzB,IAAI,CAAC,sBAAsB,CAAC,kBAAkB,CAAC,CAAC;YAChD,OAAO,IAAI,CAAC;QACd,CAAC;QACD,IAAI,CAAC,uBAAuB,CAAC,kBAAkB,EAAE,GAAG,CAAC,CAAC;QACtD,MAAM,GAAG,CAAC;IACZ,CAAC;AACH,CAAC;AAEM,KAAK,+BACV,MAA0B,EAC1B,MAA6C;IAE7C,IAAI,CAAC,oBAAoB,EAAE,CAAC;IAC5B,MAAM,CAAC,OAAO,CAAC,GAAG,MAAM,MAAM,CAAC,oBAAoB,CACjD,2BAA2B,CAAC,MAAM,EAAE,MAAM,CAAC,cAAc,CAAC,CAC3D,CAAC;IACF,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAClB,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC;IAC5E,CAAC;IACD,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IACzC,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,iFAAiF;AAC1E,KAAK,+CACV,MAA0B,EAC1B,kBAA0B,EAC1B,MAA6C;IAE7C,MAAM,cAAc,GAAG,MAAM,iBAAiB,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAC;IAC3E,IAAI,CAAC,cAAc;QAAE,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAElE,MAAM,MAAM,GAAG,oBAAoB,CAAC,cAAc,CAAC,CAAC;IACpD,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAC9B,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,CACb,CAAC;IACpB,MAAM,aAAa,GAAG,oBAAoB,CAAC,aAAa,CAAC,CAAC;IAC1D,MAAM,UAAU,GAAa,EAAE,CAAC;IAEhC,IAAI,MAAM,CAAC,SAAS,KAAK,cAAc,CAAC,oBAAoB,EAAE,CAAC;QAC7D,UAAU,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;QAC1C,aAAa,CAAC,oBAAoB,GAAG,MAAM,CAAC,SAAS,CAAC;IACxD,CAAC;IACD,IAAI,MAAM,CAAC,WAAW,KAAK,MAAM,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC;QACpD,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC1B,aAAa,CAAC,KAAK,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;IACvD,CAAC;IAED,MAAM,aAAa,GAAG,GAAG,MAAM,CAAC,SAAS,sBAAsB,CAAC;IAChE,IAAI,aAAa,KAAK,MAAM,CAAC,+BAA+B,CAAC,WAAW,EAAE,CAAC;QACzE,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC1B,aAAa,CAAC,+BAA+B,CAAC,WAAW,GAAG,aAAa,CAAC;IAC5E,CAAC;IAED,MAAM,yBAAyB,GAC7B,MAAM,CAAC,kBAAkB,EAAE,WAAW,IAAI,EAAE,CAAC;IAC/C,MAAM,oBAAoB,GAAG,MAAM,CAAC,iBAAiB,IAAI,EAAE,CAAC;IAC5D,IAAI,oBAAoB,KAAK,yBAAyB,EAAE,CAAC;QACvD,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAC1B,IAAI,CAAC,iCAAiC,CACpC,kBAAkB,EAClB,yBAAyB,CAC1B,CAAC;YACF,MAAM,IAAI,KAAK,CAAC,QAAA,gCAAgC,CAAC,CAAC;QACpD,CAAC;QACD,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC1B,aAAa,CAAC,kBAAkB,KAAK,EAAE,CAAC;QACxC,aAAa,CAAC,kBAAkB,CAAC,WAAW,GAAG,oBAAoB,CAAC;IACtE,CAAC;IAED,IAAI,MAAM,CAAC,QAAQ,KAAK,cAAc,CAAC,QAAQ,EAAE,CAAC;QAChD,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAC5B,aAAa,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;IAC3C,CAAC;IAED,MAAM,aAAa,GAAG,YAAY,MAAM,CAAC,SAAS,WAAW,MAAM,CAAC,WAAW,EAAE,CAAC;IAClF,IAAI,aAAa,KAAK,cAAc,CAAC,uBAAuB,EAAE,CAAC;QAC7D,UAAU,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;QAC7C,aAAa,CAAC,uBAAuB,GAAG,aAAa,CAAC;IACxD,CAAC;IAED,OAAO;QACL,cAAc,EAAE,aAAa;QAC7B,UAAU,EAAE,EAAE,KAAK,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC,EAAE;KAChD,CAAC;AACJ,CAAC;AAEM,KAAK,+BACV,MAA0B,EAC1B,kBAA0B,EAC1B,MAA6C;IAE7C,MAAM,OAAO,GAAG,MAAM,oCAAoC,CACxD,MAAM,EACN,kBAAkB,EAClB,MAAM,CACP,CAAC;IACF,IAAI,CAAC,oBAAoB,CAAC,kBAAkB,CAAC,CAAC;IAC9C,MAAM,SAAS,GACb,oBAAoB,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC,2BAA2B,CAAC,UAAU,CACtG,OAAO,CACR,CAAC;IACJ,MAAM,CAAC,OAAO,CAAC,GAAG,MAAM,MAAM,CAAC,oBAAoB,CAAC,SAAS,CAAC,CAAC;IAC/D,IAAI,CAAC,qBAAqB,CAAC,kBAAkB,CAAC,CAAC;IAC/C,OAAO,OAAO,CAAC;AACjB,CAAC"}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import type { Expression } from "firebase-functions/params";
|
|
2
|
+
/** Log levels supported by the original extension. */
|
|
3
|
+
export type LogLevel = "debug" | "info" | "warn" | "error" | "silent";
|
|
4
|
+
/** Public configuration for the BigQuery-to-Firestore functions. */
|
|
5
|
+
export interface BigqueryFirestoreExportConfig {
|
|
6
|
+
/** Location of the BigQuery destination dataset, for example `US`. */
|
|
7
|
+
bigqueryDatasetLocation: string;
|
|
8
|
+
/** Google Cloud project id. */
|
|
9
|
+
projectId: string;
|
|
10
|
+
/** Stable id used to associate a DTS config with this deployment. */
|
|
11
|
+
instanceId: string;
|
|
12
|
+
/** Link this existing DTS config instead of creating one. */
|
|
13
|
+
transferConfigName?: string;
|
|
14
|
+
/** BigQuery destination dataset id. */
|
|
15
|
+
datasetId: string;
|
|
16
|
+
/** Prefix for per-run destination tables. */
|
|
17
|
+
tableName: string;
|
|
18
|
+
/** Scheduled query text. */
|
|
19
|
+
queryString: string;
|
|
20
|
+
/** Human-readable DTS scheduled-query name. */
|
|
21
|
+
displayName: string;
|
|
22
|
+
/** Optional destination-table partitioning field. */
|
|
23
|
+
partitioningField?: string;
|
|
24
|
+
/** BigQuery Data Transfer schedule, for example `every 15 minutes`. */
|
|
25
|
+
schedule: string;
|
|
26
|
+
/** Pub/Sub topic receiving DTS completion notifications. */
|
|
27
|
+
pubSubTopic?: string;
|
|
28
|
+
/** Root Firestore collection for configs and run output. */
|
|
29
|
+
firestoreCollection?: string;
|
|
30
|
+
/** Runtime identity used when creating a DTS config. */
|
|
31
|
+
serviceAccount?: string;
|
|
32
|
+
/** Log verbosity. Defaults to `info`. */
|
|
33
|
+
logLevel?: LogLevel;
|
|
34
|
+
}
|
|
35
|
+
/** Configuration after defaults and validation have been applied. */
|
|
36
|
+
export interface ResolvedBigqueryFirestoreExportConfig {
|
|
37
|
+
bigqueryDatasetLocation: string;
|
|
38
|
+
projectId: string;
|
|
39
|
+
instanceId: string;
|
|
40
|
+
transferConfigName?: string;
|
|
41
|
+
datasetId: string;
|
|
42
|
+
tableName: string;
|
|
43
|
+
queryString: string;
|
|
44
|
+
displayName: string;
|
|
45
|
+
partitioningField?: string;
|
|
46
|
+
schedule: string;
|
|
47
|
+
pubSubTopic: string;
|
|
48
|
+
firestoreCollection: string;
|
|
49
|
+
serviceAccount?: string;
|
|
50
|
+
logLevel: LogLevel;
|
|
51
|
+
}
|
|
52
|
+
/** Deploy-time values used to construct the v2 triggers. */
|
|
53
|
+
export interface DeployTimeOptions {
|
|
54
|
+
pubSubTopic: string | Expression<string>;
|
|
55
|
+
}
|
|
56
|
+
/** Applies package defaults and validates a user-supplied configuration. */
|
|
57
|
+
export declare function resolveConfig(config: BigqueryFirestoreExportConfig): ResolvedBigqueryFirestoreExportConfig;
|
|
58
|
+
//# sourceMappingURL=export-config.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"export-config.d.ts","sourceRoot":"","sources":["../src/export-config.ts"],"names":[],"mappings":"AAgBA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,2BAA2B,CAAC;AAE5D,sDAAsD;AACtD,MAAM,MAAM,QAAQ,GAAG,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,QAAQ,CAAC;AAEtE,oEAAoE;AACpE,MAAM,WAAW,6BAA6B;IAC5C,sEAAsE;IACtE,uBAAuB,EAAE,MAAM,CAAC;IAChC,+BAA+B;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,qEAAqE;IACrE,UAAU,EAAE,MAAM,CAAC;IACnB,6DAA6D;IAC7D,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,uCAAuC;IACvC,SAAS,EAAE,MAAM,CAAC;IAClB,6CAA6C;IAC7C,SAAS,EAAE,MAAM,CAAC;IAClB,4BAA4B;IAC5B,WAAW,EAAE,MAAM,CAAC;IACpB,+CAA+C;IAC/C,WAAW,EAAE,MAAM,CAAC;IACpB,qDAAqD;IACrD,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,uEAAuE;IACvE,QAAQ,EAAE,MAAM,CAAC;IACjB,4DAA4D;IAC5D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,4DAA4D;IAC5D,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,wDAAwD;IACxD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,yCAAyC;IACzC,QAAQ,CAAC,EAAE,QAAQ,CAAC;CACrB;AAED,qEAAqE;AACrE,MAAM,WAAW,qCAAqC;IACpD,uBAAuB,EAAE,MAAM,CAAC;IAChC,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,EAAE,QAAQ,CAAC;CACpB;AAED,4DAA4D;AAC5D,MAAM,WAAW,iBAAiB;IAChC,WAAW,EAAE,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;CAC1C;AAgBD,4EAA4E;AAC5E,wBAAgB,aAAa,CAC3B,MAAM,EAAE,6BAA6B,GACpC,qCAAqC,CA6BvC"}
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/*
|
|
3
|
+
* Copyright 2026 Google LLC
|
|
4
|
+
*
|
|
5
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
6
|
+
* you may not use this file except in compliance with the License.
|
|
7
|
+
* You may obtain a copy of the License at
|
|
8
|
+
*
|
|
9
|
+
* https://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
*
|
|
11
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
12
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
13
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
14
|
+
* See the License for the specific language governing permissions and
|
|
15
|
+
* limitations under the License.
|
|
16
|
+
*/
|
|
17
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
18
|
+
exports.resolveConfig = resolveConfig;
|
|
19
|
+
function required(value, field) {
|
|
20
|
+
const normalized = value.trim();
|
|
21
|
+
if (!normalized) {
|
|
22
|
+
throw new Error(`${field} must be a non-empty string.`);
|
|
23
|
+
}
|
|
24
|
+
return normalized;
|
|
25
|
+
}
|
|
26
|
+
function optional(value) {
|
|
27
|
+
if (value === undefined)
|
|
28
|
+
return undefined;
|
|
29
|
+
const normalized = value.trim();
|
|
30
|
+
return normalized || undefined;
|
|
31
|
+
}
|
|
32
|
+
/** Applies package defaults and validates a user-supplied configuration. */
|
|
33
|
+
function resolveConfig(config) {
|
|
34
|
+
const instanceId = required(config.instanceId, "instanceId");
|
|
35
|
+
const logLevel = config.logLevel ?? "info";
|
|
36
|
+
if (!["debug", "info", "warn", "error", "silent"].includes(logLevel)) {
|
|
37
|
+
throw new Error(`Unsupported logLevel: ${logLevel}`);
|
|
38
|
+
}
|
|
39
|
+
return {
|
|
40
|
+
bigqueryDatasetLocation: required(config.bigqueryDatasetLocation, "bigqueryDatasetLocation"),
|
|
41
|
+
projectId: required(config.projectId, "projectId"),
|
|
42
|
+
instanceId,
|
|
43
|
+
transferConfigName: optional(config.transferConfigName),
|
|
44
|
+
datasetId: required(config.datasetId, "datasetId"),
|
|
45
|
+
tableName: required(config.tableName, "tableName"),
|
|
46
|
+
queryString: required(config.queryString, "queryString"),
|
|
47
|
+
displayName: required(config.displayName, "displayName"),
|
|
48
|
+
partitioningField: optional(config.partitioningField),
|
|
49
|
+
schedule: required(config.schedule, "schedule"),
|
|
50
|
+
pubSubTopic: optional(config.pubSubTopic) ?? `kit-${instanceId}-processMessages`,
|
|
51
|
+
firestoreCollection: optional(config.firestoreCollection) ?? "transferConfigs",
|
|
52
|
+
serviceAccount: optional(config.serviceAccount),
|
|
53
|
+
logLevel,
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
//# sourceMappingURL=export-config.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"export-config.js","sourceRoot":"","sources":["../src/export-config.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;GAcG;;;AA8DH,SAAS,QAAQ,CAAC,KAAa,EAAE,KAAa;IAC5C,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;IAChC,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,8BAA8B,CAAC,CAAC;IAC1D,CAAC;IACD,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,SAAS,QAAQ,CAAC,KAAyB;IACzC,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAC1C,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;IAChC,OAAO,UAAU,IAAI,SAAS,CAAC;AACjC,CAAC;AAED,4EAA4E;AAC5E,uBACE,MAAqC;IAErC,MAAM,UAAU,GAAG,QAAQ,CAAC,MAAM,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC;IAC7D,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC;IAE3C,IAAI,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QACrE,MAAM,IAAI,KAAK,CAAC,yBAAyB,QAAQ,EAAE,CAAC,CAAC;IACvD,CAAC;IAED,OAAO;QACL,uBAAuB,EAAE,QAAQ,CAC/B,MAAM,CAAC,uBAAuB,EAC9B,yBAAyB,CAC1B;QACD,SAAS,EAAE,QAAQ,CAAC,MAAM,CAAC,SAAS,EAAE,WAAW,CAAC;QAClD,UAAU;QACV,kBAAkB,EAAE,QAAQ,CAAC,MAAM,CAAC,kBAAkB,CAAC;QACvD,SAAS,EAAE,QAAQ,CAAC,MAAM,CAAC,SAAS,EAAE,WAAW,CAAC;QAClD,SAAS,EAAE,QAAQ,CAAC,MAAM,CAAC,SAAS,EAAE,WAAW,CAAC;QAClD,WAAW,EAAE,QAAQ,CAAC,MAAM,CAAC,WAAW,EAAE,aAAa,CAAC;QACxD,WAAW,EAAE,QAAQ,CAAC,MAAM,CAAC,WAAW,EAAE,aAAa,CAAC;QACxD,iBAAiB,EAAE,QAAQ,CAAC,MAAM,CAAC,iBAAiB,CAAC;QACrD,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,QAAQ,EAAE,UAAU,CAAC;QAC/C,WAAW,EACT,QAAQ,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI,OAAO,UAAU,kBAAkB;QACrE,mBAAmB,EACjB,QAAQ,CAAC,MAAM,CAAC,mBAAmB,CAAC,IAAI,iBAAiB;QAC3D,cAAc,EAAE,QAAQ,CAAC,MAAM,CAAC,cAAc,CAAC;QAC/C,QAAQ;KACT,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import type { BigQuery } from "@google-cloud/bigquery";
|
|
2
|
+
import type { PubSub } from "@google-cloud/pubsub";
|
|
3
|
+
import type { Firestore } from "firebase-admin/firestore";
|
|
4
|
+
import type { CloudEvent } from "firebase-functions/v2";
|
|
5
|
+
import type { MessagePublishedData } from "firebase-functions/v2/pubsub";
|
|
6
|
+
import { type DataTransferClient } from "./dts";
|
|
7
|
+
import type { ResolvedBigqueryFirestoreExportConfig } from "./export-config";
|
|
8
|
+
import type { TransferRunPayload } from "./types";
|
|
9
|
+
export type TransferRunEvent = CloudEvent<MessagePublishedData<TransferRunPayload>>;
|
|
10
|
+
/** External services used by both handlers, injected for testability. */
|
|
11
|
+
export interface HandlerContext {
|
|
12
|
+
db: Firestore;
|
|
13
|
+
bigquery: BigQuery;
|
|
14
|
+
dataTransfer: DataTransferClient;
|
|
15
|
+
pubsub: PubSub;
|
|
16
|
+
config: ResolvedBigqueryFirestoreExportConfig;
|
|
17
|
+
}
|
|
18
|
+
/** Handles a v2 Pub/Sub completion notification from BigQuery DTS. */
|
|
19
|
+
export declare function handleMessagePublished(event: TransferRunEvent, ctx: HandlerContext): Promise<void>;
|
|
20
|
+
/** Idempotently creates, links, or updates this deployment's DTS config. */
|
|
21
|
+
export declare function handleUpsertTransferConfig(ctx: HandlerContext): Promise<void>;
|
|
22
|
+
//# sourceMappingURL=handlers.d.ts.map
|