@firebase-function-kits/bigquery-firestore-export 0.0.1 → 0.0.2-rc.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. package/CHANGELOG.md +1 -0
  2. package/README.md +188 -0
  3. package/lib/config.d.ts +5 -0
  4. package/lib/config.d.ts.map +1 -0
  5. package/lib/config.js +75 -0
  6. package/lib/config.js.map +1 -0
  7. package/lib/dts.d.ts +14 -0
  8. package/lib/dts.d.ts.map +1 -0
  9. package/lib/dts.js +189 -0
  10. package/lib/dts.js.map +1 -0
  11. package/lib/export-config.d.ts +58 -0
  12. package/lib/export-config.d.ts.map +1 -0
  13. package/lib/export-config.js +56 -0
  14. package/lib/export-config.js.map +1 -0
  15. package/lib/handlers.d.ts +22 -0
  16. package/lib/handlers.d.ts.map +1 -0
  17. package/lib/handlers.js +125 -0
  18. package/lib/handlers.js.map +1 -0
  19. package/lib/helper.d.ts +37 -0
  20. package/lib/helper.d.ts.map +1 -0
  21. package/lib/helper.js +219 -0
  22. package/lib/helper.js.map +1 -0
  23. package/lib/index.d.ts +7 -0
  24. package/lib/index.d.ts.map +1 -0
  25. package/lib/index.js +137 -0
  26. package/lib/index.js.map +1 -0
  27. package/lib/lib.d.ts +11 -0
  28. package/lib/lib.d.ts.map +1 -0
  29. package/lib/lib.js +47 -0
  30. package/lib/lib.js.map +1 -0
  31. package/lib/logs.d.ts +21 -0
  32. package/lib/logs.d.ts.map +1 -0
  33. package/lib/logs.js +122 -0
  34. package/lib/logs.js.map +1 -0
  35. package/lib/metadata.d.ts +6 -0
  36. package/lib/metadata.d.ts.map +1 -0
  37. package/lib/metadata.js +34 -0
  38. package/lib/metadata.js.map +1 -0
  39. package/lib/types.d.ts +44 -0
  40. package/lib/types.d.ts.map +1 -0
  41. package/lib/types.js +18 -0
  42. package/lib/types.js.map +1 -0
  43. package/package.json +40 -4
  44. package/src/config.ts +88 -0
  45. package/src/dts.ts +214 -0
  46. package/src/export-config.ts +123 -0
  47. package/src/handlers.ts +141 -0
  48. package/src/helper.ts +320 -0
  49. package/src/index.ts +126 -0
  50. package/src/lib.ts +69 -0
  51. package/src/logs.ts +145 -0
  52. package/src/metadata.ts +31 -0
  53. package/src/types.ts +100 -0
  54. package/tests/config.test.ts +61 -0
  55. package/tests/dts.test.ts +124 -0
  56. package/tests/export-config.test.ts +78 -0
  57. package/tests/handlers.test.ts +170 -0
  58. package/tests/helper.test.ts +74 -0
  59. package/tests/lib.test.ts +43 -0
  60. package/tsconfig.json +18 -0
  61. package/tsconfig.tsbuildinfo +1 -0
  62. package/index.js +0 -0
package/lib/logs.d.ts ADDED
@@ -0,0 +1,21 @@
1
+ import type { ResolvedBigqueryFirestoreExportConfig } from "./export-config";
2
+ export declare function init(config: ResolvedBigqueryFirestoreExportConfig): void;
3
+ export declare function start(): void;
4
+ export declare function complete(): void;
5
+ export declare function error(err: unknown): void;
6
+ export declare function errorWritingToFirestore(err: unknown): void;
7
+ export declare function bigqueryJobStarted(jobId: string | undefined): void;
8
+ export declare function bigqueryResultsRowCount(transferConfigId: string, runId: string, count: number): void;
9
+ export declare function bigqueryQueryFailed(transferConfigId: string, runId: string, tableName: string, err: unknown): void;
10
+ export declare function runResultsWrittenToFirestore(runId: string, successCount: number, totalCount: number): void;
11
+ export declare function latestDocUpdateSkipped(transferConfigId: string, runId: string, reason: string): void;
12
+ export declare function handlingNonSuccessRun(transferConfigId: string, runId: string, state: string): void;
13
+ export declare function createTransferConfig(): void;
14
+ export declare function transferConfigCreated(name: string): void;
15
+ export declare function updateTransferConfig(name: string): void;
16
+ export declare function transferConfigUpdated(name: string): void;
17
+ export declare function transferConfigNotFound(name: string): void;
18
+ export declare function getTransferConfigFailed(name: string, err: unknown): void;
19
+ export declare function partitioningFieldRemovalAttempted(name: string, existingField: string): void;
20
+ export declare function topicCreated(name: string): void;
21
+ //# sourceMappingURL=logs.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"logs.d.ts","sourceRoot":"","sources":["../src/logs.ts"],"names":[],"mappings":"AAiBA,OAAO,KAAK,EAAE,qCAAqC,EAAE,MAAM,iBAAiB,CAAC;AAE7E,wBAAgB,IAAI,CAAC,MAAM,EAAE,qCAAqC,GAAG,IAAI,CAKxE;AAED,wBAAgB,KAAK,IAAI,IAAI,CAE5B;AAED,wBAAgB,QAAQ,IAAI,IAAI,CAE/B;AAED,wBAAgB,KAAK,CAAC,GAAG,EAAE,OAAO,GAAG,IAAI,CAExC;AAED,wBAAgB,uBAAuB,CAAC,GAAG,EAAE,OAAO,GAAG,IAAI,CAE1D;AAED,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI,CAElE;AAED,wBAAgB,uBAAuB,CACrC,gBAAgB,EAAE,MAAM,EACxB,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,GACZ,IAAI,CAMN;AAED,wBAAgB,mBAAmB,CACjC,gBAAgB,EAAE,MAAM,EACxB,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,MAAM,EACjB,GAAG,EAAE,OAAO,GACX,IAAI,CAON;AAED,wBAAgB,4BAA4B,CAC1C,KAAK,EAAE,MAAM,EACb,YAAY,EAAE,MAAM,EACpB,UAAU,EAAE,MAAM,GACjB,IAAI,CAMN;AAED,wBAAgB,sBAAsB,CACpC,gBAAgB,EAAE,MAAM,EACxB,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,GACb,IAAI,CAMN;AAED,wBAAgB,qBAAqB,CACnC,gBAAgB,EAAE,MAAM,EACxB,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,GACZ,IAAI,CAMN;AAED,wBAAgB,oBAAoB,IAAI,IAAI,CAE3C;AAED,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAExD;AAED,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAEvD;AAED,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAExD;AAED,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAEzD;AAED,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,GAAG,IAAI,CAExE;AAED,wBAAgB,iCAAiC,CAC/C,IAAI,EAAE,MAAM,EACZ,aAAa,EAAE,MAAM,GACpB,IAAI,CAKN;AAED,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAE/C"}
package/lib/logs.js ADDED
@@ -0,0 +1,122 @@
1
+ "use strict";
2
+ /*
3
+ * Copyright 2025 Google LLC
4
+ *
5
+ * Licensed under the Apache License, Version 2.0 (the "License");
6
+ * you may not use this file except in compliance with the License.
7
+ * You may obtain a copy of the License at
8
+ *
9
+ * https://www.apache.org/licenses/LICENSE-2.0
10
+ *
11
+ * Unless required by applicable law or agreed to in writing, software
12
+ * distributed under the License is distributed on an "AS IS" BASIS,
13
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
+ * See the License for the specific language governing permissions and
15
+ * limitations under the License.
16
+ */
17
+ Object.defineProperty(exports, "__esModule", { value: true });
18
+ exports.init = init;
19
+ exports.start = start;
20
+ exports.complete = complete;
21
+ exports.error = error;
22
+ exports.errorWritingToFirestore = errorWritingToFirestore;
23
+ exports.bigqueryJobStarted = bigqueryJobStarted;
24
+ exports.bigqueryResultsRowCount = bigqueryResultsRowCount;
25
+ exports.bigqueryQueryFailed = bigqueryQueryFailed;
26
+ exports.runResultsWrittenToFirestore = runResultsWrittenToFirestore;
27
+ exports.latestDocUpdateSkipped = latestDocUpdateSkipped;
28
+ exports.handlingNonSuccessRun = handlingNonSuccessRun;
29
+ exports.createTransferConfig = createTransferConfig;
30
+ exports.transferConfigCreated = transferConfigCreated;
31
+ exports.updateTransferConfig = updateTransferConfig;
32
+ exports.transferConfigUpdated = transferConfigUpdated;
33
+ exports.transferConfigNotFound = transferConfigNotFound;
34
+ exports.getTransferConfigFailed = getTransferConfigFailed;
35
+ exports.partitioningFieldRemovalAttempted = partitioningFieldRemovalAttempted;
36
+ exports.topicCreated = topicCreated;
37
+ const firebase_functions_1 = require("firebase-functions");
38
+ function init(config) {
39
+ firebase_functions_1.logger.info("Initializing BigQuery Firestore export", {
40
+ ...config,
41
+ queryString: "<omitted>",
42
+ });
43
+ }
44
+ function start() {
45
+ firebase_functions_1.logger.info("Started BigQuery Firestore export execution");
46
+ }
47
+ function complete() {
48
+ firebase_functions_1.logger.info("Completed BigQuery Firestore export execution");
49
+ }
50
+ function error(err) {
51
+ firebase_functions_1.logger.error("Unhandled error during BigQuery Firestore export", err);
52
+ }
53
+ function errorWritingToFirestore(err) {
54
+ firebase_functions_1.logger.error("Error writing a BigQuery row to Firestore", err);
55
+ }
56
+ function bigqueryJobStarted(jobId) {
57
+ firebase_functions_1.logger.info(`BigQuery job ${jobId ?? "<unknown>"} started`);
58
+ }
59
+ function bigqueryResultsRowCount(transferConfigId, runId, count) {
60
+ firebase_functions_1.logger.debug("Read BigQuery transfer results", {
61
+ transferConfigId,
62
+ runId,
63
+ count,
64
+ });
65
+ }
66
+ function bigqueryQueryFailed(transferConfigId, runId, tableName, err) {
67
+ firebase_functions_1.logger.error("BigQuery result query failed", {
68
+ transferConfigId,
69
+ runId,
70
+ tableName,
71
+ err,
72
+ });
73
+ }
74
+ function runResultsWrittenToFirestore(runId, successCount, totalCount) {
75
+ firebase_functions_1.logger.info("Finished writing BigQuery results to Firestore", {
76
+ runId,
77
+ successCount,
78
+ totalCount,
79
+ });
80
+ }
81
+ function latestDocUpdateSkipped(transferConfigId, runId, reason) {
82
+ firebase_functions_1.logger.debug("Skipped updating latest transfer-run document", {
83
+ transferConfigId,
84
+ runId,
85
+ reason,
86
+ });
87
+ }
88
+ function handlingNonSuccessRun(transferConfigId, runId, state) {
89
+ firebase_functions_1.logger.info("Recording non-successful BigQuery transfer run", {
90
+ transferConfigId,
91
+ runId,
92
+ state,
93
+ });
94
+ }
95
+ function createTransferConfig() {
96
+ firebase_functions_1.logger.info("Creating BigQuery Data Transfer config");
97
+ }
98
+ function transferConfigCreated(name) {
99
+ firebase_functions_1.logger.info("Created BigQuery Data Transfer config", { name });
100
+ }
101
+ function updateTransferConfig(name) {
102
+ firebase_functions_1.logger.info("Updating BigQuery Data Transfer config", { name });
103
+ }
104
+ function transferConfigUpdated(name) {
105
+ firebase_functions_1.logger.info("Updated BigQuery Data Transfer config", { name });
106
+ }
107
+ function transferConfigNotFound(name) {
108
+ firebase_functions_1.logger.error("BigQuery Data Transfer config not found", { name });
109
+ }
110
+ function getTransferConfigFailed(name, err) {
111
+ firebase_functions_1.logger.error("Failed to read BigQuery Data Transfer config", { name, err });
112
+ }
113
+ function partitioningFieldRemovalAttempted(name, existingField) {
114
+ firebase_functions_1.logger.warn("Cannot remove an existing DTS partitioning field", {
115
+ name,
116
+ existingField,
117
+ });
118
+ }
119
+ function topicCreated(name) {
120
+ firebase_functions_1.logger.info("Created Pub/Sub topic for transfer notifications", { name });
121
+ }
122
+ //# sourceMappingURL=logs.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"logs.js","sourceRoot":"","sources":["../src/logs.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;GAcG;;;;;;;;;;;;;;;;;;;;;AAEH,2DAA4C;AAG5C,cAAqB,MAA6C;IAChE,2BAAM,CAAC,IAAI,CAAC,wCAAwC,EAAE;QACpD,GAAG,MAAM;QACT,WAAW,EAAE,WAAW;KACzB,CAAC,CAAC;AACL,CAAC;AAED;IACE,2BAAM,CAAC,IAAI,CAAC,6CAA6C,CAAC,CAAC;AAC7D,CAAC;AAED;IACE,2BAAM,CAAC,IAAI,CAAC,+CAA+C,CAAC,CAAC;AAC/D,CAAC;AAED,eAAsB,GAAY;IAChC,2BAAM,CAAC,KAAK,CAAC,kDAAkD,EAAE,GAAG,CAAC,CAAC;AACxE,CAAC;AAED,iCAAwC,GAAY;IAClD,2BAAM,CAAC,KAAK,CAAC,2CAA2C,EAAE,GAAG,CAAC,CAAC;AACjE,CAAC;AAED,4BAAmC,KAAyB;IAC1D,2BAAM,CAAC,IAAI,CAAC,gBAAgB,KAAK,IAAI,WAAW,UAAU,CAAC,CAAC;AAC9D,CAAC;AAED,iCACE,gBAAwB,EACxB,KAAa,EACb,KAAa;IAEb,2BAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE;QAC7C,gBAAgB;QAChB,KAAK;QACL,KAAK;KACN,CAAC,CAAC;AACL,CAAC;AAED,6BACE,gBAAwB,EACxB,KAAa,EACb,SAAiB,EACjB,GAAY;IAEZ,2BAAM,CAAC,KAAK,CAAC,8BAA8B,EAAE;QAC3C,gBAAgB;QAChB,KAAK;QACL,SAAS;QACT,GAAG;KACJ,CAAC,CAAC;AACL,CAAC;AAED,sCACE,KAAa,EACb,YAAoB,EACpB,UAAkB;IAElB,2BAAM,CAAC,IAAI,CAAC,gDAAgD,EAAE;QAC5D,KAAK;QACL,YAAY;QACZ,UAAU;KACX,CAAC,CAAC;AACL,CAAC;AAED,gCACE,gBAAwB,EACxB,KAAa,EACb,MAAc;IAEd,2BAAM,CAAC,KAAK,CAAC,+CAA+C,EAAE;QAC5D,gBAAgB;QAChB,KAAK;QACL,MAAM;KACP,CAAC,CAAC;AACL,CAAC;AAED,+BACE,gBAAwB,EACxB,KAAa,EACb,KAAa;IAEb,2BAAM,CAAC,IAAI,CAAC,gDAAgD,EAAE;QAC5D,gBAAgB;QAChB,KAAK;QACL,KAAK;KACN,CAAC,CAAC;AACL,CAAC;AAED;IACE,2BAAM,CAAC,IAAI,CAAC,wCAAwC,CAAC,CAAC;AACxD,CAAC;AAED,+BAAsC,IAAY;IAChD,2BAAM,CAAC,IAAI,CAAC,uCAAuC,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;AACjE,CAAC;AAED,8BAAqC,IAAY;IAC/C,2BAAM,CAAC,IAAI,CAAC,wCAAwC,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;AAClE,CAAC;AAED,+BAAsC,IAAY;IAChD,2BAAM,CAAC,IAAI,CAAC,uCAAuC,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;AACjE,CAAC;AAED,gCAAuC,IAAY;IACjD,2BAAM,CAAC,KAAK,CAAC,yCAAyC,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;AACpE,CAAC;AAED,iCAAwC,IAAY,EAAE,GAAY;IAChE,2BAAM,CAAC,KAAK,CAAC,8CAA8C,EAAE,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC;AAC9E,CAAC;AAED,2CACE,IAAY,EACZ,aAAqB;IAErB,2BAAM,CAAC,IAAI,CAAC,kDAAkD,EAAE;QAC9D,IAAI;QACJ,aAAa;KACd,CAAC,CAAC;AACL,CAAC;AAED,sBAA6B,IAAY;IACvC,2BAAM,CAAC,IAAI,CAAC,kDAAkD,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;AAC5E,CAAC"}
@@ -0,0 +1,6 @@
1
+ export declare const metadata: {
2
+ readonly roles: readonly ["datastore.user", "bigquery.admin", "pubsub.admin", "eventarc.eventReceiver", "run.invoker"];
3
+ readonly apis: readonly ["bigquery.googleapis.com", "bigquerydatatransfer.googleapis.com", "pubsub.googleapis.com"];
4
+ readonly functionNames: readonly ["processMessages", "upsertTransferConfig"];
5
+ };
6
+ //# sourceMappingURL=metadata.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"metadata.d.ts","sourceRoot":"","sources":["../src/metadata.ts"],"names":[],"mappings":"AAgBA,eAAO,MAAM,QAAQ;aACnB,KAAK,YACH,gBAAgB,EAChB,gBAAgB,EAChB,cAAc,EACd,wBAAwB,EACxB,aAAa;aAEf,IAAI,YACF,yBAAyB,EACzB,qCAAqC,EACrC,uBAAuB;aAEzB,aAAa,YAAG,iBAAiB,EAAE,sBAAsB;CACjD,CAAC"}
@@ -0,0 +1,34 @@
1
+ "use strict";
2
+ /*
3
+ * Copyright 2026 Google LLC
4
+ *
5
+ * Licensed under the Apache License, Version 2.0 (the "License");
6
+ * you may not use this file except in compliance with the License.
7
+ * You may obtain a copy of the License at
8
+ *
9
+ * https://www.apache.org/licenses/LICENSE-2.0
10
+ *
11
+ * Unless required by applicable law or agreed to in writing, software
12
+ * distributed under the License is distributed on an "AS IS" BASIS,
13
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
+ * See the License for the specific language governing permissions and
15
+ * limitations under the License.
16
+ */
17
+ Object.defineProperty(exports, "__esModule", { value: true });
18
+ exports.metadata = void 0;
19
+ exports.metadata = {
20
+ roles: [
21
+ "datastore.user",
22
+ "bigquery.admin",
23
+ "pubsub.admin",
24
+ "eventarc.eventReceiver",
25
+ "run.invoker",
26
+ ],
27
+ apis: [
28
+ "bigquery.googleapis.com",
29
+ "bigquerydatatransfer.googleapis.com",
30
+ "pubsub.googleapis.com",
31
+ ],
32
+ functionNames: ["processMessages", "upsertTransferConfig"],
33
+ };
34
+ //# sourceMappingURL=metadata.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"metadata.js","sourceRoot":"","sources":["../src/metadata.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;GAcG;;;AAEU,QAAA,QAAQ,GAAG;IACtB,KAAK,EAAE;QACL,gBAAgB;QAChB,gBAAgB;QAChB,cAAc;QACd,wBAAwB;QACxB,aAAa;KACd;IACD,IAAI,EAAE;QACJ,yBAAyB;QACzB,qCAAqC;QACrC,uBAAuB;KACxB;IACD,aAAa,EAAE,CAAC,iBAAiB,EAAE,sBAAsB,CAAC;CAClD,CAAC"}
package/lib/types.d.ts ADDED
@@ -0,0 +1,44 @@
1
+ import type { BigQueryDate, BigQueryDatetime, BigQueryTime, BigQueryTimestamp, Geography } from "@google-cloud/bigquery";
2
+ import type { Timestamp } from "firebase-admin/firestore";
3
+ /** Parameters attached to a BigQuery scheduled-query transfer run. */
4
+ export interface TransferRunParams {
5
+ destination_table_name_template: string;
6
+ partitioning_field: string;
7
+ query: string;
8
+ write_disposition: string;
9
+ }
10
+ /** State of a BigQuery Data Transfer run. */
11
+ export type TransferRunState = "SUCCEEDED" | "FAILED" | "CANCELLED" | "PENDING" | "RUNNING" | "TRANSFER_STATE_UNSPECIFIED";
12
+ /** JSON payload from a BigQuery Data Transfer Pub/Sub notification. */
13
+ export interface TransferRunPayload {
14
+ name: string;
15
+ runTime: string;
16
+ state: TransferRunState;
17
+ destinationDatasetId: string;
18
+ dataSourceId: string;
19
+ schedule: string;
20
+ scheduleTime: string;
21
+ startTime: string;
22
+ endTime: string;
23
+ updateTime: string;
24
+ userId: string;
25
+ notificationPubsubTopic: string;
26
+ params: TransferRunParams;
27
+ emailPreferences: Record<string, unknown>;
28
+ errorStatus: Record<string, unknown>;
29
+ }
30
+ /** Message-shaped wrapper retained for the core handler boundary. */
31
+ export interface TransferRunMessage {
32
+ json: TransferRunPayload;
33
+ }
34
+ /** Values returned in a BigQuery row before conversion. */
35
+ export type BigQueryRowValue = string | number | boolean | null | Date | Buffer | BigQueryTimestamp | BigQueryDate | BigQueryTime | BigQueryDatetime | Geography | BigQueryRow | BigQueryRowValue[];
36
+ export interface BigQueryRow {
37
+ [key: string]: BigQueryRowValue;
38
+ }
39
+ /** Firestore-compatible equivalent of a BigQuery row value. */
40
+ export type FirestoreRowValue = string | number | boolean | null | Timestamp | Uint8Array | FirestoreRow | FirestoreRowValue[];
41
+ export interface FirestoreRow {
42
+ [key: string]: FirestoreRowValue;
43
+ }
44
+ //# sourceMappingURL=types.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAgBA,OAAO,KAAK,EACV,YAAY,EACZ,gBAAgB,EAChB,YAAY,EACZ,iBAAiB,EACjB,SAAS,EACV,MAAM,wBAAwB,CAAC;AAChC,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AAE1D,sEAAsE;AACtE,MAAM,WAAW,iBAAiB;IAChC,+BAA+B,EAAE,MAAM,CAAC;IACxC,kBAAkB,EAAE,MAAM,CAAC;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,iBAAiB,EAAE,MAAM,CAAC;CAC3B;AAED,6CAA6C;AAC7C,MAAM,MAAM,gBAAgB,GACxB,WAAW,GACX,QAAQ,GACR,WAAW,GACX,SAAS,GACT,SAAS,GACT,4BAA4B,CAAC;AAEjC,uEAAuE;AACvE,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,gBAAgB,CAAC;IACxB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,uBAAuB,EAAE,MAAM,CAAC;IAChC,MAAM,EAAE,iBAAiB,CAAC;IAC1B,gBAAgB,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC1C,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACtC;AAED,qEAAqE;AACrE,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,kBAAkB,CAAC;CAC1B;AAED,2DAA2D;AAC3D,MAAM,MAAM,gBAAgB,GACxB,MAAM,GACN,MAAM,GACN,OAAO,GACP,IAAI,GACJ,IAAI,GACJ,MAAM,GACN,iBAAiB,GACjB,YAAY,GACZ,YAAY,GACZ,gBAAgB,GAChB,SAAS,GACT,WAAW,GACX,gBAAgB,EAAE,CAAC;AAEvB,MAAM,WAAW,WAAW;IAC1B,CAAC,GAAG,EAAE,MAAM,GAAG,gBAAgB,CAAC;CACjC;AAED,+DAA+D;AAC/D,MAAM,MAAM,iBAAiB,GACzB,MAAM,GACN,MAAM,GACN,OAAO,GACP,IAAI,GACJ,SAAS,GACT,UAAU,GACV,YAAY,GACZ,iBAAiB,EAAE,CAAC;AAExB,MAAM,WAAW,YAAY;IAC3B,CAAC,GAAG,EAAE,MAAM,GAAG,iBAAiB,CAAC;CAClC"}
package/lib/types.js ADDED
@@ -0,0 +1,18 @@
1
+ "use strict";
2
+ /*
3
+ * Copyright 2025 Google LLC
4
+ *
5
+ * Licensed under the Apache License, Version 2.0 (the "License");
6
+ * you may not use this file except in compliance with the License.
7
+ * You may obtain a copy of the License at
8
+ *
9
+ * https://www.apache.org/licenses/LICENSE-2.0
10
+ *
11
+ * Unless required by applicable law or agreed to in writing, software
12
+ * distributed under the License is distributed on an "AS IS" BASIS,
13
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
+ * See the License for the specific language governing permissions and
15
+ * limitations under the License.
16
+ */
17
+ Object.defineProperty(exports, "__esModule", { value: true });
18
+ //# sourceMappingURL=types.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;GAcG"}
package/package.json CHANGED
@@ -1,12 +1,48 @@
1
1
  {
2
2
  "name": "@firebase-function-kits/bigquery-firestore-export",
3
- "version": "0.0.1",
3
+ "version": "0.0.2-rc.0",
4
+ "repository": {
5
+ "type": "git",
6
+ "url": "https://github.com/firebase/extensions.git",
7
+ "directory": "kits/bigquery-firestore-export"
8
+ },
9
+ "private": false,
4
10
  "description": "Reverse-sync rows from BigQuery into Firestore",
5
11
  "license": "Apache-2.0",
6
- "main": "index.js",
7
- "private": false,
8
- "repository": "firebase/extensions",
12
+ "main": "lib/index.js",
13
+ "types": "lib/index.d.ts",
14
+ "exports": {
15
+ ".": {
16
+ "types": "./lib/index.d.ts",
17
+ "default": "./lib/index.js"
18
+ },
19
+ "./lib": {
20
+ "types": "./lib/lib.d.ts",
21
+ "default": "./lib/lib.js"
22
+ }
23
+ },
9
24
  "engines": {
10
25
  "node": ">=22"
26
+ },
27
+ "scripts": {
28
+ "build": "tsc -b",
29
+ "clean": "tsc -b --clean",
30
+ "test": "vitest run",
31
+ "deploy": "pnpm build && firebase deploy --only functions",
32
+ "serve": "firebase emulators:start --only functions"
33
+ },
34
+ "dependencies": {
35
+ "@firebase/app": "^0.16.0",
36
+ "@google-cloud/bigquery": "^8.3.1",
37
+ "@google-cloud/bigquery-data-transfer": "^5.0.1",
38
+ "@google-cloud/pubsub": "^4.11.0",
39
+ "firebase-admin": "^14.1.0",
40
+ "firebase-functions": "7.3.2"
41
+ },
42
+ "devDependencies": {
43
+ "vitest": "^4.1.10"
44
+ },
45
+ "overrides": {
46
+ "uuid@9.0.1": "11.1.1"
11
47
  }
12
48
  }
package/src/config.ts ADDED
@@ -0,0 +1,88 @@
1
+ /*
2
+ * Copyright 2026 Google LLC
3
+ *
4
+ * Licensed under the Apache License, Version 2.0 (the "License");
5
+ * you may not use this file except in compliance with the License.
6
+ * You may obtain a copy of the License at
7
+ *
8
+ * https://www.apache.org/licenses/LICENSE-2.0
9
+ *
10
+ * Unless required by applicable law or agreed to in writing, software
11
+ * distributed under the License is distributed on an "AS IS" BASIS,
12
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ * See the License for the specific language governing permissions and
14
+ * limitations under the License.
15
+ */
16
+
17
+ import {
18
+ defineString,
19
+ expr,
20
+ projectID,
21
+ select,
22
+ } from "firebase-functions/params";
23
+ import type {
24
+ BigqueryFirestoreExportConfig,
25
+ DeployTimeOptions,
26
+ LogLevel,
27
+ } from "./export-config";
28
+
29
+ const LOG_LEVEL_OPTIONS = ["debug", "info", "warn", "error", "silent"] as const;
30
+ const instanceId = defineString("INSTANCE_ID");
31
+
32
+ const params = {
33
+ instanceId,
34
+ bigqueryDatasetLocation: defineString("BIGQUERY_DATASET_LOCATION", {
35
+ default: "US",
36
+ }),
37
+ transferConfigName: defineString("TRANSFER_CONFIG_NAME", { default: "" }),
38
+ datasetId: defineString("DATASET_ID"),
39
+ tableName: defineString("TABLE_NAME"),
40
+ queryString: defineString("QUERY_STRING"),
41
+ displayName: defineString("DISPLAY_NAME"),
42
+ partitioningField: defineString("PARTITIONING_FIELD", { default: "" }),
43
+ schedule: defineString("SCHEDULE"),
44
+ firestoreCollection: defineString("COLLECTION_PATH", {
45
+ default: "transferConfigs",
46
+ }),
47
+ logLevel: defineString("LOG_LEVEL", {
48
+ default: "info",
49
+ input: select([...LOG_LEVEL_OPTIONS]),
50
+ }),
51
+ };
52
+
53
+ export const CONFIG_EXPRESSIONS: DeployTimeOptions = {
54
+ pubSubTopic: expr`kit-${instanceId}-processMessages`,
55
+ };
56
+
57
+ function optional(value: string): string | undefined {
58
+ const normalized = value.trim();
59
+ return normalized.length > 0 ? normalized : undefined;
60
+ }
61
+
62
+ function normalizeLogLevel(value: string): LogLevel {
63
+ const normalized = value.toLowerCase();
64
+ return LOG_LEVEL_OPTIONS.includes(normalized as LogLevel)
65
+ ? (normalized as LogLevel)
66
+ : "info";
67
+ }
68
+
69
+ /** Reads runtime values from Firebase deploy-time parameters. */
70
+ export function configFromEnv(): BigqueryFirestoreExportConfig {
71
+ const resolvedInstanceId = params.instanceId.value();
72
+
73
+ return {
74
+ bigqueryDatasetLocation: params.bigqueryDatasetLocation.value(),
75
+ projectId: projectID.value(),
76
+ instanceId: resolvedInstanceId,
77
+ transferConfigName: optional(params.transferConfigName.value()),
78
+ datasetId: params.datasetId.value(),
79
+ tableName: params.tableName.value(),
80
+ queryString: params.queryString.value(),
81
+ displayName: params.displayName.value(),
82
+ partitioningField: optional(params.partitioningField.value()),
83
+ schedule: params.schedule.value(),
84
+ pubSubTopic: `kit-${resolvedInstanceId}-processMessages`,
85
+ firestoreCollection: params.firestoreCollection.value(),
86
+ logLevel: normalizeLogLevel(params.logLevel.value()),
87
+ };
88
+ }
package/src/dts.ts ADDED
@@ -0,0 +1,214 @@
1
+ /*
2
+ * Copyright 2025 Google LLC
3
+ *
4
+ * Licensed under the Apache License, Version 2.0 (the "License");
5
+ * you may not use this file except in compliance with the License.
6
+ * You may obtain a copy of the License at
7
+ *
8
+ * https://www.apache.org/licenses/LICENSE-2.0
9
+ *
10
+ * Unless required by applicable law or agreed to in writing, software
11
+ * distributed under the License is distributed on an "AS IS" BASIS,
12
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ * See the License for the specific language governing permissions and
14
+ * limitations under the License.
15
+ */
16
+
17
+ import * as bigqueryDataTransfer from "@google-cloud/bigquery-data-transfer";
18
+ import type { ResolvedBigqueryFirestoreExportConfig } from "./export-config";
19
+ import * as logs from "./logs";
20
+
21
+ export type DataTransferClient =
22
+ bigqueryDataTransfer.v1.DataTransferServiceClient;
23
+ export type TransferConfig =
24
+ bigqueryDataTransfer.protos.google.cloud.bigquery.datatransfer.v1.ITransferConfig;
25
+
26
+ const GRPC_NOT_FOUND = 5;
27
+ const PACKAGE_PARTITIONING_ERROR =
28
+ "Cannot remove partitioning_field from an existing transfer config";
29
+
30
+ export const PARTITIONING_FIELD_REMOVAL_ERROR_PREFIX =
31
+ PACKAGE_PARTITIONING_ERROR;
32
+ export const PARTITIONING_FIELD_REMOVAL_ERROR = `${PACKAGE_PARTITIONING_ERROR}. The BigQuery Data Transfer API does not support clearing this parameter once it has been set. To change partitioning, create a new transfer config with the desired setting.`;
33
+
34
+ function isNotFoundError(err: unknown): boolean {
35
+ return (
36
+ typeof err === "object" &&
37
+ err !== null &&
38
+ "code" in err &&
39
+ err.code === GRPC_NOT_FOUND
40
+ );
41
+ }
42
+
43
+ function transferConfigFields(config: TransferConfig) {
44
+ const fields = config.params?.fields;
45
+ if (!fields) {
46
+ throw new Error(
47
+ "Transfer config has invalid structure: missing params.fields"
48
+ );
49
+ }
50
+ if (!fields.query) {
51
+ throw new Error(
52
+ "Transfer config has invalid structure: missing params.fields.query"
53
+ );
54
+ }
55
+ if (!fields.destination_table_name_template) {
56
+ throw new Error(
57
+ "Transfer config has invalid structure: missing params.fields.destination_table_name_template"
58
+ );
59
+ }
60
+
61
+ return fields;
62
+ }
63
+
64
+ function stringField(value: string | undefined): { stringValue: string } {
65
+ return { stringValue: value ?? "" };
66
+ }
67
+
68
+ /** Creates the protobuf-shaped request used for a scheduled query. */
69
+ export function createTransferConfigRequest(
70
+ config: ResolvedBigqueryFirestoreExportConfig,
71
+ serviceAccountEmail?: string
72
+ ): bigqueryDataTransfer.protos.google.cloud.bigquery.datatransfer.v1.ICreateTransferConfigRequest {
73
+ return {
74
+ parent: `projects/${config.projectId}`,
75
+ transferConfig: {
76
+ destinationDatasetId: config.datasetId,
77
+ displayName: config.displayName,
78
+ dataSourceId: "scheduled_query",
79
+ params: {
80
+ fields: {
81
+ query: stringField(config.queryString),
82
+ destination_table_name_template: stringField(
83
+ `${config.tableName}_{run_time|"%H%M%S"}`
84
+ ),
85
+ write_disposition: stringField("WRITE_TRUNCATE"),
86
+ partitioning_field: stringField(config.partitioningField),
87
+ },
88
+ },
89
+ schedule: config.schedule,
90
+ notificationPubsubTopic: `projects/${config.projectId}/topics/${config.pubSubTopic}`,
91
+ ...(serviceAccountEmail
92
+ ? { serviceAccountName: serviceAccountEmail }
93
+ : {}),
94
+ },
95
+ };
96
+ }
97
+
98
+ export async function getTransferConfig(
99
+ client: DataTransferClient,
100
+ transferConfigName: string
101
+ ): Promise<TransferConfig | null> {
102
+ try {
103
+ const [config] = await client.getTransferConfig({
104
+ name: transferConfigName,
105
+ });
106
+ return config;
107
+ } catch (err) {
108
+ if (isNotFoundError(err)) {
109
+ logs.transferConfigNotFound(transferConfigName);
110
+ return null;
111
+ }
112
+ logs.getTransferConfigFailed(transferConfigName, err);
113
+ throw err;
114
+ }
115
+ }
116
+
117
+ export async function createTransferConfig(
118
+ client: DataTransferClient,
119
+ config: ResolvedBigqueryFirestoreExportConfig
120
+ ): Promise<TransferConfig> {
121
+ logs.createTransferConfig();
122
+ const [created] = await client.createTransferConfig(
123
+ createTransferConfigRequest(config, config.serviceAccount)
124
+ );
125
+ if (!created.name) {
126
+ throw new Error("BigQuery API returned a transfer config without a name");
127
+ }
128
+ logs.transferConfigCreated(created.name);
129
+ return created;
130
+ }
131
+
132
+ /** Builds a minimal update mask while retaining unsupported immutable fields. */
133
+ export async function constructUpdateTransferConfigRequest(
134
+ client: DataTransferClient,
135
+ transferConfigName: string,
136
+ config: ResolvedBigqueryFirestoreExportConfig
137
+ ): Promise<bigqueryDataTransfer.protos.google.cloud.bigquery.datatransfer.v1.IUpdateTransferConfigRequest> {
138
+ const transferConfig = await getTransferConfig(client, transferConfigName);
139
+ if (!transferConfig) throw new Error("Transfer config not found");
140
+
141
+ const fields = transferConfigFields(transferConfig);
142
+ const updatedConfig = JSON.parse(
143
+ JSON.stringify(transferConfig)
144
+ ) as TransferConfig;
145
+ const updatedFields = transferConfigFields(updatedConfig);
146
+ const updateMask: string[] = [];
147
+
148
+ if (config.datasetId !== transferConfig.destinationDatasetId) {
149
+ updateMask.push("destination_dataset_id");
150
+ updatedConfig.destinationDatasetId = config.datasetId;
151
+ }
152
+ if (config.queryString !== fields.query.stringValue) {
153
+ updateMask.push("params");
154
+ updatedFields.query.stringValue = config.queryString;
155
+ }
156
+
157
+ const tableTemplate = `${config.tableName}_{run_time|"%H%M%S"}`;
158
+ if (tableTemplate !== fields.destination_table_name_template.stringValue) {
159
+ updateMask.push("params");
160
+ updatedFields.destination_table_name_template.stringValue = tableTemplate;
161
+ }
162
+
163
+ const existingPartitioningField =
164
+ fields.partitioning_field?.stringValue ?? "";
165
+ const newPartitioningField = config.partitioningField ?? "";
166
+ if (newPartitioningField !== existingPartitioningField) {
167
+ if (!newPartitioningField) {
168
+ logs.partitioningFieldRemovalAttempted(
169
+ transferConfigName,
170
+ existingPartitioningField
171
+ );
172
+ throw new Error(PARTITIONING_FIELD_REMOVAL_ERROR);
173
+ }
174
+ updateMask.push("params");
175
+ updatedFields.partitioning_field ??= {};
176
+ updatedFields.partitioning_field.stringValue = newPartitioningField;
177
+ }
178
+
179
+ if (config.schedule !== transferConfig.schedule) {
180
+ updateMask.push("schedule");
181
+ updatedConfig.schedule = config.schedule;
182
+ }
183
+
184
+ const expectedTopic = `projects/${config.projectId}/topics/${config.pubSubTopic}`;
185
+ if (expectedTopic !== transferConfig.notificationPubsubTopic) {
186
+ updateMask.push("notification_pubsub_topic");
187
+ updatedConfig.notificationPubsubTopic = expectedTopic;
188
+ }
189
+
190
+ return {
191
+ transferConfig: updatedConfig,
192
+ updateMask: { paths: [...new Set(updateMask)] },
193
+ };
194
+ }
195
+
196
+ export async function updateTransferConfig(
197
+ client: DataTransferClient,
198
+ transferConfigName: string,
199
+ config: ResolvedBigqueryFirestoreExportConfig
200
+ ): Promise<TransferConfig> {
201
+ const request = await constructUpdateTransferConfigRequest(
202
+ client,
203
+ transferConfigName,
204
+ config
205
+ );
206
+ logs.updateTransferConfig(transferConfigName);
207
+ const converted =
208
+ bigqueryDataTransfer.protos.google.cloud.bigquery.datatransfer.v1.UpdateTransferConfigRequest.fromObject(
209
+ request
210
+ );
211
+ const [updated] = await client.updateTransferConfig(converted);
212
+ logs.transferConfigUpdated(transferConfigName);
213
+ return updated;
214
+ }