@firebase-function-kits/firestore-bigquery-export 0.0.1 → 0.0.2-rc.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +1 -0
- package/README.md +234 -0
- package/lib/config.d.ts +33 -0
- package/lib/config.d.ts.map +1 -0
- package/lib/config.js +345 -0
- package/lib/config.js.map +1 -0
- package/lib/events.d.ts +45 -0
- package/lib/events.d.ts.map +1 -0
- package/lib/events.js +154 -0
- package/lib/events.js.map +1 -0
- package/lib/export-config.d.ts +96 -0
- package/lib/export-config.d.ts.map +1 -0
- package/lib/export-config.js +77 -0
- package/lib/export-config.js.map +1 -0
- package/lib/handlers.d.ts +29 -0
- package/lib/handlers.d.ts.map +1 -0
- package/lib/handlers.js +165 -0
- package/lib/handlers.js.map +1 -0
- package/lib/index.d.ts +23 -0
- package/lib/index.d.ts.map +1 -0
- package/lib/index.js +179 -0
- package/lib/index.js.map +1 -0
- package/lib/init.d.ts +16 -0
- package/lib/init.d.ts.map +1 -0
- package/lib/init.js +44 -0
- package/lib/init.js.map +1 -0
- package/lib/lib.d.ts +20 -0
- package/lib/lib.d.ts.map +1 -0
- package/lib/lib.js +47 -0
- package/lib/lib.js.map +1 -0
- package/lib/logs.d.ts +39 -0
- package/lib/logs.d.ts.map +1 -0
- package/lib/logs.js +186 -0
- package/lib/logs.js.map +1 -0
- package/lib/util.d.ts +23 -0
- package/lib/util.d.ts.map +1 -0
- package/lib/util.js +66 -0
- package/lib/util.js.map +1 -0
- package/package.json +33 -3
- package/src/config.ts +420 -0
- package/src/events.ts +146 -0
- package/src/export-config.ts +205 -0
- package/src/handlers.ts +211 -0
- package/src/index.ts +174 -0
- package/src/init.ts +45 -0
- package/src/lib.ts +55 -0
- package/src/logs.ts +233 -0
- package/src/util.ts +64 -0
- package/tests/config.test.ts +182 -0
- package/tests/events.test.ts +84 -0
- package/tests/export-config.test.ts +114 -0
- package/tests/handlers.test.ts +220 -0
- package/tests/init.test.ts +76 -0
- package/tests/util.test.ts +102 -0
- package/tsconfig.json +18 -0
- package/tsconfig.tsbuildinfo +1 -0
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Copyright 2019 Google LLC
|
|
3
|
+
*
|
|
4
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
* you may not use this file except in compliance with the License.
|
|
6
|
+
* You may obtain a copy of the License at
|
|
7
|
+
*
|
|
8
|
+
* https://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
*
|
|
10
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
* See the License for the specific language governing permissions and
|
|
14
|
+
* limitations under the License.
|
|
15
|
+
*/
|
|
16
|
+
import type {
|
|
17
|
+
ChangeTrackerConfig,
|
|
18
|
+
LogLevel,
|
|
19
|
+
} from "@firebaseextensions/firestore-bigquery-change-tracker";
|
|
20
|
+
import type { Expression } from "firebase-functions/params";
|
|
21
|
+
|
|
22
|
+
type TrackerLogLevel = "debug" | "info" | "warn" | "error" | "silent";
|
|
23
|
+
type ConfigValue<T extends string | number | boolean | string[]> =
|
|
24
|
+
| T
|
|
25
|
+
| Expression<T>;
|
|
26
|
+
|
|
27
|
+
/** How the BigQuery changelog view is materialized. */
|
|
28
|
+
export type ViewType =
|
|
29
|
+
| "view"
|
|
30
|
+
| "materialized_incremental"
|
|
31
|
+
| "materialized_non_incremental";
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* The export configuration. The main entry point builds it from deploy-time
|
|
35
|
+
* params; handler consumers can construct it directly.
|
|
36
|
+
*
|
|
37
|
+
* `collectionPath`, `datasetId`, `tableId`, `location`, and `projectId` are
|
|
38
|
+
* required; everything else has a sensible default.
|
|
39
|
+
*/
|
|
40
|
+
export interface ExportConfig {
|
|
41
|
+
/** Firestore collection (or collection group) path to mirror, e.g. `users`
|
|
42
|
+
* or `regions/{regionId}/users`. */
|
|
43
|
+
collectionPath: ConfigValue<string>;
|
|
44
|
+
/** BigQuery dataset id to write into. */
|
|
45
|
+
datasetId: ConfigValue<string>;
|
|
46
|
+
/** BigQuery changelog table id. */
|
|
47
|
+
tableId: ConfigValue<string>;
|
|
48
|
+
|
|
49
|
+
/** Region for the trigger and task queue. */
|
|
50
|
+
location: ConfigValue<string>;
|
|
51
|
+
/** BigQuery dataset location, e.g. `us`, `eu`. Defaults to `us`. */
|
|
52
|
+
datasetLocation?: ConfigValue<string>;
|
|
53
|
+
/** Project that owns the BigQuery dataset, if different from the function's
|
|
54
|
+
* project. */
|
|
55
|
+
bqProjectId?: ConfigValue<string>;
|
|
56
|
+
/** GCP project id. */
|
|
57
|
+
projectId: ConfigValue<string>;
|
|
58
|
+
/** Firestore database id. Defaults to `(default)`. */
|
|
59
|
+
databaseId?: ConfigValue<string>;
|
|
60
|
+
|
|
61
|
+
/** Include path-param values as columns. Defaults to `false`. */
|
|
62
|
+
wildcardIds?: ConfigValue<boolean>;
|
|
63
|
+
/** Skip writing the previous document state on updates. Defaults to `false`. */
|
|
64
|
+
excludeOldData?: ConfigValue<boolean>;
|
|
65
|
+
/** Use the newer snapshot view query syntax. Defaults to `false`. */
|
|
66
|
+
useNewSnapshotQuerySyntax?: ConfigValue<boolean>;
|
|
67
|
+
/** Changelog view strategy. Defaults to `view`. */
|
|
68
|
+
viewType?: ConfigValue<ViewType>;
|
|
69
|
+
|
|
70
|
+
/** Partitioning strategy for the changelog table. */
|
|
71
|
+
partitioning?: ChangeTrackerConfig["partitioning"];
|
|
72
|
+
/** Clustering columns (max 4). */
|
|
73
|
+
clustering?: string[] | null;
|
|
74
|
+
/** Materialized-view max staleness interval, e.g. `0-0 0 4:0:0`. */
|
|
75
|
+
maxStaleness?: ConfigValue<string>;
|
|
76
|
+
/** Incremental materialized-view refresh interval in minutes. */
|
|
77
|
+
refreshIntervalMinutes?: ConfigValue<number>;
|
|
78
|
+
|
|
79
|
+
/** Collection id used for the changelog backup table. */
|
|
80
|
+
backupCollectionId?: ConfigValue<string>;
|
|
81
|
+
/** Name of a transform function to apply before writing. */
|
|
82
|
+
transformFunction?: ConfigValue<string>;
|
|
83
|
+
/** Customer-managed encryption key for the dataset. */
|
|
84
|
+
kmsKeyName?: ConfigValue<string>;
|
|
85
|
+
|
|
86
|
+
/** Log verbosity. Defaults to `info`. */
|
|
87
|
+
logLevel?: ConfigValue<TrackerLogLevel | LogLevel>;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/** {@link ExportConfig} with all defaults applied. */
|
|
91
|
+
export interface ResolvedExportConfig {
|
|
92
|
+
collectionPath: string;
|
|
93
|
+
datasetId: string;
|
|
94
|
+
tableId: string;
|
|
95
|
+
location: string;
|
|
96
|
+
datasetLocation: string;
|
|
97
|
+
bqProjectId?: string;
|
|
98
|
+
projectId: string;
|
|
99
|
+
databaseId: string;
|
|
100
|
+
wildcardIds: boolean;
|
|
101
|
+
excludeOldData: boolean;
|
|
102
|
+
useNewSnapshotQuerySyntax: boolean;
|
|
103
|
+
viewType: ViewType;
|
|
104
|
+
partitioning?: ChangeTrackerConfig["partitioning"];
|
|
105
|
+
clustering: string[] | null;
|
|
106
|
+
maxStaleness?: string;
|
|
107
|
+
refreshIntervalMinutes?: number;
|
|
108
|
+
backupCollectionId?: string;
|
|
109
|
+
transformFunction?: string;
|
|
110
|
+
kmsKeyName?: string;
|
|
111
|
+
logLevel: TrackerLogLevel;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
function isExpression<T extends string | number | boolean | string[]>(
|
|
115
|
+
value: ConfigValue<T>
|
|
116
|
+
): value is Expression<T> {
|
|
117
|
+
return typeof value === "object" && value !== null && "value" in value;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
function resolveConfigValue<T extends string | number | boolean | string[]>(
|
|
121
|
+
value: ConfigValue<T>
|
|
122
|
+
): T {
|
|
123
|
+
return isExpression(value) ? value.value() : value;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
function resolveOptionalConfigValue<
|
|
127
|
+
T extends string | number | boolean | string[]
|
|
128
|
+
>(value: ConfigValue<T> | undefined): T | undefined {
|
|
129
|
+
return value === undefined ? undefined : resolveConfigValue(value);
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* Applies defaults to an {@link ExportConfig}.
|
|
134
|
+
*
|
|
135
|
+
* @param config - The user-supplied configuration.
|
|
136
|
+
* @returns The configuration with every field populated.
|
|
137
|
+
*/
|
|
138
|
+
export function resolveExportConfig(
|
|
139
|
+
config: ExportConfig
|
|
140
|
+
): ResolvedExportConfig {
|
|
141
|
+
const projectId = resolveConfigValue(config.projectId);
|
|
142
|
+
const viewType = resolveOptionalConfigValue(config.viewType);
|
|
143
|
+
const logLevel = resolveOptionalConfigValue(config.logLevel);
|
|
144
|
+
|
|
145
|
+
return {
|
|
146
|
+
collectionPath: resolveConfigValue(config.collectionPath),
|
|
147
|
+
datasetId: resolveConfigValue(config.datasetId),
|
|
148
|
+
tableId: resolveConfigValue(config.tableId),
|
|
149
|
+
location: resolveConfigValue(config.location),
|
|
150
|
+
datasetLocation: resolveOptionalConfigValue(config.datasetLocation) ?? "us",
|
|
151
|
+
bqProjectId: resolveOptionalConfigValue(config.bqProjectId),
|
|
152
|
+
projectId,
|
|
153
|
+
databaseId: resolveOptionalConfigValue(config.databaseId) ?? "(default)",
|
|
154
|
+
wildcardIds: resolveOptionalConfigValue(config.wildcardIds) ?? false,
|
|
155
|
+
excludeOldData: resolveOptionalConfigValue(config.excludeOldData) ?? false,
|
|
156
|
+
useNewSnapshotQuerySyntax:
|
|
157
|
+
resolveOptionalConfigValue(config.useNewSnapshotQuerySyntax) ?? false,
|
|
158
|
+
viewType: viewType ?? "view",
|
|
159
|
+
partitioning: config.partitioning,
|
|
160
|
+
clustering: config.clustering ?? null,
|
|
161
|
+
maxStaleness: resolveOptionalConfigValue(config.maxStaleness),
|
|
162
|
+
refreshIntervalMinutes: resolveOptionalConfigValue(
|
|
163
|
+
config.refreshIntervalMinutes
|
|
164
|
+
),
|
|
165
|
+
backupCollectionId: resolveOptionalConfigValue(config.backupCollectionId),
|
|
166
|
+
transformFunction: resolveOptionalConfigValue(config.transformFunction),
|
|
167
|
+
kmsKeyName: resolveOptionalConfigValue(config.kmsKeyName),
|
|
168
|
+
logLevel: (logLevel as TrackerLogLevel) ?? "info",
|
|
169
|
+
};
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/**
|
|
173
|
+
* Maps a resolved export config onto the change-tracker's config shape.
|
|
174
|
+
*
|
|
175
|
+
* @param config - The resolved export configuration.
|
|
176
|
+
* @returns The {@link ChangeTrackerConfig} for the event history tracker.
|
|
177
|
+
*/
|
|
178
|
+
export function toTrackerConfig(
|
|
179
|
+
config: ResolvedExportConfig
|
|
180
|
+
): ChangeTrackerConfig {
|
|
181
|
+
return {
|
|
182
|
+
firestoreInstanceId: config.databaseId,
|
|
183
|
+
tableId: config.tableId,
|
|
184
|
+
datasetId: config.datasetId,
|
|
185
|
+
datasetLocation: config.datasetLocation,
|
|
186
|
+
backupTableId: config.backupCollectionId,
|
|
187
|
+
transformFunction: config.transformFunction,
|
|
188
|
+
partitioning: config.partitioning,
|
|
189
|
+
databaseId: config.databaseId,
|
|
190
|
+
clustering: config.clustering,
|
|
191
|
+
wildcardIds: config.wildcardIds,
|
|
192
|
+
bqProjectId: config.bqProjectId ?? config.projectId,
|
|
193
|
+
useNewSnapshotQuerySyntax: config.useNewSnapshotQuerySyntax,
|
|
194
|
+
skipInit: true,
|
|
195
|
+
kmsKeyName: config.kmsKeyName,
|
|
196
|
+
useMaterializedView:
|
|
197
|
+
config.viewType === "materialized_incremental" ||
|
|
198
|
+
config.viewType === "materialized_non_incremental",
|
|
199
|
+
useIncrementalMaterializedView:
|
|
200
|
+
config.viewType === "materialized_incremental",
|
|
201
|
+
maxStaleness: config.maxStaleness,
|
|
202
|
+
refreshIntervalMinutes: config.refreshIntervalMinutes,
|
|
203
|
+
logLevel: config.logLevel,
|
|
204
|
+
};
|
|
205
|
+
}
|
package/src/handlers.ts
ADDED
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Copyright 2019 Google LLC
|
|
3
|
+
*
|
|
4
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
* you may not use this file except in compliance with the License.
|
|
6
|
+
* You may obtain a copy of the License at
|
|
7
|
+
*
|
|
8
|
+
* https://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
*
|
|
10
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
* See the License for the specific language governing permissions and
|
|
14
|
+
* limitations under the License.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import {
|
|
18
|
+
ChangeType,
|
|
19
|
+
type FirestoreBigQueryEventHistoryTracker,
|
|
20
|
+
type FirestoreDocumentChangeEvent,
|
|
21
|
+
} from "@firebaseextensions/firestore-bigquery-change-tracker";
|
|
22
|
+
import type {
|
|
23
|
+
Change,
|
|
24
|
+
DocumentSnapshot,
|
|
25
|
+
FirestoreEvent,
|
|
26
|
+
} from "firebase-functions/firestore";
|
|
27
|
+
import * as events from "./events";
|
|
28
|
+
import type { ResolvedExportConfig } from "./export-config";
|
|
29
|
+
import * as logs from "./logs";
|
|
30
|
+
import { getChangeType, getDocumentId } from "./util";
|
|
31
|
+
|
|
32
|
+
/** Serialized Firestore change ready to write to BigQuery. */
|
|
33
|
+
interface SerializedDocumentChange {
|
|
34
|
+
timestamp: string;
|
|
35
|
+
eventId: string;
|
|
36
|
+
fullResourceName: string;
|
|
37
|
+
changeType: ChangeType;
|
|
38
|
+
documentId: string;
|
|
39
|
+
params: FirestoreDocumentChangeEvent["pathParams"] | null;
|
|
40
|
+
data: FirestoreDocumentChangeEvent["data"];
|
|
41
|
+
oldData: FirestoreDocumentChangeEvent["oldData"];
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/** The Firestore document-write event passed to {@link handleDocumentWrite}. */
|
|
45
|
+
export type DocumentWriteEvent = FirestoreEvent<
|
|
46
|
+
Change<DocumentSnapshot> | undefined,
|
|
47
|
+
Record<string, string>
|
|
48
|
+
>;
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Everything a handler needs to do its work, injected by the caller so the
|
|
52
|
+
* handlers stay free of global state.
|
|
53
|
+
*/
|
|
54
|
+
export interface HandlerContext {
|
|
55
|
+
tracker: FirestoreBigQueryEventHistoryTracker;
|
|
56
|
+
config: ResolvedExportConfig;
|
|
57
|
+
/**
|
|
58
|
+
* Provisions the BigQuery dataset/table/views once per instance. Only called
|
|
59
|
+
* after an inline write failure as a self-heal; the hot path relies on
|
|
60
|
+
* out-of-band provisioning (`initBigQuerySync` / `setupBigQuerySync`).
|
|
61
|
+
*/
|
|
62
|
+
ensureInitialized: () => Promise<void>;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Records a Firestore document change to BigQuery.
|
|
67
|
+
*
|
|
68
|
+
* @param change - Serialized change metadata and payload.
|
|
69
|
+
* @param tracker - The event history tracker to write through.
|
|
70
|
+
*/
|
|
71
|
+
async function recordEventToBigQuery(
|
|
72
|
+
change: SerializedDocumentChange,
|
|
73
|
+
tracker: FirestoreBigQueryEventHistoryTracker
|
|
74
|
+
): Promise<void> {
|
|
75
|
+
const event: FirestoreDocumentChangeEvent = {
|
|
76
|
+
timestamp: change.timestamp,
|
|
77
|
+
operation: change.changeType,
|
|
78
|
+
documentName: change.fullResourceName,
|
|
79
|
+
documentId: change.documentId,
|
|
80
|
+
pathParams: change.params,
|
|
81
|
+
eventId: change.eventId,
|
|
82
|
+
data: change.data,
|
|
83
|
+
oldData: change.oldData,
|
|
84
|
+
};
|
|
85
|
+
|
|
86
|
+
await tracker.record([event]);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* Gives a failed inline write one self-heal attempt before surfacing it to the
|
|
91
|
+
* Firestore trigger retry policy.
|
|
92
|
+
*
|
|
93
|
+
* @param change - The serialized change to write.
|
|
94
|
+
* @param ctx - The handler context.
|
|
95
|
+
*/
|
|
96
|
+
async function retryAfterSelfHeal(
|
|
97
|
+
change: SerializedDocumentChange,
|
|
98
|
+
ctx: HandlerContext
|
|
99
|
+
): Promise<void> {
|
|
100
|
+
try {
|
|
101
|
+
await ctx.ensureInitialized();
|
|
102
|
+
await recordEventToBigQuery(change, ctx.tracker);
|
|
103
|
+
} catch (retryErr) {
|
|
104
|
+
await events.recordErrorEvent(retryErr as Error);
|
|
105
|
+
|
|
106
|
+
logs.logFailedEventAction(
|
|
107
|
+
"Failed to write event to BigQuery from onWrite handler after self-heal",
|
|
108
|
+
change.fullResourceName,
|
|
109
|
+
change.eventId,
|
|
110
|
+
change.changeType,
|
|
111
|
+
retryErr as Error
|
|
112
|
+
);
|
|
113
|
+
|
|
114
|
+
throw retryErr;
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* Handles a Firestore document write: serializes the change and writes it to
|
|
120
|
+
* BigQuery. Failed writes are surfaced to the trigger retry policy after one
|
|
121
|
+
* self-heal attempt.
|
|
122
|
+
*
|
|
123
|
+
* @param event - The Firestore document-write event.
|
|
124
|
+
* @param ctx - The handler context.
|
|
125
|
+
*/
|
|
126
|
+
export async function handleDocumentWrite(
|
|
127
|
+
event: DocumentWriteEvent,
|
|
128
|
+
ctx: HandlerContext
|
|
129
|
+
): Promise<void> {
|
|
130
|
+
const { data, ...context } = event;
|
|
131
|
+
if (!data) return;
|
|
132
|
+
|
|
133
|
+
logs.start();
|
|
134
|
+
|
|
135
|
+
// No provisioning on the hot path: BigQuery resources are provisioned
|
|
136
|
+
// out-of-band (afterFirstDeploy / afterRedeploy tasks). If they are missing,
|
|
137
|
+
// the inline write fails, self-heals once, then falls back to the trigger
|
|
138
|
+
// retry policy.
|
|
139
|
+
const { config, tracker } = ctx;
|
|
140
|
+
const changeType = getChangeType(data);
|
|
141
|
+
const documentId = getDocumentId(data);
|
|
142
|
+
const isCreated = changeType === ChangeType.CREATE;
|
|
143
|
+
const isDeleted = changeType === ChangeType.DELETE;
|
|
144
|
+
|
|
145
|
+
const newData = isDeleted ? undefined : data.after.data();
|
|
146
|
+
const oldData =
|
|
147
|
+
isCreated || config.excludeOldData ? undefined : data.before.data();
|
|
148
|
+
|
|
149
|
+
const relativeName = context.document;
|
|
150
|
+
const projectId = config.projectId;
|
|
151
|
+
const fullResourceName = `projects/${projectId}/databases/${config.databaseId}/documents/${relativeName}`;
|
|
152
|
+
const eventId = context.id;
|
|
153
|
+
const operation = changeType;
|
|
154
|
+
|
|
155
|
+
logs.logEventAction(
|
|
156
|
+
"Firestore event received by onDocumentWritten trigger",
|
|
157
|
+
fullResourceName,
|
|
158
|
+
eventId,
|
|
159
|
+
operation
|
|
160
|
+
);
|
|
161
|
+
|
|
162
|
+
let serializedData: FirestoreDocumentChangeEvent["data"];
|
|
163
|
+
let serializedOldData: FirestoreDocumentChangeEvent["oldData"];
|
|
164
|
+
|
|
165
|
+
try {
|
|
166
|
+
serializedData = tracker.serializeData(newData);
|
|
167
|
+
serializedOldData = tracker.serializeData(oldData);
|
|
168
|
+
} catch (err) {
|
|
169
|
+
logs.logFailedEventAction(
|
|
170
|
+
"Failed to serialize data",
|
|
171
|
+
fullResourceName,
|
|
172
|
+
eventId,
|
|
173
|
+
operation,
|
|
174
|
+
err as Error
|
|
175
|
+
);
|
|
176
|
+
throw err;
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
try {
|
|
180
|
+
await events.recordStartEvent({
|
|
181
|
+
documentId,
|
|
182
|
+
changeType,
|
|
183
|
+
before: { data: data.before.data() },
|
|
184
|
+
after: { data: data.after.data() },
|
|
185
|
+
context,
|
|
186
|
+
});
|
|
187
|
+
} catch (err) {
|
|
188
|
+
logs.error(false, "Failed to record start event", err);
|
|
189
|
+
throw err;
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
const change: SerializedDocumentChange = {
|
|
193
|
+
timestamp: context.time,
|
|
194
|
+
eventId: context.id,
|
|
195
|
+
fullResourceName,
|
|
196
|
+
changeType,
|
|
197
|
+
documentId,
|
|
198
|
+
params: config.wildcardIds ? { ...context.params, documentId } : null,
|
|
199
|
+
data: serializedData,
|
|
200
|
+
oldData: serializedOldData,
|
|
201
|
+
};
|
|
202
|
+
|
|
203
|
+
try {
|
|
204
|
+
await recordEventToBigQuery(change, tracker);
|
|
205
|
+
} catch (err) {
|
|
206
|
+
logs.failedToWriteToBigQueryImmediately(err as Error);
|
|
207
|
+
await retryAfterSelfHeal(change, ctx);
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
logs.complete();
|
|
211
|
+
}
|
package/src/index.ts
ADDED
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Copyright 2019 Google LLC
|
|
3
|
+
*
|
|
4
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
* you may not use this file except in compliance with the License.
|
|
6
|
+
* You may obtain a copy of the License at
|
|
7
|
+
*
|
|
8
|
+
* https://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
*
|
|
10
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
* See the License for the specific language governing permissions and
|
|
14
|
+
* limitations under the License.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Main entry point. Exports the wired functions with deploy-time param
|
|
19
|
+
* expressions, then resolves concrete config lazily at runtime. Re-export
|
|
20
|
+
* `fsexportbigquery` and `initBigQuerySync` from your own functions codebase
|
|
21
|
+
* entry; configuration comes from a `.env` (or
|
|
22
|
+
* `.env.<projectId>`), which the Firebase CLI loads at deploy.
|
|
23
|
+
*
|
|
24
|
+
* Because this module initializes runtime dependencies lazily, deploy discovery
|
|
25
|
+
* can analyze it without resolving params too early.
|
|
26
|
+
* For side-effect-free imports (the handlers and config types), import from
|
|
27
|
+
* `./lib` instead.
|
|
28
|
+
*/
|
|
29
|
+
|
|
30
|
+
import { FirestoreBigQueryEventHistoryTracker } from "@firebaseextensions/firestore-bigquery-change-tracker";
|
|
31
|
+
import { getApps, initializeApp } from "firebase-admin/app";
|
|
32
|
+
import { onDocumentWritten } from "firebase-functions/firestore";
|
|
33
|
+
import { expr } from "firebase-functions/params";
|
|
34
|
+
import { onTaskDispatched } from "firebase-functions/tasks";
|
|
35
|
+
import type { Role } from "firebase-functions/v2";
|
|
36
|
+
import { requiresAPI, requiresRole } from "firebase-functions/v2";
|
|
37
|
+
import {
|
|
38
|
+
afterFirstDeploy,
|
|
39
|
+
afterRedeploy,
|
|
40
|
+
} from "firebase-functions/v2/lifecycle";
|
|
41
|
+
import { CONFIG_EXPRESSIONS, configFromEnv } from "./config";
|
|
42
|
+
import * as events from "./events";
|
|
43
|
+
import { resolveExportConfig, toTrackerConfig } from "./export-config";
|
|
44
|
+
import { type HandlerContext, handleDocumentWrite } from "./handlers";
|
|
45
|
+
import { createEnsureInitialized } from "./init";
|
|
46
|
+
import * as logs from "./logs";
|
|
47
|
+
|
|
48
|
+
// Re-export the side-effect-free library surface (handlers and config types).
|
|
49
|
+
export * from "./lib";
|
|
50
|
+
|
|
51
|
+
const INIT_BIGQUERY_SYNC_FUNCTION = "initBigQuerySync";
|
|
52
|
+
const SETUP_BIGQUERY_SYNC_FUNCTION = "setupBigQuerySync";
|
|
53
|
+
const LIFECYCLE_RETRY_CONFIG = {
|
|
54
|
+
maxAttempts: 15,
|
|
55
|
+
minBackoffSeconds: 60,
|
|
56
|
+
} as const;
|
|
57
|
+
const REQUIRED_ROLES: ReadonlyArray<Role> = [
|
|
58
|
+
"roles/bigquery.dataEditor",
|
|
59
|
+
"roles/datastore.user",
|
|
60
|
+
"roles/bigquery.user",
|
|
61
|
+
// Gen2 Firestore triggers need Eventarc receive and run.invoker on the function SA.
|
|
62
|
+
"roles/eventarc.eventReceiver",
|
|
63
|
+
"roles/run.invoker",
|
|
64
|
+
];
|
|
65
|
+
const REQUIRED_APIS = [
|
|
66
|
+
{
|
|
67
|
+
api: "bigquery.googleapis.com",
|
|
68
|
+
reason: "Mirrors data from your Cloud Firestore collection in BigQuery.",
|
|
69
|
+
},
|
|
70
|
+
] as const;
|
|
71
|
+
|
|
72
|
+
for (const role of REQUIRED_ROLES) {
|
|
73
|
+
requiresRole(role);
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
for (const { api, reason } of REQUIRED_APIS) {
|
|
77
|
+
requiresAPI(api, reason);
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
afterFirstDeploy({
|
|
81
|
+
task: { function: INIT_BIGQUERY_SYNC_FUNCTION, body: { data: {} } },
|
|
82
|
+
});
|
|
83
|
+
afterRedeploy({
|
|
84
|
+
task: { function: SETUP_BIGQUERY_SYNC_FUNCTION, body: { data: {} } },
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
let ctx: HandlerContext | undefined;
|
|
88
|
+
|
|
89
|
+
function getHandlerContext(): HandlerContext {
|
|
90
|
+
if (ctx) {
|
|
91
|
+
return ctx;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
const config = resolveExportConfig(configFromEnv());
|
|
95
|
+
const tracker = new FirestoreBigQueryEventHistoryTracker(
|
|
96
|
+
toTrackerConfig(config)
|
|
97
|
+
);
|
|
98
|
+
|
|
99
|
+
logs.logger.setLogLevel(config.logLevel);
|
|
100
|
+
logs.init(config);
|
|
101
|
+
|
|
102
|
+
if (getApps().length === 0) {
|
|
103
|
+
initializeApp();
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
events.setupEventChannel();
|
|
107
|
+
|
|
108
|
+
const ensureInitialized = createEnsureInitialized(tracker);
|
|
109
|
+
|
|
110
|
+
ctx = {
|
|
111
|
+
tracker,
|
|
112
|
+
config,
|
|
113
|
+
ensureInitialized,
|
|
114
|
+
};
|
|
115
|
+
|
|
116
|
+
return ctx;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
const functionOptions = {
|
|
120
|
+
region: CONFIG_EXPRESSIONS.location,
|
|
121
|
+
};
|
|
122
|
+
|
|
123
|
+
/**
|
|
124
|
+
* Firestore trigger: streams document writes on the watched collection into the
|
|
125
|
+
* BigQuery changelog table. Failed executions are retried by the Firebase
|
|
126
|
+
* Functions runtime.
|
|
127
|
+
*/
|
|
128
|
+
export const fsexportbigquery = onDocumentWritten(
|
|
129
|
+
{
|
|
130
|
+
...functionOptions,
|
|
131
|
+
document: expr`${CONFIG_EXPRESSIONS.collectionPath}/{documentId}`,
|
|
132
|
+
database: CONFIG_EXPRESSIONS.database,
|
|
133
|
+
retry: true,
|
|
134
|
+
},
|
|
135
|
+
(event) => handleDocumentWrite(event, getHandlerContext())
|
|
136
|
+
);
|
|
137
|
+
|
|
138
|
+
async function handleBigQuerySyncInitialization(): Promise<void> {
|
|
139
|
+
try {
|
|
140
|
+
await getHandlerContext().ensureInitialized();
|
|
141
|
+
} catch (err) {
|
|
142
|
+
logs.error(false, "Failed to initialize BigQuery resources", err);
|
|
143
|
+
throw err;
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/**
|
|
148
|
+
* First-deploy provisioning task (`afterFirstDeploy`). Runs in the function's
|
|
149
|
+
* own identity, so it has the runtime service account and bound secrets that
|
|
150
|
+
* creating the dataset/table/views needs (e.g. CMEK). Cloud Tasks retries a
|
|
151
|
+
* failed initialization on its own schedule, so a transient BigQuery error does
|
|
152
|
+
* not leave the resources unprovisioned. It is idempotent (`initialize()`
|
|
153
|
+
* no-ops when resources already exist) and can also be invoked directly as an
|
|
154
|
+
* authenticated HTTP POST, without queue retries.
|
|
155
|
+
*/
|
|
156
|
+
export const initBigQuerySync = onTaskDispatched(
|
|
157
|
+
{
|
|
158
|
+
...functionOptions,
|
|
159
|
+
retryConfig: LIFECYCLE_RETRY_CONFIG,
|
|
160
|
+
},
|
|
161
|
+
handleBigQuerySyncInitialization
|
|
162
|
+
);
|
|
163
|
+
|
|
164
|
+
/**
|
|
165
|
+
* Update/configure lifecycle task. Uses the same idempotent provisioning path
|
|
166
|
+
* so BigQuery resources are reconciled after parameter changes.
|
|
167
|
+
*/
|
|
168
|
+
export const setupBigQuerySync = onTaskDispatched(
|
|
169
|
+
{
|
|
170
|
+
...functionOptions,
|
|
171
|
+
retryConfig: LIFECYCLE_RETRY_CONFIG,
|
|
172
|
+
},
|
|
173
|
+
handleBigQuerySyncInitialization
|
|
174
|
+
);
|
package/src/init.ts
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Copyright 2019 Google LLC
|
|
3
|
+
*
|
|
4
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
* you may not use this file except in compliance with the License.
|
|
6
|
+
* You may obtain a copy of the License at
|
|
7
|
+
*
|
|
8
|
+
* https://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
*
|
|
10
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
* See the License for the specific language governing permissions and
|
|
14
|
+
* limitations under the License.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import type { FirestoreBigQueryEventHistoryTracker } from "@firebaseextensions/firestore-bigquery-change-tracker";
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Builds the provisioning guard used by the `initBigQuerySync` endpoint and the
|
|
21
|
+
* retry-path self-heal. The hot write path never calls it.
|
|
22
|
+
*
|
|
23
|
+
* The returned function runs `tracker.initialize()` at most once per instance:
|
|
24
|
+
* concurrent invocations on a cold instance share a single in-flight promise. A
|
|
25
|
+
* failed initialization resets the guard so a later invocation can retry after
|
|
26
|
+
* a transient error.
|
|
27
|
+
*
|
|
28
|
+
* @param tracker - The event history tracker whose `initialize()` provisions
|
|
29
|
+
* the BigQuery dataset, table, and views.
|
|
30
|
+
* @returns A function that resolves once the resources are provisioned.
|
|
31
|
+
*/
|
|
32
|
+
export function createEnsureInitialized(
|
|
33
|
+
tracker: FirestoreBigQueryEventHistoryTracker
|
|
34
|
+
): () => Promise<void> {
|
|
35
|
+
let initialization: Promise<void> | null = null;
|
|
36
|
+
return () => {
|
|
37
|
+
if (!initialization) {
|
|
38
|
+
initialization = tracker.initialize().catch((err) => {
|
|
39
|
+
initialization = null;
|
|
40
|
+
throw err;
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
return initialization;
|
|
44
|
+
};
|
|
45
|
+
}
|
package/src/lib.ts
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Copyright 2019 Google LLC
|
|
3
|
+
*
|
|
4
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
* you may not use this file except in compliance with the License.
|
|
6
|
+
* You may obtain a copy of the License at
|
|
7
|
+
*
|
|
8
|
+
* https://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
*
|
|
10
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
* See the License for the specific language governing permissions and
|
|
14
|
+
* limitations under the License.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Side-effect-free library surface:
|
|
19
|
+
*
|
|
20
|
+
* - {@link handleDocumentWrite} — the handler, for consumers who want to own
|
|
21
|
+
* trigger registration themselves. It takes an injected
|
|
22
|
+
* {@link HandlerContext}.
|
|
23
|
+
* - Config types and helpers ({@link ExportConfig}, {@link resolveExportConfig},
|
|
24
|
+
* {@link toTrackerConfig}) for building that context.
|
|
25
|
+
*
|
|
26
|
+
* Importing this module has no side effects (it reads no environment), so it is
|
|
27
|
+
* safe to import anywhere. The main entry point (`./index`) is the one that
|
|
28
|
+
* reads env params and exports the wired functions.
|
|
29
|
+
*
|
|
30
|
+
* The change-tracker engine is an internal dependency and is deliberately not
|
|
31
|
+
* re-exported; only the types the handler signatures need are surfaced.
|
|
32
|
+
*/
|
|
33
|
+
|
|
34
|
+
// Engine types referenced by the handler and config signatures. Type-only —
|
|
35
|
+
// the engine itself is internal. `ChangeType` is exported for handler tests and
|
|
36
|
+
// advanced consumers that build Firestore events directly.
|
|
37
|
+
export {
|
|
38
|
+
type ChangeTrackerConfig,
|
|
39
|
+
ChangeType,
|
|
40
|
+
type FirestoreBigQueryEventHistoryTracker,
|
|
41
|
+
} from "@firebaseextensions/firestore-bigquery-change-tracker";
|
|
42
|
+
// Config types and helpers
|
|
43
|
+
export {
|
|
44
|
+
type ExportConfig,
|
|
45
|
+
type ResolvedExportConfig,
|
|
46
|
+
resolveExportConfig,
|
|
47
|
+
toTrackerConfig,
|
|
48
|
+
type ViewType,
|
|
49
|
+
} from "./export-config";
|
|
50
|
+
// Handlers
|
|
51
|
+
export {
|
|
52
|
+
type DocumentWriteEvent,
|
|
53
|
+
type HandlerContext,
|
|
54
|
+
handleDocumentWrite,
|
|
55
|
+
} from "./handlers";
|