@firebaseextensions/firestore-bigquery-change-tracker 1.1.22 → 1.1.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/bigquery/index.js +7 -7
- package/lib/bigquery/partitioning.js +5 -2
- package/lib/bigquery/schema.js +2 -1
- package/lib/bigquery/snapshot.js +2 -1
- package/lib/bigquery/validateProject.js +2 -1
- package/lib/errors.js +4 -2
- package/lib/index.js +1 -0
- package/lib/logs.js +86 -43
- package/package.json +4 -3
package/lib/bigquery/index.js
CHANGED
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
* limitations under the License.
|
|
16
16
|
*/
|
|
17
17
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
18
|
-
exports.FirestoreBigQueryEventHistoryTracker = void 0;
|
|
18
|
+
exports.FirestoreBigQueryEventHistoryTracker = exports.RawChangelogViewSchema = exports.RawChangelogSchema = void 0;
|
|
19
19
|
const bigquery = require("@google-cloud/bigquery");
|
|
20
20
|
const firebase = require("firebase-admin");
|
|
21
21
|
const traverse = require("traverse");
|
|
@@ -79,7 +79,7 @@ class FirestoreBigQueryEventHistoryTracker {
|
|
|
79
79
|
}
|
|
80
80
|
async transformRows(rows) {
|
|
81
81
|
if (this.config.transformFunction && this.config.transformFunction !== "") {
|
|
82
|
-
const response = await node_fetch_1.default(this.config.transformFunction, {
|
|
82
|
+
const response = await (0, node_fetch_1.default)(this.config.transformFunction, {
|
|
83
83
|
method: "post",
|
|
84
84
|
body: JSON.stringify({ data: rows }),
|
|
85
85
|
headers: { "Content-Type": "application/json" },
|
|
@@ -188,7 +188,7 @@ class FirestoreBigQueryEventHistoryTracker {
|
|
|
188
188
|
}
|
|
189
189
|
// Exceeded number of retries, save in failed collection
|
|
190
190
|
if (!retry && this.config.backupTableId) {
|
|
191
|
-
await handleFailedTransactions_1.default(rows, this.config, e);
|
|
191
|
+
await (0, handleFailedTransactions_1.default)(rows, this.config, e);
|
|
192
192
|
}
|
|
193
193
|
// Reinitializing in case the destintation table is modified.
|
|
194
194
|
this._initialized = false;
|
|
@@ -267,7 +267,7 @@ class FirestoreBigQueryEventHistoryTracker {
|
|
|
267
267
|
logs.addNewColumn(this.rawChangeLogTableName(), schema_1.documentPathParams.name);
|
|
268
268
|
}
|
|
269
269
|
/** Updated table metadata if required */
|
|
270
|
-
const shouldUpdate = await checkUpdates_1.tableRequiresUpdate({
|
|
270
|
+
const shouldUpdate = await (0, checkUpdates_1.tableRequiresUpdate)({
|
|
271
271
|
table,
|
|
272
272
|
config: this.config,
|
|
273
273
|
documentIdColExists,
|
|
@@ -331,7 +331,7 @@ class FirestoreBigQueryEventHistoryTracker {
|
|
|
331
331
|
const pathParamsColExists = columnNames.includes("path_params");
|
|
332
332
|
const oldDataColExists = columnNames.includes("old_data");
|
|
333
333
|
/** If new view or opt-in to new query syntax **/
|
|
334
|
-
const updateView = checkUpdates_1.viewRequiresUpdate({
|
|
334
|
+
const updateView = (0, checkUpdates_1.viewRequiresUpdate)({
|
|
335
335
|
metadata,
|
|
336
336
|
config: this.config,
|
|
337
337
|
documentIdColExists,
|
|
@@ -339,7 +339,7 @@ class FirestoreBigQueryEventHistoryTracker {
|
|
|
339
339
|
oldDataColExists,
|
|
340
340
|
});
|
|
341
341
|
if (updateView) {
|
|
342
|
-
metadata.view = snapshot_1.latestConsistentSnapshotView({
|
|
342
|
+
metadata.view = (0, snapshot_1.latestConsistentSnapshotView)({
|
|
343
343
|
datasetId: this.config.datasetId,
|
|
344
344
|
tableName: this.rawChangeLogTableName(),
|
|
345
345
|
schema,
|
|
@@ -362,7 +362,7 @@ class FirestoreBigQueryEventHistoryTracker {
|
|
|
362
362
|
if (this.config.wildcardIds) {
|
|
363
363
|
schema.fields.push(schema_1.documentPathParams);
|
|
364
364
|
}
|
|
365
|
-
const latestSnapshot = snapshot_1.latestConsistentSnapshotView({
|
|
365
|
+
const latestSnapshot = (0, snapshot_1.latestConsistentSnapshotView)({
|
|
366
366
|
datasetId: this.config.datasetId,
|
|
367
367
|
tableName: this.rawChangeLogTableName(),
|
|
368
368
|
schema,
|
|
@@ -37,7 +37,10 @@ class Partitioning {
|
|
|
37
37
|
/* Check if valid timestamp value from sdk */
|
|
38
38
|
if (value instanceof firebase.firestore.Timestamp)
|
|
39
39
|
return true;
|
|
40
|
-
/* Check if valid date/
|
|
40
|
+
/* Check if valid date/timstemap, expedted result from production */
|
|
41
|
+
if (value && value.toDate && value.toDate())
|
|
42
|
+
return true;
|
|
43
|
+
/* Check if valid date/time value from the console, expected result from testing locally */
|
|
41
44
|
return Object.prototype.toString.call(value) === "[object Date]";
|
|
42
45
|
}
|
|
43
46
|
hasHourAndDatePartitionConfig() {
|
|
@@ -196,7 +199,7 @@ class Partitioning {
|
|
|
196
199
|
// Field already exists on schema, skip
|
|
197
200
|
if (this.customFieldExists(fields))
|
|
198
201
|
return Promise.resolve();
|
|
199
|
-
fields.push(schema_1.getNewPartitionField(this.config));
|
|
202
|
+
fields.push((0, schema_1.getNewPartitionField)(this.config));
|
|
200
203
|
/** log successful addition of partition column */
|
|
201
204
|
logs.addPartitionFieldColumn(this.table.id, this.config.timePartitioningField);
|
|
202
205
|
return Promise.resolve();
|
package/lib/bigquery/schema.js
CHANGED
|
@@ -136,7 +136,7 @@ exports.RawChangelogSchema = {
|
|
|
136
136
|
],
|
|
137
137
|
};
|
|
138
138
|
// Helper function for Partitioned Changelogs field
|
|
139
|
-
|
|
139
|
+
const getNewPartitionField = (config) => {
|
|
140
140
|
const { timePartitioningField, timePartitioningFieldType } = config;
|
|
141
141
|
return {
|
|
142
142
|
name: timePartitioningField,
|
|
@@ -145,3 +145,4 @@ exports.getNewPartitionField = (config) => {
|
|
|
145
145
|
description: "The document TimePartition partition field selected by user",
|
|
146
146
|
};
|
|
147
147
|
};
|
|
148
|
+
exports.getNewPartitionField = getNewPartitionField;
|
package/lib/bigquery/snapshot.js
CHANGED
|
@@ -19,7 +19,7 @@ exports.buildLatestSnapshotViewQuery = exports.latestConsistentSnapshotView = vo
|
|
|
19
19
|
const sqlFormatter = require("sql-formatter");
|
|
20
20
|
const schema_1 = require("./schema");
|
|
21
21
|
const excludeFields = ["document_name", "document_id"];
|
|
22
|
-
|
|
22
|
+
const latestConsistentSnapshotView = ({ datasetId, tableName, schema, bqProjectId, useLegacyQuery = false, }) => ({
|
|
23
23
|
query: buildLatestSnapshotViewQuery({
|
|
24
24
|
datasetId,
|
|
25
25
|
tableName,
|
|
@@ -32,6 +32,7 @@ exports.latestConsistentSnapshotView = ({ datasetId, tableName, schema, bqProjec
|
|
|
32
32
|
}),
|
|
33
33
|
useLegacySql: false,
|
|
34
34
|
});
|
|
35
|
+
exports.latestConsistentSnapshotView = latestConsistentSnapshotView;
|
|
35
36
|
function buildLatestSnapshotViewQuery({ datasetId, tableName, timestampColumnName, groupByColumns, bqProjectId, useLegacyQuery = true, }) {
|
|
36
37
|
if (datasetId === "" || tableName === "" || timestampColumnName === "") {
|
|
37
38
|
throw Error(`Missing some query parameters!`);
|
|
@@ -5,7 +5,7 @@ const { ProjectsClient } = require("@google-cloud/resource-manager");
|
|
|
5
5
|
/* TODO: searchProjectsAsync sometimes returns {}.
|
|
6
6
|
* Could be resource intensive, if checked on every records insert.
|
|
7
7
|
*/
|
|
8
|
-
|
|
8
|
+
const validateProject = async (id) => {
|
|
9
9
|
let isValid = false;
|
|
10
10
|
const client = new ProjectsClient();
|
|
11
11
|
const projects = client.searchProjectsAsync();
|
|
@@ -16,3 +16,4 @@ exports.validateProject = async (id) => {
|
|
|
16
16
|
}
|
|
17
17
|
return isValid;
|
|
18
18
|
};
|
|
19
|
+
exports.validateProject = validateProject;
|
package/lib/errors.js
CHANGED
|
@@ -16,5 +16,7 @@
|
|
|
16
16
|
*/
|
|
17
17
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
18
18
|
exports.changedFieldType = exports.changedFieldMode = void 0;
|
|
19
|
-
|
|
20
|
-
exports.
|
|
19
|
+
const changedFieldMode = (fieldName, bqMode, schemaMode) => new Error(`Field ${fieldName} has different field mode. BigQuery mode: ${bqMode}; Schema mode: ${schemaMode}`);
|
|
20
|
+
exports.changedFieldMode = changedFieldMode;
|
|
21
|
+
const changedFieldType = (fieldName, bqType, schemaType) => new Error(`Field: ${fieldName} has changed field type. BigQuery type: ${bqType}; Schema type: ${schemaType}`);
|
|
22
|
+
exports.changedFieldType = changedFieldType;
|
package/lib/index.js
CHANGED
|
@@ -15,6 +15,7 @@
|
|
|
15
15
|
* limitations under the License.
|
|
16
16
|
*/
|
|
17
17
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
18
|
+
exports.ChangeType = exports.RawChangelogViewSchema = exports.RawChangelogSchema = exports.FirestoreBigQueryEventHistoryTracker = void 0;
|
|
18
19
|
var bigquery_1 = require("./bigquery");
|
|
19
20
|
Object.defineProperty(exports, "FirestoreBigQueryEventHistoryTracker", { enumerable: true, get: function () { return bigquery_1.FirestoreBigQueryEventHistoryTracker; } });
|
|
20
21
|
Object.defineProperty(exports, "RawChangelogSchema", { enumerable: true, get: function () { return bigquery_1.RawChangelogSchema; } });
|
package/lib/logs.js
CHANGED
|
@@ -17,113 +17,148 @@
|
|
|
17
17
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
18
18
|
exports.updatingMetadata = exports.failedToInitializeWait = exports.tableCreationError = exports.invalidClustering = exports.hourAndDatePartitioningWarning = exports.invalidTableReference = exports.invalidProjectIdWarning = exports.cannotPartitionExistingTable = exports.removedClustering = exports.updatedClustering = exports.bigQueryTableInsertErrors = exports.firestoreTimePartitioningParametersWarning = exports.firestoreTimePartitionFieldError = exports.addPartitionFieldColumn = exports.addNewColumn = exports.timestampMissingValue = exports.error = exports.dataTypeInvalid = exports.dataInserting = exports.dataInsertRetried = exports.dataInserted = exports.complete = exports.bigQueryViewValidating = exports.bigQueryViewValidated = exports.bigQueryViewUpToDate = exports.bigQueryViewUpdating = exports.bigQueryViewUpdated = exports.bigQueryViewAlreadyExists = exports.bigQueryViewCreating = exports.bigQueryViewCreated = exports.bigQueryUserDefinedFunctionCreated = exports.bigQueryUserDefinedFunctionCreating = exports.bigQueryTableValidating = exports.bigQueryTableValidated = exports.bigQueryTableUpToDate = exports.bigQueryTableUpdating = exports.bigQueryTableUpdated = exports.bigQueryTableCreating = exports.bigQueryTableCreated = exports.bigQueryTableAlreadyExists = exports.bigQuerySchemaViewCreated = exports.bigQueryLatestSnapshotViewQueryCreated = exports.bigQueryErrorRecordingDocumentChange = exports.bigQueryDatasetExists = exports.bigQueryDatasetCreating = exports.bigQueryDatasetCreated = exports.arrayFieldInvalid = void 0;
|
|
19
19
|
const firebase_functions_1 = require("firebase-functions");
|
|
20
|
-
|
|
20
|
+
const arrayFieldInvalid = (fieldName) => {
|
|
21
21
|
firebase_functions_1.logger.warn(`Array field '${fieldName}' does not contain an array, skipping`);
|
|
22
22
|
};
|
|
23
|
-
exports.
|
|
23
|
+
exports.arrayFieldInvalid = arrayFieldInvalid;
|
|
24
|
+
const bigQueryDatasetCreated = (datasetId) => {
|
|
24
25
|
firebase_functions_1.logger.log(`Created BigQuery dataset: ${datasetId}`);
|
|
25
26
|
};
|
|
26
|
-
exports.
|
|
27
|
+
exports.bigQueryDatasetCreated = bigQueryDatasetCreated;
|
|
28
|
+
const bigQueryDatasetCreating = (datasetId) => {
|
|
27
29
|
firebase_functions_1.logger.log(`Creating BigQuery dataset: ${datasetId}`);
|
|
28
30
|
};
|
|
29
|
-
exports.
|
|
31
|
+
exports.bigQueryDatasetCreating = bigQueryDatasetCreating;
|
|
32
|
+
const bigQueryDatasetExists = (datasetId) => {
|
|
30
33
|
firebase_functions_1.logger.log(`BigQuery dataset already exists: ${datasetId}`);
|
|
31
34
|
};
|
|
32
|
-
exports.
|
|
35
|
+
exports.bigQueryDatasetExists = bigQueryDatasetExists;
|
|
36
|
+
const bigQueryErrorRecordingDocumentChange = (e) => {
|
|
33
37
|
firebase_functions_1.logger.error(`Error recording document changes.`, e);
|
|
34
38
|
};
|
|
35
|
-
exports.
|
|
39
|
+
exports.bigQueryErrorRecordingDocumentChange = bigQueryErrorRecordingDocumentChange;
|
|
40
|
+
const bigQueryLatestSnapshotViewQueryCreated = (query) => {
|
|
36
41
|
firebase_functions_1.logger.log(`BigQuery latest snapshot view query:\n${query}`);
|
|
37
42
|
};
|
|
38
|
-
exports.
|
|
43
|
+
exports.bigQueryLatestSnapshotViewQueryCreated = bigQueryLatestSnapshotViewQueryCreated;
|
|
44
|
+
const bigQuerySchemaViewCreated = (name) => {
|
|
39
45
|
firebase_functions_1.logger.log(`BigQuery created schema view ${name}\n`);
|
|
40
46
|
};
|
|
41
|
-
exports.
|
|
47
|
+
exports.bigQuerySchemaViewCreated = bigQuerySchemaViewCreated;
|
|
48
|
+
const bigQueryTableAlreadyExists = (tableName, datasetName) => {
|
|
42
49
|
firebase_functions_1.logger.log(`BigQuery table with name ${tableName} already ` +
|
|
43
50
|
`exists in dataset ${datasetName}!`);
|
|
44
51
|
};
|
|
45
|
-
exports.
|
|
52
|
+
exports.bigQueryTableAlreadyExists = bigQueryTableAlreadyExists;
|
|
53
|
+
const bigQueryTableCreated = (tableName) => {
|
|
46
54
|
firebase_functions_1.logger.log(`Created BigQuery table: ${tableName}`);
|
|
47
55
|
};
|
|
48
|
-
exports.
|
|
56
|
+
exports.bigQueryTableCreated = bigQueryTableCreated;
|
|
57
|
+
const bigQueryTableCreating = (tableName) => {
|
|
49
58
|
firebase_functions_1.logger.log(`Creating BigQuery table: ${tableName}`);
|
|
50
59
|
};
|
|
51
|
-
exports.
|
|
60
|
+
exports.bigQueryTableCreating = bigQueryTableCreating;
|
|
61
|
+
const bigQueryTableUpdated = (tableName) => {
|
|
52
62
|
firebase_functions_1.logger.log(`Updated existing BigQuery table: ${tableName}`);
|
|
53
63
|
};
|
|
54
|
-
exports.
|
|
64
|
+
exports.bigQueryTableUpdated = bigQueryTableUpdated;
|
|
65
|
+
const bigQueryTableUpdating = (tableName) => {
|
|
55
66
|
firebase_functions_1.logger.log(`Updating existing BigQuery table: ${tableName}`);
|
|
56
67
|
};
|
|
57
|
-
exports.
|
|
68
|
+
exports.bigQueryTableUpdating = bigQueryTableUpdating;
|
|
69
|
+
const bigQueryTableUpToDate = (tableName) => {
|
|
58
70
|
firebase_functions_1.logger.log(`BigQuery table: ${tableName} is up to date`);
|
|
59
71
|
};
|
|
60
|
-
exports.
|
|
72
|
+
exports.bigQueryTableUpToDate = bigQueryTableUpToDate;
|
|
73
|
+
const bigQueryTableValidated = (tableName) => {
|
|
61
74
|
firebase_functions_1.logger.log(`Validated existing BigQuery table: ${tableName}`);
|
|
62
75
|
};
|
|
63
|
-
exports.
|
|
76
|
+
exports.bigQueryTableValidated = bigQueryTableValidated;
|
|
77
|
+
const bigQueryTableValidating = (tableName) => {
|
|
64
78
|
firebase_functions_1.logger.log(`Validating existing BigQuery table: ${tableName}`);
|
|
65
79
|
};
|
|
66
|
-
exports.
|
|
80
|
+
exports.bigQueryTableValidating = bigQueryTableValidating;
|
|
81
|
+
const bigQueryUserDefinedFunctionCreating = (functionName) => {
|
|
67
82
|
firebase_functions_1.logger.log(`Creating BigQuery user-defined function ${functionName}`);
|
|
68
83
|
};
|
|
69
|
-
exports.
|
|
84
|
+
exports.bigQueryUserDefinedFunctionCreating = bigQueryUserDefinedFunctionCreating;
|
|
85
|
+
const bigQueryUserDefinedFunctionCreated = (functionName) => {
|
|
70
86
|
firebase_functions_1.logger.log(`Created BigQuery user-defined function ${functionName}`);
|
|
71
87
|
};
|
|
72
|
-
exports.
|
|
88
|
+
exports.bigQueryUserDefinedFunctionCreated = bigQueryUserDefinedFunctionCreated;
|
|
89
|
+
const bigQueryViewCreated = (viewName) => {
|
|
73
90
|
firebase_functions_1.logger.log(`Created BigQuery view: ${viewName}`);
|
|
74
91
|
};
|
|
75
|
-
exports.
|
|
92
|
+
exports.bigQueryViewCreated = bigQueryViewCreated;
|
|
93
|
+
const bigQueryViewCreating = (viewName, query) => {
|
|
76
94
|
firebase_functions_1.logger.log(`Creating BigQuery view: ${viewName}\nQuery:\n${query}`);
|
|
77
95
|
};
|
|
78
|
-
exports.
|
|
96
|
+
exports.bigQueryViewCreating = bigQueryViewCreating;
|
|
97
|
+
const bigQueryViewAlreadyExists = (viewName, datasetName) => {
|
|
79
98
|
firebase_functions_1.logger.log(`View with id ${viewName} already exists in dataset ${datasetName}.`);
|
|
80
99
|
};
|
|
81
|
-
exports.
|
|
100
|
+
exports.bigQueryViewAlreadyExists = bigQueryViewAlreadyExists;
|
|
101
|
+
const bigQueryViewUpdated = (viewName) => {
|
|
82
102
|
firebase_functions_1.logger.log(`Updated existing BigQuery view: ${viewName}`);
|
|
83
103
|
};
|
|
84
|
-
exports.
|
|
104
|
+
exports.bigQueryViewUpdated = bigQueryViewUpdated;
|
|
105
|
+
const bigQueryViewUpdating = (viewName) => {
|
|
85
106
|
firebase_functions_1.logger.log(`Updating existing BigQuery view: ${viewName}`);
|
|
86
107
|
};
|
|
87
|
-
exports.
|
|
108
|
+
exports.bigQueryViewUpdating = bigQueryViewUpdating;
|
|
109
|
+
const bigQueryViewUpToDate = (viewName) => {
|
|
88
110
|
firebase_functions_1.logger.log(`BigQuery view: ${viewName} is up to date`);
|
|
89
111
|
};
|
|
90
|
-
exports.
|
|
112
|
+
exports.bigQueryViewUpToDate = bigQueryViewUpToDate;
|
|
113
|
+
const bigQueryViewValidated = (viewName) => {
|
|
91
114
|
firebase_functions_1.logger.log(`Validated existing BigQuery view: ${viewName}`);
|
|
92
115
|
};
|
|
93
|
-
exports.
|
|
116
|
+
exports.bigQueryViewValidated = bigQueryViewValidated;
|
|
117
|
+
const bigQueryViewValidating = (viewName) => {
|
|
94
118
|
firebase_functions_1.logger.log(`Validating existing BigQuery view: ${viewName}`);
|
|
95
119
|
};
|
|
96
|
-
exports.
|
|
120
|
+
exports.bigQueryViewValidating = bigQueryViewValidating;
|
|
121
|
+
const complete = () => {
|
|
97
122
|
firebase_functions_1.logger.log("Completed mod execution");
|
|
98
123
|
};
|
|
99
|
-
exports.
|
|
124
|
+
exports.complete = complete;
|
|
125
|
+
const dataInserted = (rowCount) => {
|
|
100
126
|
firebase_functions_1.logger.log(`Inserted ${rowCount} row(s) of data into BigQuery`);
|
|
101
127
|
};
|
|
102
|
-
exports.
|
|
128
|
+
exports.dataInserted = dataInserted;
|
|
129
|
+
const dataInsertRetried = (rowCount) => {
|
|
103
130
|
firebase_functions_1.logger.log(`Retried to insert ${rowCount} row(s) of data into BigQuery (ignoring unknown columns)`);
|
|
104
131
|
};
|
|
105
|
-
exports.
|
|
132
|
+
exports.dataInsertRetried = dataInsertRetried;
|
|
133
|
+
const dataInserting = (rowCount) => {
|
|
106
134
|
firebase_functions_1.logger.log(`Inserting ${rowCount} row(s) of data into BigQuery`);
|
|
107
135
|
};
|
|
108
|
-
exports.
|
|
136
|
+
exports.dataInserting = dataInserting;
|
|
137
|
+
const dataTypeInvalid = (fieldName, fieldType, dataType) => {
|
|
109
138
|
firebase_functions_1.logger.warn(`Field '${fieldName}' has invalid data. Expected: ${fieldType}, received: ${dataType}`);
|
|
110
139
|
};
|
|
111
|
-
exports.
|
|
140
|
+
exports.dataTypeInvalid = dataTypeInvalid;
|
|
141
|
+
const error = (err) => {
|
|
112
142
|
firebase_functions_1.logger.error("Error when mirroring data to BigQuery", err);
|
|
113
143
|
};
|
|
114
|
-
exports.
|
|
144
|
+
exports.error = error;
|
|
145
|
+
const timestampMissingValue = (fieldName) => {
|
|
115
146
|
firebase_functions_1.logger.warn(`Missing value for timestamp field: ${fieldName}, using default timestamp instead.`);
|
|
116
147
|
};
|
|
117
|
-
exports.
|
|
148
|
+
exports.timestampMissingValue = timestampMissingValue;
|
|
149
|
+
const addNewColumn = (table, field) => {
|
|
118
150
|
firebase_functions_1.logger.log(`Updated '${table}' table with a '${field}' column`);
|
|
119
151
|
};
|
|
120
|
-
exports.
|
|
152
|
+
exports.addNewColumn = addNewColumn;
|
|
153
|
+
const addPartitionFieldColumn = (table, field) => {
|
|
121
154
|
firebase_functions_1.logger.log(`Updated '${table}' table with a partition field '${field}' column`);
|
|
122
155
|
};
|
|
123
|
-
exports.
|
|
156
|
+
exports.addPartitionFieldColumn = addPartitionFieldColumn;
|
|
157
|
+
const firestoreTimePartitionFieldError = (documentName, fieldName, firestoreFieldName, firestoreFieldData) => {
|
|
124
158
|
firebase_functions_1.logger.warn(`Wrong type of Firestore Field for TimePartitioning. Accepts only strings in BigQuery format (DATE, DATETIME, TIMESTAMP) and Firestore Timestamp. Firestore Document field path: ${documentName}. Field name: ${firestoreFieldName}. Field data: ${firestoreFieldData}. Schema field "${fieldName}" value will be null.`);
|
|
125
159
|
};
|
|
126
|
-
exports.
|
|
160
|
+
exports.firestoreTimePartitionFieldError = firestoreTimePartitionFieldError;
|
|
161
|
+
const firestoreTimePartitioningParametersWarning = (fieldName, fieldType, firestoreFieldName, dataFirestoreField) => {
|
|
127
162
|
firebase_functions_1.logger.warn("All TimePartitioning option parameters need to be available to create new custom schema field");
|
|
128
163
|
!fieldName && firebase_functions_1.logger.warn(`Parameter missing: TIME_PARTITIONING_FIELD`);
|
|
129
164
|
!fieldType && firebase_functions_1.logger.warn(`Parameter missing: TIME_PARTITIONING_FIELD_TYPE`);
|
|
@@ -132,7 +167,8 @@ exports.firestoreTimePartitioningParametersWarning = (fieldName, fieldType, fire
|
|
|
132
167
|
!dataFirestoreField &&
|
|
133
168
|
firebase_functions_1.logger.warn(`No data found in Firestore Document under selected field: "${firestoreFieldName}"`);
|
|
134
169
|
};
|
|
135
|
-
exports.
|
|
170
|
+
exports.firestoreTimePartitioningParametersWarning = firestoreTimePartitioningParametersWarning;
|
|
171
|
+
const bigQueryTableInsertErrors = (insertErrors) => {
|
|
136
172
|
firebase_functions_1.logger.warn(`Error when inserting data to table.`);
|
|
137
173
|
insertErrors?.forEach((error) => {
|
|
138
174
|
firebase_functions_1.logger.warn("ROW DATA JSON:");
|
|
@@ -140,15 +176,19 @@ exports.bigQueryTableInsertErrors = (insertErrors) => {
|
|
|
140
176
|
error.errors?.forEach((error) => firebase_functions_1.logger.warn(`ROW ERROR MESSAGE: ${error.message}`));
|
|
141
177
|
});
|
|
142
178
|
};
|
|
143
|
-
exports.
|
|
179
|
+
exports.bigQueryTableInsertErrors = bigQueryTableInsertErrors;
|
|
180
|
+
const updatedClustering = (fields) => {
|
|
144
181
|
firebase_functions_1.logger.info(`Clustering updated with new settings fields: ${fields}`);
|
|
145
182
|
};
|
|
146
|
-
exports.
|
|
183
|
+
exports.updatedClustering = updatedClustering;
|
|
184
|
+
const removedClustering = (tableName) => {
|
|
147
185
|
firebase_functions_1.logger.info(`Clustering removed on ${tableName}`);
|
|
148
186
|
};
|
|
149
|
-
exports.
|
|
187
|
+
exports.removedClustering = removedClustering;
|
|
188
|
+
const cannotPartitionExistingTable = (table) => {
|
|
150
189
|
firebase_functions_1.logger.warn(`Cannot partition an existing table ${table.dataset.id}_${table.id}`);
|
|
151
190
|
};
|
|
191
|
+
exports.cannotPartitionExistingTable = cannotPartitionExistingTable;
|
|
152
192
|
function invalidProjectIdWarning(bqProjectId) {
|
|
153
193
|
firebase_functions_1.logger.warn(`Invalid project Id ${bqProjectId}, data cannot be synchronized`);
|
|
154
194
|
}
|
|
@@ -165,12 +205,15 @@ function invalidClustering(fields) {
|
|
|
165
205
|
firebase_functions_1.logger.warn(`Unable to add clustering, field(s) ${fields} do not exist on the expected table`);
|
|
166
206
|
}
|
|
167
207
|
exports.invalidClustering = invalidClustering;
|
|
168
|
-
|
|
208
|
+
const tableCreationError = (table, message) => {
|
|
169
209
|
firebase_functions_1.logger.warn(`Error caught creating table`, message);
|
|
170
210
|
};
|
|
171
|
-
exports.
|
|
211
|
+
exports.tableCreationError = tableCreationError;
|
|
212
|
+
const failedToInitializeWait = (message) => {
|
|
172
213
|
firebase_functions_1.logger.warn(`Failed while waiting to initialize.`, message);
|
|
173
214
|
};
|
|
174
|
-
exports.
|
|
215
|
+
exports.failedToInitializeWait = failedToInitializeWait;
|
|
216
|
+
const updatingMetadata = (tableName, resources) => {
|
|
175
217
|
firebase_functions_1.logger.info(`Updated Metadata on ${tableName}, ${JSON.stringify(resources)})`);
|
|
176
218
|
};
|
|
219
|
+
exports.updatingMetadata = updatingMetadata;
|
package/package.json
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
"url": "github.com/firebase/extensions.git",
|
|
6
6
|
"directory": "firestore-bigquery-export/firestore-bigquery-change-tracker"
|
|
7
7
|
},
|
|
8
|
-
"version": "1.1.
|
|
8
|
+
"version": "1.1.23",
|
|
9
9
|
"description": "Core change-tracker library for Cloud Firestore Collection BigQuery Exports",
|
|
10
10
|
"main": "./lib/index.js",
|
|
11
11
|
"scripts": {
|
|
@@ -25,7 +25,7 @@
|
|
|
25
25
|
"dependencies": {
|
|
26
26
|
"@google-cloud/bigquery": "^4.7.0",
|
|
27
27
|
"@google-cloud/resource-manager": "^3.0.0",
|
|
28
|
-
"firebase-admin": "^
|
|
28
|
+
"firebase-admin": "^11.4.1",
|
|
29
29
|
"firebase-functions": "^3.13.2",
|
|
30
30
|
"generate-schema": "^2.6.0",
|
|
31
31
|
"inquirer": "^6.4.0",
|
|
@@ -35,8 +35,9 @@
|
|
|
35
35
|
"traverse": "^0.6.6"
|
|
36
36
|
},
|
|
37
37
|
"devDependencies": {
|
|
38
|
+
"@types/node": "14.18.34",
|
|
38
39
|
"@types/traverse": "^0.6.32",
|
|
39
|
-
"typescript": "^
|
|
40
|
+
"typescript": "^4.9.4",
|
|
40
41
|
"rimraf": "^2.6.3",
|
|
41
42
|
"nyc": "^14.0.0",
|
|
42
43
|
"jest": "^24.9.0",
|