@firebaseextensions/firestore-bigquery-change-tracker 1.1.17 → 1.1.19

package/lib/bigquery/checkUpdates.js

@@ -2,7 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.viewRequiresUpdate = exports.tableRequiresUpdate = void 0;
 const partitioning_1 = require("./partitioning");
-async function tableRequiresUpdate({ table, config, documentIdColExists, pathParamsColExists, }) {
+async function tableRequiresUpdate({ table, config, documentIdColExists, pathParamsColExists, oldDataColExists, }) {
     /* Setup checks */
     const { metadata } = table;
     /** Check clustering */
@@ -16,6 +16,9 @@ async function tableRequiresUpdate({ table, config, documentIdColExists, pathPar
     /** Check document id column */
     if (!documentIdColExists)
         return true;
+    /** Check old_data column exists */
+    if (!oldDataColExists)
+        return true;
     /** Check partitioning */
     const partitioning = new partitioning_1.Partitioning(config, table);
     const isValidPartition = await partitioning.isValidPartitionForExistingTable();
@@ -25,16 +28,16 @@ async function tableRequiresUpdate({ table, config, documentIdColExists, pathPar
     return false;
 }
 exports.tableRequiresUpdate = tableRequiresUpdate;
-function viewRequiresUpdate({ metadata, config, documentIdColExists, pathParamsColExists, }) {
-    /** Check if documentId column exists */
-    if (!documentIdColExists)
-        return true;
+function viewRequiresUpdate({ metadata, config, documentIdColExists, pathParamsColExists, oldDataColExists, }) {
     /** Check wildcards */
     if (!!config.wildcardIds !== pathParamsColExists)
         return true;
     /** Check document id column */
     if (!documentIdColExists)
         return true;
+    /** Check old_data column exists */
+    if (!oldDataColExists)
+        return true;
     /* Using the new query syntax for snapshots */
     if (metadata) {
         const query = metadata.view?.query || "";

package/lib/bigquery/index.js

@@ -253,6 +253,11 @@ class FirestoreBigQueryEventHistoryTracker {
             await clustering.updateClustering(metadata);
             const documentIdColExists = fields.find((column) => column.name === "document_id");
             const pathParamsColExists = fields.find((column) => column.name === "path_params");
+            const oldDataColExists = fields.find((column) => column.name === "old_data");
+            if (!oldDataColExists) {
+                fields.push(schema_1.oldDataField);
+                logs.addNewColumn(this.rawChangeLogTableName(), schema_1.oldDataField.name);
+            }
             if (!documentIdColExists) {
                 fields.push(schema_1.documentIdField);
                 logs.addNewColumn(this.rawChangeLogTableName(), schema_1.documentIdField.name);
@@ -261,16 +266,26 @@ class FirestoreBigQueryEventHistoryTracker {
                 fields.push(schema_1.documentPathParams);
                 logs.addNewColumn(this.rawChangeLogTableName(), schema_1.documentPathParams.name);
             }
-            await partitioning.addPartitioningToSchema(metadata.schema.fields);
             /** Updated table metadata if required */
             const shouldUpdate = await checkUpdates_1.tableRequiresUpdate({
                 table,
                 config: this.config,
                 documentIdColExists,
                 pathParamsColExists,
+                oldDataColExists,
             });
             if (shouldUpdate) {
+                /** set partitioning */
+                await partitioning.addPartitioningToSchema(metadata.schema.fields);
+                /** update table metadata with changes. */
                 await table.setMetadata(metadata);
+                logs.updatingMetadata(this.rawChangeLogTableName(), {
+                    table,
+                    config: this.config,
+                    documentIdColExists,
+                    pathParamsColExists,
+                    oldDataColExists,
+                });
             }
         }
         else {
@@ -307,18 +322,22 @@ class FirestoreBigQueryEventHistoryTracker {
         if (viewExists) {
             logs.bigQueryViewAlreadyExists(view.id, dataset.id);
             const [metadata] = await view.getMetadata();
-            const fields = metadata.schema ? metadata.schema.fields : [];
+            // TODO: just casting this for now, needs properly fixing
+            const fields = (metadata.schema ? metadata.schema.fields : []);
             if (this.config.wildcardIds) {
                 schema.fields.push(schema_1.documentPathParams);
             }
-            const documentIdColExists = fields.find((column) => column.name === "document_id");
-            const pathParamsColExists = fields.find((column) => column.name === "path_params");
+            const columnNames = fields.map((field) => field.name);
+            const documentIdColExists = columnNames.includes("document_id");
+            const pathParamsColExists = columnNames.includes("path_params");
+            const oldDataColExists = columnNames.includes("old_data");
             /** If new view or opt-in to new query syntax **/
             const updateView = checkUpdates_1.viewRequiresUpdate({
                 metadata,
                 config: this.config,
                 documentIdColExists,
                 pathParamsColExists,
+                oldDataColExists,
             });
             if (updateView) {
                 metadata.view = snapshot_1.latestConsistentSnapshotView({
@@ -327,8 +346,17 @@ class FirestoreBigQueryEventHistoryTracker {
                     schema,
                     useLegacyQuery: !this.config.useNewSnapshotQuerySyntax,
                 });
-                logs.addNewColumn(this.rawLatestView(), schema_1.documentIdField.name);
+                if (!documentIdColExists) {
+                    logs.addNewColumn(this.rawLatestView(), schema_1.documentIdField.name);
+                }
                 await view.setMetadata(metadata);
+                logs.updatingMetadata(this.rawLatestView(), {
+                    metadata,
+                    config: this.config,
+                    documentIdColExists,
+                    pathParamsColExists,
+                    oldDataColExists,
+                });
             }
         }
         else {

package/lib/bigquery/schema.js

@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getNewPartitionField = exports.RawChangelogSchema = exports.RawChangelogViewSchema = exports.documentPathParams = exports.documentIdField = exports.longitudeField = exports.latitudeField = exports.timestampField = exports.operationField = exports.eventIdField = exports.documentNameField = exports.dataField = void 0;
+exports.getNewPartitionField = exports.RawChangelogSchema = exports.RawChangelogViewSchema = exports.oldDataField = exports.documentPathParams = exports.documentIdField = exports.longitudeField = exports.latitudeField = exports.timestampField = exports.operationField = exports.eventIdField = exports.documentNameField = exports.dataField = void 0;
 const bigQueryField = (name, type, mode, fields) => ({
     fields,
     mode: mode || "NULLABLE",
@@ -43,6 +43,12 @@ exports.documentPathParams = {
     type: "STRING",
     description: "JSON string representing wildcard params with Firestore Document ids",
 };
+exports.oldDataField = {
+    name: "old_data",
+    mode: "NULLABLE",
+    type: "STRING",
+    description: "The full JSON representation of the document state before the indicated operation is applied. This field will be null for CREATE operations.",
+};
 /*
  * We cannot specify a schema for view creation, and all view columns default
  * to the NULLABLE mode.

package/lib/logs.js

@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.failedToInitializeWait = exports.tableCreationError = exports.invalidClustering = exports.hourAndDatePartitioningWarning = exports.invalidTableReference = exports.invalidProjectIdWarning = exports.cannotPartitionExistingTable = exports.removedClustering = exports.updatedClustering = exports.bigQueryTableInsertErrors = exports.firestoreTimePartitioningParametersWarning = exports.firestoreTimePartitionFieldError = exports.addPartitionFieldColumn = exports.addNewColumn = exports.timestampMissingValue = exports.error = exports.dataTypeInvalid = exports.dataInserting = exports.dataInsertRetried = exports.dataInserted = exports.complete = exports.bigQueryViewValidating = exports.bigQueryViewValidated = exports.bigQueryViewUpToDate = exports.bigQueryViewUpdating = exports.bigQueryViewUpdated = exports.bigQueryViewAlreadyExists = exports.bigQueryViewCreating = exports.bigQueryViewCreated = exports.bigQueryUserDefinedFunctionCreated = exports.bigQueryUserDefinedFunctionCreating = exports.bigQueryTableValidating = exports.bigQueryTableValidated = exports.bigQueryTableUpToDate = exports.bigQueryTableUpdating = exports.bigQueryTableUpdated = exports.bigQueryTableCreating = exports.bigQueryTableCreated = exports.bigQueryTableAlreadyExists = exports.bigQuerySchemaViewCreated = exports.bigQueryLatestSnapshotViewQueryCreated = exports.bigQueryErrorRecordingDocumentChange = exports.bigQueryDatasetExists = exports.bigQueryDatasetCreating = exports.bigQueryDatasetCreated = exports.arrayFieldInvalid = void 0;
+exports.updatingMetadata = exports.failedToInitializeWait = exports.tableCreationError = exports.invalidClustering = exports.hourAndDatePartitioningWarning = exports.invalidTableReference = exports.invalidProjectIdWarning = exports.cannotPartitionExistingTable = exports.removedClustering = exports.updatedClustering = exports.bigQueryTableInsertErrors = exports.firestoreTimePartitioningParametersWarning = exports.firestoreTimePartitionFieldError = exports.addPartitionFieldColumn = exports.addNewColumn = exports.timestampMissingValue = exports.error = exports.dataTypeInvalid = exports.dataInserting = exports.dataInsertRetried = exports.dataInserted = exports.complete = exports.bigQueryViewValidating = exports.bigQueryViewValidated = exports.bigQueryViewUpToDate = exports.bigQueryViewUpdating = exports.bigQueryViewUpdated = exports.bigQueryViewAlreadyExists = exports.bigQueryViewCreating = exports.bigQueryViewCreated = exports.bigQueryUserDefinedFunctionCreated = exports.bigQueryUserDefinedFunctionCreating = exports.bigQueryTableValidating = exports.bigQueryTableValidated = exports.bigQueryTableUpToDate = exports.bigQueryTableUpdating = exports.bigQueryTableUpdated = exports.bigQueryTableCreating = exports.bigQueryTableCreated = exports.bigQueryTableAlreadyExists = exports.bigQuerySchemaViewCreated = exports.bigQueryLatestSnapshotViewQueryCreated = exports.bigQueryErrorRecordingDocumentChange = exports.bigQueryDatasetExists = exports.bigQueryDatasetCreating = exports.bigQueryDatasetCreated = exports.arrayFieldInvalid = void 0;
 const firebase_functions_1 = require("firebase-functions");
 exports.arrayFieldInvalid = (fieldName) => {
     firebase_functions_1.logger.warn(`Array field '${fieldName}' does not contain an array, skipping`);
@@ -173,3 +173,6 @@ exports.tableCreationError = (table, message) => {
 exports.failedToInitializeWait = (message) => {
     firebase_functions_1.logger.warn(`Failed while waiting to initialize.`, message);
 };
+exports.updatingMetadata = (tableName, resources) => {
+    firebase_functions_1.logger.info(`Updated Metadata on ${tableName}, ${JSON.stringify(resources)})`);
+};

package/package.json

@@ -5,7 +5,7 @@
     "url": "github.com/firebase/extensions.git",
     "directory": "firestore-bigquery-export/firestore-bigquery-change-tracker"
   },
-  "version": "1.1.17",
+  "version": "1.1.19",
   "description": "Core change-tracker library for Cloud Firestore Collection BigQuery Exports",
   "main": "./lib/index.js",
   "scripts": {