@firebaseextensions/firestore-bigquery-change-tracker 1.1.15 → 1.1.17

package/lib/bigquery/checkUpdates.js

@@ -0,0 +1,53 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.viewRequiresUpdate = exports.tableRequiresUpdate = void 0;
+const partitioning_1 = require("./partitioning");
+async function tableRequiresUpdate({ table, config, documentIdColExists, pathParamsColExists, }) {
+    /* Setup checks */
+    const { metadata } = table;
+    /** Check clustering */
+    const configCluster = JSON.stringify(config.clustering);
+    const tableCluster = JSON.stringify(metadata.clustering?.fields || []);
+    if (configCluster !== tableCluster)
+        return true;
+    /** Check wildcards */
+    if (!!config.wildcardIds !== pathParamsColExists)
+        return true;
+    /** Check document id column */
+    if (!documentIdColExists)
+        return true;
+    /** Check partitioning */
+    const partitioning = new partitioning_1.Partitioning(config, table);
+    const isValidPartition = await partitioning.isValidPartitionForExistingTable();
+    if (isValidPartition)
+        return true;
+    // No updates have occured.
+    return false;
+}
+exports.tableRequiresUpdate = tableRequiresUpdate;
+function viewRequiresUpdate({ metadata, config, documentIdColExists, pathParamsColExists, }) {
+    /** Check if documentId column exists */
+    if (!documentIdColExists)
+        return true;
+    /** Check wildcards */
+    if (!!config.wildcardIds !== pathParamsColExists)
+        return true;
+    /** Check document id column */
+    if (!documentIdColExists)
+        return true;
+    /* Using the new query syntax for snapshots */
+    if (metadata) {
+        const query = metadata.view?.query || "";
+        const hasLegacyQuery = query.includes("FIRST_VALUE");
+        const { useNewSnapshotQuerySyntax } = config;
+        /** If enabled and has legacy query, can update */
+        if (useNewSnapshotQuerySyntax && hasLegacyQuery)
+            return true;
+        /** If not enabled and has an updated query, can update */
+        if (!useNewSnapshotQuerySyntax && !hasLegacyQuery)
+            return true;
+    }
+    // No updates have occured.
+    return false;
+}
+exports.viewRequiresUpdate = viewRequiresUpdate;

package/lib/bigquery/index.js

@@ -27,6 +27,7 @@ const tracker_1 = require("../tracker");
 const logs = require("../logs");
 const partitioning_1 = require("./partitioning");
 const clustering_1 = require("./clustering");
+const checkUpdates_1 = require("./checkUpdates");
 var schema_2 = require("./schema");
 Object.defineProperty(exports, "RawChangelogSchema", { enumerable: true, get: function () { return schema_2.RawChangelogSchema; } });
 Object.defineProperty(exports, "RawChangelogViewSchema", { enumerable: true, get: function () { return schema_2.RawChangelogViewSchema; } });
@@ -42,7 +43,7 @@ Object.defineProperty(exports, "RawChangelogViewSchema", { enumerable: true, get
 class FirestoreBigQueryEventHistoryTracker {
     constructor(config) {
         this.config = config;
-        this.initialized = false;
+        this._initialized = false;
         this.bq = new bigquery.BigQuery();
         this.bq.projectId = config.bqProjectId || process.env.PROJECT_ID;
         if (!this.config.datasetLocation) {
@@ -64,6 +65,9 @@ class FirestoreBigQueryEventHistoryTracker {
                     document_id: event.documentId,
                     operation: tracker_1.ChangeType[event.operation],
                     data: JSON.stringify(this.serializeData(event.data)),
+                    old_data: event.oldData
+                        ? JSON.stringify(this.serializeData(event.oldData))
+                        : null,
                     ...partitionValue,
                     ...(this.config.wildcardIds &&
                         event.pathParams && { path_params: JSON.stringify(pathParams) }),
@@ -138,16 +142,25 @@ class FirestoreBigQueryEventHistoryTracker {
      * A half a second delay is added per check while the function
      * continually re-checks until the referenced dataset and table become available.
      */
-    async waitForInitialization(dataset, table) {
+    async waitForInitialization() {
         return new Promise((resolve) => {
             let handle = setInterval(async () => {
-                const [datasetExists] = await dataset.exists();
-                const [tableExists] = await table.exists();
-                if (datasetExists && tableExists) {
+                try {
+                    const dataset = this.bigqueryDataset();
+                    const changelogName = this.rawChangeLogTableName();
+                    const table = dataset.table(changelogName);
+                    const [datasetExists] = await dataset.exists();
+                    const [tableExists] = await table.exists();
+                    if (datasetExists && tableExists) {
+                        clearInterval(handle);
+                        return resolve(table);
+                    }
+                }
+                catch (ex) {
                     clearInterval(handle);
-                    return resolve(table);
+                    logs.failedToInitializeWait(ex.message);
                 }
-            }, 500);
+            }, 5000);
         });
     }
     /**
@@ -163,7 +176,6 @@ class FirestoreBigQueryEventHistoryTracker {
         try {
             const dataset = this.bigqueryDataset();
             const table = dataset.table(this.rawChangeLogTableName());
-            await this.waitForInitialization(dataset, table);
             logs.dataInserting(rows.length);
             await table.insert(rows, options);
             logs.dataInserted(rows.length);
@@ -179,7 +191,7 @@ class FirestoreBigQueryEventHistoryTracker {
                 await handleFailedTransactions_1.default(rows, this.config, e);
             }
             // Reinitializing in case the destintation table is modified.
-            this.initialized = false;
+            this._initialized = false;
             logs.bigQueryTableInsertErrors(e.errors);
             throw e;
         }
@@ -189,13 +201,19 @@ class FirestoreBigQueryEventHistoryTracker {
      * After the first invokation, it skips initialization assuming these resources are still there.
      */
     async initialize() {
-        if (this.initialized) {
-            return;
+        try {
+            if (this._initialized) {
+                return;
+            }
+            await this.initializeDataset();
+            await this.initializeRawChangeLogTable();
+            await this.initializeLatestView();
+            this._initialized = true;
+        }
+        catch (ex) {
+            await this.waitForInitialization();
+            this._initialized = true;
         }
-        await this.initializeDataset();
-        await this.initializeRawChangeLogTable();
-        await this.initializeLatestView();
-        this.initialized = true;
     }
     /**
      * Creates the specified dataset if it doesn't already exists.
@@ -244,7 +262,14 @@ class FirestoreBigQueryEventHistoryTracker {
                 logs.addNewColumn(this.rawChangeLogTableName(), schema_1.documentPathParams.name);
             }
             await partitioning.addPartitioningToSchema(metadata.schema.fields);
-            if (!documentIdColExists || !pathParamsColExists) {
+            /** Updated table metadata if required */
+            const shouldUpdate = await checkUpdates_1.tableRequiresUpdate({
+                table,
+                config: this.config,
+                documentIdColExists,
+                pathParamsColExists,
+            });
+            if (shouldUpdate) {
                 await table.setMetadata(metadata);
             }
         }
@@ -279,7 +304,6 @@ class FirestoreBigQueryEventHistoryTracker {
         const view = dataset.table(this.rawLatestView());
         const [viewExists] = await view.exists();
         const schema = schema_1.RawChangelogViewSchema;
-        const partitioning = new partitioning_1.Partitioning(this.config, view);
         if (viewExists) {
             logs.bigQueryViewAlreadyExists(view.id, dataset.id);
             const [metadata] = await view.getMetadata();
@@ -289,45 +313,41 @@ class FirestoreBigQueryEventHistoryTracker {
             }
             const documentIdColExists = fields.find((column) => column.name === "document_id");
             const pathParamsColExists = fields.find((column) => column.name === "path_params");
-            if (!documentIdColExists) {
-                metadata.view = snapshot_1.latestConsistentSnapshotView(this.config.datasetId, this.rawChangeLogTableName(), schema);
+            /** If new view or opt-in to new query syntax **/
+            const updateView = checkUpdates_1.viewRequiresUpdate({
+                metadata,
+                config: this.config,
+                documentIdColExists,
+                pathParamsColExists,
+            });
+            if (updateView) {
+                metadata.view = snapshot_1.latestConsistentSnapshotView({
+                    datasetId: this.config.datasetId,
+                    tableName: this.rawChangeLogTableName(),
+                    schema,
+                    useLegacyQuery: !this.config.useNewSnapshotQuerySyntax,
+                });
                 logs.addNewColumn(this.rawLatestView(), schema_1.documentIdField.name);
+                await view.setMetadata(metadata);
             }
-            if (!pathParamsColExists && this.config.wildcardIds) {
-                metadata.view = snapshot_1.latestConsistentSnapshotView(this.config.datasetId, this.rawChangeLogTableName(), schema);
-                logs.addNewColumn(this.rawLatestView(), schema_1.documentPathParams.name);
-            }
-            //Add partitioning
-            await partitioning.addPartitioningToSchema(schema.fields);
-            //TODO: Tidy up and format / add test cases?
-            // if (
-            //   !documentIdColExists ||
-            //   (!pathParamsColExists && this.config.wildcardIds) ||
-            //   partition.isValidPartitionForExistingTable(partitionColExists)
-            // ) {
-            await view.setMetadata(metadata);
-            // }
         }
         else {
             const schema = { fields: [...schema_1.RawChangelogViewSchema.fields] };
-            //Add partitioning field
-            await partitioning.addPartitioningToSchema(schema.fields);
-            //TODO Create notification for a user that View cannot be Time Partitioned by the field.
-            // await partitioning.updateTableMetadata(options);
             if (this.config.wildcardIds) {
                 schema.fields.push(schema_1.documentPathParams);
             }
-            const latestSnapshot = snapshot_1.latestConsistentSnapshotView(this.config.datasetId, this.rawChangeLogTableName(), schema, this.bq.projectId);
+            const latestSnapshot = snapshot_1.latestConsistentSnapshotView({
+                datasetId: this.config.datasetId,
+                tableName: this.rawChangeLogTableName(),
+                schema,
+                bqProjectId: this.bq.projectId,
+                useLegacyQuery: !this.config.useNewSnapshotQuerySyntax,
+            });
             logs.bigQueryViewCreating(this.rawLatestView(), latestSnapshot.query);
             const options = {
                 friendlyName: this.rawLatestView(),
                 view: latestSnapshot,
             };
-            if (this.config.timePartitioning) {
-                options.timePartitioning = {
-                    type: this.config.timePartitioning,
-                };
-            }
             try {
                 await view.create(options);
                 await view.setMetadata({ schema: schema_1.RawChangelogViewSchema });

package/lib/bigquery/partitioning.js

@@ -56,7 +56,7 @@ class Partitioning {
         const hasNoCustomOptions = !timePartitioningField &&
             !timePartitioningFieldType &&
             !timePartitioningFirestoreField;
-        /* No custom congig has been set, use partition value option only */
+        /* No custom config has been set, use partition value option only */
         if (hasNoCustomOptions)
             return true;
         /* check if all options have been provided to be  */
@@ -107,8 +107,9 @@ class Partitioning {
         return schemaFields.some((column) => column.name === this.config.timePartitioningField);
     }
     async isValidPartitionForExistingTable() {
-        if (this.isTablePartitioned())
-            return false;
+        const isPartitioned = await this.isTablePartitioned();
+        if (isPartitioned)
+            return Promise.resolve(false);
         return this.hasValidCustomPartitionConfig();
     }
     isValidPartitionForNewTable() {

package/lib/bigquery/schema.js

@@ -79,6 +79,12 @@ exports.RawChangelogViewSchema = {
             type: "STRING",
             description: "The full JSON representation of the current document state.",
         },
+        {
+            name: "old_data",
+            mode: "NULLABLE",
+            type: "STRING",
+            description: "The full JSON representation of the document state before the indicated operation is applied.",
+        },
         exports.documentIdField,
     ],
 };
@@ -114,6 +120,12 @@ exports.RawChangelogSchema = {
             type: "STRING",
             description: "The full JSON representation of the document state after the indicated operation is applied. This field will be null for DELETE operations.",
         },
+        {
+            name: "old_data",
+            mode: "NULLABLE",
+            type: "STRING",
+            description: "The full JSON representation of the document state before the indicated operation is applied. This field will be null for CREATE operations.",
+        },
         exports.documentIdField,
     ],
 };

package/lib/bigquery/snapshot.js

@@ -19,49 +19,86 @@ exports.buildLatestSnapshotViewQuery = exports.latestConsistentSnapshotView = vo
 const sqlFormatter = require("sql-formatter");
 const schema_1 = require("./schema");
 const excludeFields = ["document_name", "document_id"];
-exports.latestConsistentSnapshotView = (datasetId, tableName, schema, bqProjectId) => ({
-    query: buildLatestSnapshotViewQuery(datasetId, tableName, schema_1.timestampField.name, schema["fields"]
-        .map((field) => field.name)
-        .filter((name) => excludeFields.indexOf(name) === -1), bqProjectId),
+exports.latestConsistentSnapshotView = ({ datasetId, tableName, schema, bqProjectId, useLegacyQuery = false, }) => ({
+    query: buildLatestSnapshotViewQuery({
+        datasetId,
+        tableName,
+        timestampColumnName: schema_1.timestampField.name,
+        groupByColumns: schema["fields"]
+            .map((field) => field.name)
+            .filter((name) => excludeFields.indexOf(name) === -1),
+        bqProjectId,
+        useLegacyQuery,
+    }),
     useLegacySql: false,
 });
-function buildLatestSnapshotViewQuery(datasetId, tableName, timestampColumnName, groupByColumns, bqProjectId) {
+function buildLatestSnapshotViewQuery({ datasetId, tableName, timestampColumnName, groupByColumns, bqProjectId, useLegacyQuery = true, }) {
     if (datasetId === "" || tableName === "" || timestampColumnName === "") {
         throw Error(`Missing some query parameters!`);
     }
-    for (let columnName in groupByColumns) {
+    for (let columnName of groupByColumns) {
         if (columnName === "") {
             throw Error(`Found empty group by column!`);
         }
     }
+    const legacyQuery = sqlFormatter.format(` -- Retrieves the latest document change events for all live documents.
+  --   timestamp: The Firestore timestamp at which the event took place.
+  --   operation: One of INSERT, UPDATE, DELETE, IMPORT.
+  --   event_id: The id of the event that triggered the cloud function mirrored the event.
+  --   data: A raw JSON payload of the current state of the document.
+  --   document_id: The document id as defined in the Firestore database
+  SELECT
+  document_name,
+  document_id${groupByColumns.length > 0 ? `,` : ``}
+    ${groupByColumns.join(",")}
+  FROM (
+    SELECT
+      document_name,
+      document_id,
+      ${groupByColumns
+        .map((columnName) => `FIRST_VALUE(${columnName})
+          OVER(PARTITION BY document_name ORDER BY ${timestampColumnName} DESC)
+          AS ${columnName}`)
+        .join(",")}${groupByColumns.length > 0 ? `,` : ``}
+      FIRST_VALUE(operation)
+        OVER(PARTITION BY document_name ORDER BY ${timestampColumnName} DESC) = "DELETE"
+        AS is_deleted
+    FROM \`${bqProjectId || process.env.PROJECT_ID}.${datasetId}.${tableName}\`
+    ORDER BY document_name, ${timestampColumnName} DESC
+  )
+  WHERE NOT is_deleted
+  GROUP BY document_name, document_id${groupByColumns.length > 0 ? `, ` : ``}${groupByColumns.join(",")}`);
+    const nonGroupFields = ["event_id", "data", "old_data"];
+    const joinFields = ["document_name"];
+    const addSelectField = (field) => {
+        if (joinFields.includes(field))
+            return `t.${field}`;
+        return nonGroupFields.includes(field)
+            ? `ANY_VALUE(${field}) as ${field}`
+            : `${field} as ${field}`;
+    };
+    const filterGroupField = (field) => {
+        return nonGroupFields.includes(field);
+    };
     const query = sqlFormatter.format(` -- Retrieves the latest document change events for all live documents.
     --   timestamp: The Firestore timestamp at which the event took place.
     --   operation: One of INSERT, UPDATE, DELETE, IMPORT.
     --   event_id: The id of the event that triggered the cloud function mirrored the event.
     --   data: A raw JSON payload of the current state of the document.
     --   document_id: The document id as defined in the Firestore database
+    WITH latest AS (
+      SELECT max(${timestampColumnName}) as latest_timestamp, document_name
+      FROM \`${bqProjectId || process.env.PROJECT_ID}.${datasetId}.${tableName}\`
+      GROUP BY document_name
+    )
     SELECT
-    document_name,
+    t.document_name,
     document_id${groupByColumns.length > 0 ? `,` : ``}
-      ${groupByColumns.join(",")}
-    FROM (
-      SELECT
-        document_name,
-        document_id,
-        ${groupByColumns
-        .map((columnName) => `FIRST_VALUE(${columnName})
-            OVER(PARTITION BY document_name ORDER BY ${timestampColumnName} DESC)
-            AS ${columnName}`)
-        .join(",")}${groupByColumns.length > 0 ? `,` : ``}
-        FIRST_VALUE(operation)
-          OVER(PARTITION BY document_name ORDER BY ${timestampColumnName} DESC) = "DELETE"
-          AS is_deleted
-      FROM \`${bqProjectId ||
-        process.env.PROJECT_ID}.${datasetId}.${tableName}\`
-      ORDER BY document_name, ${timestampColumnName} DESC
-    )
-    WHERE NOT is_deleted
-    GROUP BY document_name, document_id${groupByColumns.length > 0 ? `, ` : ``}${groupByColumns.join(",")}`);
-    return query;
+      ${groupByColumns.map((f) => addSelectField(f)).join(",")}
+    FROM \`${bqProjectId || process.env.PROJECT_ID}.${datasetId}.${tableName}\` AS t
+    JOIN latest ON (t.document_name = latest.document_name AND (IFNULL(t.${timestampColumnName}, timestamp("1970-01-01 00:00:00+00"))) = (IFNULL(latest.latest_timestamp, timestamp("1970-01-01 00:00:00+00"))))
+    WHERE operation != "DELETE"
+    GROUP BY document_name, document_id${groupByColumns.length > 0 ? `, ` : ``}${groupByColumns.filter((c) => !filterGroupField(c)).join(",")}`);
+    return useLegacyQuery ? legacyQuery : query;
 }
 exports.buildLatestSnapshotViewQuery = buildLatestSnapshotViewQuery;

package/lib/logs.js

@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.tableCreationError = exports.invalidClustering = exports.hourAndDatePartitioningWarning = exports.invalidTableReference = exports.invalidProjectIdWarning = exports.cannotPartitionExistingTable = exports.removedClustering = exports.updatedClustering = exports.bigQueryTableInsertErrors = exports.firestoreTimePartitioningParametersWarning = exports.firestoreTimePartitionFieldError = exports.addPartitionFieldColumn = exports.addNewColumn = exports.timestampMissingValue = exports.error = exports.dataTypeInvalid = exports.dataInserting = exports.dataInsertRetried = exports.dataInserted = exports.complete = exports.bigQueryViewValidating = exports.bigQueryViewValidated = exports.bigQueryViewUpToDate = exports.bigQueryViewUpdating = exports.bigQueryViewUpdated = exports.bigQueryViewAlreadyExists = exports.bigQueryViewCreating = exports.bigQueryViewCreated = exports.bigQueryUserDefinedFunctionCreated = exports.bigQueryUserDefinedFunctionCreating = exports.bigQueryTableValidating = exports.bigQueryTableValidated = exports.bigQueryTableUpToDate = exports.bigQueryTableUpdating = exports.bigQueryTableUpdated = exports.bigQueryTableCreating = exports.bigQueryTableCreated = exports.bigQueryTableAlreadyExists = exports.bigQuerySchemaViewCreated = exports.bigQueryLatestSnapshotViewQueryCreated = exports.bigQueryErrorRecordingDocumentChange = exports.bigQueryDatasetExists = exports.bigQueryDatasetCreating = exports.bigQueryDatasetCreated = exports.arrayFieldInvalid = void 0;
+exports.failedToInitializeWait = exports.tableCreationError = exports.invalidClustering = exports.hourAndDatePartitioningWarning = exports.invalidTableReference = exports.invalidProjectIdWarning = exports.cannotPartitionExistingTable = exports.removedClustering = exports.updatedClustering = exports.bigQueryTableInsertErrors = exports.firestoreTimePartitioningParametersWarning = exports.firestoreTimePartitionFieldError = exports.addPartitionFieldColumn = exports.addNewColumn = exports.timestampMissingValue = exports.error = exports.dataTypeInvalid = exports.dataInserting = exports.dataInsertRetried = exports.dataInserted = exports.complete = exports.bigQueryViewValidating = exports.bigQueryViewValidated = exports.bigQueryViewUpToDate = exports.bigQueryViewUpdating = exports.bigQueryViewUpdated = exports.bigQueryViewAlreadyExists = exports.bigQueryViewCreating = exports.bigQueryViewCreated = exports.bigQueryUserDefinedFunctionCreated = exports.bigQueryUserDefinedFunctionCreating = exports.bigQueryTableValidating = exports.bigQueryTableValidated = exports.bigQueryTableUpToDate = exports.bigQueryTableUpdating = exports.bigQueryTableUpdated = exports.bigQueryTableCreating = exports.bigQueryTableCreated = exports.bigQueryTableAlreadyExists = exports.bigQuerySchemaViewCreated = exports.bigQueryLatestSnapshotViewQueryCreated = exports.bigQueryErrorRecordingDocumentChange = exports.bigQueryDatasetExists = exports.bigQueryDatasetCreating = exports.bigQueryDatasetCreated = exports.arrayFieldInvalid = void 0;
 const firebase_functions_1 = require("firebase-functions");
 exports.arrayFieldInvalid = (fieldName) => {
     firebase_functions_1.logger.warn(`Array field '${fieldName}' does not contain an array, skipping`);
@@ -137,7 +137,9 @@ exports.bigQueryTableInsertErrors = (insertErrors) => {
     insertErrors.forEach((error) => {
         firebase_functions_1.logger.warn("ROW DATA JSON:");
         firebase_functions_1.logger.warn(error.row);
-        error.errors.forEach((error) => firebase_functions_1.logger.warn(`ROW ERROR MESSAGE: ${error.message}`));
+        if (error && error.errors) {
+            error.errors.forEach((error) => firebase_functions_1.logger.warn(`ROW ERROR MESSAGE: ${error.message}`));
+        }
     });
 };
 exports.updatedClustering = (fields) => {
@@ -168,3 +170,6 @@ exports.invalidClustering = invalidClustering;
 exports.tableCreationError = (table, message) => {
     firebase_functions_1.logger.warn(`Error caught creating table`, message);
 };
+exports.failedToInitializeWait = (message) => {
+    firebase_functions_1.logger.warn(`Failed while waiting to initialize.`, message);
+};

package/package.json

@@ -5,7 +5,7 @@
     "url": "github.com/firebase/extensions.git",
     "directory": "firestore-bigquery-export/firestore-bigquery-change-tracker"
   },
-  "version": "1.1.15",
+  "version": "1.1.17",
   "description": "Core change-tracker library for Cloud Firestore Collection BigQuery Exports",
   "main": "./lib/index.js",
   "scripts": {
@@ -13,7 +13,8 @@
     "clean": "rimraf lib",
     "compile": "tsc",
     "test:local": "firebase ext:dev:emulators:exec ./node_modules/.bin/jest --test-params=./src/__tests__/emulator-params.env --project=extensions-testing --config=./src/__tests__/firebase.json",
-    "prepare": "npm run build"
+    "prepare": "npm run build",
+    "generate-stresstest-table": "bq query --project_id=extensions-testing  --use_legacy_sql=false < ./src/__tests__/fixtures/sql/generateSnapshotStresstestTable.sql"
   },
   "files": [
     "lib/*.js",