@firebaseextensions/firestore-bigquery-change-tracker 1.1.16 → 1.1.18

package/lib/bigquery/checkUpdates.js

@@ -0,0 +1,56 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.viewRequiresUpdate = exports.tableRequiresUpdate = void 0;
+const partitioning_1 = require("./partitioning");
+async function tableRequiresUpdate({ table, config, documentIdColExists, pathParamsColExists, oldDataColExists, }) {
+    /* Setup checks */
+    const { metadata } = table;
+    /** Check clustering */
+    const configCluster = JSON.stringify(config.clustering);
+    const tableCluster = JSON.stringify(metadata.clustering?.fields || []);
+    if (configCluster !== tableCluster)
+        return true;
+    /** Check wildcards */
+    if (!!config.wildcardIds !== pathParamsColExists)
+        return true;
+    /** Check document id column */
+    if (!documentIdColExists)
+        return true;
+    /** Check old_data column exists */
+    if (!oldDataColExists)
+        return true;
+    /** Check partitioning */
+    const partitioning = new partitioning_1.Partitioning(config, table);
+    const isValidPartition = await partitioning.isValidPartitionForExistingTable();
+    if (isValidPartition)
+        return true;
+    // No updates have occured.
+    return false;
+}
+exports.tableRequiresUpdate = tableRequiresUpdate;
+function viewRequiresUpdate({ metadata, config, documentIdColExists, pathParamsColExists, oldDataColExists, }) {
+    /** Check wildcards */
+    if (!!config.wildcardIds !== pathParamsColExists)
+        return true;
+    /** Check document id column */
+    if (!documentIdColExists)
+        return true;
+    /** Check old_data column exists */
+    if (!oldDataColExists)
+        return true;
+    /* Using the new query syntax for snapshots */
+    if (metadata) {
+        const query = metadata.view?.query || "";
+        const hasLegacyQuery = query.includes("FIRST_VALUE");
+        const { useNewSnapshotQuerySyntax } = config;
+        /** If enabled and has legacy query, can update */
+        if (useNewSnapshotQuerySyntax && hasLegacyQuery)
+            return true;
+        /** If not enabled and has an updated query, can update */
+        if (!useNewSnapshotQuerySyntax && !hasLegacyQuery)
+            return true;
+    }
+    // No updates have occured.
+    return false;
+}
+exports.viewRequiresUpdate = viewRequiresUpdate;

package/lib/bigquery/index.js

@@ -27,6 +27,7 @@ const tracker_1 = require("../tracker");
 const logs = require("../logs");
 const partitioning_1 = require("./partitioning");
 const clustering_1 = require("./clustering");
+const checkUpdates_1 = require("./checkUpdates");
 var schema_2 = require("./schema");
 Object.defineProperty(exports, "RawChangelogSchema", { enumerable: true, get: function () { return schema_2.RawChangelogSchema; } });
 Object.defineProperty(exports, "RawChangelogViewSchema", { enumerable: true, get: function () { return schema_2.RawChangelogViewSchema; } });
@@ -64,6 +65,9 @@ class FirestoreBigQueryEventHistoryTracker {
                     document_id: event.documentId,
                     operation: tracker_1.ChangeType[event.operation],
                     data: JSON.stringify(this.serializeData(event.data)),
+                    old_data: event.oldData
+                        ? JSON.stringify(this.serializeData(event.oldData))
+                        : null,
                     ...partitionValue,
                     ...(this.config.wildcardIds &&
                         event.pathParams && { path_params: JSON.stringify(pathParams) }),
@@ -249,6 +253,7 @@ class FirestoreBigQueryEventHistoryTracker {
             await clustering.updateClustering(metadata);
             const documentIdColExists = fields.find((column) => column.name === "document_id");
             const pathParamsColExists = fields.find((column) => column.name === "path_params");
+            const oldDataColExists = fields.find((column) => column.name === "old_data");
             if (!documentIdColExists) {
                 fields.push(schema_1.documentIdField);
                 logs.addNewColumn(this.rawChangeLogTableName(), schema_1.documentIdField.name);
@@ -258,7 +263,15 @@ class FirestoreBigQueryEventHistoryTracker {
                 logs.addNewColumn(this.rawChangeLogTableName(), schema_1.documentPathParams.name);
             }
             await partitioning.addPartitioningToSchema(metadata.schema.fields);
-            if (!documentIdColExists || !pathParamsColExists) {
+            /** Updated table metadata if required */
+            const shouldUpdate = await checkUpdates_1.tableRequiresUpdate({
+                table,
+                config: this.config,
+                documentIdColExists,
+                pathParamsColExists,
+                oldDataColExists,
+            });
+            if (shouldUpdate) {
                 await table.setMetadata(metadata);
             }
         }
@@ -293,55 +306,54 @@ class FirestoreBigQueryEventHistoryTracker {
         const view = dataset.table(this.rawLatestView());
         const [viewExists] = await view.exists();
         const schema = schema_1.RawChangelogViewSchema;
-        const partitioning = new partitioning_1.Partitioning(this.config, view);
         if (viewExists) {
             logs.bigQueryViewAlreadyExists(view.id, dataset.id);
             const [metadata] = await view.getMetadata();
-            const fields = metadata.schema ? metadata.schema.fields : [];
+            // TODO: just casting this for now, needs properly fixing
+            const fields = (metadata.schema ? metadata.schema.fields : []);
             if (this.config.wildcardIds) {
                 schema.fields.push(schema_1.documentPathParams);
             }
-            const documentIdColExists = fields.find((column) => column.name === "document_id");
-            const pathParamsColExists = fields.find((column) => column.name === "path_params");
-            if (!documentIdColExists) {
-                metadata.view = snapshot_1.latestConsistentSnapshotView(this.config.datasetId, this.rawChangeLogTableName(), schema);
+            const columnNames = fields.map((field) => field.name);
+            const documentIdColExists = columnNames.includes("document_id");
+            const pathParamsColExists = columnNames.includes("path_params");
+            const oldDataColExists = columnNames.includes("old_data");
+            /** If new view or opt-in to new query syntax **/
+            const updateView = checkUpdates_1.viewRequiresUpdate({
+                metadata,
+                config: this.config,
+                documentIdColExists,
+                pathParamsColExists,
+                oldDataColExists,
+            });
+            if (updateView) {
+                metadata.view = snapshot_1.latestConsistentSnapshotView({
+                    datasetId: this.config.datasetId,
+                    tableName: this.rawChangeLogTableName(),
+                    schema,
+                    useLegacyQuery: !this.config.useNewSnapshotQuerySyntax,
+                });
                 logs.addNewColumn(this.rawLatestView(), schema_1.documentIdField.name);
+                await view.setMetadata(metadata);
             }
-            if (!pathParamsColExists && this.config.wildcardIds) {
-                metadata.view = snapshot_1.latestConsistentSnapshotView(this.config.datasetId, this.rawChangeLogTableName(), schema);
-                logs.addNewColumn(this.rawLatestView(), schema_1.documentPathParams.name);
-            }
-            //Add partitioning
-            await partitioning.addPartitioningToSchema(schema.fields);
-            //TODO: Tidy up and format / add test cases?
-            // if (
-            //   !documentIdColExists ||
-            //   (!pathParamsColExists && this.config.wildcardIds) ||
-            //   partition.isValidPartitionForExistingTable(partitionColExists)
-            // ) {
-            await view.setMetadata(metadata);
-            // }
         }
         else {
             const schema = { fields: [...schema_1.RawChangelogViewSchema.fields] };
-            //Add partitioning field
-            await partitioning.addPartitioningToSchema(schema.fields);
-            //TODO Create notification for a user that View cannot be Time Partitioned by the field.
-            // await partitioning.updateTableMetadata(options);
             if (this.config.wildcardIds) {
                 schema.fields.push(schema_1.documentPathParams);
             }
-            const latestSnapshot = snapshot_1.latestConsistentSnapshotView(this.config.datasetId, this.rawChangeLogTableName(), schema, this.bq.projectId);
+            const latestSnapshot = snapshot_1.latestConsistentSnapshotView({
+                datasetId: this.config.datasetId,
+                tableName: this.rawChangeLogTableName(),
+                schema,
+                bqProjectId: this.bq.projectId,
+                useLegacyQuery: !this.config.useNewSnapshotQuerySyntax,
+            });
             logs.bigQueryViewCreating(this.rawLatestView(), latestSnapshot.query);
             const options = {
                 friendlyName: this.rawLatestView(),
                 view: latestSnapshot,
             };
-            if (this.config.timePartitioning) {
-                options.timePartitioning = {
-                    type: this.config.timePartitioning,
-                };
-            }
             try {
                 await view.create(options);
                 await view.setMetadata({ schema: schema_1.RawChangelogViewSchema });

package/lib/bigquery/partitioning.js

@@ -56,7 +56,7 @@ class Partitioning {
         const hasNoCustomOptions = !timePartitioningField &&
             !timePartitioningFieldType &&
             !timePartitioningFirestoreField;
-        /* No custom congig has been set, use partition value option only */
+        /* No custom config has been set, use partition value option only */
         if (hasNoCustomOptions)
             return true;
         /* check if all options have been provided to be  */
@@ -107,8 +107,9 @@ class Partitioning {
         return schemaFields.some((column) => column.name === this.config.timePartitioningField);
     }
     async isValidPartitionForExistingTable() {
-        if (this.isTablePartitioned())
-            return false;
+        const isPartitioned = await this.isTablePartitioned();
+        if (isPartitioned)
+            return Promise.resolve(false);
         return this.hasValidCustomPartitionConfig();
     }
     isValidPartitionForNewTable() {

package/lib/bigquery/schema.js

@@ -79,6 +79,12 @@ exports.RawChangelogViewSchema = {
             type: "STRING",
             description: "The full JSON representation of the current document state.",
         },
+        {
+            name: "old_data",
+            mode: "NULLABLE",
+            type: "STRING",
+            description: "The full JSON representation of the document state before the indicated operation is applied.",
+        },
         exports.documentIdField,
     ],
 };
@@ -114,6 +120,12 @@ exports.RawChangelogSchema = {
             type: "STRING",
             description: "The full JSON representation of the document state after the indicated operation is applied. This field will be null for DELETE operations.",
         },
+        {
+            name: "old_data",
+            mode: "NULLABLE",
+            type: "STRING",
+            description: "The full JSON representation of the document state before the indicated operation is applied. This field will be null for CREATE operations.",
+        },
         exports.documentIdField,
     ],
 };

package/lib/bigquery/snapshot.js

@@ -19,49 +19,86 @@ exports.buildLatestSnapshotViewQuery = exports.latestConsistentSnapshotView = vo
 const sqlFormatter = require("sql-formatter");
 const schema_1 = require("./schema");
 const excludeFields = ["document_name", "document_id"];
-exports.latestConsistentSnapshotView = (datasetId, tableName, schema, bqProjectId) => ({
-    query: buildLatestSnapshotViewQuery(datasetId, tableName, schema_1.timestampField.name, schema["fields"]
-        .map((field) => field.name)
-        .filter((name) => excludeFields.indexOf(name) === -1), bqProjectId),
+exports.latestConsistentSnapshotView = ({ datasetId, tableName, schema, bqProjectId, useLegacyQuery = false, }) => ({
+    query: buildLatestSnapshotViewQuery({
+        datasetId,
+        tableName,
+        timestampColumnName: schema_1.timestampField.name,
+        groupByColumns: schema["fields"]
+            .map((field) => field.name)
+            .filter((name) => excludeFields.indexOf(name) === -1),
+        bqProjectId,
+        useLegacyQuery,
+    }),
     useLegacySql: false,
 });
-function buildLatestSnapshotViewQuery(datasetId, tableName, timestampColumnName, groupByColumns, bqProjectId) {
+function buildLatestSnapshotViewQuery({ datasetId, tableName, timestampColumnName, groupByColumns, bqProjectId, useLegacyQuery = true, }) {
     if (datasetId === "" || tableName === "" || timestampColumnName === "") {
         throw Error(`Missing some query parameters!`);
     }
-    for (let columnName in groupByColumns) {
+    for (let columnName of groupByColumns) {
         if (columnName === "") {
             throw Error(`Found empty group by column!`);
         }
     }
+    const legacyQuery = sqlFormatter.format(` -- Retrieves the latest document change events for all live documents.
+  --   timestamp: The Firestore timestamp at which the event took place.
+  --   operation: One of INSERT, UPDATE, DELETE, IMPORT.
+  --   event_id: The id of the event that triggered the cloud function mirrored the event.
+  --   data: A raw JSON payload of the current state of the document.
+  --   document_id: The document id as defined in the Firestore database
+  SELECT
+  document_name,
+  document_id${groupByColumns.length > 0 ? `,` : ``}
+    ${groupByColumns.join(",")}
+  FROM (
+    SELECT
+      document_name,
+      document_id,
+      ${groupByColumns
+        .map((columnName) => `FIRST_VALUE(${columnName})
+          OVER(PARTITION BY document_name ORDER BY ${timestampColumnName} DESC)
+          AS ${columnName}`)
+        .join(",")}${groupByColumns.length > 0 ? `,` : ``}
+      FIRST_VALUE(operation)
+        OVER(PARTITION BY document_name ORDER BY ${timestampColumnName} DESC) = "DELETE"
+        AS is_deleted
+    FROM \`${bqProjectId || process.env.PROJECT_ID}.${datasetId}.${tableName}\`
+    ORDER BY document_name, ${timestampColumnName} DESC
+  )
+  WHERE NOT is_deleted
+  GROUP BY document_name, document_id${groupByColumns.length > 0 ? `, ` : ``}${groupByColumns.join(",")}`);
+    const nonGroupFields = ["event_id", "data", "old_data"];
+    const joinFields = ["document_name"];
+    const addSelectField = (field) => {
+        if (joinFields.includes(field))
+            return `t.${field}`;
+        return nonGroupFields.includes(field)
+            ? `ANY_VALUE(${field}) as ${field}`
+            : `${field} as ${field}`;
+    };
+    const filterGroupField = (field) => {
+        return nonGroupFields.includes(field);
+    };
     const query = sqlFormatter.format(` -- Retrieves the latest document change events for all live documents.
     --   timestamp: The Firestore timestamp at which the event took place.
     --   operation: One of INSERT, UPDATE, DELETE, IMPORT.
     --   event_id: The id of the event that triggered the cloud function mirrored the event.
     --   data: A raw JSON payload of the current state of the document.
     --   document_id: The document id as defined in the Firestore database
+    WITH latest AS (
+      SELECT max(${timestampColumnName}) as latest_timestamp, document_name
+      FROM \`${bqProjectId || process.env.PROJECT_ID}.${datasetId}.${tableName}\`
+      GROUP BY document_name
+    )
     SELECT
-    document_name,
+    t.document_name,
     document_id${groupByColumns.length > 0 ? `,` : ``}
-      ${groupByColumns.join(",")}
-    FROM (
-      SELECT
-        document_name,
-        document_id,
-        ${groupByColumns
-        .map((columnName) => `FIRST_VALUE(${columnName})
-            OVER(PARTITION BY document_name ORDER BY ${timestampColumnName} DESC)
-            AS ${columnName}`)
-        .join(",")}${groupByColumns.length > 0 ? `,` : ``}
-        FIRST_VALUE(operation)
-          OVER(PARTITION BY document_name ORDER BY ${timestampColumnName} DESC) = "DELETE"
-          AS is_deleted
-      FROM \`${bqProjectId ||
-        process.env.PROJECT_ID}.${datasetId}.${tableName}\`
-      ORDER BY document_name, ${timestampColumnName} DESC
-    )
-    WHERE NOT is_deleted
-    GROUP BY document_name, document_id${groupByColumns.length > 0 ? `, ` : ``}${groupByColumns.join(",")}`);
-    return query;
+      ${groupByColumns.map((f) => addSelectField(f)).join(",")}
+    FROM \`${bqProjectId || process.env.PROJECT_ID}.${datasetId}.${tableName}\` AS t
+    JOIN latest ON (t.document_name = latest.document_name AND (IFNULL(t.${timestampColumnName}, timestamp("1970-01-01 00:00:00+00"))) = (IFNULL(latest.latest_timestamp, timestamp("1970-01-01 00:00:00+00"))))
+    WHERE operation != "DELETE"
+    GROUP BY document_name, document_id${groupByColumns.length > 0 ? `, ` : ``}${groupByColumns.filter((c) => !filterGroupField(c)).join(",")}`);
+    return useLegacyQuery ? legacyQuery : query;
 }
 exports.buildLatestSnapshotViewQuery = buildLatestSnapshotViewQuery;

package/package.json

@@ -5,7 +5,7 @@
     "url": "github.com/firebase/extensions.git",
     "directory": "firestore-bigquery-export/firestore-bigquery-change-tracker"
   },
-  "version": "1.1.16",
+  "version": "1.1.18",
   "description": "Core change-tracker library for Cloud Firestore Collection BigQuery Exports",
   "main": "./lib/index.js",
   "scripts": {
@@ -13,7 +13,8 @@
     "clean": "rimraf lib",
     "compile": "tsc",
     "test:local": "firebase ext:dev:emulators:exec ./node_modules/.bin/jest --test-params=./src/__tests__/emulator-params.env --project=extensions-testing --config=./src/__tests__/firebase.json",
-    "prepare": "npm run build"
+    "prepare": "npm run build",
+    "generate-stresstest-table": "bq query --project_id=extensions-testing  --use_legacy_sql=false < ./src/__tests__/fixtures/sql/generateSnapshotStresstestTable.sql"
   },
   "files": [
     "lib/*.js",