@firebaseextensions/firestore-bigquery-change-tracker 1.1.38 → 1.1.39

package/lib/bigquery/clustering.js

@@ -2,6 +2,18 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Clustering = void 0;
 const logs = require("../logs");
+const VALID_CLUSTERING_TYPES = [
+    "BIGNUMERIC",
+    "BOOL",
+    "DATE",
+    "DATETIME",
+    "GEOGRAPHY",
+    "INT64",
+    "NUMERIC",
+    "RANGE",
+    "STRING",
+    "TIMESTAMP",
+];
 class Clustering {
     constructor(config, table, schema) {
         this.updateCluster = async (metaData) => {
@@ -33,17 +45,37 @@ class Clustering {
     }
     async hasInvalidFields(metaData) {
         const { clustering = [] } = this.config;
-        if (!clustering)
-            return Promise.resolve(false);
-        const fieldNames = metaData
-            ? metaData.schema.fields.map(($) => $.name)
-            : [];
-        const invalidFields = clustering.filter(($) => !fieldNames.includes($));
-        if (invalidFields.length) {
-            logs.invalidClustering(invalidFields.join(","));
-            return Promise.resolve(true);
+        if (!clustering) {
+            return false;
         }
-        return Promise.resolve(false);
+        if (!clustering.length) {
+            return false;
+        }
+        if (!metaData?.schema.fields.length) {
+            return false;
+        }
+        const fields = metaData.schema.fields;
+        const fieldNameToType = new Map(fields.map((field) => [field.name, field.type]));
+        // First check if all clustering fields exist in the schema
+        const nonExistentFields = clustering.filter((fieldName) => !fieldNameToType.has(fieldName));
+        if (nonExistentFields.length) {
+            logs.invalidClustering(nonExistentFields.join(","));
+            return true;
+        }
+        // Then check for invalid types among existing clustering fields
+        const invalidFieldTypes = clustering
+            .map((fieldName) => ({
+            fieldName,
+            type: fieldNameToType.get(fieldName),
+        }))
+            .filter(({ type }) => !VALID_CLUSTERING_TYPES.includes(type));
+        if (invalidFieldTypes.length) {
+            logs.invalidClusteringTypes(invalidFieldTypes
+                .map(({ fieldName, type }) => `${fieldName} (${type})`)
+                .join(", "));
+            return true;
+        }
+        return false;
     }
 }
 exports.Clustering = Clustering;

package/lib/bigquery/index.d.ts

@@ -18,6 +18,10 @@ export interface FirestoreBigQueryEventHistoryTrackerConfig {
     useNewSnapshotQuerySyntax?: boolean;
     skipInit?: boolean;
     kmsKeyName?: string | undefined;
+    useMaterializedView?: boolean;
+    useIncrementalMaterializedView?: boolean;
+    maxStaleness?: string;
+    refreshIntervalMinutes?: number;
 }
 /**
  * An FirestoreEventHistoryTracker that exports data to BigQuery.
@@ -70,7 +74,7 @@ export declare class FirestoreBigQueryEventHistoryTracker implements FirestoreEv
      * Creates the latest snapshot view, which returns only latest operations
      * of all existing documents over the raw change log table.
      */
-    private initializeLatestView;
+    private _initializeLatestView;
     bigqueryDataset(): bigquery.Dataset;
     private rawChangeLogTableName;
     private rawLatestView;

package/lib/bigquery/index.js

@@ -6,7 +6,6 @@ const firestore_1 = require("firebase-admin/firestore");
 const traverse = require("traverse");
 const node_fetch_1 = require("node-fetch");
 const schema_1 = require("./schema");
-const snapshot_1 = require("./snapshot");
 const handleFailedTransactions_1 = require("./handleFailedTransactions");
 const tracker_1 = require("../tracker");
 const logs = require("../logs");
@@ -14,6 +13,7 @@ const partitioning_1 = require("./partitioning");
 const clustering_1 = require("./clustering");
 const checkUpdates_1 = require("./checkUpdates");
 const utils_1 = require("./utils");
+const initializeLatestView_1 = require("./initializeLatestView");
 var schema_2 = require("./schema");
 Object.defineProperty(exports, "RawChangelogSchema", { enumerable: true, get: function () { return schema_2.RawChangelogSchema; } });
 Object.defineProperty(exports, "RawChangelogViewSchema", { enumerable: true, get: function () { return schema_2.RawChangelogViewSchema; } });
@@ -133,7 +133,15 @@ class FirestoreBigQueryEventHistoryTracker {
     async _waitForInitialization() {
         const dataset = this.bigqueryDataset();
         const changelogName = this.rawChangeLogTableName();
-        return (0, utils_1.waitForInitialization)({ dataset, changelogName });
+        let materializedViewName;
+        if (this.config.useMaterializedView) {
+            materializedViewName = this.rawLatestView();
+        }
+        return (0, utils_1.waitForInitialization)({
+            dataset,
+            changelogName,
+            materializedViewName,
+        });
     }
     /**
      * Inserts rows of data into the BigQuery raw change log table.
@@ -192,7 +200,7 @@ class FirestoreBigQueryEventHistoryTracker {
                 throw new Error(`Error initializing raw change log table: ${message}`);
             }
             try {
-                await this.initializeLatestView();
+                await this._initializeLatestView();
             }
             catch (error) {
                 const message = (0, utils_1.parseErrorMessage)(error, "initializing latest view");
@@ -310,77 +318,19 @@ class FirestoreBigQueryEventHistoryTracker {
      * Creates the latest snapshot view, which returns only latest operations
      * of all existing documents over the raw change log table.
      */
-    async initializeLatestView() {
+    async _initializeLatestView() {
         const dataset = this.bigqueryDataset();
         const view = dataset.table(this.rawLatestView());
         const [viewExists] = await view.exists();
-        const schema = schema_1.RawChangelogViewSchema;
-        if (viewExists) {
-            logs.bigQueryViewAlreadyExists(view.id, dataset.id);
-            const [metadata] = await view.getMetadata();
-            // TODO: just casting this for now, needs properly fixing
-            const fields = (metadata.schema ? metadata.schema.fields : []);
-            if (this.config.wildcardIds) {
-                schema.fields.push(schema_1.documentPathParams);
-            }
-            const columnNames = fields.map((field) => field.name);
-            const documentIdColExists = columnNames.includes("document_id");
-            const pathParamsColExists = columnNames.includes("path_params");
-            const oldDataColExists = columnNames.includes("old_data");
-            /** If new view or opt-in to new query syntax **/
-            const updateView = (0, checkUpdates_1.viewRequiresUpdate)({
-                metadata,
-                config: this.config,
-                documentIdColExists,
-                pathParamsColExists,
-                oldDataColExists,
-            });
-            if (updateView) {
-                metadata.view = (0, snapshot_1.latestConsistentSnapshotView)({
-                    datasetId: this.config.datasetId,
-                    tableName: this.rawChangeLogTableName(),
-                    schema,
-                    useLegacyQuery: !this.config.useNewSnapshotQuerySyntax,
-                });
-                if (!documentIdColExists) {
-                    logs.addNewColumn(this.rawLatestView(), schema_1.documentIdField.name);
-                }
-                await view.setMetadata(metadata);
-                logs.updatingMetadata(this.rawLatestView(), {
-                    config: this.config,
-                    documentIdColExists,
-                    pathParamsColExists,
-                    oldDataColExists,
-                });
-            }
-        }
-        else {
-            const schema = { fields: [...schema_1.RawChangelogViewSchema.fields] };
-            if (this.config.wildcardIds) {
-                schema.fields.push(schema_1.documentPathParams);
-            }
-            const latestSnapshot = (0, snapshot_1.latestConsistentSnapshotView)({
-                datasetId: this.config.datasetId,
-                tableName: this.rawChangeLogTableName(),
-                schema,
-                bqProjectId: this.bq.projectId,
-                useLegacyQuery: !this.config.useNewSnapshotQuerySyntax,
-            });
-            logs.bigQueryViewCreating(this.rawLatestView(), latestSnapshot.query);
-            const options = {
-                friendlyName: this.rawLatestView(),
-                view: latestSnapshot,
-            };
-            try {
-                await view.create(options);
-                await view.setMetadata({ schema: schema_1.RawChangelogViewSchema });
-                logs.bigQueryViewCreated(this.rawLatestView());
-            }
-            catch (ex) {
-                logs.tableCreationError(this.rawLatestView(), ex.message);
-            }
-        }
-        return view;
+        return await (0, initializeLatestView_1.initializeLatestView)({
+            bq: this.bq,
+            changeTrackerConfig: this.config,
+            dataset,
+            view,
+            viewExists,
+            rawChangeLogTableName: this.rawChangeLogTableName(),
+            rawLatestViewName: this.rawLatestView(),
+        });
     }
     bigqueryDataset() {
         return this.bq.dataset(this.config.datasetId, {

package/lib/bigquery/initializeLatestMaterializedView.d.ts

@@ -0,0 +1,17 @@
+import { BigQuery, Table } from "@google-cloud/bigquery";
+import { FirestoreBigQueryEventHistoryTrackerConfig } from ".";
+interface InitializeLatestMaterializedViewOptions {
+    bq: BigQuery;
+    changeTrackerConfig: FirestoreBigQueryEventHistoryTrackerConfig;
+    view: Table;
+    viewExists: boolean;
+    rawChangeLogTableName: string;
+    rawLatestViewName: string;
+    schema?: any;
+}
+export declare function shouldRecreateMaterializedView(view: Table, config: FirestoreBigQueryEventHistoryTrackerConfig, source: string): Promise<boolean>;
+/**
+ * Creates the latest materialized view.
+ */
+export declare function initializeLatestMaterializedView({ bq, changeTrackerConfig: config, view, viewExists, rawChangeLogTableName, rawLatestViewName, schema, }: InitializeLatestMaterializedViewOptions): Promise<Table>;
+export {};

package/lib/bigquery/initializeLatestMaterializedView.js

@@ -0,0 +1,71 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.initializeLatestMaterializedView = exports.shouldRecreateMaterializedView = void 0;
+const logs = require("../logs");
+const snapshot_1 = require("./snapshot");
+const firebase_functions_1 = require("firebase-functions");
+const sqlFormatter = require("sql-formatter");
+async function shouldRecreateMaterializedView(view, config, source) {
+    const [viewMetadata] = await view.getMetadata();
+    const isIncremental = !viewMetadata.materializedView
+        ?.allowNonIncrementalDefinition;
+    const incrementalMatch = isIncremental === !!config.useIncrementalMaterializedView;
+    const viewQuery = viewMetadata.materializedView?.query || "";
+    const queryMatch = sqlFormatter.format(viewQuery) === sqlFormatter.format(source);
+    return !queryMatch || !incrementalMatch;
+}
+exports.shouldRecreateMaterializedView = shouldRecreateMaterializedView;
+/**
+ * Creates the latest materialized view.
+ */
+async function initializeLatestMaterializedView({ bq, changeTrackerConfig: config, view, viewExists, rawChangeLogTableName, rawLatestViewName, schema, }) {
+    try {
+        const { query, source } = config.useIncrementalMaterializedView
+            ? (0, snapshot_1.buildMaterializedViewQuery)({
+                projectId: bq.projectId,
+                datasetId: config.datasetId,
+                tableName: rawChangeLogTableName,
+                rawLatestViewName,
+                schema,
+            })
+            : (0, snapshot_1.buildNonIncrementalMaterializedViewQuery)({
+                projectId: bq.projectId,
+                datasetId: config.datasetId,
+                tableName: rawChangeLogTableName,
+                maxStaleness: config.maxStaleness,
+                refreshIntervalMinutes: config.refreshIntervalMinutes,
+                rawLatestViewName,
+                enableRefresh: true,
+                schema,
+            });
+        const desiredQuery = sqlFormatter.format(query);
+        if (viewExists) {
+            const shouldRecreate = await shouldRecreateMaterializedView(view, config, source);
+            if (!shouldRecreate) {
+                firebase_functions_1.logger.warn(`Materialized view requested, but a view with matching configuration exists. Skipping creation.`);
+                return view;
+            }
+            firebase_functions_1.logger.warn(`Configuration mismatch detected for ${rawLatestViewName} ` +
+                `Recreating view...`);
+            await view.delete();
+            return await initializeLatestMaterializedView({
+                bq,
+                changeTrackerConfig: config,
+                view,
+                viewExists: false,
+                rawChangeLogTableName,
+                rawLatestViewName,
+                schema,
+            });
+        }
+        logs.bigQueryViewCreating(rawLatestViewName, desiredQuery);
+        await bq.query(desiredQuery);
+        logs.bigQueryViewCreated(rawLatestViewName);
+    }
+    catch (error) {
+        logs.tableCreationError(rawLatestViewName, error.message);
+        throw error;
+    }
+    return view;
+}
+exports.initializeLatestMaterializedView = initializeLatestMaterializedView;

package/lib/bigquery/initializeLatestView.d.ts

@@ -0,0 +1,21 @@
+import { BigQuery, Dataset, Table } from "@google-cloud/bigquery";
+import { FirestoreBigQueryEventHistoryTrackerConfig } from ".";
+interface InitializeLatestViewOptions {
+    bq: BigQuery;
+    changeTrackerConfig: FirestoreBigQueryEventHistoryTrackerConfig;
+    dataset: Dataset;
+    view: Table;
+    viewExists: boolean;
+    rawChangeLogTableName: string;
+    rawLatestViewName: string;
+    useMaterializedView?: boolean;
+    useIncrementalMaterializedView?: boolean;
+    useLegacyQuery?: boolean;
+    refreshIntervalMinutes?: number;
+    maxStaleness?: string;
+}
+/**
+ * Creates the latest snapshot view or materialized view.
+ */
+export declare function initializeLatestView({ changeTrackerConfig: config, dataset, view, viewExists, rawChangeLogTableName, rawLatestViewName, bq, }: InitializeLatestViewOptions): Promise<Table>;
+export {};

package/lib/bigquery/initializeLatestView.js

@@ -0,0 +1,94 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.initializeLatestView = void 0;
+const schema_1 = require("./schema");
+const logs = require("../logs");
+const snapshot_1 = require("./snapshot");
+const checkUpdates_1 = require("./checkUpdates");
+const initializeLatestMaterializedView_1 = require("./initializeLatestMaterializedView");
+/**
+ * Creates the latest snapshot view or materialized view.
+ */
+async function initializeLatestView({ changeTrackerConfig: config, dataset, view, viewExists, rawChangeLogTableName, rawLatestViewName, bq, }) {
+    if (config.useMaterializedView) {
+        const schema = { fields: [...schema_1.RawChangelogViewSchema.fields] };
+        if (config.wildcardIds) {
+            schema.fields.push(schema_1.documentPathParams);
+        }
+        return (0, initializeLatestMaterializedView_1.initializeLatestMaterializedView)({
+            bq,
+            changeTrackerConfig: config,
+            view,
+            viewExists,
+            rawChangeLogTableName,
+            rawLatestViewName,
+            schema,
+        });
+    }
+    const schema = schema_1.RawChangelogViewSchema;
+    if (viewExists) {
+        logs.bigQueryViewAlreadyExists(view.id, dataset.id);
+        const [metadata] = await view.getMetadata();
+        const fields = (metadata.schema ? metadata.schema.fields : []);
+        if (config.wildcardIds) {
+            schema.fields.push(schema_1.documentPathParams);
+        }
+        const columnNames = fields.map((field) => field.name);
+        const documentIdColExists = columnNames.includes("document_id");
+        const pathParamsColExists = columnNames.includes("path_params");
+        const oldDataColExists = columnNames.includes("old_data");
+        const updateView = (0, checkUpdates_1.viewRequiresUpdate)({
+            metadata,
+            config,
+            documentIdColExists,
+            pathParamsColExists,
+            oldDataColExists,
+        });
+        if (updateView) {
+            metadata.view = (0, snapshot_1.latestConsistentSnapshotView)({
+                datasetId: config.datasetId,
+                tableName: rawChangeLogTableName,
+                schema,
+                useLegacyQuery: !config.useNewSnapshotQuerySyntax,
+            });
+            if (!documentIdColExists) {
+                logs.addNewColumn(rawLatestViewName, schema_1.documentIdField.name);
+            }
+            await view.setMetadata(metadata);
+            logs.updatingMetadata(rawLatestViewName, {
+                config,
+                documentIdColExists,
+                pathParamsColExists,
+                oldDataColExists,
+            });
+        }
+    }
+    else {
+        const schema = { fields: [...schema_1.RawChangelogViewSchema.fields] };
+        if (config.wildcardIds) {
+            schema.fields.push(schema_1.documentPathParams);
+        }
+        const latestSnapshot = (0, snapshot_1.latestConsistentSnapshotView)({
+            datasetId: config.datasetId,
+            tableName: rawChangeLogTableName,
+            schema,
+            bqProjectId: bq.projectId,
+            useLegacyQuery: !config.useNewSnapshotQuerySyntax,
+        });
+        logs.bigQueryViewCreating(rawLatestViewName, latestSnapshot.query);
+        const options = {
+            friendlyName: rawLatestViewName,
+            view: latestSnapshot,
+        };
+        try {
+            await view.create(options);
+            await view.setMetadata({ schema: schema_1.RawChangelogViewSchema });
+            logs.bigQueryViewCreated(rawLatestViewName);
+        }
+        catch (error) {
+            logs.tableCreationError(rawLatestViewName, error.message);
+        }
+    }
+    return view;
+}
+exports.initializeLatestView = initializeLatestView;

package/lib/bigquery/snapshot.d.ts

@@ -9,7 +9,7 @@ export declare const latestConsistentSnapshotView: ({ datasetId, tableName, sche
     query: string;
     useLegacySql: boolean;
 };
-interface buildLatestSnapshotViewQueryOptions {
+interface BuildLatestSnapshotViewQueryOptions {
     datasetId: string;
     tableName: string;
     timestampColumnName: string;
@@ -17,5 +17,27 @@ interface buildLatestSnapshotViewQueryOptions {
     bqProjectId?: string;
     useLegacyQuery?: boolean;
 }
-export declare function buildLatestSnapshotViewQuery({ datasetId, tableName, timestampColumnName, groupByColumns, bqProjectId, useLegacyQuery, }: buildLatestSnapshotViewQueryOptions): string;
+export declare function buildLatestSnapshotViewQuery({ datasetId, tableName, timestampColumnName, groupByColumns, bqProjectId, useLegacyQuery, }: BuildLatestSnapshotViewQueryOptions): string;
+interface MaterializedViewOptions {
+    projectId: string;
+    datasetId: string;
+    tableName: string;
+    rawLatestViewName: string;
+    schema: any;
+    refreshIntervalMinutes?: number;
+    maxStaleness?: string;
+}
+interface NonIncrementalMaterializedViewOptions extends MaterializedViewOptions {
+    enableRefresh?: boolean;
+}
+export declare function buildMaterializedViewQuery({ projectId, datasetId, tableName, rawLatestViewName, schema, refreshIntervalMinutes, maxStaleness, }: NonIncrementalMaterializedViewOptions): {
+    target: string;
+    source: string;
+    query: string;
+};
+export declare function buildNonIncrementalMaterializedViewQuery({ projectId, datasetId, tableName, rawLatestViewName, schema, refreshIntervalMinutes, maxStaleness, enableRefresh, }: NonIncrementalMaterializedViewOptions): {
+    target: string;
+    source: string;
+    query: string;
+};
 export {};

package/lib/bigquery/snapshot.js

@@ -15,18 +15,17 @@
  * limitations under the License.
  */
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.buildLatestSnapshotViewQuery = exports.latestConsistentSnapshotView = void 0;
+exports.buildNonIncrementalMaterializedViewQuery = exports.buildMaterializedViewQuery = exports.buildLatestSnapshotViewQuery = exports.latestConsistentSnapshotView = void 0;
 const sqlFormatter = require("sql-formatter");
 const schema_1 = require("./schema");
 const excludeFields = ["document_name", "document_id"];
+const nonGroupFields = ["event_id", "data", "old_data"];
 const latestConsistentSnapshotView = ({ datasetId, tableName, schema, bqProjectId, useLegacyQuery = false, }) => ({
     query: buildLatestSnapshotViewQuery({
         datasetId,
         tableName,
         timestampColumnName: schema_1.timestampField.name,
-        groupByColumns: schema["fields"]
-            .map((field) => field.name)
-            .filter((name) => excludeFields.indexOf(name) === -1),
+        groupByColumns: extractGroupByColumns(schema),
         bqProjectId,
         useLegacyQuery,
     }),
@@ -34,72 +33,187 @@ const latestConsistentSnapshotView = ({ datasetId, tableName, schema, bqProjectI
 });
 exports.latestConsistentSnapshotView = latestConsistentSnapshotView;
 function buildLatestSnapshotViewQuery({ datasetId, tableName, timestampColumnName, groupByColumns, bqProjectId, useLegacyQuery = true, }) {
-    if (datasetId === "" || tableName === "" || timestampColumnName === "") {
-        throw Error(`Missing some query parameters!`);
+    validateInputs({ datasetId, tableName, timestampColumnName, groupByColumns });
+    const projectId = bqProjectId || process.env.PROJECT_ID;
+    return useLegacyQuery
+        ? buildLegacyQuery(projectId, datasetId, tableName, timestampColumnName, groupByColumns)
+        : buildStandardQuery(projectId, datasetId, tableName, timestampColumnName, groupByColumns);
+}
+exports.buildLatestSnapshotViewQuery = buildLatestSnapshotViewQuery;
+function extractGroupByColumns(schema) {
+    return schema["fields"]
+        .map((field) => field.name)
+        .filter((name) => !excludeFields.includes(name));
+}
+function validateInputs({ datasetId, tableName, timestampColumnName, groupByColumns, }) {
+    if (!datasetId || !tableName || !timestampColumnName) {
+        throw new Error("Missing required query parameters!");
     }
-    for (let columnName of groupByColumns) {
-        if (columnName === "") {
-            throw Error(`Found empty group by column!`);
-        }
+    if (groupByColumns.some((columnName) => !columnName)) {
+        throw new Error("Group by columns must not contain empty values!");
     }
-    const legacyQuery = sqlFormatter.format(` -- Retrieves the latest document change events for all live documents.
-  --   timestamp: The Firestore timestamp at which the event took place.
-  --   operation: One of INSERT, UPDATE, DELETE, IMPORT.
-  --   event_id: The id of the event that triggered the cloud function mirrored the event.
-  --   data: A raw JSON payload of the current state of the document.
-  --   document_id: The document id as defined in the Firestore database
-  SELECT
-  document_name,
-  document_id${groupByColumns.length > 0 ? `,` : ``}
-    ${groupByColumns.join(",")}
-  FROM (
+}
+function buildLegacyQuery(projectId, datasetId, tableName, timestampColumnName, groupByColumns) {
+    return sqlFormatter.format(`
+    -- Retrieves the latest document change events for all live documents.
+    --   timestamp: The Firestore timestamp at which the event took place.
+    --   operation: One of INSERT, UPDATE, DELETE, IMPORT.
+    --   event_id: The id of the event that triggered the cloud function mirrored the event.
+    --   data: A raw JSON payload of the current state of the document.
+    --   document_id: The document id as defined in the Firestore database
     SELECT
       document_name,
-      document_id,
-      ${groupByColumns
-        .map((columnName) => `FIRST_VALUE(${columnName})
-          OVER(PARTITION BY document_name ORDER BY ${timestampColumnName} DESC)
-          AS ${columnName}`)
-        .join(",")}${groupByColumns.length > 0 ? `,` : ``}
-      FIRST_VALUE(operation)
-        OVER(PARTITION BY document_name ORDER BY ${timestampColumnName} DESC) = "DELETE"
-        AS is_deleted
-    FROM \`${bqProjectId || process.env.PROJECT_ID}.${datasetId}.${tableName}\`
-    ORDER BY document_name, ${timestampColumnName} DESC
-  )
-  WHERE NOT is_deleted
-  GROUP BY document_name, document_id${groupByColumns.length > 0 ? `, ` : ``}${groupByColumns.join(",")}`);
-    const nonGroupFields = ["event_id", "data", "old_data"];
-    const joinFields = ["document_name"];
-    const addSelectField = (field) => {
-        if (joinFields.includes(field))
-            return `t.${field}`;
-        return nonGroupFields.includes(field)
-            ? `ANY_VALUE(${field}) as ${field}`
-            : `${field} as ${field}`;
-    };
-    const filterGroupField = (field) => {
-        return nonGroupFields.includes(field);
-    };
-    const query = sqlFormatter.format(` -- Retrieves the latest document change events for all live documents.
+      document_id${groupByColumns.length > 0 ? `,` : ``}
+      ${groupByColumns.join(",")}
+    FROM (
+      SELECT
+        document_name,
+        document_id,
+        ${groupByColumns
+        .map((columnName) => `FIRST_VALUE(${columnName}) OVER (
+              PARTITION BY document_name
+              ORDER BY ${timestampColumnName} DESC
+            ) AS ${columnName}`)
+        .join(",")}${groupByColumns.length > 0 ? "," : ""}
+        FIRST_VALUE(operation) OVER (
+          PARTITION BY document_name
+          ORDER BY ${timestampColumnName} DESC
+        ) = "DELETE" AS is_deleted
+      FROM \`${projectId}.${datasetId}.${tableName}\`
+      ORDER BY document_name, ${timestampColumnName} DESC
+    )
+    WHERE NOT is_deleted
+    GROUP BY document_name, document_id${groupByColumns.length > 0 ? ", " : ""}${groupByColumns.join(",")}`);
+}
+function buildStandardQuery(projectId, datasetId, tableName, timestampColumnName, groupByColumns) {
+    return sqlFormatter.format(`
+    -- Retrieves the latest document change events for all live documents.
     --   timestamp: The Firestore timestamp at which the event took place.
     --   operation: One of INSERT, UPDATE, DELETE, IMPORT.
     --   event_id: The id of the event that triggered the cloud function mirrored the event.
     --   data: A raw JSON payload of the current state of the document.
     --   document_id: The document id as defined in the Firestore database
     WITH latest AS (
-      SELECT max(${timestampColumnName}) as latest_timestamp, document_name
-      FROM \`${bqProjectId || process.env.PROJECT_ID}.${datasetId}.${tableName}\`
+      SELECT MAX(${timestampColumnName}) AS latest_timestamp, document_name
+      FROM \`${projectId}.${datasetId}.${tableName}\`
       GROUP BY document_name
     )
     SELECT
-    t.document_name,
-    document_id${groupByColumns.length > 0 ? `,` : ``}
-      ${groupByColumns.map((f) => addSelectField(f)).join(",")}
-    FROM \`${bqProjectId || process.env.PROJECT_ID}.${datasetId}.${tableName}\` AS t
-    JOIN latest ON (t.document_name = latest.document_name AND (IFNULL(t.${timestampColumnName}, timestamp("1970-01-01 00:00:00+00"))) = (IFNULL(latest.latest_timestamp, timestamp("1970-01-01 00:00:00+00"))))
+      t.document_name,
+      document_id${groupByColumns.length > 0 ? "," : ""}
+      ${groupByColumns
+        .map((field) => nonGroupFields.includes(field)
+        ? `ANY_VALUE(${field}) AS ${field}`
+        : `${field} AS ${field}`)
+        .join(",")}
+    FROM \`${projectId}.${datasetId}.${tableName}\` AS t
+    JOIN latest ON (
+      t.document_name = latest.document_name AND
+      IFNULL(t.${timestampColumnName}, TIMESTAMP("1970-01-01 00:00:00+00")) =
+      IFNULL(latest.latest_timestamp, TIMESTAMP("1970-01-01 00:00:00+00"))
+    )
     WHERE operation != "DELETE"
-    GROUP BY document_name, document_id${groupByColumns.length > 0 ? `, ` : ``}${groupByColumns.filter((c) => !filterGroupField(c)).join(",")}`);
-    return useLegacyQuery ? legacyQuery : query;
+    GROUP BY document_name, document_id${groupByColumns.length > 0 ? ", " : ""}${groupByColumns
+        .filter((field) => !nonGroupFields.includes(field))
+        .join(",")}`);
 }
-exports.buildLatestSnapshotViewQuery = buildLatestSnapshotViewQuery;
+// Helper function to extract fields from schema
+function extractFieldsFromSchema(schema) {
+    if (!schema || !schema.fields) {
+        throw new Error("Invalid schema: must contain fields array");
+    }
+    return schema.fields.map((field) => field.name);
+}
+function buildMaterializedViewQuery({ projectId, datasetId, tableName, rawLatestViewName, schema, refreshIntervalMinutes, maxStaleness, }) {
+    // Build the options string
+    const options = [];
+    if (refreshIntervalMinutes !== undefined) {
+        options.push(`refresh_interval_minutes = ${refreshIntervalMinutes}`);
+    }
+    if (maxStaleness) {
+        options.push(`max_staleness = ${maxStaleness}`);
+    }
+    const optionsString = options.length > 0
+        ? `OPTIONS (
+    ${options.join(",\n  ")}
+  )`
+        : "";
+    // Extract fields from schema
+    const fields = extractFieldsFromSchema(schema);
+    // Build the aggregated fields for the CTE
+    const aggregatedFields = fields
+        .map((fieldName) => {
+        if (fieldName === "document_name") {
+            return "  document_name";
+        }
+        if (fieldName === "timestamp") {
+            return "  MAX(timestamp) AS timestamp";
+        }
+        return `  MAX_BY(${fieldName}, timestamp) AS ${fieldName}`;
+    })
+        .join(",\n        ");
+    const target = `CREATE MATERIALIZED VIEW \`${projectId}.${datasetId}.${rawLatestViewName}\` ${optionsString}`;
+    const source = `
+    WITH latests AS (
+      SELECT 
+      ${aggregatedFields}
+      FROM \`${projectId}.${datasetId}.${tableName}\`
+      GROUP BY document_name
+    )
+    SELECT *
+    FROM latests
+  `;
+    // Combine all parts with options before AS
+    const fullQuery = sqlFormatter.format(`${target} AS (${source})`);
+    return { target, source, query: fullQuery };
+}
+exports.buildMaterializedViewQuery = buildMaterializedViewQuery;
+function buildNonIncrementalMaterializedViewQuery({ projectId, datasetId, tableName, rawLatestViewName, schema, refreshIntervalMinutes, maxStaleness, enableRefresh = true, }) {
+    // Build the options string
+    const options = [];
+    options.push("allow_non_incremental_definition = true");
+    if (enableRefresh !== undefined) {
+        options.push(`enable_refresh = ${enableRefresh}`);
+    }
+    if (refreshIntervalMinutes !== undefined) {
+        options.push(`refresh_interval_minutes = ${refreshIntervalMinutes}`);
+    }
+    if (maxStaleness) {
+        options.push(`max_staleness = ${maxStaleness}`);
+    }
+    const optionsString = options.length > 0
+        ? `OPTIONS (
+    ${options.join(",\n  ")}
+  )`
+        : "";
+    // Extract fields from schema
+    const fields = extractFieldsFromSchema(schema);
+    // Build the aggregated fields for the CTE
+    const aggregatedFields = fields
+        .map((fieldName) => {
+        if (fieldName === "document_name") {
+            return "  document_name";
+        }
+        if (fieldName === "timestamp") {
+            return "  MAX(timestamp) AS timestamp";
+        }
+        return `  MAX_BY(${fieldName}, timestamp) AS ${fieldName}`;
+    })
+        .join(",\n        ");
+    const target = `CREATE MATERIALIZED VIEW \`${projectId}.${datasetId}.${rawLatestViewName}\` ${optionsString}`;
+    const source = `
+    WITH latests AS (
+      SELECT 
+      ${aggregatedFields}
+      FROM \`${projectId}.${datasetId}.${tableName}\`
+      GROUP BY document_name
+    )
+    SELECT *
+    FROM latests
+    WHERE operation != "DELETE"
+  `;
+    // Combine all parts with options before AS
+    const fullQuery = sqlFormatter.format(`${target} AS (${source})`);
+    return { target, source, query: fullQuery };
+}
+exports.buildNonIncrementalMaterializedViewQuery = buildNonIncrementalMaterializedViewQuery;

package/lib/bigquery/utils.d.ts

@@ -2,6 +2,7 @@ import { Dataset, Table } from "@google-cloud/bigquery";
 interface WaitForInitializationParams {
     dataset: Dataset;
     changelogName: string;
+    materializedViewName?: string;
 }
 /**
  * Periodically checks for the existence of a dataset and table until both are found or a maximum number of attempts is reached.
@@ -10,6 +11,6 @@ interface WaitForInitializationParams {
  * @returns {Promise<Table>} A promise that resolves with the Table if it exists, or rejects if it doesn't exist after maxAttempts or an error occurs.
  * @throws {Error} Throws an error if the dataset or table cannot be verified to exist after multiple attempts or if an unexpected error occurs.
  */
-export declare function waitForInitialization({ dataset, changelogName }: WaitForInitializationParams, maxAttempts?: number): Promise<Table>;
+export declare function waitForInitialization({ dataset, changelogName, materializedViewName }: WaitForInitializationParams, maxAttempts?: number): Promise<Table>;
 export declare function parseErrorMessage(error: unknown, process?: string): string;
 export {};

package/lib/bigquery/utils.js

@@ -9,7 +9,7 @@ const logs = require("../logs");
  * @returns {Promise<Table>} A promise that resolves with the Table if it exists, or rejects if it doesn't exist after maxAttempts or an error occurs.
  * @throws {Error} Throws an error if the dataset or table cannot be verified to exist after multiple attempts or if an unexpected error occurs.
  */
-async function waitForInitialization({ dataset, changelogName }, maxAttempts = 12) {
+async function waitForInitialization({ dataset, changelogName, materializedViewName }, maxAttempts = 12) {
     return new Promise((resolve, reject) => {
         let attempts = 0;
         let handle = setInterval(async () => {
@@ -17,7 +17,13 @@ async function waitForInitialization({ dataset, changelogName }, maxAttempts = 1
                 const [datasetExists] = await dataset.exists();
                 const table = dataset.table(changelogName);
                 const [tableExists] = await table.exists();
-                if (datasetExists && tableExists) {
+                let waitingForMaterializedView = false;
+                if (materializedViewName) {
+                    const materializedView = dataset.table(materializedViewName);
+                    const [materializedViewExists] = await materializedView.exists();
+                    waitingForMaterializedView = !materializedViewExists;
+                }
+                if (datasetExists && tableExists && !waitingForMaterializedView) {
                     clearInterval(handle);
                     resolve(table);
                 }

package/lib/bigquery/utils.test.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/bigquery/utils.test.js

@@ -0,0 +1,65 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const utils_1 = require("./utils");
+const logs = require("../logs");
+jest.mock("@google-cloud/bigquery");
+jest.mock("../../logs");
+const dataset = {
+    exists: jest.fn(),
+    table: jest.fn(),
+};
+const table = {
+    exists: jest.fn(),
+};
+const changelogName = "testTable";
+describe("waitForInitialization", () => {
+    beforeEach(() => {
+        jest.clearAllMocks();
+        dataset.table.mockReturnValue(table);
+    });
+    test("should successfully find the dataset and table", async () => {
+        dataset.exists.mockResolvedValue([true]);
+        table.exists.mockResolvedValue([true]);
+        const result = await (0, utils_1.waitForInitialization)({
+            dataset: dataset,
+            changelogName,
+        });
+        expect(result).toBe(table);
+        expect(dataset.exists).toHaveBeenCalledTimes(1);
+        expect(table.exists).toHaveBeenCalledTimes(1);
+    });
+    test("should fail after max attempts if table does not exist", async () => {
+        dataset.exists.mockResolvedValue([true]);
+        table.exists.mockResolvedValue([false]);
+        await expect((0, utils_1.waitForInitialization)({ dataset: dataset, changelogName }, 3)).rejects.toThrow("Initialization timed out. Dataset or table could not be verified to exist after multiple attempts.");
+        expect(dataset.exists).toHaveBeenCalledTimes(3);
+        expect(table.exists).toHaveBeenCalledTimes(3);
+    });
+    test("should handle and throw an error if dataset.exists throws", async () => {
+        const error = new Error("Access denied");
+        dataset.exists.mockRejectedValue(error);
+        await expect((0, utils_1.waitForInitialization)({
+            dataset: dataset,
+            changelogName,
+        })).rejects.toThrow("Access denied");
+        expect(logs.failedToInitializeWait).toHaveBeenCalledWith(error.message);
+    });
+    test("should handle and throw an error if table.exists throws", async () => {
+        dataset.exists.mockResolvedValue([true]);
+        const error = new Error("Table error");
+        table.exists.mockRejectedValue(error);
+        await expect((0, utils_1.waitForInitialization)({
+            dataset: dataset,
+            changelogName,
+        })).rejects.toThrow("Table error");
+        expect(logs.failedToInitializeWait).toHaveBeenCalledWith(error.message);
+    });
+    test("should handle unexpected error types gracefully", async () => {
+        dataset.exists.mockRejectedValue("String error");
+        await expect((0, utils_1.waitForInitialization)({
+            dataset: dataset,
+            changelogName,
+        })).rejects.toThrow("An unexpected error occurred");
+        expect(logs.failedToInitializeWait).toHaveBeenCalledWith("An unexpected error occurred");
+    });
+});

package/lib/logs.d.ts

@@ -50,6 +50,7 @@ export declare const cannotPartitionExistingTable: (table: Table) => void;
 export declare function invalidProjectIdWarning(bqProjectId: string): void;
 export declare function invalidTableReference(): void;
 export declare function hourAndDatePartitioningWarning(): void;
+export declare function invalidClusteringTypes(fields: string): void;
 export declare function invalidClustering(fields: string): void;
 export declare const tableCreationError: (table: any, message: any) => void;
 export declare const failedToInitializeWait: (message: any) => void;

package/lib/logs.js

@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.updatingMetadata = exports.failedToInitializeWait = exports.tableCreationError = exports.invalidClustering = exports.hourAndDatePartitioningWarning = exports.invalidTableReference = exports.invalidProjectIdWarning = exports.cannotPartitionExistingTable = exports.removedClustering = exports.updatedClustering = exports.bigQueryTableInsertErrors = exports.firestoreTimePartitioningParametersWarning = exports.firestoreTimePartitionFieldError = exports.addPartitionFieldColumn = exports.addNewColumn = exports.timestampMissingValue = exports.error = exports.dataTypeInvalid = exports.dataInserting = exports.dataInsertRetried = exports.dataInserted = exports.complete = exports.bigQueryViewValidating = exports.bigQueryViewValidated = exports.bigQueryViewUpToDate = exports.bigQueryViewUpdating = exports.bigQueryViewUpdated = exports.bigQueryViewAlreadyExists = exports.bigQueryViewCreating = exports.bigQueryViewCreated = exports.bigQueryUserDefinedFunctionCreated = exports.bigQueryUserDefinedFunctionCreating = exports.bigQueryTableValidating = exports.bigQueryTableValidated = exports.bigQueryTableUpToDate = exports.bigQueryTableUpdating = exports.bigQueryTableUpdated = exports.bigQueryTableCreating = exports.bigQueryTableCreated = exports.bigQueryTableAlreadyExists = exports.bigQuerySchemaViewCreated = exports.bigQueryLatestSnapshotViewQueryCreated = exports.bigQueryErrorRecordingDocumentChange = exports.bigQueryDatasetExists = exports.bigQueryDatasetCreating = exports.bigQueryDatasetCreated = exports.arrayFieldInvalid = void 0;
+exports.updatingMetadata = exports.failedToInitializeWait = exports.tableCreationError = exports.invalidClustering = exports.invalidClusteringTypes = exports.hourAndDatePartitioningWarning = exports.invalidTableReference = exports.invalidProjectIdWarning = exports.cannotPartitionExistingTable = exports.removedClustering = exports.updatedClustering = exports.bigQueryTableInsertErrors = exports.firestoreTimePartitioningParametersWarning = exports.firestoreTimePartitionFieldError = exports.addPartitionFieldColumn = exports.addNewColumn = exports.timestampMissingValue = exports.error = exports.dataTypeInvalid = exports.dataInserting = exports.dataInsertRetried = exports.dataInserted = exports.complete = exports.bigQueryViewValidating = exports.bigQueryViewValidated = exports.bigQueryViewUpToDate = exports.bigQueryViewUpdating = exports.bigQueryViewUpdated = exports.bigQueryViewAlreadyExists = exports.bigQueryViewCreating = exports.bigQueryViewCreated = exports.bigQueryUserDefinedFunctionCreated = exports.bigQueryUserDefinedFunctionCreating = exports.bigQueryTableValidating = exports.bigQueryTableValidated = exports.bigQueryTableUpToDate = exports.bigQueryTableUpdating = exports.bigQueryTableUpdated = exports.bigQueryTableCreating = exports.bigQueryTableCreated = exports.bigQueryTableAlreadyExists = exports.bigQuerySchemaViewCreated = exports.bigQueryLatestSnapshotViewQueryCreated = exports.bigQueryErrorRecordingDocumentChange = exports.bigQueryDatasetExists = exports.bigQueryDatasetCreating = exports.bigQueryDatasetCreated = exports.arrayFieldInvalid = void 0;
 const firebase_functions_1 = require("firebase-functions");
 const arrayFieldInvalid = (fieldName) => {
     firebase_functions_1.logger.warn(`Array field '${fieldName}' does not contain an array, skipping`);
@@ -201,6 +201,10 @@ function hourAndDatePartitioningWarning() {
     firebase_functions_1.logger.warn(`Cannot partition table with hour partitioning and Date. For DATE columns, the partitions can have daily, monthly, or yearly granularity. Skipping partitioning`);
 }
 exports.hourAndDatePartitioningWarning = hourAndDatePartitioningWarning;
+function invalidClusteringTypes(fields) {
+    firebase_functions_1.logger.warn(`Unable to add clustering, field(s) ${fields} have invalid types.`);
+}
+exports.invalidClusteringTypes = invalidClusteringTypes;
 function invalidClustering(fields) {
     firebase_functions_1.logger.warn(`Unable to add clustering, field(s) ${fields} do not exist on the expected table`);
 }

package/package.json

@@ -5,7 +5,7 @@
     "url": "github.com/firebase/extensions.git",
     "directory": "firestore-bigquery-export/firestore-bigquery-change-tracker"
   },
-  "version": "1.1.38",
+  "version": "1.1.39",
   "description": "Core change-tracker library for Cloud Firestore Collection BigQuery Exports",
   "main": "./lib/index.js",
   "scripts": {
@@ -38,17 +38,18 @@
   },
   "devDependencies": {
     "@types/chai": "^4.1.6",
-    "@types/jest": "29.5.0",
+    "@types/jest": "^29.5.14",
     "@types/node": "14.18.34",
     "@types/traverse": "^0.6.32",
     "chai": "^4.2.0",
-    "nyc": "^14.0.0",
-    "rimraf": "^2.6.3",
-    "typescript": "^4.9.4",
     "jest": "29.5.0",
+    "jest-config": "29.5.0",
     "jest-environment-node": "29.5.0",
+    "jest-summarizing-reporter": "^1.1.4",
     "mocked-env": "^1.3.2",
+    "nyc": "^17.1.0",
+    "rimraf": "^2.6.3",
     "ts-jest": "29.1.2",
-    "jest-config": "29.5.0"
+    "typescript": "^4.9.4"
   }
 }