@firebaseextensions/firestore-bigquery-change-tracker 1.1.13 → 1.1.16

package/lib/bigquery/clustering.js

@@ -0,0 +1,49 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Clustering = void 0;
+const logs = require("../logs");
+class Clustering {
+    constructor(config, table, schema) {
+        this.updateCluster = async (metaData) => {
+            const clustering = { fields: this.config.clustering };
+            metaData.clustering = clustering;
+            logs.updatedClustering(this.config.clustering.join(","));
+            return Promise.resolve();
+        };
+        this.removeCluster = async (metaData) => {
+            metaData.clustering = null;
+            logs.removedClustering(this.table.id);
+            return Promise.resolve();
+        };
+        this.updateClustering = async (metaData) => {
+            /** Return if invalid config */
+            if (await this.hasInvalidFields(metaData))
+                return Promise.resolve();
+            return !!this.config.clustering && !!this.config.clustering.length
+                ? this.updateCluster(metaData)
+                : this.removeCluster(metaData);
+        };
+        this.config = config;
+        this.table = table;
+        this.schema = schema;
+    }
+    hasValidTableReference() {
+        logs.invalidTableReference();
+        return !!this.table;
+    }
+    async hasInvalidFields(metaData) {
+        const { clustering = [] } = this.config;
+        if (!clustering)
+            return Promise.resolve(false);
+        const fieldNames = metaData
+            ? metaData.schema.fields.map(($) => $.name)
+            : [];
+        const invalidFields = clustering.filter(($) => !fieldNames.includes($));
+        if (invalidFields.length) {
+            logs.invalidClustering(invalidFields.join(","));
+            return Promise.resolve(true);
+        }
+        return Promise.resolve(false);
+    }
+}
+exports.Clustering = Clustering;

package/lib/bigquery/handleFailedTransactions.js

@@ -0,0 +1,31 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const firebase = require("firebase-admin");
+if (!firebase.apps.length) {
+    firebase.initializeApp();
+    firebase.firestore().settings({ ignoreUndefinedProperties: true });
+}
+exports.default = async (rows, config, e) => {
+    const db = firebase.firestore();
+    const batchArray = [db.batch()];
+    let operationCounter = 0;
+    let batchIndex = 0;
+    rows.forEach((row) => {
+        var ref = db.collection(config.backupTableId).doc(row.insertId);
+        batchArray[batchIndex].set(ref, {
+            ...row,
+            error_details: e.message,
+        });
+        operationCounter++;
+        // Check if max limit for batch has been met.
+        if (operationCounter === 499) {
+            batchArray.push(db.batch());
+            batchIndex++;
+            operationCounter = 0;
+        }
+    });
+    for (let batch of batchArray) {
+        await batch.commit();
+    }
+    return Promise.resolve();
+};

package/lib/bigquery/index.js

@@ -19,10 +19,14 @@ exports.FirestoreBigQueryEventHistoryTracker = void 0;
 const bigquery = require("@google-cloud/bigquery");
 const firebase = require("firebase-admin");
 const traverse = require("traverse");
+const node_fetch_1 = require("node-fetch");
 const schema_1 = require("./schema");
 const snapshot_1 = require("./snapshot");
+const handleFailedTransactions_1 = require("./handleFailedTransactions");
 const tracker_1 = require("../tracker");
 const logs = require("../logs");
+const partitioning_1 = require("./partitioning");
+const clustering_1 = require("./clustering");
 var schema_2 = require("./schema");
 Object.defineProperty(exports, "RawChangelogSchema", { enumerable: true, get: function () { return schema_2.RawChangelogSchema; } });
 Object.defineProperty(exports, "RawChangelogViewSchema", { enumerable: true, get: function () { return schema_2.RawChangelogViewSchema; } });
@@ -38,15 +42,19 @@ Object.defineProperty(exports, "RawChangelogViewSchema", { enumerable: true, get
 class FirestoreBigQueryEventHistoryTracker {
     constructor(config) {
         this.config = config;
-        this.initialized = false;
+        this._initialized = false;
         this.bq = new bigquery.BigQuery();
+        this.bq.projectId = config.bqProjectId || process.env.PROJECT_ID;
         if (!this.config.datasetLocation) {
             this.config.datasetLocation = "us";
         }
     }
     async record(events) {
         await this.initialize();
+        const partitionHandler = new partitioning_1.Partitioning(this.config);
         const rows = events.map((event) => {
+            const partitionValue = partitionHandler.getPartitionValue(event);
+            const { documentId, ...pathParams } = event.pathParams || {};
             return {
                 insertId: event.eventId,
                 json: {
@@ -56,10 +64,26 @@ class FirestoreBigQueryEventHistoryTracker {
                     document_id: event.documentId,
                     operation: tracker_1.ChangeType[event.operation],
                     data: JSON.stringify(this.serializeData(event.data)),
+                    ...partitionValue,
+                    ...(this.config.wildcardIds &&
+                        event.pathParams && { path_params: JSON.stringify(pathParams) }),
                 },
             };
         });
-        await this.insertData(rows);
+        const transformedRows = await this.transformRows(rows);
+        await this.insertData(transformedRows);
+    }
+    async transformRows(rows) {
+        if (this.config.transformFunction && this.config.transformFunction !== "") {
+            const response = await node_fetch_1.default(this.config.transformFunction, {
+                method: "post",
+                body: JSON.stringify({ data: rows }),
+                headers: { "Content-Type": "application/json" },
+            });
+            const responseJson = await response.json();
+            return responseJson.data;
+        }
+        return rows;
     }
     serializeData(eventData) {
         if (typeof eventData === "undefined") {
@@ -87,10 +111,8 @@ class FirestoreBigQueryEventHistoryTracker {
     async isRetryableInsertionError(e) {
         let isRetryable = true;
         const expectedErrors = [
-            {
-                message: "no such field.",
-                location: "document_id",
-            },
+            { message: "no such field.", location: schema_1.documentIdField.name },
+            { message: "no such field.", location: schema_1.documentPathParams.name },
         ];
         if (e.response &&
             e.response.insertErrors &&
@@ -111,6 +133,32 @@ class FirestoreBigQueryEventHistoryTracker {
         }
         return isRetryable;
     }
+    /**
+     * Tables can often take time to create and propagate.
+     * A half a second delay is added per check while the function
+     * continually re-checks until the referenced dataset and table become available.
+     */
+    async waitForInitialization() {
+        return new Promise((resolve) => {
+            let handle = setInterval(async () => {
+                try {
+                    const dataset = this.bigqueryDataset();
+                    const changelogName = this.rawChangeLogTableName();
+                    const table = dataset.table(changelogName);
+                    const [datasetExists] = await dataset.exists();
+                    const [tableExists] = await table.exists();
+                    if (datasetExists && tableExists) {
+                        clearInterval(handle);
+                        return resolve(table);
+                    }
+                }
+                catch (ex) {
+                    clearInterval(handle);
+                    logs.failedToInitializeWait(ex.message);
+                }
+            }, 5000);
+        });
+    }
     /**
      * Inserts rows of data into the BigQuery raw change log table.
      */
@@ -134,8 +182,13 @@ class FirestoreBigQueryEventHistoryTracker {
                 logs.dataInsertRetried(rows.length);
                 return this.insertData(rows, { ...overrideOptions, ignoreUnknownValues: true }, retry);
             }
+            // Exceeded number of retires, save in failed collection
+            if (!retry && this.config.backupTableId) {
+                await handleFailedTransactions_1.default(rows, this.config, e);
+            }
             // Reinitializing in case the destintation table is modified.
-            this.initialized = false;
+            this._initialized = false;
+            logs.bigQueryTableInsertErrors(e.errors);
             throw e;
         }
     }
@@ -144,13 +197,19 @@ class FirestoreBigQueryEventHistoryTracker {
      * After the first invokation, it skips initialization assuming these resources are still there.
      */
     async initialize() {
-        if (this.initialized) {
-            return;
+        try {
+            if (this._initialized) {
+                return;
+            }
+            await this.initializeDataset();
+            await this.initializeRawChangeLogTable();
+            await this.initializeLatestView();
+            this._initialized = true;
+        }
+        catch (ex) {
+            await this.waitForInitialization();
+            this._initialized = true;
         }
-        await this.initializeDataset();
-        await this.initializeRawChangeLogTable();
-        await this.initializeLatestView();
-        this.initialized = true;
     }
     /**
      * Creates the specified dataset if it doesn't already exists.
@@ -162,9 +221,14 @@ class FirestoreBigQueryEventHistoryTracker {
             logs.bigQueryDatasetExists(this.config.datasetId);
         }
         else {
-            logs.bigQueryDatasetCreating(this.config.datasetId);
-            await dataset.create();
-            logs.bigQueryDatasetCreated(this.config.datasetId);
+            try {
+                logs.bigQueryDatasetCreating(this.config.datasetId);
+                await dataset.create();
+                logs.bigQueryDatasetCreated(this.config.datasetId);
+            }
+            catch (ex) {
+                logs.tableCreationError(this.config.datasetId, ex.message);
+            }
         }
         return dataset;
     }
@@ -176,30 +240,47 @@ class FirestoreBigQueryEventHistoryTracker {
         const dataset = this.bigqueryDataset();
         const table = dataset.table(changelogName);
         const [tableExists] = await table.exists();
+        const partitioning = new partitioning_1.Partitioning(this.config, table);
+        const clustering = new clustering_1.Clustering(this.config, table);
         if (tableExists) {
             logs.bigQueryTableAlreadyExists(table.id, dataset.id);
             const [metadata] = await table.getMetadata();
-            const fields = metadata.schema.fields;
+            const fields = metadata.schema ? metadata.schema.fields : [];
+            await clustering.updateClustering(metadata);
             const documentIdColExists = fields.find((column) => column.name === "document_id");
+            const pathParamsColExists = fields.find((column) => column.name === "path_params");
             if (!documentIdColExists) {
                 fields.push(schema_1.documentIdField);
+                logs.addNewColumn(this.rawChangeLogTableName(), schema_1.documentIdField.name);
+            }
+            if (!pathParamsColExists && this.config.wildcardIds) {
+                fields.push(schema_1.documentPathParams);
+                logs.addNewColumn(this.rawChangeLogTableName(), schema_1.documentPathParams.name);
+            }
+            await partitioning.addPartitioningToSchema(metadata.schema.fields);
+            if (!documentIdColExists || !pathParamsColExists) {
                 await table.setMetadata(metadata);
-                logs.addDocumentIdColumn(this.rawChangeLogTableName());
             }
         }
         else {
             logs.bigQueryTableCreating(changelogName);
-            const options = {
-                friendlyName: changelogName,
-                schema: schema_1.RawChangelogSchema,
-            };
-            if (this.config.tablePartitioning) {
-                options.timePartitioning = {
-                    type: this.config.tablePartitioning,
-                };
+            const schema = { fields: [...schema_1.RawChangelogSchema.fields] };
+            if (this.config.wildcardIds) {
+                schema.fields.push(schema_1.documentPathParams);
+            }
+            const options = { friendlyName: changelogName, schema };
+            //Add partitioning
+            await partitioning.addPartitioningToSchema(schema.fields);
+            await partitioning.updateTableMetadata(options);
+            // Add clustering
+            await clustering.updateClustering(options);
+            try {
+                await table.create(options);
+                logs.bigQueryTableCreated(changelogName);
+            }
+            catch (ex) {
+                logs.tableCreationError(changelogName, ex.message);
             }
-            await table.create(options);
-            logs.bigQueryTableCreated(changelogName);
         }
         return table;
     }
@@ -211,32 +292,64 @@ class FirestoreBigQueryEventHistoryTracker {
         const dataset = this.bigqueryDataset();
         const view = dataset.table(this.rawLatestView());
         const [viewExists] = await view.exists();
+        const schema = schema_1.RawChangelogViewSchema;
+        const partitioning = new partitioning_1.Partitioning(this.config, view);
         if (viewExists) {
             logs.bigQueryViewAlreadyExists(view.id, dataset.id);
             const [metadata] = await view.getMetadata();
-            const fields = metadata.schema.fields;
+            const fields = metadata.schema ? metadata.schema.fields : [];
+            if (this.config.wildcardIds) {
+                schema.fields.push(schema_1.documentPathParams);
+            }
             const documentIdColExists = fields.find((column) => column.name === "document_id");
+            const pathParamsColExists = fields.find((column) => column.name === "path_params");
             if (!documentIdColExists) {
-                metadata.view = snapshot_1.latestConsistentSnapshotView(this.config.datasetId, this.rawChangeLogTableName());
-                await view.setMetadata(metadata);
-                logs.addDocumentIdColumn(this.rawLatestView());
+                metadata.view = snapshot_1.latestConsistentSnapshotView(this.config.datasetId, this.rawChangeLogTableName(), schema);
+                logs.addNewColumn(this.rawLatestView(), schema_1.documentIdField.name);
+            }
+            if (!pathParamsColExists && this.config.wildcardIds) {
+                metadata.view = snapshot_1.latestConsistentSnapshotView(this.config.datasetId, this.rawChangeLogTableName(), schema);
+                logs.addNewColumn(this.rawLatestView(), schema_1.documentPathParams.name);
             }
+            //Add partitioning
+            await partitioning.addPartitioningToSchema(schema.fields);
+            //TODO: Tidy up and format / add test cases?
+            // if (
+            //   !documentIdColExists ||
+            //   (!pathParamsColExists && this.config.wildcardIds) ||
+            //   partition.isValidPartitionForExistingTable(partitionColExists)
+            // ) {
+            await view.setMetadata(metadata);
+            // }
         }
         else {
-            const latestSnapshot = snapshot_1.latestConsistentSnapshotView(this.config.datasetId, this.rawChangeLogTableName());
+            const schema = { fields: [...schema_1.RawChangelogViewSchema.fields] };
+            //Add partitioning field
+            await partitioning.addPartitioningToSchema(schema.fields);
+            //TODO Create notification for a user that View cannot be Time Partitioned by the field.
+            // await partitioning.updateTableMetadata(options);
+            if (this.config.wildcardIds) {
+                schema.fields.push(schema_1.documentPathParams);
+            }
+            const latestSnapshot = snapshot_1.latestConsistentSnapshotView(this.config.datasetId, this.rawChangeLogTableName(), schema, this.bq.projectId);
             logs.bigQueryViewCreating(this.rawLatestView(), latestSnapshot.query);
             const options = {
                 friendlyName: this.rawLatestView(),
                 view: latestSnapshot,
             };
-            if (this.config.tablePartitioning) {
+            if (this.config.timePartitioning) {
                 options.timePartitioning = {
-                    type: this.config.tablePartitioning,
+                    type: this.config.timePartitioning,
                 };
             }
-            await view.create(options);
-            await view.setMetadata({ schema: schema_1.RawChangelogViewSchema });
-            logs.bigQueryViewCreated(this.rawLatestView());
+            try {
+                await view.create(options);
+                await view.setMetadata({ schema: schema_1.RawChangelogViewSchema });
+                logs.bigQueryViewCreated(this.rawLatestView());
+            }
+            catch (ex) {
+                logs.tableCreationError(this.rawLatestView(), ex.message);
+            }
         }
         return view;
     }

package/lib/bigquery/partitioning.js

@@ -0,0 +1,232 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Partitioning = void 0;
+const firebase = require("firebase-admin");
+const logs = require("../logs");
+const schema_1 = require("./schema");
+const bigquery_1 = require("@google-cloud/bigquery");
+const types_1 = require("../types");
+class Partitioning {
+    constructor(config, table, schema) {
+        this.config = config;
+        this.table = table;
+        this.schema = schema;
+    }
+    isPartitioningEnabled() {
+        const { timePartitioning } = this.config;
+        return !!timePartitioning;
+    }
+    isValidPartitionTypeString(value) {
+        return typeof value === "string";
+    }
+    async metaDataSchemaFields() {
+        let metadata;
+        try {
+            [metadata] = await this.table.getMetadata();
+        }
+        catch {
+            console.log("No metadata found");
+            return null;
+        }
+        /** Return null if no valid schema on table **/
+        if (!metadata.schema)
+            return null;
+        return metadata.schema.fields;
+    }
+    isValidPartitionTypeDate(value) {
+        /* Check if valid timestamp value from sdk */
+        if (value instanceof firebase.firestore.Timestamp)
+            return true;
+        /* Check if valid date/time value from console */
+        return Object.prototype.toString.call(value) === "[object Date]";
+    }
+    hasHourAndDatePartitionConfig() {
+        if (this.config.timePartitioning === "HOUR" &&
+            this.config.timePartitioningFieldType === "DATE") {
+            logs.hourAndDatePartitioningWarning();
+            return true;
+        }
+        return false;
+    }
+    hasValidCustomPartitionConfig() {
+        /* Return false if partition type option has not been set*/
+        if (!this.isPartitioningEnabled())
+            return false;
+        const { timePartitioningField, timePartitioningFieldType, timePartitioningFirestoreField, } = this.config;
+        const hasNoCustomOptions = !timePartitioningField &&
+            !timePartitioningFieldType &&
+            !timePartitioningFirestoreField;
+        /* No custom congig has been set, use partition value option only */
+        if (hasNoCustomOptions)
+            return true;
+        /* check if all options have been provided to be  */
+        return (!!timePartitioningField &&
+            !!timePartitioningFieldType &&
+            !!timePartitioningFirestoreField);
+    }
+    hasValidTimePartitionOption() {
+        const { timePartitioning } = this.config;
+        return ["HOUR", "DAY", "MONTH", "YEAR"].includes(timePartitioning);
+    }
+    hasValidTimePartitionType() {
+        const { timePartitioningFieldType } = this.config;
+        if (!timePartitioningFieldType || timePartitioningFieldType === undefined)
+            return true;
+        return ["TIMESTAMP", "DATE", "DATETIME"].includes(timePartitioningFieldType);
+    }
+    async hasExistingSchema() {
+        const [metadata] = await this.table.getMetadata();
+        return !!metadata.schema;
+    }
+    hasValidTableReference() {
+        logs.invalidTableReference();
+        return !!this.table;
+    }
+    async isTablePartitioned() {
+        if (!this.table)
+            return Promise.resolve(false);
+        // No table provided, cannot evaluate
+        if (this.table.exists()) {
+            logs.cannotPartitionExistingTable(this.table);
+            return Promise.resolve(false);
+        }
+        /*** No table exists, return */
+        const [tableExists] = await this.table.exists();
+        if (!tableExists)
+            return Promise.resolve(false);
+        /* Check if partition metadata already exists */
+        const [metadata] = await this.table.getMetadata();
+        if (!!metadata.timePartitioning)
+            return Promise.resolve(true);
+        /** Find schema fields **/
+        const schemaFields = await this.metaDataSchemaFields();
+        /** No Schema exists, return */
+        if (!schemaFields)
+            return Promise.resolve(false);
+        /* Return false if time partition field not found */
+        return schemaFields.some((column) => column.name === this.config.timePartitioningField);
+    }
+    async isValidPartitionForExistingTable() {
+        if (this.isTablePartitioned())
+            return false;
+        return this.hasValidCustomPartitionConfig();
+    }
+    isValidPartitionForNewTable() {
+        if (!this.isPartitioningEnabled())
+            return false;
+        return this.hasValidCustomPartitionConfig();
+    }
+    convertDateValue(fieldValue) {
+        const { timePartitioningFieldType } = this.config;
+        /* Return as Datetime value */
+        if (timePartitioningFieldType === types_1.PartitionFieldType.DATETIME) {
+            return bigquery_1.BigQuery.datetime(fieldValue.toISOString()).value;
+        }
+        /* Return as Date value */
+        if (timePartitioningFieldType === types_1.PartitionFieldType.DATE) {
+            return bigquery_1.BigQuery.date(fieldValue.toISOString().substring(0, 10)).value;
+        }
+        /* Return as Timestamp  */
+        return bigquery_1.BigQuery.timestamp(fieldValue).value;
+    }
+    /*
+      Extracts a valid Partition field from the Document Change Event.
+      Matches result based on a pre-defined Firestore field matching the event data object.
+      Return an empty object if no field name or value provided.
+      Returns empty object if not a string or timestamp
+      Logs warning if not a valid datatype
+      Delete changes events have no data, return early as cannot partition on empty data.
+    **/
+    getPartitionValue(event) {
+        if (!event.data)
+            return {};
+        const firestoreFieldName = this.config.timePartitioningFirestoreField;
+        const fieldName = this.config.timePartitioningField;
+        const fieldValue = event.data[firestoreFieldName];
+        if (!fieldName || !fieldValue) {
+            return {};
+        }
+        if (this.isValidPartitionTypeString(fieldValue)) {
+            return { [fieldName]: fieldValue };
+        }
+        if (this.isValidPartitionTypeDate(fieldValue)) {
+            /* Return converted console value */
+            if (fieldValue.toDate) {
+                return { [fieldName]: this.convertDateValue(fieldValue.toDate()) };
+            }
+            /* Return standard date value */
+            return { [fieldName]: fieldValue };
+        }
+        logs.firestoreTimePartitionFieldError(event.documentName, fieldName, firestoreFieldName, fieldValue);
+        return {};
+    }
+    customFieldExists(fields = []) {
+        if (!fields.length)
+            return false;
+        const { timePartitioningField } = this.config;
+        return fields.map(($) => $.name).includes(timePartitioningField);
+    }
+    async addPartitioningToSchema(fields = []) {
+        /** check if class has valid table reference */
+        if (!this.hasValidTableReference())
+            return Promise.resolve();
+        /** return if table is already partitioned **/
+        if (await this.isTablePartitioned())
+            return Promise.resolve();
+        /** return if an invalid partition type has been requested**/
+        if (!this.hasValidTimePartitionType())
+            return Promise.resolve();
+        /** Return if invalid partitioning and field type combination */
+        if (this.hasHourAndDatePartitionConfig())
+            return Promise.resolve();
+        /** return if an invalid partition type has been requested**/
+        if (!this.hasValidCustomPartitionConfig())
+            return Promise.resolve();
+        /** return if an invalid partition type has been requested**/
+        if (!this.hasValidCustomPartitionConfig())
+            return Promise.resolve();
+        /** update fields with new schema option ** */
+        if (!this.hasValidTimePartitionOption())
+            return Promise.resolve();
+        /* Check if partition field has been provided */
+        if (!this.config.timePartitioningField)
+            return Promise.resolve();
+        // if (await !this.hasExistingSchema) return Promise.resolve();
+        // Field already exists on schema, skip
+        if (this.customFieldExists(fields))
+            return Promise.resolve();
+        fields.push(schema_1.getNewPartitionField(this.config));
+        /** log successful addition of partition column */
+        logs.addPartitionFieldColumn(this.table.id, this.config.timePartitioningField);
+        return Promise.resolve();
+    }
+    async updateTableMetadata(options) {
+        /** return if table is already partitioned **/
+        if (await this.isTablePartitioned())
+            return Promise.resolve();
+        /** return if an invalid partition type has been requested**/
+        if (!this.hasValidTimePartitionType())
+            return Promise.resolve();
+        /** update fields with new schema option ** */
+        if (!this.hasValidTimePartitionOption())
+            return Promise.resolve();
+        /** Return if invalid partitioning and field type combination */
+        if (this.hasHourAndDatePartitionConfig())
+            return Promise.resolve();
+        /** return if an invalid partition type has been requested**/
+        if (!this.hasValidCustomPartitionConfig())
+            return Promise.resolve();
+        // if (await !this.hasExistingSchema) return Promise.resolve();
+        if (this.config.timePartitioning) {
+            options.timePartitioning = { type: this.config.timePartitioning };
+        }
+        //TODO: Add check for skipping adding views partition field, this is not a feature that can be added .
+        if (this.config.timePartitioningField) {
+            options.timePartitioning = {
+                ...options.timePartitioning,
+                field: this.config.timePartitioningField,
+            };
+        }
+    }
+}
+exports.Partitioning = Partitioning;

package/lib/bigquery/schema.js

@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.RawChangelogSchema = exports.RawChangelogViewSchema = exports.documentIdField = exports.longitudeField = exports.latitudeField = exports.timestampField = exports.operationField = exports.eventIdField = exports.documentNameField = exports.dataField = void 0;
+exports.getNewPartitionField = exports.RawChangelogSchema = exports.RawChangelogViewSchema = exports.documentPathParams = exports.documentIdField = exports.longitudeField = exports.latitudeField = exports.timestampField = exports.operationField = exports.eventIdField = exports.documentNameField = exports.dataField = void 0;
 const bigQueryField = (name, type, mode, fields) => ({
     fields,
     mode: mode || "NULLABLE",
@@ -37,6 +37,12 @@ exports.documentIdField = {
     type: "STRING",
     description: "The document id as defined in the firestore database.",
 };
+exports.documentPathParams = {
+    name: "path_params",
+    mode: "NULLABLE",
+    type: "STRING",
+    description: "JSON string representing wildcard params with Firestore Document ids",
+};
 /*
  * We cannot specify a schema for view creation, and all view columns default
  * to the NULLABLE mode.
@@ -111,3 +117,13 @@ exports.RawChangelogSchema = {
         exports.documentIdField,
     ],
 };
+// Helper function for Partitioned Changelogs field
+exports.getNewPartitionField = (config) => {
+    const { timePartitioningField, timePartitioningFieldType } = config;
+    return {
+        name: timePartitioningField,
+        mode: "NULLABLE",
+        type: timePartitioningFieldType,
+        description: "The document TimePartition partition field selected by user",
+    };
+};

package/lib/bigquery/snapshot.js

@@ -19,13 +19,13 @@ exports.buildLatestSnapshotViewQuery = exports.latestConsistentSnapshotView = vo
 const sqlFormatter = require("sql-formatter");
 const schema_1 = require("./schema");
 const excludeFields = ["document_name", "document_id"];
-exports.latestConsistentSnapshotView = (datasetId, tableName) => ({
-    query: buildLatestSnapshotViewQuery(datasetId, tableName, schema_1.timestampField.name, schema_1.RawChangelogViewSchema["fields"]
+exports.latestConsistentSnapshotView = (datasetId, tableName, schema, bqProjectId) => ({
+    query: buildLatestSnapshotViewQuery(datasetId, tableName, schema_1.timestampField.name, schema["fields"]
         .map((field) => field.name)
-        .filter((name) => excludeFields.indexOf(name) === -1)),
+        .filter((name) => excludeFields.indexOf(name) === -1), bqProjectId),
     useLegacySql: false,
 });
-function buildLatestSnapshotViewQuery(datasetId, tableName, timestampColumnName, groupByColumns) {
+function buildLatestSnapshotViewQuery(datasetId, tableName, timestampColumnName, groupByColumns, bqProjectId) {
     if (datasetId === "" || tableName === "" || timestampColumnName === "") {
         throw Error(`Missing some query parameters!`);
     }
@@ -56,7 +56,8 @@ function buildLatestSnapshotViewQuery(datasetId, tableName, timestampColumnName,
         FIRST_VALUE(operation)
           OVER(PARTITION BY document_name ORDER BY ${timestampColumnName} DESC) = "DELETE"
           AS is_deleted
-      FROM \`${process.env.PROJECT_ID}.${datasetId}.${tableName}\`
+      FROM \`${bqProjectId ||
+        process.env.PROJECT_ID}.${datasetId}.${tableName}\`
       ORDER BY document_name, ${timestampColumnName} DESC
     )
     WHERE NOT is_deleted

package/lib/bigquery/validateProject.js

@@ -0,0 +1,18 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.validateProject = void 0;
+const { ProjectsClient } = require("@google-cloud/resource-manager");
+/* TODO: searchProjectsAsync sometimes returns {}.
+ * Could be resource intensive, if checked on every records insert.
+ */
+exports.validateProject = async (id) => {
+    let isValid = false;
+    const client = new ProjectsClient();
+    const projects = client.searchProjectsAsync();
+    for await (const project of projects) {
+        if (project.projectId === id) {
+            isValid = true;
+        }
+    }
+    return isValid;
+};

package/lib/logs.js

@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.addDocumentIdColumn = exports.timestampMissingValue = exports.error = exports.dataTypeInvalid = exports.dataInserting = exports.dataInsertRetried = exports.dataInserted = exports.complete = exports.bigQueryViewValidating = exports.bigQueryViewValidated = exports.bigQueryViewUpToDate = exports.bigQueryViewUpdating = exports.bigQueryViewUpdated = exports.bigQueryViewAlreadyExists = exports.bigQueryViewCreating = exports.bigQueryViewCreated = exports.bigQueryUserDefinedFunctionCreated = exports.bigQueryUserDefinedFunctionCreating = exports.bigQueryTableValidating = exports.bigQueryTableValidated = exports.bigQueryTableUpToDate = exports.bigQueryTableUpdating = exports.bigQueryTableUpdated = exports.bigQueryTableCreating = exports.bigQueryTableCreated = exports.bigQueryTableAlreadyExists = exports.bigQuerySchemaViewCreated = exports.bigQueryLatestSnapshotViewQueryCreated = exports.bigQueryErrorRecordingDocumentChange = exports.bigQueryDatasetExists = exports.bigQueryDatasetCreating = exports.bigQueryDatasetCreated = exports.arrayFieldInvalid = void 0;
+exports.failedToInitializeWait = exports.tableCreationError = exports.invalidClustering = exports.hourAndDatePartitioningWarning = exports.invalidTableReference = exports.invalidProjectIdWarning = exports.cannotPartitionExistingTable = exports.removedClustering = exports.updatedClustering = exports.bigQueryTableInsertErrors = exports.firestoreTimePartitioningParametersWarning = exports.firestoreTimePartitionFieldError = exports.addPartitionFieldColumn = exports.addNewColumn = exports.timestampMissingValue = exports.error = exports.dataTypeInvalid = exports.dataInserting = exports.dataInsertRetried = exports.dataInserted = exports.complete = exports.bigQueryViewValidating = exports.bigQueryViewValidated = exports.bigQueryViewUpToDate = exports.bigQueryViewUpdating = exports.bigQueryViewUpdated = exports.bigQueryViewAlreadyExists = exports.bigQueryViewCreating = exports.bigQueryViewCreated = exports.bigQueryUserDefinedFunctionCreated = exports.bigQueryUserDefinedFunctionCreating = exports.bigQueryTableValidating = exports.bigQueryTableValidated = exports.bigQueryTableUpToDate = exports.bigQueryTableUpdating = exports.bigQueryTableUpdated = exports.bigQueryTableCreating = exports.bigQueryTableCreated = exports.bigQueryTableAlreadyExists = exports.bigQuerySchemaViewCreated = exports.bigQueryLatestSnapshotViewQueryCreated = exports.bigQueryErrorRecordingDocumentChange = exports.bigQueryDatasetExists = exports.bigQueryDatasetCreating = exports.bigQueryDatasetCreated = exports.arrayFieldInvalid = void 0;
 const firebase_functions_1 = require("firebase-functions");
 exports.arrayFieldInvalid = (fieldName) => {
     firebase_functions_1.logger.warn(`Array field '${fieldName}' does not contain an array, skipping`);
@@ -114,6 +114,62 @@ exports.error = (err) => {
 exports.timestampMissingValue = (fieldName) => {
     firebase_functions_1.logger.warn(`Missing value for timestamp field: ${fieldName}, using default timestamp instead.`);
 };
-exports.addDocumentIdColumn = (table) => {
-    firebase_functions_1.logger.log(`Updated '${table}' table with a 'document_id' column`);
+exports.addNewColumn = (table, field) => {
+    firebase_functions_1.logger.log(`Updated '${table}' table with a '${field}' column`);
+};
+exports.addPartitionFieldColumn = (table, field) => {
+    firebase_functions_1.logger.log(`Updated '${table}' table with a partition field '${field}' column`);
+};
+exports.firestoreTimePartitionFieldError = (documentName, fieldName, firestoreFieldName, firestoreFieldData) => {
+    firebase_functions_1.logger.warn(`Wrong type of Firestore Field for TimePartitioning. Accepts only strings in BigQuery format (DATE, DATETIME, TIMESTAMP) and Firestore Timestamp. Firestore Document field path: ${documentName}. Field name: ${firestoreFieldName}. Field data: ${firestoreFieldData}. Schema field "${fieldName}" value will be null.`);
+};
+exports.firestoreTimePartitioningParametersWarning = (fieldName, fieldType, firestoreFieldName, dataFirestoreField) => {
+    firebase_functions_1.logger.warn("All TimePartitioning option parameters need to be available to create new custom schema field");
+    !fieldName && firebase_functions_1.logger.warn(`Parameter missing: TIME_PARTITIONING_FIELD`);
+    !fieldType && firebase_functions_1.logger.warn(`Parameter missing: TIME_PARTITIONING_FIELD_TYPE`);
+    !firestoreFieldName &&
+        firebase_functions_1.logger.warn(`Parameter missing: TIME_PARTITIONING_FIRESTORE_FIELD`);
+    !dataFirestoreField &&
+        firebase_functions_1.logger.warn(`No data found in Firestore Document under selected field: "${firestoreFieldName}"`);
+};
+exports.bigQueryTableInsertErrors = (insertErrors) => {
+    firebase_functions_1.logger.warn(`Error when inserting data to table.`);
+    insertErrors.forEach((error) => {
+        firebase_functions_1.logger.warn("ROW DATA JSON:");
+        firebase_functions_1.logger.warn(error.row);
+        if (error && error.errors) {
+            error.errors.forEach((error) => firebase_functions_1.logger.warn(`ROW ERROR MESSAGE: ${error.message}`));
+        }
+    });
+};
+exports.updatedClustering = (fields) => {
+    firebase_functions_1.logger.info(`Clustering updated with new settings fields: ${fields}`);
+};
+exports.removedClustering = (tableName) => {
+    firebase_functions_1.logger.info(`Clustering removed on ${tableName}`);
+};
+exports.cannotPartitionExistingTable = (table) => {
+    firebase_functions_1.logger.warn(`Cannot partition an existing table ${table.dataset.id}_${table.id}`);
+};
+function invalidProjectIdWarning(bqProjectId) {
+    firebase_functions_1.logger.warn(`Invalid project Id ${bqProjectId}, data cannot be synchronized`);
+}
+exports.invalidProjectIdWarning = invalidProjectIdWarning;
+function invalidTableReference() {
+    firebase_functions_1.logger.warn(`No valid table reference is available. Skipping partitioning`);
+}
+exports.invalidTableReference = invalidTableReference;
+function hourAndDatePartitioningWarning() {
+    firebase_functions_1.logger.warn(`Cannot partition table with hour partitioning and Date. For DATE columns, the partitions can have daily, monthly, or yearly granularity. Skipping partitioning`);
+}
+exports.hourAndDatePartitioningWarning = hourAndDatePartitioningWarning;
+function invalidClustering(fields) {
+    firebase_functions_1.logger.warn(`Unable to add clustering, field(s) ${fields} do not exist on the expected table`);
+}
+exports.invalidClustering = invalidClustering;
+exports.tableCreationError = (table, message) => {
+    firebase_functions_1.logger.warn(`Error caught creating table`, message);
+};
+exports.failedToInitializeWait = (message) => {
+    firebase_functions_1.logger.warn(`Failed while waiting to initialize.`, message);
 };

package/lib/types.js

@@ -0,0 +1,9 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PartitionFieldType = void 0;
+var PartitionFieldType;
+(function (PartitionFieldType) {
+    PartitionFieldType["DATE"] = "DATE";
+    PartitionFieldType["DATETIME"] = "DATETIME";
+    PartitionFieldType["TIMESTAMP"] = "TIMESTAMP";
+})(PartitionFieldType = exports.PartitionFieldType || (exports.PartitionFieldType = {}));

package/package.json

@@ -5,14 +5,14 @@
     "url": "github.com/firebase/extensions.git",
     "directory": "firestore-bigquery-export/firestore-bigquery-change-tracker"
   },
-  "version": "1.1.13",
+  "version": "1.1.16",
   "description": "Core change-tracker library for Cloud Firestore Collection BigQuery Exports",
   "main": "./lib/index.js",
   "scripts": {
     "build": "npm run clean && npm run compile",
     "clean": "rimraf lib",
     "compile": "tsc",
-    "test": "jest",
+    "test:local": "firebase ext:dev:emulators:exec ./node_modules/.bin/jest --test-params=./src/__tests__/emulator-params.env --project=extensions-testing --config=./src/__tests__/firebase.json",
     "prepare": "npm run build"
   },
   "files": [
@@ -23,11 +23,13 @@
   "license": "Apache-2.0",
   "dependencies": {
     "@google-cloud/bigquery": "^4.7.0",
+    "@google-cloud/resource-manager": "^3.0.0",
     "firebase-admin": "^8.0.0",
     "firebase-functions": "^3.13.2",
     "generate-schema": "^2.6.0",
     "inquirer": "^6.4.0",
     "lodash": "^4.17.14",
+    "node-fetch": "^2.6.1",
     "sql-formatter": "^2.3.3",
     "traverse": "^0.6.6"
   },