@firebaseextensions/firestore-bigquery-change-tracker 1.1.32 → 1.1.34

package/lib/bigquery/index.js

@@ -13,6 +13,7 @@ const logs = require("../logs");
 const partitioning_1 = require("./partitioning");
 const clustering_1 = require("./clustering");
 const checkUpdates_1 = require("./checkUpdates");
+const utils_1 = require("./utils");
 var schema_2 = require("./schema");
 Object.defineProperty(exports, "RawChangelogSchema", { enumerable: true, get: function () { return schema_2.RawChangelogSchema; } });
 Object.defineProperty(exports, "RawChangelogViewSchema", { enumerable: true, get: function () { return schema_2.RawChangelogViewSchema; } });
@@ -129,26 +130,10 @@ class FirestoreBigQueryEventHistoryTracker {
      * A half a second delay is added per check while the function
      * continually re-checks until the referenced dataset and table become available.
      */
-    async waitForInitialization() {
-        return new Promise((resolve) => {
-            let handle = setInterval(async () => {
-                try {
-                    const dataset = this.bigqueryDataset();
-                    const changelogName = this.rawChangeLogTableName();
-                    const table = dataset.table(changelogName);
-                    const [datasetExists] = await dataset.exists();
-                    const [tableExists] = await table.exists();
-                    if (datasetExists && tableExists) {
-                        clearInterval(handle);
-                        return resolve(table);
-                    }
-                }
-                catch (ex) {
-                    clearInterval(handle);
-                    logs.failedToInitializeWait(ex.message);
-                }
-            }, 5000);
-        });
+    async _waitForInitialization() {
+        const dataset = this.bigqueryDataset();
+        const changelogName = this.rawChangeLogTableName();
+        return (0, utils_1.waitForInitialization)({ dataset, changelogName });
     }
     /**
      * Inserts rows of data into the BigQuery raw change log table.
@@ -192,14 +177,34 @@ class FirestoreBigQueryEventHistoryTracker {
             if (this._initialized) {
                 return;
             }
-            await this.initializeDataset();
-            await this.initializeRawChangeLogTable();
-            await this.initializeLatestView();
+            try {
+                await this.initializeDataset();
+            }
+            catch (error) {
+                const message = (0, utils_1.parseErrorMessage)(error, "initializing dataset");
+                throw new Error(`Error initializing dataset: ${message}`);
+            }
+            try {
+                await this.initializeRawChangeLogTable();
+            }
+            catch (error) {
+                const message = (0, utils_1.parseErrorMessage)(error, "initializing raw change log table");
+                throw new Error(`Error initializing raw change log table: ${message}`);
+            }
+            try {
+                await this.initializeLatestView();
+            }
+            catch (error) {
+                const message = (0, utils_1.parseErrorMessage)(error, "initializing latest view");
+                throw new Error(`Error initializing latest view: ${message}`);
+            }
+            await this._waitForInitialization();
             this._initialized = true;
         }
-        catch (ex) {
-            await this.waitForInitialization();
-            this._initialized = true;
+        catch (error) {
+            const message = (0, utils_1.parseErrorMessage)(error, "initializing BigQuery resources");
+            console.error("Error initializing BigQuery resources: ", message);
+            throw error;
         }
     }
     /**

package/lib/bigquery/partitioning.js

@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.Partitioning = void 0;
 const firebase = require("firebase-admin");
 const logs = require("../logs");
+const functions = require("firebase-functions");
 const schema_1 = require("./schema");
 const bigquery_1 = require("@google-cloud/bigquery");
 const types_1 = require("../types");
@@ -93,28 +94,31 @@ class Partitioning {
         return !!this.table;
     }
     async isTablePartitioned() {
+        const [tableExists] = await this.table.exists();
+        if (!this.table || !tableExists)
+            return false;
         /* Return true if partition metadata already exists */
         const [metadata] = await this.table.getMetadata();
-        if (!!metadata.timePartitioning) {
+        if (metadata.timePartitioning) {
             logs.cannotPartitionExistingTable(this.table);
-            return Promise.resolve(true);
+            return true;
         }
         /** Find schema fields **/
         const schemaFields = await this.metaDataSchemaFields();
         /** Return false if no schema exists */
         if (!schemaFields)
-            return Promise.resolve(false);
+            return false;
         /* Return false if time partition field not found */
         return schemaFields.some((column) => column.name === this.config.timePartitioningField);
     }
     async isValidPartitionForExistingTable() {
         /** Return false if partition type option has not been set */
         if (!this.isPartitioningEnabled())
-            return Promise.resolve(false);
+            return false;
         /* Return false if table is already partitioned */
         const isPartitioned = await this.isTablePartitioned();
         if (isPartitioned)
-            return Promise.resolve(false);
+            return false;
         return this.hasValidCustomPartitionConfig();
     }
     convertDateValue(fieldValue) {
@@ -168,44 +172,53 @@ class Partitioning {
         return {};
     }
     customFieldExists(fields = []) {
-        if (!fields.length)
-            return false;
+        /** Extract the time partioning field name */
         const { timePartitioningField } = this.config;
+        /** Return based the field already exist */
         return fields.map(($) => $.name).includes(timePartitioningField);
     }
+    async shouldAddPartitioningToSchema(fields) {
+        if (!this.isPartitioningEnabled()) {
+            return { proceed: false, message: "Partitioning not enabled" };
+        }
+        if (!this.hasValidTableReference()) {
+            return { proceed: false, message: "Invalid table reference" };
+        }
+        if (!this.hasValidCustomPartitionConfig()) {
+            return { proceed: false, message: "Invalid partition config" };
+        }
+        if (!this.hasValidTimePartitionType()) {
+            return { proceed: false, message: "Invalid partition type" };
+        }
+        if (!this.hasValidTimePartitionOption()) {
+            return { proceed: false, message: "Invalid partition option" };
+        }
+        if (this.hasHourAndDatePartitionConfig()) {
+            return {
+                proceed: false,
+                message: "Invalid partitioning and field type combination",
+            };
+        }
+        if (this.customFieldExists(fields)) {
+            return { proceed: false, message: "Field already exists on schema" };
+        }
+        if (await this.isTablePartitioned()) {
+            return { proceed: false, message: "Table is already partitioned" };
+        }
+        if (!this.config.timePartitioningField) {
+            return { proceed: false, message: "Partition field not provided" };
+        }
+        return { proceed: true, message: "" };
+    }
     async addPartitioningToSchema(fields = []) {
-        /** Return if partition type option has not been set */
-        if (!this.isPartitioningEnabled())
+        const { proceed, message } = await this.shouldAddPartitioningToSchema(fields);
+        if (!proceed) {
+            functions.logger.warn(`Did not add partitioning to schema: ${message}`);
             return;
-        /** Return if class has invalid table reference */
-        if (!this.hasValidTableReference())
-            return;
-        /** Return if table is already partitioned **/
-        if (await this.isTablePartitioned())
-            return;
-        /** Return if partition config is invalid */
-        if (!this.hasValidCustomPartitionConfig())
-            return;
-        /** Return if an invalid partition type has been requested */
-        if (!this.hasValidTimePartitionType())
-            return;
-        /** Return if an invalid partition option has been requested */
-        if (!this.hasValidTimePartitionOption())
-            return;
-        /** Return if invalid partitioning and field type combination */
-        if (this.hasHourAndDatePartitionConfig())
-            return;
-        /** Return if partition field has not been provided */
-        if (!this.config.timePartitioningField)
-            return;
-        /** Return if field already exists on schema */
-        if (this.customFieldExists(fields))
-            return;
-        /** Add new partitioning field **/
+        }
+        // Add new partitioning field
         fields.push((0, schema_1.getNewPartitionField)(this.config));
-        /** Log successful addition of partition column */
-        logs.addPartitionFieldColumn(this.table.id, this.config.timePartitioningField);
-        return;
+        functions.logger.log(`Added new partition field: ${this.config.timePartitioningField} to table ID: ${this.table.id}`);
     }
     async updateTableMetadata(options) {
         /** Return if partition type option has not been set */

package/lib/bigquery/utils.js

@@ -0,0 +1,62 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseErrorMessage = exports.waitForInitialization = void 0;
+const logs = require("../logs");
+/**
+ * Periodically checks for the existence of a dataset and table until both are found or a maximum number of attempts is reached.
+ * @param {WaitForInitializationParams} params - Parameters for initialization including the dataset and the table name to check.
+ * @param {number} maxAttempts - Maximum number of attempts before giving up (defaults to 12).
+ * @returns {Promise<Table>} A promise that resolves with the Table if it exists, or rejects if it doesn't exist after maxAttempts or an error occurs.
+ * @throws {Error} Throws an error if the dataset or table cannot be verified to exist after multiple attempts or if an unexpected error occurs.
+ */
+async function waitForInitialization({ dataset, changelogName }, maxAttempts = 12) {
+    return new Promise((resolve, reject) => {
+        let attempts = 0;
+        let handle = setInterval(async () => {
+            try {
+                const [datasetExists] = await dataset.exists();
+                const table = dataset.table(changelogName);
+                const [tableExists] = await table.exists();
+                if (datasetExists && tableExists) {
+                    clearInterval(handle);
+                    resolve(table);
+                }
+                else {
+                    attempts++;
+                    if (attempts >= maxAttempts) {
+                        clearInterval(handle);
+                        reject(new Error("Initialization timed out. Dataset or table could not be verified to exist after multiple attempts."));
+                    }
+                }
+            }
+            catch (error) {
+                clearInterval(handle);
+                const message = error instanceof Error
+                    ? error.message
+                    : "An unexpected error occurred";
+                logs.failedToInitializeWait(message);
+                reject(new Error(message));
+            }
+        }, 500);
+    });
+}
+exports.waitForInitialization = waitForInitialization;
+function parseErrorMessage(error, process = "") {
+    let message;
+    if (error instanceof Error) {
+        // Standard error handling
+        message = error.message;
+    }
+    else if (typeof error === "object" &&
+        error !== null &&
+        "message" in error) {
+        // Handling errors from APIs or other sources that are not native Error objects
+        message = error.message;
+    }
+    else {
+        // Fallback for when error is neither an Error object nor an expected structured object
+        message = `An unexpected error occurred${process ? ` during ${process}` : ""}.`;
+    }
+    return message;
+}
+exports.parseErrorMessage = parseErrorMessage;

package/package.json

@@ -5,7 +5,7 @@
     "url": "github.com/firebase/extensions.git",
     "directory": "firestore-bigquery-export/firestore-bigquery-change-tracker"
   },
-  "version": "1.1.32",
+  "version": "1.1.34",
   "description": "Core change-tracker library for Cloud Firestore Collection BigQuery Exports",
   "main": "./lib/index.js",
   "scripts": {
@@ -23,10 +23,10 @@
   "author": "Jan Wyszynski <wyszynski@google.com>",
   "license": "Apache-2.0",
   "dependencies": {
-    "@google-cloud/bigquery": "^4.7.0",
-    "@google-cloud/resource-manager": "^3.0.0",
-    "firebase-admin": "^11.11.1",
-    "firebase-functions": "^4.6.0",
+    "@google-cloud/bigquery": "^7.6.0",
+    "@google-cloud/resource-manager": "^5.1.0",
+    "firebase-admin": "^12.0.0",
+    "firebase-functions": "^4.9.0",
     "generate-schema": "^2.6.0",
     "inquirer": "^6.4.0",
     "lodash": "^4.17.14",
@@ -36,15 +36,17 @@
   },
   "devDependencies": {
     "@types/chai": "^4.1.6",
-    "@types/jest": "^24.0.18",
+    "@types/jest": "29.5.0",
     "@types/node": "14.18.34",
     "@types/traverse": "^0.6.32",
     "chai": "^4.2.0",
-    "jest": "^24.9.0",
     "nyc": "^14.0.0",
     "rimraf": "^2.6.3",
-    "ts-jest": "^24.1.0",
-    "ts-node": "^7.0.1",
-    "typescript": "^4.9.4"
+    "typescript": "^4.9.4",
+    "jest": "29.5.0",
+    "jest-environment-node": "29.5.0",
+    "mocked-env": "^1.3.2",
+    "ts-jest": "29.1.2",
+    "jest-config": "29.5.0"
   }
 }