@firebaseextensions/firestore-bigquery-change-tracker 1.1.31 → 1.1.33

package/lib/bigquery/handleFailedTransactions.js

@@ -1,12 +1,13 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 const firebase = require("firebase-admin");
+const firestore_1 = require("firebase-admin/firestore");
 if (!firebase.apps.length) {
     firebase.initializeApp();
     firebase.firestore().settings({ ignoreUndefinedProperties: true });
 }
 exports.default = async (rows, config, e) => {
-    const db = firebase.firestore();
+    const db = (0, firestore_1.getFirestore)();
     const batchArray = [db.batch()];
     let operationCounter = 0;
     let batchIndex = 0;

package/lib/bigquery/index.js

@@ -13,6 +13,7 @@ const logs = require("../logs");
 const partitioning_1 = require("./partitioning");
 const clustering_1 = require("./clustering");
 const checkUpdates_1 = require("./checkUpdates");
+const utils_1 = require("./utils");
 var schema_2 = require("./schema");
 Object.defineProperty(exports, "RawChangelogSchema", { enumerable: true, get: function () { return schema_2.RawChangelogSchema; } });
 Object.defineProperty(exports, "RawChangelogViewSchema", { enumerable: true, get: function () { return schema_2.RawChangelogViewSchema; } });
@@ -129,26 +130,10 @@ class FirestoreBigQueryEventHistoryTracker {
      * A half a second delay is added per check while the function
      * continually re-checks until the referenced dataset and table become available.
      */
-    async waitForInitialization() {
-        return new Promise((resolve) => {
-            let handle = setInterval(async () => {
-                try {
-                    const dataset = this.bigqueryDataset();
-                    const changelogName = this.rawChangeLogTableName();
-                    const table = dataset.table(changelogName);
-                    const [datasetExists] = await dataset.exists();
-                    const [tableExists] = await table.exists();
-                    if (datasetExists && tableExists) {
-                        clearInterval(handle);
-                        return resolve(table);
-                    }
-                }
-                catch (ex) {
-                    clearInterval(handle);
-                    logs.failedToInitializeWait(ex.message);
-                }
-            }, 5000);
-        });
+    async _waitForInitialization() {
+        const dataset = this.bigqueryDataset();
+        const changelogName = this.rawChangeLogTableName();
+        return (0, utils_1.waitForInitialization)({ dataset, changelogName });
     }
     /**
      * Inserts rows of data into the BigQuery raw change log table.
@@ -192,14 +177,34 @@ class FirestoreBigQueryEventHistoryTracker {
             if (this._initialized) {
                 return;
             }
-            await this.initializeDataset();
-            await this.initializeRawChangeLogTable();
-            await this.initializeLatestView();
+            try {
+                await this.initializeDataset();
+            }
+            catch (error) {
+                const message = (0, utils_1.parseErrorMessage)(error, "initializing dataset");
+                throw new Error(`Error initializing dataset: ${message}`);
+            }
+            try {
+                await this.initializeRawChangeLogTable();
+            }
+            catch (error) {
+                const message = (0, utils_1.parseErrorMessage)(error, "initializing raw change log table");
+                throw new Error(`Error initializing raw change log table: ${message}`);
+            }
+            try {
+                await this.initializeLatestView();
+            }
+            catch (error) {
+                const message = (0, utils_1.parseErrorMessage)(error, "initializing latest view");
+                throw new Error(`Error initializing latest view: ${message}`);
+            }
+            await this._waitForInitialization();
             this._initialized = true;
         }
-        catch (ex) {
-            await this.waitForInitialization();
-            this._initialized = true;
+        catch (error) {
+            const message = (0, utils_1.parseErrorMessage)(error, "initializing BigQuery resources");
+            console.error("Error initializing BigQuery resources: ", message);
+            throw error;
         }
     }
     /**

package/lib/bigquery/partitioning.js

@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.Partitioning = void 0;
 const firebase = require("firebase-admin");
 const logs = require("../logs");
+const functions = require("firebase-functions");
 const schema_1 = require("./schema");
 const bigquery_1 = require("@google-cloud/bigquery");
 const types_1 = require("../types");
@@ -87,41 +88,36 @@ class Partitioning {
         return !!metadata.schema;
     }
     hasValidTableReference() {
-        logs.invalidTableReference();
+        if (!this.table) {
+            logs.invalidTableReference();
+        }
         return !!this.table;
     }
     async isTablePartitioned() {
-        if (!this.table)
-            return Promise.resolve(false);
-        // No table provided, cannot evaluate
-        if (this.table.exists()) {
-            logs.cannotPartitionExistingTable(this.table);
-            return Promise.resolve(false);
-        }
-        /*** No table exists, return */
         const [tableExists] = await this.table.exists();
-        if (!tableExists)
-            return Promise.resolve(false);
-        /* Check if partition metadata already exists */
+        if (!this.table || !tableExists)
+            return false;
+        /* Return true if partition metadata already exists */
         const [metadata] = await this.table.getMetadata();
-        if (!!metadata.timePartitioning)
-            return Promise.resolve(true);
+        if (metadata.timePartitioning) {
+            logs.cannotPartitionExistingTable(this.table);
+            return true;
+        }
         /** Find schema fields **/
         const schemaFields = await this.metaDataSchemaFields();
-        /** No Schema exists, return */
+        /** Return false if no schema exists */
         if (!schemaFields)
-            return Promise.resolve(false);
+            return false;
         /* Return false if time partition field not found */
         return schemaFields.some((column) => column.name === this.config.timePartitioningField);
     }
     async isValidPartitionForExistingTable() {
+        /** Return false if partition type option has not been set */
+        if (!this.isPartitioningEnabled())
+            return false;
+        /* Return false if table is already partitioned */
         const isPartitioned = await this.isTablePartitioned();
         if (isPartitioned)
-            return Promise.resolve(false);
-        return this.hasValidCustomPartitionConfig();
-    }
-    isValidPartitionForNewTable() {
-        if (!this.isPartitioningEnabled())
             return false;
         return this.hasValidCustomPartitionConfig();
     }
@@ -181,57 +177,71 @@ class Partitioning {
         const { timePartitioningField } = this.config;
         return fields.map(($) => $.name).includes(timePartitioningField);
     }
+    async shouldAddPartitioningToSchema() {
+        if (!this.isPartitioningEnabled()) {
+            return { proceed: false, message: "Partitioning not enabled" };
+        }
+        if (!this.hasValidTableReference()) {
+            return { proceed: false, message: "Invalid table reference" };
+        }
+        if (!this.hasValidCustomPartitionConfig()) {
+            return { proceed: false, message: "Invalid partition config" };
+        }
+        if (!this.hasValidTimePartitionType()) {
+            return { proceed: false, message: "Invalid partition type" };
+        }
+        if (!this.hasValidTimePartitionOption()) {
+            return { proceed: false, message: "Invalid partition option" };
+        }
+        if (this.hasHourAndDatePartitionConfig()) {
+            return {
+                proceed: false,
+                message: "Invalid partitioning and field type combination",
+            };
+        }
+        if (this.customFieldExists()) {
+            return { proceed: false, message: "Field already exists on schema" };
+        }
+        if (await this.isTablePartitioned()) {
+            return { proceed: false, message: "Table is already partitioned" };
+        }
+        if (!this.config.timePartitioningField) {
+            return { proceed: false, message: "Partition field not provided" };
+        }
+        return { proceed: true, message: "" };
+    }
     async addPartitioningToSchema(fields = []) {
-        /** check if class has valid table reference */
-        if (!this.hasValidTableReference())
-            return;
-        /** return if table is already partitioned **/
-        if (await this.isTablePartitioned())
-            return;
-        /** return if an invalid partition type has been requested**/
-        if (!this.hasValidTimePartitionType())
-            return;
-        /** Return if invalid partitioning and field type combination */
-        if (this.hasHourAndDatePartitionConfig())
-            return;
-        /** return if an invalid partition type has been requested**/
-        if (!this.hasValidCustomPartitionConfig())
-            return;
-        /** return if an invalid partition type has been requested**/
-        if (!this.hasValidCustomPartitionConfig())
-            return;
-        /** update fields with new schema option ** */
-        if (!this.hasValidTimePartitionOption())
-            return;
-        /* Check if partition field has been provided */
-        if (!this.config.timePartitioningField)
-            return;
-        // if (await !this.hasExistingSchema) return Promise.resolve();
-        // Field already exists on schema, skip
-        if (this.customFieldExists(fields))
+        const { proceed, message } = await this.shouldAddPartitioningToSchema();
+        if (!proceed) {
+            functions.logger.warn(`Did not add partitioning to schema: ${message}`);
             return;
+        }
+        // Add new partitioning field
         fields.push((0, schema_1.getNewPartitionField)(this.config));
-        /** log successful addition of partition column */
-        logs.addPartitionFieldColumn(this.table.id, this.config.timePartitioningField);
-        return;
+        functions.logger.log(`Added new partition field: ${this.config.timePartitioningField} to table ID: ${this.table.id}`);
     }
     async updateTableMetadata(options) {
-        /** return if table is already partitioned **/
+        /** Return if partition type option has not been set */
+        if (!this.isPartitioningEnabled())
+            return;
+        /** Return if class has invalid table reference */
+        if (!this.hasValidTableReference())
+            return;
+        /** Return if table is already partitioned **/
         if (await this.isTablePartitioned())
             return;
-        /** return if an invalid partition type has been requested**/
+        /** Return if an invalid partition type has been requested**/
+        if (!this.hasValidCustomPartitionConfig())
+            return;
+        /** Return if an invalid partition type has been requested**/
         if (!this.hasValidTimePartitionType())
             return;
-        /** update fields with new schema option ** */
+        /** Update fields with new schema option ** */
         if (!this.hasValidTimePartitionOption())
             return;
         /** Return if invalid partitioning and field type combination */
         if (this.hasHourAndDatePartitionConfig())
             return;
-        /** return if an invalid partition type has been requested**/
-        if (!this.hasValidCustomPartitionConfig())
-            return;
-        // if (await !this.hasExistingSchema) return Promise.resolve();
         if (this.config.timePartitioning) {
             options.timePartitioning = { type: this.config.timePartitioning };
         }

package/lib/bigquery/utils.js

@@ -0,0 +1,62 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseErrorMessage = exports.waitForInitialization = void 0;
+const logs = require("../logs");
+/**
+ * Periodically checks for the existence of a dataset and table until both are found or a maximum number of attempts is reached.
+ * @param {WaitForInitializationParams} params - Parameters for initialization including the dataset and the table name to check.
+ * @param {number} maxAttempts - Maximum number of attempts before giving up (defaults to 12).
+ * @returns {Promise<Table>} A promise that resolves with the Table if it exists, or rejects if it doesn't exist after maxAttempts or an error occurs.
+ * @throws {Error} Throws an error if the dataset or table cannot be verified to exist after multiple attempts or if an unexpected error occurs.
+ */
+async function waitForInitialization({ dataset, changelogName }, maxAttempts = 12) {
+    return new Promise((resolve, reject) => {
+        let attempts = 0;
+        let handle = setInterval(async () => {
+            try {
+                const [datasetExists] = await dataset.exists();
+                const table = dataset.table(changelogName);
+                const [tableExists] = await table.exists();
+                if (datasetExists && tableExists) {
+                    clearInterval(handle);
+                    resolve(table);
+                }
+                else {
+                    attempts++;
+                    if (attempts >= maxAttempts) {
+                        clearInterval(handle);
+                        reject(new Error("Initialization timed out. Dataset or table could not be verified to exist after multiple attempts."));
+                    }
+                }
+            }
+            catch (error) {
+                clearInterval(handle);
+                const message = error instanceof Error
+                    ? error.message
+                    : "An unexpected error occurred";
+                logs.failedToInitializeWait(message);
+                reject(new Error(message));
+            }
+        }, 500);
+    });
+}
+exports.waitForInitialization = waitForInitialization;
+function parseErrorMessage(error, process = "") {
+    let message;
+    if (error instanceof Error) {
+        // Standard error handling
+        message = error.message;
+    }
+    else if (typeof error === "object" &&
+        error !== null &&
+        "message" in error) {
+        // Handling errors from APIs or other sources that are not native Error objects
+        message = error.message;
+    }
+    else {
+        // Fallback for when error is neither an Error object nor an expected structured object
+        message = `An unexpected error occurred${process ? ` during ${process}` : ""}.`;
+    }
+    return message;
+}
+exports.parseErrorMessage = parseErrorMessage;

package/lib/bigquery/utils.test.js

@@ -0,0 +1,65 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const utils_1 = require("./utils");
+const logs = require("../logs");
+jest.mock("@google-cloud/bigquery");
+jest.mock("../../logs");
+const dataset = {
+    exists: jest.fn(),
+    table: jest.fn(),
+};
+const table = {
+    exists: jest.fn(),
+};
+const changelogName = "testTable";
+describe("waitForInitialization", () => {
+    beforeEach(() => {
+        jest.clearAllMocks();
+        dataset.table.mockReturnValue(table);
+    });
+    test("should successfully find the dataset and table", async () => {
+        dataset.exists.mockResolvedValue([true]);
+        table.exists.mockResolvedValue([true]);
+        const result = await (0, utils_1.waitForInitialization)({
+            dataset: dataset,
+            changelogName,
+        });
+        expect(result).toBe(table);
+        expect(dataset.exists).toHaveBeenCalledTimes(1);
+        expect(table.exists).toHaveBeenCalledTimes(1);
+    });
+    test("should fail after max attempts if table does not exist", async () => {
+        dataset.exists.mockResolvedValue([true]);
+        table.exists.mockResolvedValue([false]);
+        await expect((0, utils_1.waitForInitialization)({ dataset: dataset, changelogName }, 3)).rejects.toThrow("Initialization timed out. Dataset or table could not be verified to exist after multiple attempts.");
+        expect(dataset.exists).toHaveBeenCalledTimes(3);
+        expect(table.exists).toHaveBeenCalledTimes(3);
+    });
+    test("should handle and throw an error if dataset.exists throws", async () => {
+        const error = new Error("Access denied");
+        dataset.exists.mockRejectedValue(error);
+        await expect((0, utils_1.waitForInitialization)({
+            dataset: dataset,
+            changelogName,
+        })).rejects.toThrow("Access denied");
+        expect(logs.failedToInitializeWait).toHaveBeenCalledWith(error.message);
+    });
+    test("should handle and throw an error if table.exists throws", async () => {
+        dataset.exists.mockResolvedValue([true]);
+        const error = new Error("Table error");
+        table.exists.mockRejectedValue(error);
+        await expect((0, utils_1.waitForInitialization)({
+            dataset: dataset,
+            changelogName,
+        })).rejects.toThrow("Table error");
+        expect(logs.failedToInitializeWait).toHaveBeenCalledWith(error.message);
+    });
+    test("should handle unexpected error types gracefully", async () => {
+        dataset.exists.mockRejectedValue("String error");
+        await expect((0, utils_1.waitForInitialization)({
+            dataset: dataset,
+            changelogName,
+        })).rejects.toThrow("An unexpected error occurred");
+        expect(logs.failedToInitializeWait).toHaveBeenCalledWith("An unexpected error occurred");
+    });
+});

package/package.json

@@ -5,7 +5,7 @@
     "url": "github.com/firebase/extensions.git",
     "directory": "firestore-bigquery-export/firestore-bigquery-change-tracker"
   },
-  "version": "1.1.31",
+  "version": "1.1.33",
   "description": "Core change-tracker library for Cloud Firestore Collection BigQuery Exports",
   "main": "./lib/index.js",
   "scripts": {
@@ -23,10 +23,10 @@
   "author": "Jan Wyszynski <wyszynski@google.com>",
   "license": "Apache-2.0",
   "dependencies": {
-    "@google-cloud/bigquery": "^4.7.0",
-    "@google-cloud/resource-manager": "^3.0.0",
-    "firebase-admin": "^11.4.1",
-    "firebase-functions": "^3.13.2",
+    "@google-cloud/bigquery": "^7.6.0",
+    "@google-cloud/resource-manager": "^5.1.0",
+    "firebase-admin": "^12.0.0",
+    "firebase-functions": "^4.9.0",
     "generate-schema": "^2.6.0",
     "inquirer": "^6.4.0",
     "lodash": "^4.17.14",
@@ -36,15 +36,17 @@
   },
   "devDependencies": {
     "@types/chai": "^4.1.6",
-    "@types/jest": "^24.0.18",
+    "@types/jest": "29.5.0",
     "@types/node": "14.18.34",
     "@types/traverse": "^0.6.32",
     "chai": "^4.2.0",
-    "jest": "^24.9.0",
     "nyc": "^14.0.0",
     "rimraf": "^2.6.3",
-    "ts-jest": "^24.1.0",
-    "ts-node": "^7.0.1",
-    "typescript": "^4.9.4"
+    "typescript": "^4.9.4",
+    "jest": "29.5.0",
+    "jest-environment-node": "29.5.0",
+    "mocked-env": "^1.3.2",
+    "ts-jest": "29.1.2",
+    "jest-config": "29.5.0"
   }
 }