bulltrackers-module 1.0.1069 → 1.0.1071

package/functions/api-v4/services/write/WatchlistWriteService.js

@@ -1,4 +1,5 @@
 const { recordAnalyticsWrite } = require('../../../core/utils/analytics_registry');
+const { ensureWatchlistsFirestoreBigQueryExport } = require('../../../core/utils/bigquery_utils');
 const { resolveFirestorePaths } = require('../../config/firestorePaths');
 const {
   validateUserId,
@@ -90,6 +91,8 @@ class WatchlistWriteService {
     if (firestoreData.tags !== undefined && Array.isArray(firestoreData.tags)) firestoreData.tags = firestoreData.tags.map(t => sanitizeString(t));
     Object.keys(firestoreData).forEach(k => { if (firestoreData[k] === undefined) delete firestoreData[k]; });
 
+    await ensureWatchlistsFirestoreBigQueryExport();
+
     const docRef = this.db.collection(this.c.signedInUsers).doc(validUserId).collection(this.sc.watchlists).doc(watchlistId);
     const oldDoc = await docRef.get();
     const created = !oldDoc.exists;
@@ -105,6 +108,7 @@ class WatchlistWriteService {
     const validUserId = validateUserId(userId);
     const validWatchlistId = validateId(watchlistId, 'Watchlist ID');
     await assertUserOwnsDocument(this.db, this.c, this.sc, validUserId, 'watchlists', validWatchlistId);
+    await ensureWatchlistsFirestoreBigQueryExport();
     const docRef = this.db.collection(this.c.signedInUsers).doc(validUserId).collection(this.sc.watchlists).doc(validWatchlistId);
     const oldDoc = await docRef.get();
     const oldItems = oldDoc.exists ? (oldDoc.data().items || []) : [];

package/functions/core/utils/bigquery_utils.js

@@ -11,17 +11,121 @@ const os = require('os');
 // Singleton BigQuery client
 let bigqueryClient = null;
 
+/** After a successful ensure, skip repeated BigQuery metadata checks on watchlist writes. */
+let watchlistsFirestoreExportEnsured = false;
+
 /**
  * Get or create BigQuery client
  */
 function getBigQueryClient() {
     if (!bigqueryClient) {
-        const projectId = process.env.GCP_PROJECT_ID || 'stocks-12345';
+        const projectId = process.env.GCP_PROJECT_ID || process.env.GOOGLE_CLOUD_PROJECT || 'stocks-12345';
         bigqueryClient = new BigQuery({ projectId });
     }
     return bigqueryClient;
 }
 
+/** Dataset where Firebase `firestore-bigquery-export` instances write (`bt_core` in prod). */
+function getFirestoreBigQueryExportDatasetId() {
+    return process.env.BQ_BT_CORE_DATASET_ID || 'bt_core';
+}
+
+async function runBqDdl(sql) {
+    const bigquery = getBigQueryClient();
+    const [job] = await bigquery.createQueryJob({ query: sql, location: 'europe-west1' });
+    await job.getQueryResults();
+}
+
+/**
+ * If the extension never created `watchlists_sync_raw_changelog` / `_raw_latest`, create them by
+ * cloning schema from another working export in the same dataset (same extension version/settings).
+ * Call before Firestore writes so the extension trigger finds the changelog table.
+ *
+ * Never throws — logs and returns so user writes still succeed.
+ */
+async function ensureWatchlistsFirestoreBigQueryExport(logger = null) {
+    if (watchlistsFirestoreExportEnsured) return;
+    if (process.env.SKIP_FIRESTORE_BQ_EXPORT_ENSURE === '1' || process.env.NODE_ENV === 'test') return;
+    if (process.env.BIGQUERY_ENABLED === 'false') return;
+
+    const projectId = process.env.GCP_PROJECT_ID || process.env.GOOGLE_CLOUD_PROJECT || 'stocks-12345';
+    const datasetId = getFirestoreBigQueryExportDatasetId();
+    const targetBase = 'watchlists_sync';
+    const changelogTarget = `${targetBase}_raw_changelog`;
+    const latestTarget = `${targetBase}_raw_latest`;
+
+    const referenceBases = [
+        'alert_subscriptions_sync',
+        'user_settings_sync',
+        'fcm_tokens_sync',
+        'reviews_sync',
+    ];
+
+    const log = (level, msg, extra) => {
+        if (logger && typeof logger.log === 'function') logger.log(level, msg, extra);
+        else if (level === 'WARN' || level === 'ERROR') console.warn(msg, extra || '');
+    };
+
+    try {
+        const bigquery = getBigQueryClient();
+        const dataset = bigquery.dataset(datasetId);
+
+        const [changelogExists] = await dataset.table(changelogTarget).exists();
+        const [latestExists] = await dataset.table(latestTarget).exists();
+        if (changelogExists && latestExists) {
+            watchlistsFirestoreExportEnsured = true;
+            return;
+        }
+
+        let referenceBase = null;
+        for (const base of referenceBases) {
+            const [refExists] = await dataset.table(`${base}_raw_changelog`).exists();
+            if (refExists) {
+                referenceBase = base;
+                break;
+            }
+        }
+        if (!referenceBase) {
+            log('WARN', `[BigQuery] ensureWatchlistsFirestoreBigQueryExport: no reference *_raw_changelog in ${datasetId} (skipped)`);
+            return;
+        }
+
+        const refChangelog = `${referenceBase}_raw_changelog`;
+        const fq = (tid) => `\`${projectId}.${datasetId}.${tid}\``;
+
+        if (!changelogExists) {
+            await runBqDdl(`CREATE TABLE ${fq(changelogTarget)} LIKE ${fq(refChangelog)}`);
+            log('INFO', `[BigQuery] Created ${datasetId}.${changelogTarget} LIKE ${refChangelog}`);
+        }
+
+        const [latestExistsAfter] = await dataset.table(latestTarget).exists();
+        if (!latestExistsAfter) {
+            const viewSql = `
+CREATE VIEW ${fq(latestTarget)} AS
+SELECT * EXCEPT (rank)
+FROM (
+  SELECT
+    *,
+    ROW_NUMBER() OVER (PARTITION BY document_name ORDER BY timestamp DESC) AS rank
+  FROM ${fq(changelogTarget)}
+)
+WHERE rank = 1
+`;
+            await runBqDdl(viewSql);
+            log('INFO', `[BigQuery] Created view ${datasetId}.${latestTarget}`);
+        }
+
+        watchlistsFirestoreExportEnsured = true;
+    } catch (err) {
+        const msg = err && err.message ? err.message : String(err);
+        if (/Already exists|duplicate/i.test(msg)) {
+            watchlistsFirestoreExportEnsured = true;
+            return;
+        }
+        log('WARN', `[BigQuery] ensureWatchlistsFirestoreBigQueryExport failed (watchlist write continues): ${msg}`);
+    }
+}
+
 /**
  * Get dataset reference, creating it if it doesn't exist
  * @param {string} datasetId - Dataset ID (e.g., 'bulltrackers_data')
@@ -2422,6 +2526,8 @@ async function queryPIAlertHistory(dateStr, logger = null) {
 
 module.exports = {
     getBigQueryClient,
+    getFirestoreBigQueryExportDatasetId,
+    ensureWatchlistsFirestoreBigQueryExport,
     getOrCreateDataset,
     ensureTableExists,
     insertRows,

package/functions/maintenance/firestore-bq-extension-setup/extension-instances.md

@@ -19,6 +19,26 @@ or CLI (`firebase ext:install firebase/firestore-bigquery-export`).
 | Max enqueue attempts       | `3`                       |
 | Log level                  | `Info`                    |
 
+## How BigQuery objects get created
+
+On each Firestore write, the extension’s Cloud Function runs **`initialize()` first** (unless disabled). That path creates the dataset if needed, the **`{TABLE_ID}_raw_changelog`** table, and the **`{TABLE_ID}_raw_latest`** view, then streams the row. So you should **not** need to create `watchlists_sync_raw_changelog` by hand.
+
+If you see **`Not found: Table …watchlists_sync_raw_changelog`** while **`alert_subscriptions_sync_raw_changelog`** (etc.) already exist, initialization **never completed successfully for that instance**—not “the extension skips creating missing tables.” Typical causes:
+
+1. **BigQuery IAM for this instance’s service account**  
+   Each extension instance uses its **own** service account (Firebase Console → Extensions → instance → **Managed service account**). It must be able to create tables in `bt_core` (usually **`roles/bigquery.dataEditor`** on dataset `bt_core`, or project-level editor). Other instances can work while watchlists fails if this SA was never granted or was recreated.
+
+2. **`initialize()` failed earlier**  
+   Check **Logs** for the same function **before** your latest write (permission errors, API errors, wrong `BIGQUERY_PROJECT_ID` / `DATASET_ID` / `TABLE_ID`). Fix config or IAM, then trigger another write.
+
+3. **Wrong `TABLE_ID` in the Console**  
+   Must be exactly `watchlists_sync` so BigQuery names are `watchlists_sync_raw_changelog` and `watchlists_sync_raw_latest`.
+
+4. **Changelog removed manually**  
+   Recreate by reinstalling the extension or running the backfill tool below (it creates the destination schema).
+
+**Recovery:** Grant the instance SA **`roles/bigquery.dataEditor`** on `bt_core`, then save a watchlist again **or** run the backfill command for watchlists (creates/aligns tables). If still broken, uninstall and reinstall the watchlists extension with the same parameters as this doc.
+
 ## Instance 1: bt-watchlists
 
 | Parameter            | Value                                      |

package/functions/version.js

@@ -1 +1 @@
-const version = 23.0
+const version = 24.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.1069",
+  "version": "1.0.1071",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [