bulltrackers-module 1.0.993 → 1.0.995

package/functions/api-v3/routes/popular-investors.js

@@ -9,6 +9,7 @@ const { requireFirebaseAuth } = require('../middleware/auth');
 const { requireVerifiedUser, sanitizeCid } = require('../middleware/identity');
 
 const { EtoroApiService } = require('../services/EtoroApiService');
+const { recordAnalyticsWrite } = require('../../core/utils/analytics_registry');
 
 const router = express.Router();
 
@@ -137,7 +138,7 @@ router.get('/search', async (req, res, next) => {
                 query: validated.query,
                 resultCount: results.length,
                 createdAt: new Date()
-            }).catch(() => {});
+            }).then(() => recordAnalyticsWrite(db, 'analytics_events_search_query').catch(() => {})).catch(() => {});
         }
 
         res.json({ success: true, count: results.length, data: results });
@@ -295,7 +296,7 @@ router.post('/request-addition', async (req, res, next) => {
                     requestedAt: new Date(),
                     status: 'rejected_not_pi',
                     etoroCid: info.realCID || info.gcid || null
-                }).catch(() => {});
+                }).then(() => recordAnalyticsWrite(db, 'pi_addition_requests').catch(() => {})).catch(() => {});
             }
 
             return res.status(400).json({
@@ -355,7 +356,7 @@ router.post('/request-addition', async (req, res, next) => {
                 requestedAt: new Date(),
                 status: 'accepted',
                 isPi: true
-            }).catch(() => {});
+            }).then(() => recordAnalyticsWrite(db, 'pi_addition_requests').catch(() => {})).catch(() => {});
         }
 
         return res.json({

package/functions/api-v3/routes/tutorial.js

@@ -27,6 +27,59 @@ function getTutorialProgressRef(db, cid) {
     .doc('progress');
 }
 
+async function getDerivedCompletions(db, cid) {
+  const completed = new Set();
+  const userDocRef = db.collection('SignedInUsers').doc(String(cid));
+
+  try {
+    const userDoc = await userDocRef.get();
+    if (userDoc.exists) {
+      completed.add('account_create');
+    }
+  } catch (e) {
+    // Non-fatal; ignore and continue.
+  }
+
+  try {
+    const verificationRef = userDocRef.collection('verification').doc('data');
+    const verificationSnap = await verificationRef.get();
+    if (verificationSnap.exists) {
+      const vData = verificationSnap.data() || {};
+      if (vData.etoroCID || vData.etoroUsername || vData.accountSetupComplete) {
+        completed.add('account_link_etoro');
+      }
+    }
+  } catch (e) {
+    // Non-fatal; ignore and continue.
+  }
+
+  try {
+    const subSnap = await userDocRef
+      .collection('alert_subscriptions')
+      .limit(1)
+      .get();
+    if (!subSnap.empty) {
+      completed.add('alerts_create_threshold');
+    }
+  } catch (e) {
+    // Non-fatal; ignore and continue.
+  }
+
+  try {
+    const alertsSnap = await userDocRef
+      .collection('alerts')
+      .limit(1)
+      .get();
+    if (!alertsSnap.empty) {
+      completed.add('alerts_receive_notification');
+    }
+  } catch (e) {
+    // Non-fatal; ignore and continue.
+  }
+
+  return completed;
+}
+
 async function getTutorialConfig(req) {
   const config = (req.config && req.config.tutorial) || {};
   return {
@@ -40,15 +93,31 @@ router.get('/progress', requireVerifiedUser, async (req, res, next) => {
     const { db } = req.services;
     const cid = req.targetUserId;
 
-    const doc = await getTutorialProgressRef(db, cid).get();
-    const data = doc.exists ? doc.data() : {};
+    const ref = getTutorialProgressRef(db, cid);
+    const doc = await ref.get();
+    const data = doc.exists ? doc.data() || {} : {};
+
+    const persistedCompleted = new Set(
+      Array.isArray(data.completedTaskIds) ? data.completedTaskIds : []
+    );
+    const derivedCompleted = await getDerivedCompletions(db, cid);
+    const mergedCompleted = Array.from(
+      new Set([...persistedCompleted, ...derivedCompleted])
+    );
+
+    if (mergedCompleted.length !== persistedCompleted.size) {
+      await ref.set(
+        {
+          completedTaskIds: mergedCompleted,
+        },
+        { merge: true }
+      );
+    }
 
     res.json({
       success: true,
       data: {
-        completedTaskIds: Array.isArray(data.completedTaskIds)
-          ? data.completedTaskIds
-          : [],
+        completedTaskIds: mergedCompleted,
         explicitlyDismissedTaskIds: Array.isArray(
           data.explicitlyDismissedTaskIds
         )

package/functions/api-v3/services/WriteService.js

@@ -11,6 +11,7 @@ const {
 } = require('../../shared/upstashRedis');
 
 const { VALID_WATCHLIST_TAGS } = require('../constants');
+const { recordAnalyticsWrite } = require('../../core/utils/analytics_registry');
 
 const LIMITS = {
     WATCHLIST_NAME_MAX: 200,
@@ -349,6 +350,7 @@ class WriteService {
 
             await this._updateWatchlistMembership(validUserId, watchlistId, oldItems, firestoreData.items || []);
 
+            recordAnalyticsWrite(this.db, 'watchlists').catch(() => {});
             firestoreSuccess = true;
         } catch (error) {
             console.error('[WriteService] Firestore write failed for watchlist:', error.message);
@@ -381,6 +383,7 @@ class WriteService {
         // Delete from Firestore only. Hourly analytics-load job syncs watchlist state to BigQuery.
         await docRef.delete();
 
+        recordAnalyticsWrite(this.db, 'watchlists').catch(() => {});
         await this._updateWatchlistMembership(validUserId, validWatchlistId, oldItems, []);
     }
 
@@ -438,6 +441,7 @@ class WriteService {
                     .doc(validUserId);
                 transaction.set(piReviewRef, review);
             });
+            recordAnalyticsWrite(this.db, 'reviews').catch(() => {});
         } catch (error) {
             // Rethrow explicit errors, log others
             if (error.message === 'Already reviewed') {
@@ -520,6 +524,7 @@ class WriteService {
                 viewedAt: now,
                 createdAt: now
             });
+            recordAnalyticsWrite(this.db, 'analytics_events_pi_page_view').catch(() => {});
         } catch (error) {
             console.error('[WriteService] analytics_events pi_page_view write failed:', error.message);
         }
@@ -544,6 +549,7 @@ class WriteService {
                 new_watchlist_id: this._validateId(newWatchlistId, 'New watchlist ID'),
                 createdAt: now
             });
+            recordAnalyticsWrite(this.db, 'analytics_events_watchlist_copied').catch(() => {});
         } catch (error) {
             console.error('[WriteService] analytics_events watchlist_copied write failed:', error.message);
         }
@@ -689,6 +695,7 @@ class WriteService {
                 ...settings,
                 updatedAt: new Date()
             }, { merge: true });
+        recordAnalyticsWrite(this.db, 'alert_subscriptions').catch(() => {});
     }
 
     /**
@@ -707,6 +714,7 @@ class WriteService {
             .collection('alert_subscriptions')
             .doc(validPiCid)
             .delete();
+        recordAnalyticsWrite(this.db, 'alert_subscriptions').catch(() => {});
     }
 
     // =========================================================================
@@ -743,6 +751,8 @@ class WriteService {
 
         await firestoreRef.set({ investors }, { merge: true });
 
+        recordAnalyticsWrite(this.db, 'pi_master_list').catch(() => {});
+
         // Invalidate API cache for PI master list so new PI appears quickly.
         if (this.redisEnabled) {
             const cacheKey = `api:pi-master-list:${coll}:${docId}`;

package/functions/core/utils/analytics_registry.js

@@ -0,0 +1,116 @@
+/**
+ * @fileoverview Analytics load registry: time-bucketed Firestore docs for "what to sync".
+ * API calls recordAnalyticsWrite(db, sourceKey) after writing analytics data; the loader
+ * reads period docs to discover which sources had writes in the run window.
+ * Documents are named by time period (e.g. YYYY-MM-DD-HH for 2h buckets) to avoid 1MB limit;
+ * each doc has sources: { [sourceKey]: lastWrittenAt } and expire_at for TTL (30 days).
+ */
+
+const BUCKET_HOURS = parseInt(process.env.REGISTRY_BUCKET_HOURS || '2', 10) || 2;
+const TTL_DAYS = parseInt(process.env.REGISTRY_TTL_DAYS || '30', 10) || 30;
+const REGISTRY_COLLECTION = process.env.FIRESTORE_ANALYTICS_REGISTRY_COLLECTION || 'analytics_load_registry';
+
+/**
+ * Get the period document ID for a given date (e.g. "2025-03-13-00" for 2h buckets).
+ * @param {Date} date - Reference time
+ * @returns {string} Period doc ID (YYYY-MM-DD-HH)
+ */
+function getPeriodDocId(date = new Date()) {
+    const y = date.getUTCFullYear();
+    const m = String(date.getUTCMonth() + 1).padStart(2, '0');
+    const d = String(date.getUTCDate()).padStart(2, '0');
+    const hour = date.getUTCHours();
+    const bucketHour = Math.floor(hour / BUCKET_HOURS) * BUCKET_HOURS;
+    const h = String(bucketHour).padStart(2, '0');
+    return `${y}-${m}-${d}-${h}`;
+}
+
+/**
+ * Get period start date (UTC) for expire_at calculation.
+ * @param {string} periodDocId - e.g. "2025-03-13-00"
+ * @returns {Date} Start of that period (UTC)
+ */
+function getPeriodStartFromDocId(periodDocId) {
+    const [y, m, d, h] = periodDocId.split('-').map(Number);
+    return new Date(Date.UTC(y, m - 1, d, h, 0, 0, 0));
+}
+
+/**
+ * Record that a source was written to (for the analytics loader). Fire-and-forget; logs errors only.
+ * Updates the current period's registry doc: merge source key + writtenAt into sources, set expire_at.
+ * @param {object} db - Firestore instance
+ * @param {string} sourceKey - e.g. 'watchlists', 'analytics_events_pi_page_view'
+ */
+async function recordAnalyticsWrite(db, sourceKey) {
+    const now = new Date();
+    const periodDocId = getPeriodDocId(now);
+    const periodStart = getPeriodStartFromDocId(periodDocId);
+    const expireAt = new Date(periodStart);
+    expireAt.setUTCDate(expireAt.getUTCDate() + TTL_DAYS);
+
+    try {
+        const col = db.collection(REGISTRY_COLLECTION);
+        const ref = col.doc(periodDocId);
+        await ref.set(
+            {
+                [`sources.${sourceKey}`]: now,
+                expire_at: expireAt
+            },
+            { merge: true }
+        );
+    } catch (error) {
+        console.error('[analytics_registry] recordAnalyticsWrite failed:', error.message);
+    }
+}
+
+/**
+ * List period doc IDs that overlap the given time window [since, now].
+ * @param {Date} since - Start of window
+ * @param {Date} end - End of window (default now)
+ * @returns {string[]} Period doc IDs in range
+ */
+function getPeriodDocIdsInWindow(since, end = new Date()) {
+    const ids = [];
+    const cursor = new Date(since);
+    cursor.setUTCMinutes(0, 0, 0);
+    const hour = cursor.getUTCHours();
+    const bucketHour = Math.floor(hour / BUCKET_HOURS) * BUCKET_HOURS;
+    cursor.setUTCHours(bucketHour, 0, 0, 0);
+
+    while (cursor <= end) {
+        ids.push(getPeriodDocId(cursor));
+        cursor.setUTCHours(cursor.getUTCHours() + BUCKET_HOURS);
+    }
+    return [...new Set(ids)];
+}
+
+/**
+ * Read registry docs for the given period IDs and return the union of source keys that had writes.
+ * @param {object} db - Firestore instance
+ * @param {string[]} periodDocIds - Period document IDs to read
+ * @returns {Promise<Set<string>>} Set of source keys
+ */
+async function getSourceKeysFromRegistry(db, periodDocIds) {
+    const keys = new Set();
+    const col = db.collection(REGISTRY_COLLECTION);
+    await Promise.all(
+        periodDocIds.map(async (id) => {
+            const snap = await col.doc(id).get();
+            if (!snap.exists) return;
+            const data = snap.data();
+            const sources = data && data.sources && typeof data.sources === 'object' ? data.sources : {};
+            Object.keys(sources).forEach((k) => keys.add(k));
+        })
+    );
+    return keys;
+}
+
+module.exports = {
+    recordAnalyticsWrite,
+    getPeriodDocId,
+    getPeriodDocIdsInWindow,
+    getSourceKeysFromRegistry,
+    REGISTRY_COLLECTION,
+    BUCKET_HOURS,
+    TTL_DAYS
+};

package/functions/maintenance/analytics-load-firestore-to-bq/ANALYTICS_SYSTEM_DOCUMENTATION.md

@@ -82,11 +82,49 @@ Full schema definitions live in `functions/core/utils/bigquery_utils.js` (SCHEMA
 | **Notification preferences** | `SignedInUsers/{cid}/settings/notifications` | Preference analytics | Firestore only; not in loader. |
 | **User computation create/update** | `users/{userId}/computations/{id}` | Feature usage | Firestore only; not in loader. |
 
-To add any of these: (1) ensure the action is written to Firestore (if not already), (2) add a loader step that queries that Firestore source and maps to a `bt_core.*` table (with a defined schema in `bigquery_utils.js`), and (3) run the loader on the same schedule.
+To add any of these: (1) ensure the action is written to Firestore (if not already), (2) call `recordAnalyticsWrite(db, sourceKey)` after the write, (3) add a source entry in `sourceConfig.js` and ensure the BQ table/schema exist in `bigquery_utils.js`, and (4) run the loader on the same schedule.
 
 ---
 
-## 5. Firestore → BigQuery mapping (load job)
+## 5. Analytics load registry (what to sync)
+
+The loader is **registry-driven**: the API records which analytics sources were written to in **time-bucketed** Firestore documents, and the loader reads those to decide which sources to sync for the run window. This avoids hardcoding paths in the loader and makes adding a new path a single config entry + API registry write.
+
+### 5.1 Registry collection
+
+- **Collection:** `analytics_load_registry` (override: `FIRESTORE_ANALYTICS_REGISTRY_COLLECTION`).
+- **Document ID:** Time-period based so a **new document is created every few hours** (e.g. 2h), avoiding the 1MB-per-document limit. Format: `YYYY-MM-DD-HH` (e.g. `2025-03-13-00`, `2025-03-13-02`). Bucket length: **`REGISTRY_BUCKET_HOURS`** (default `2`).
+- **Document fields:**
+  - `sources` – map of source key → Timestamp of last write in that period.
+  - **`expire_at`** – Timestamp set to **period start + 30 days** (configurable via **`REGISTRY_TTL_DAYS`**). Used by Firestore TTL to auto-delete old docs.
+- **TTL policy:** A **Firestore TTL policy must be created** on the registry collection (Firestore console or IaC) that deletes documents when `expire_at` is in the past. Without it, documents will not auto-delete.
+
+### 5.2 Source keys
+
+Each analytics path is identified by a **source key**. The API calls `recordAnalyticsWrite(db, sourceKey)` after writing; the loader uses the same keys in `sourceConfig.js`.
+
+| Source key | Firestore source | BQ table |
+|------------|-------------------|----------|
+| `watchlists` | Collection group `watchlists` | `watchlists` |
+| `reviews` | Collection group `reviews` | `pi_reviews` |
+| `analytics_events_pi_page_view` | `analytics_events` (eventType `pi_page_view`) | `pi_page_view_events` |
+| `pi_master_list` | `system_state/popular_investor_master_list` | `pi_master_list` |
+| `analytics_events_search_query` | `analytics_events` (eventType `search_query`) | `search_queries` |
+| `analytics_events_watchlist_copied` | `analytics_events` (eventType `watchlist_copied`) | `watchlist_copied_events` |
+| `alert_subscriptions` | Collection group `alert_subscriptions` | `alert_subscriptions` |
+| `pi_addition_requests` | Root collection `pi_addition_requests` | `pi_addition_requests` |
+
+### 5.3 How to add a new path
+
+1. **API:** After writing to the new Firestore path, call `recordAnalyticsWrite(db, 'your_source_key')` (fire-and-forget). Helper: `core/utils/analytics_registry.js`.
+2. **Loader config:** Add an entry in `maintenance/analytics-load-firestore-to-bq/sourceConfig.js`: `sourceKey`, `ensureTable`, `fetch`, `mapRows`, `bqTable`, `mergeKeys` (or append).
+3. **BQ schema:** Ensure the table and schema exist in `core/utils/bigquery_utils.js` (SCHEMAS, ensure*Table if needed).
+
+No change to the loader loop in `index.js` is required.
+
+---
+
+## 6. Firestore → BigQuery mapping (load job)
 
 | Firestore source | How the loader reads it | BigQuery dataset.table | Load method |
 |------------------|--------------------------|-------------------------|-------------|
@@ -98,11 +136,13 @@ To add any of these: (1) ensure the action is written to Firestore (if not alrea
 | `SignedInUsers/{cid}/alert_subscriptions` | Collection group `alert_subscriptions` | `bt_core.alert_subscriptions` | insertRowsWithMerge(key: user_id, pi_cid) |
 | `pi_addition_requests` | Root collection; optional filter `requestedAt >= since` | `bt_core.pi_addition_requests` | insertRowsWithMerge(key: request_id) |
 
+Loader behaviour: **Incremental run** – loader reads registry period docs that overlap the run window (e.g. last 1–2h), collects the union of source keys that had writes, then runs the config-driven sync only for those sources. **Full sync** – runs all configured sources (no registry read). **Fallback** – if no registry docs exist for the window, loader runs all sources for the window so cold start still populates BQ.
+
 Loader defaults: **incremental window** = last 2 hours (`since`); **full sync** = no time filter. Dataset for all above tables: **`bt_core`** (override via `BQ_BT_CORE_DATASET_ID` or `BIGQUERY_DATASET_ID`).
 
 ---
 
-## 6. Access control: own-user only
+## 7. Access control: own-user only
 
 **Computation access:** When the computation system or downstream services read from `bt_core` analytics tables for a given user/owner, they must only see **that user's own** behavioural data. Users may not query other users' watchlists, reviews, search_queries, pi_page_view_events, or alert_subscriptions.
 
@@ -112,17 +152,19 @@ Loader defaults: **incremental window** = last 2 hours (`since`); **full sync**
 
 ---
 
-## 7. Related files
+## 8. Related files
 
-- **Loader:** `functions/maintenance/analytics-load-firestore-to-bq/index.js`
+- **Loader:** `functions/maintenance/analytics-load-firestore-to-bq/index.js` (registry-driven; reads `sourceConfig.js`).
+- **Source config:** `functions/maintenance/analytics-load-firestore-to-bq/sourceConfig.js` (source key → Firestore query, row mapper, BQ table).
+- **Registry helper:** `functions/core/utils/analytics_registry.js` (recordAnalyticsWrite, getPeriodDocIdsInWindow, getSourceKeysFromRegistry; time-bucketed docs + TTL).
 - **Firestore event schema (short):** `functions/maintenance/analytics-load-firestore-to-bq/ANALYTICS_EVENTS_SCHEMA.md`
 - **BQ schemas and ensure*:** `functions/core/utils/bigquery_utils.js` (SCHEMAS.watchlists, pi_reviews, pi_page_view_events, search_queries, alert_subscriptions, pi_addition_requests; getBtCoreDatasetId; ensureWatchlistsTable, etc.)
-- **API writes (Firestore-only):** `api-v3/services/WriteService.js`, `api-v3/routes/popular-investors.js` (track-view, search logging)
+- **API writes (Firestore + registry):** `api-v3/services/WriteService.js`, `api-v3/routes/popular-investors.js` (track-view, search logging, pi_addition_requests; each calls recordAnalyticsWrite with the appropriate source key).
 - **Run/schedule:** `maintenance/analytics-load-firestore-to-bq/README.md` (CLI, HTTP, env, schedule)
 
 ---
 
-## 8. Summary tables
+## 9. Summary tables
 
 **Stored in Firestore and loaded to BigQuery (monitored):**
 

package/functions/maintenance/analytics-load-firestore-to-bq/README.md

@@ -2,6 +2,8 @@
 
 Hourly job that reads user analytics from Firestore and bulk-loads into BigQuery (`bt_core`). Replaces API-time dual writes to reduce latency.
 
+**Registry-driven:** The API records which sources were written to in time-bucketed Firestore docs (`analytics_load_registry`). The loader reads those docs to decide which sources to sync for the run window. Documents are named by period (e.g. `YYYY-MM-DD-HH` for 2h buckets) to avoid the 1MB limit; each doc has `expire_at` set to 30 days. **A Firestore TTL policy must be created** on the registry collection so documents auto-delete when `expire_at` is in the past.
+
 ## What it loads
 
 - **Watchlists** – `SignedInUsers/{cid}/watchlists` → `bt_core.watchlists` (MERGE)
@@ -9,6 +11,7 @@ Hourly job that reads user analytics from Firestore and bulk-loads into BigQuery
 - **PI page view events** – `analytics_events` (eventType `pi_page_view`) → `bt_core.pi_page_view_events` (append)
 - **PI master list** – `system_state/popular_investor_master_list` → `bt_core.pi_master_list` (MERGE)
 - **Search queries** – `analytics_events` (eventType `search_query`) → `bt_core.search_queries` (append)
+- **Watchlist copied events** – `analytics_events` (eventType `watchlist_copied`) → `bt_core.watchlist_copied_events` (append)
 - **Alert subscriptions** – collection group `alert_subscriptions` → `bt_core.alert_subscriptions` (MERGE)
 - **PI addition requests** – `pi_addition_requests` → `bt_core.pi_addition_requests` (MERGE on request_id)
 
@@ -51,3 +54,6 @@ Then create a Cloud Scheduler job that POSTs to that URL hourly with no body (or
 - `BQ_BT_CORE_DATASET_ID` or `BIGQUERY_DATASET_ID` – BigQuery dataset for `bt_core` (default `bt_core`)
 - `FIRESTORE_PI_MASTER_LIST_COLLECTION` – default `system_state`
 - `FIRESTORE_PI_MASTER_LIST_DOCUMENT` – default `popular_investor_master_list`
+- **Registry:** `FIRESTORE_ANALYTICS_REGISTRY_COLLECTION` – default `analytics_load_registry`
+- `REGISTRY_BUCKET_HOURS` – period length in hours for time-bucketed doc IDs (default `2`)
+- `REGISTRY_TTL_DAYS` – days after period start before doc is eligible for TTL delete (default `30`). Create a **Firestore TTL policy** on the registry collection using the `expire_at` field so documents are auto-deleted.

package/functions/maintenance/analytics-load-firestore-to-bq/index.js

@@ -7,6 +7,11 @@
  * Trigger: HTTP (Cloud Scheduler) or onSchedule. Use window-based incremental load
  * (e.g. last 2 hours) to avoid cursor state.
  *
+ * Registry-driven: reads analytics_load_registry (time-bucketed docs) to discover which
+ * sources had writes in the run window, then runs config-driven sync for each. Full sync
+ * runs all configured sources. If no registry docs exist for the window, falls back to
+ * running all sources for the window.
+ *
  * Usage:
  *   node index.js                    (run with default 2h window)
  *   node index.js --windowHours=1    (1 hour window)
@@ -16,14 +21,11 @@
 const { Firestore } = require('@google-cloud/firestore');
 const {
     getBtCoreDatasetId,
-    ensureWatchlistsTable,
-    ensurePiReviewsTable,
-    ensurePiPageViewEventsTable,
-    ensureTableExists,
-    getSchema,
     insertRows,
     insertRowsWithMerge
 } = require('../../core/utils/bigquery_utils');
+const { getPeriodDocIdsInWindow, getSourceKeysFromRegistry } = require('../../core/utils/analytics_registry');
+const { SOURCE_CONFIG_BY_KEY, ALL_SOURCE_KEYS } = require('./sourceConfig');
 
 const db = new Firestore();
 const DEFAULT_WINDOW_HOURS = 2;
@@ -32,7 +34,7 @@ const DEFAULT_WINDOW_HOURS = 2;
  * @param {object} logger - { log(level, message) }
  * @param {Date} since - Start of time window
  * @param {boolean} fullSync - If true, ignore since and load all
- * @returns {Promise<{ watchlists: number, pi_reviews: number, pi_page_view_events: number, pi_master_list: number }>}
+ * @returns {Promise<{ watchlists: number, pi_reviews: number, pi_page_view_events: number, pi_master_list: number, search_queries: number, watchlist_copied_events: number, alert_subscriptions: number, pi_addition_requests: number }>}
  */
 async function runAnalyticsLoad(logger = console, since = null, fullSync = false) {
     const log = (level, msg, ...args) => (typeof logger.log === 'function' ? logger.log(level, msg, ...args) : console.log(`[${level}] ${msg}`, ...args));
@@ -42,220 +44,58 @@ async function runAnalyticsLoad(logger = console, since = null, fullSync = false
     const stats = { watchlists: 0, pi_reviews: 0, pi_page_view_events: 0, pi_master_list: 0, search_queries: 0, watchlist_copied_events: 0, alert_subscriptions: 0, pi_addition_requests: 0 };
 
     try {
-        // --- Watchlists (collection group, MERGE) ---
-        await ensureWatchlistsTable(logger, datasetId);
-        const watchlistsSnap = fullSync
-            ? await db.collectionGroup('watchlists').get()
-            : await db.collectionGroup('watchlists').where('updatedAt', '>=', sinceTime).get();
-
-        if (!watchlistsSnap.empty) {
-            const watchlistRows = [];
-            for (const doc of watchlistsSnap.docs) {
-                const userId = doc.ref.parent.parent.id;
-                const data = doc.data();
-                const updatedAt = data.updatedAt ? (data.updatedAt.toDate ? data.updatedAt.toDate() : data.updatedAt) : new Date();
-                watchlistRows.push({
-                    user_id: userId,
-                    watchlist_id: doc.id,
-                    name: data.name ?? null,
-                    type: data.type ?? null,
-                    visibility: data.visibility ?? null,
-                    items: typeof data.items === 'string' ? data.items : JSON.stringify(data.items || []),
-                    item_config: data.itemConfig != null ? JSON.stringify(data.itemConfig) : null,
-                    dynamic_config: data.dynamicConfig != null ? JSON.stringify(data.dynamicConfig) : null,
-                    updated_at: updatedAt.toISOString().replace(/\.\d{3}Z$/, 'Z'),
-                    deleted: false,
-                    deleted_at: null
-                });
-            }
-            if (watchlistRows.length > 0) {
-                await insertRowsWithMerge(datasetId, 'watchlists', watchlistRows, ['user_id', 'watchlist_id'], logger);
-                stats.watchlists = watchlistRows.length;
-            }
-        }
-
-        // --- Reviews (collection group; MERGE on user_cid + pi_cid) ---
-        await ensurePiReviewsTable(logger, datasetId);
-        const reviewsSnap = fullSync
-            ? await db.collectionGroup('reviews').get()
-            : await db.collectionGroup('reviews').where('createdAt', '>=', sinceTime).get();
-
-        if (!reviewsSnap.empty) {
-            const reviewRows = [];
-            const seen = new Set();
-            for (const doc of reviewsSnap.docs) {
-                const data = doc.data();
-                const userCid = String(data.userCid ?? doc.ref.parent.parent.id);
-                const piCid = String(data.piCid ?? doc.id);
-                const key = `${userCid}_${piCid}`;
-                if (seen.has(key)) continue;
-                seen.add(key);
-                const createdAt = data.createdAt ? (data.createdAt.toDate ? data.createdAt.toDate() : data.createdAt) : new Date();
-                reviewRows.push({
-                    user_cid: userCid,
-                    pi_cid: piCid,
-                    rating: data.rating ?? 0,
-                    comment: data.comment ?? '',
-                    is_anonymous: data.isAnonymous === true,
-                    created_at: createdAt.toISOString().replace(/\.\d{3}Z$/, 'Z')
-                });
-            }
-            if (reviewRows.length > 0) {
-                await insertRowsWithMerge(datasetId, 'pi_reviews', reviewRows, ['user_cid', 'pi_cid'], logger);
-                stats.pi_reviews = reviewRows.length;
-            }
-        }
-
-        // --- PI page view events (analytics_events, append) ---
-        await ensurePiPageViewEventsTable(logger, datasetId);
-        const eventsRef = db.collection('analytics_events');
-        const pageViewQuery = fullSync
-            ? eventsRef.where('eventType', '==', 'pi_page_view')
-            : eventsRef.where('eventType', '==', 'pi_page_view').where('createdAt', '>=', sinceTime);
-        const pageViewSnap = await pageViewQuery.get();
-
-        if (!pageViewSnap.empty) {
-            const eventRows = [];
-            for (const doc of pageViewSnap.docs) {
-                const d = doc.data();
-                const viewedAt = d.viewedAt ? (d.viewedAt.toDate ? d.viewedAt.toDate() : d.viewedAt) : new Date();
-                const dateStr = viewedAt.toISOString().split('T')[0];
-                eventRows.push({
-                    date: dateStr,
-                    pi_id: String(d.piId ?? ''),
-                    viewer_cid: String(d.viewerCid ?? ''),
-                    viewed_at: viewedAt.toISOString().replace(/\.\d{3}Z$/, 'Z')
-                });
-            }
-            if (eventRows.length > 0) {
-                await insertRows(datasetId, 'pi_page_view_events', eventRows, logger);
-                stats.pi_page_view_events = eventRows.length;
+        let sourceKeysToSync;
+        if (fullSync) {
+            sourceKeysToSync = new Set(ALL_SOURCE_KEYS);
+            log('INFO', '[AnalyticsLoad] Full sync: running all configured sources');
+        } else {
+            const periodIds = getPeriodDocIdsInWindow(sinceTime, new Date());
+            const keysFromRegistry = await getSourceKeysFromRegistry(db, periodIds);
+            if (keysFromRegistry.size === 0) {
+                sourceKeysToSync = new Set(ALL_SOURCE_KEYS);
+                log('INFO', '[AnalyticsLoad] No registry docs for window; running all sources for window');
+            } else {
+                sourceKeysToSync = keysFromRegistry;
+                log('INFO', '[AnalyticsLoad] Registry sources for window:', [...sourceKeysToSync].join(', '));
             }
         }
 
-        // --- PI master list (single doc → MERGE) ---
-        const coll = process.env.FIRESTORE_PI_MASTER_LIST_COLLECTION || 'system_state';
-        const docId = process.env.FIRESTORE_PI_MASTER_LIST_DOCUMENT || 'popular_investor_master_list';
-        const masterDoc = await db.collection(coll).doc(docId).get();
-        if (masterDoc.exists) {
-            const investors = masterDoc.data().investors || {};
-            const masterRows = [];
-            for (const [cid, info] of Object.entries(investors)) {
-                if (!info || typeof info !== 'object') continue;
-                const firstSeen = info.firstSeenAt?.toDate ? info.firstSeenAt.toDate() : info.firstSeenAt;
-                const lastSeen = info.lastSeenAt?.toDate ? info.lastSeenAt.toDate() : info.lastSeenAt;
-                const t = (v) => (v && v.toISOString ? v.toISOString().replace(/\.\d{3}Z$/, '') : (v ? String(v) : new Date().toISOString().replace(/\.\d{3}Z$/, '')));
-                masterRows.push({
-                    cid: parseInt(String(cid), 10) || 0,
-                    username: info.username || '',
-                    first_seen_at: t(firstSeen),
-                    last_seen_at: t(lastSeen),
-                    last_updated: t(lastSeen)
-                });
+        for (const sourceKey of sourceKeysToSync) {
+            const config = SOURCE_CONFIG_BY_KEY[sourceKey];
+            if (!config) {
+                log('WARN', `[AnalyticsLoad] No config for sourceKey ${sourceKey}, skipping`);
+                continue;
             }
-            if (masterRows.length > 0) {
-                await ensureTableExists(datasetId, 'pi_master_list', getSchema('pi_master_list'), {}, logger);
-                await insertRowsWithMerge(datasetId, 'pi_master_list', masterRows, ['cid'], logger);
-                stats.pi_master_list = masterRows.length;
-            }
-        }
-
-        // --- Search queries (analytics_events, append) ---
-        const searchQuery = fullSync
-            ? db.collection('analytics_events').where('eventType', '==', 'search_query')
-            : db.collection('analytics_events').where('eventType', '==', 'search_query').where('createdAt', '>=', sinceTime);
-        const searchSnap = await searchQuery.get();
-        if (!searchSnap.empty) {
-            const { ensureSearchQueriesTable } = require('../../core/utils/bigquery_utils');
-            await ensureSearchQueriesTable(logger, datasetId);
-            const searchRows = searchSnap.docs.map((doc) => {
-                const d = doc.data();
-                const createdAt = d.createdAt ? (d.createdAt.toDate ? d.createdAt.toDate() : d.createdAt) : new Date();
-                return {
-                    user_id: d.userId ?? null,
-                    query: d.query ?? '',
-                    result_count: d.resultCount != null ? d.resultCount : null,
-                    created_at: createdAt.toISOString().replace(/\.\d{3}Z$/, 'Z')
-                };
-            });
-            if (searchRows.length > 0) {
-                await insertRows(datasetId, 'search_queries', searchRows, logger);
-                stats.search_queries = searchRows.length;
-            }
-        }
-
-        // --- Watchlist copied events (analytics_events, append) ---
-        const copyEventsRef = db.collection('analytics_events');
-        const copyQuery = fullSync
-            ? copyEventsRef.where('eventType', '==', 'watchlist_copied')
-            : copyEventsRef.where('eventType', '==', 'watchlist_copied').where('createdAt', '>=', sinceTime);
-        const copySnap = await copyQuery.get();
-        if (!copySnap.empty) {
-            const { ensureWatchlistCopiedEventsTable } = require('../../core/utils/bigquery_utils');
-            await ensureWatchlistCopiedEventsTable(logger, datasetId);
-            const copyRows = copySnap.docs.map((doc) => {
-                const d = doc.data();
-                const createdAt = d.createdAt ? (d.createdAt.toDate ? d.createdAt.toDate() : d.createdAt) : new Date();
-                return {
-                    user_id: d.user_id ?? '',
-                    source_user_id: d.source_user_id ?? '',
-                    source_watchlist_id: d.source_watchlist_id ?? '',
-                    new_watchlist_id: d.new_watchlist_id ?? '',
-                    created_at: createdAt.toISOString().replace(/\.\d{3}Z$/, 'Z')
-                };
-            });
-            if (copyRows.length > 0) {
-                await insertRows(datasetId, 'watchlist_copied_events', copyRows, logger);
-                stats.watchlist_copied_events = copyRows.length;
-            }
-        }
-
-        // --- Alert subscriptions (collection group, MERGE) ---
-        const subsSnap = fullSync
-            ? await db.collectionGroup('alert_subscriptions').get()
-            : await db.collectionGroup('alert_subscriptions').get(); // no updatedAt filter on all; for incremental we could add updatedAt
-        if (!subsSnap.empty) {
-            const { ensureAlertSubscriptionsTable } = require('../../core/utils/bigquery_utils');
-            await ensureAlertSubscriptionsTable(logger, datasetId);
-            const subRows = [];
-            for (const doc of subsSnap.docs) {
-                const userId = doc.ref.parent.parent.id;
-                const data = doc.data();
-                const updatedAt = data.updatedAt ? (data.updatedAt.toDate ? data.updatedAt.toDate() : data.updatedAt) : new Date();
-                subRows.push({
-                    user_id: userId,
-                    pi_cid: doc.id,
-                    settings: JSON.stringify(data),
-                    updated_at: updatedAt.toISOString().replace(/\.\d{3}Z$/, 'Z')
-                });
-            }
-            if (subRows.length > 0) {
-                await insertRowsWithMerge(datasetId, 'alert_subscriptions', subRows, ['user_id', 'pi_cid'], logger);
-                stats.alert_subscriptions = subRows.length;
-            }
-        }
 
-        // --- PI addition requests (root collection, append with idempotency by request_id) ---
-        const reqSnap = fullSync
-            ? await db.collection('pi_addition_requests').get()
-            : await db.collection('pi_addition_requests').where('requestedAt', '>=', sinceTime).get();
-        if (!reqSnap.empty) {
-            const { ensurePiAdditionRequestsTable } = require('../../core/utils/bigquery_utils');
-            await ensurePiAdditionRequestsTable(logger, datasetId);
-            const reqRows = reqSnap.docs.map((doc) => {
-                const d = doc.data();
-                const requestedAt = d.requestedAt ? (d.requestedAt.toDate ? d.requestedAt.toDate() : d.requestedAt) : new Date();
-                return {
-                    request_id: doc.id,
-                    username: d.username ?? null,
-                    requested_by: d.requestedBy ?? null,
-                    requested_at: requestedAt.toISOString().replace(/\.\d{3}Z$/, 'Z'),
-                    status: d.status ?? null
-                };
-            });
-            if (reqRows.length > 0) {
-                await insertRowsWithMerge(datasetId, 'pi_addition_requests', reqRows, ['request_id'], logger);
-                stats.pi_addition_requests = reqRows.length;
+            try {
+                await config.ensureTable(logger, datasetId);
+
+                let rows = [];
+                if (config.singleDoc) {
+                    const doc = await config.fetch(db);
+                    rows = config.mapRows(doc);
+                } else {
+                    const snapshot = await config.fetch(db, sinceTime, fullSync);
+                    rows = config.mapRows(snapshot.docs || []);
+                }
+
+                if (rows.length === 0) continue;
+
+                if (config.mergeKeys) {
+                    await insertRowsWithMerge(datasetId, config.bqTable, rows, config.mergeKeys, logger);
+                } else {
+                    await insertRows(datasetId, config.bqTable, rows, logger);
+                }
+
+                const prev = stats[config.bqTable];
+                if (typeof prev === 'number') {
+                    stats[config.bqTable] = prev + rows.length;
+                } else {
+                    stats[config.bqTable] = rows.length;
+                }
+            } catch (err) {
+                log('ERROR', `[AnalyticsLoad] ${sourceKey} failed: ${err.message}`);
+                throw err;
             }
         }
 

package/functions/maintenance/analytics-load-firestore-to-bq/sourceConfig.js

@@ -0,0 +1,268 @@
+/**
+ * @fileoverview Config-driven source definitions for analytics load: Firestore → BigQuery.
+ * Each source key maps to Firestore query, row mapper, BQ table, and merge/append behavior.
+ * Add a new path: add an entry here and call recordAnalyticsWrite(db, sourceKey) from the API.
+ */
+
+const {
+    ensureWatchlistsTable,
+    ensurePiReviewsTable,
+    ensurePiPageViewEventsTable,
+    ensureSearchQueriesTable,
+    ensureWatchlistCopiedEventsTable,
+    ensureAlertSubscriptionsTable,
+    ensurePiAdditionRequestsTable,
+    ensureTableExists,
+    getSchema
+} = require('../../core/utils/bigquery_utils');
+
+const PI_MASTER_LIST_COLLECTION = process.env.FIRESTORE_PI_MASTER_LIST_COLLECTION || 'system_state';
+const PI_MASTER_LIST_DOCUMENT = process.env.FIRESTORE_PI_MASTER_LIST_DOCUMENT || 'popular_investor_master_list';
+
+/**
+ * @typedef {Object} SourceConfig
+ * @property {string} sourceKey - Registry source key (e.g. 'watchlists')
+ * @property {function(object, string): Promise<void>} ensureTable - (logger, datasetId) => ensure BQ table exists
+ * @property {function(object, Date, boolean): Promise<FirebaseFirestore.QuerySnapshot>} fetch - (db, since, fullSync) => snapshot
+ * @property {function(FirebaseFirestore.QueryDocumentSnapshot[]): Array<Object>} mapRows - (docs) => BQ rows
+ * @property {string} bqTable - BigQuery table name
+ * @property {string[]|null} mergeKeys - Key fields for MERGE; null = append only
+ */
+
+/**
+ * All configured analytics sources. Keys must match sourceKey used in recordAnalyticsWrite().
+ * @type {SourceConfig[]}
+ */
+const SOURCE_CONFIGS = [
+    {
+        sourceKey: 'watchlists',
+        ensureTable: (logger, datasetId) => ensureWatchlistsTable(logger, datasetId),
+        fetch: async (db, since, fullSync) => {
+            const q = fullSync
+                ? db.collectionGroup('watchlists')
+                : db.collectionGroup('watchlists').where('updatedAt', '>=', since);
+            return q.get();
+        },
+        mapRows: (docs) => {
+            return docs.map((doc) => {
+                const userId = doc.ref.parent.parent.id;
+                const data = doc.data();
+                const updatedAt = data.updatedAt ? (data.updatedAt.toDate ? data.updatedAt.toDate() : data.updatedAt) : new Date();
+                return {
+                    user_id: userId,
+                    watchlist_id: doc.id,
+                    name: data.name ?? null,
+                    type: data.type ?? null,
+                    visibility: data.visibility ?? null,
+                    items: typeof data.items === 'string' ? data.items : JSON.stringify(data.items || []),
+                    item_config: data.itemConfig != null ? JSON.stringify(data.itemConfig) : null,
+                    dynamic_config: data.dynamicConfig != null ? JSON.stringify(data.dynamicConfig) : null,
+                    updated_at: updatedAt.toISOString().replace(/\.\d{3}Z$/, 'Z'),
+                    deleted: false,
+                    deleted_at: null
+                };
+            });
+        },
+        bqTable: 'watchlists',
+        mergeKeys: ['user_id', 'watchlist_id']
+    },
+    {
+        sourceKey: 'reviews',
+        ensureTable: (logger, datasetId) => ensurePiReviewsTable(logger, datasetId),
+        fetch: async (db, since, fullSync) => {
+            const q = fullSync
+                ? db.collectionGroup('reviews')
+                : db.collectionGroup('reviews').where('createdAt', '>=', since);
+            return q.get();
+        },
+        mapRows: (docs) => {
+            const rows = [];
+            const seen = new Set();
+            for (const doc of docs) {
+                const data = doc.data();
+                const userCid = String(data.userCid ?? doc.ref.parent.parent.id);
+                const piCid = String(data.piCid ?? doc.id);
+                const key = `${userCid}_${piCid}`;
+                if (seen.has(key)) continue;
+                seen.add(key);
+                const createdAt = data.createdAt ? (data.createdAt.toDate ? data.createdAt.toDate() : data.createdAt) : new Date();
+                rows.push({
+                    user_cid: userCid,
+                    pi_cid: piCid,
+                    rating: data.rating ?? 0,
+                    comment: data.comment ?? '',
+                    is_anonymous: data.isAnonymous === true,
+                    created_at: createdAt.toISOString().replace(/\.\d{3}Z$/, 'Z')
+                });
+            }
+            return rows;
+        },
+        bqTable: 'pi_reviews',
+        mergeKeys: ['user_cid', 'pi_cid']
+    },
+    {
+        sourceKey: 'analytics_events_pi_page_view',
+        ensureTable: (logger, datasetId) => ensurePiPageViewEventsTable(logger, datasetId),
+        fetch: async (db, since, fullSync) => {
+            const ref = db.collection('analytics_events');
+            const q = fullSync
+                ? ref.where('eventType', '==', 'pi_page_view')
+                : ref.where('eventType', '==', 'pi_page_view').where('createdAt', '>=', since);
+            return q.get();
+        },
+        mapRows: (docs) => {
+            return docs.map((doc) => {
+                const d = doc.data();
+                const viewedAt = d.viewedAt ? (d.viewedAt.toDate ? d.viewedAt.toDate() : d.viewedAt) : new Date();
+                const dateStr = viewedAt.toISOString().split('T')[0];
+                return {
+                    date: dateStr,
+                    pi_id: String(d.piId ?? ''),
+                    viewer_cid: String(d.viewerCid ?? ''),
+                    viewed_at: viewedAt.toISOString().replace(/\.\d{3}Z$/, 'Z')
+                };
+            });
+        },
+        bqTable: 'pi_page_view_events',
+        mergeKeys: null
+    },
+    {
+        sourceKey: 'pi_master_list',
+        ensureTable: (logger, datasetId) => ensureTableExists(datasetId, 'pi_master_list', getSchema('pi_master_list'), {}, logger),
+        fetch: async (db) => {
+            return db.collection(PI_MASTER_LIST_COLLECTION).doc(PI_MASTER_LIST_DOCUMENT).get();
+        },
+        mapRows: (docOrSnapshot) => {
+            const doc = docOrSnapshot && docOrSnapshot.exists ? docOrSnapshot : (Array.isArray(docOrSnapshot) ? docOrSnapshot[0] : null);
+            if (!doc || !doc.exists) return [];
+            const investors = doc.data().investors || {};
+            const rows = [];
+            for (const [cid, info] of Object.entries(investors)) {
+                if (!info || typeof info !== 'object') continue;
+                const firstSeen = info.firstSeenAt?.toDate ? info.firstSeenAt.toDate() : info.firstSeenAt;
+                const lastSeen = info.lastSeenAt?.toDate ? info.lastSeenAt.toDate() : info.lastSeenAt;
+                const t = (v) => (v && v.toISOString ? v.toISOString().replace(/\.\d{3}Z$/, '') : (v ? String(v) : new Date().toISOString().replace(/\.\d{3}Z$/, '')));
+                rows.push({
+                    cid: parseInt(String(cid), 10) || 0,
+                    username: info.username || '',
+                    first_seen_at: t(firstSeen),
+                    last_seen_at: t(lastSeen),
+                    last_updated: t(lastSeen)
+                });
+            }
+            return rows;
+        },
+        bqTable: 'pi_master_list',
+        mergeKeys: ['cid'],
+        singleDoc: true
+    },
+    {
+        sourceKey: 'analytics_events_search_query',
+        ensureTable: (logger, datasetId) => ensureSearchQueriesTable(logger, datasetId),
+        fetch: async (db, since, fullSync) => {
+            const ref = db.collection('analytics_events');
+            const q = fullSync
+                ? ref.where('eventType', '==', 'search_query')
+                : ref.where('eventType', '==', 'search_query').where('createdAt', '>=', since);
+            return q.get();
+        },
+        mapRows: (docs) => {
+            return docs.map((doc) => {
+                const d = doc.data();
+                const createdAt = d.createdAt ? (d.createdAt.toDate ? d.createdAt.toDate() : d.createdAt) : new Date();
+                return {
+                    user_id: d.userId ?? null,
+                    query: d.query ?? '',
+                    result_count: d.resultCount != null ? d.resultCount : null,
+                    created_at: createdAt.toISOString().replace(/\.\d{3}Z$/, 'Z')
+                };
+            });
+        },
+        bqTable: 'search_queries',
+        mergeKeys: null
+    },
+    {
+        sourceKey: 'analytics_events_watchlist_copied',
+        ensureTable: (logger, datasetId) => ensureWatchlistCopiedEventsTable(logger, datasetId),
+        fetch: async (db, since, fullSync) => {
+            const ref = db.collection('analytics_events');
+            const q = fullSync
+                ? ref.where('eventType', '==', 'watchlist_copied')
+                : ref.where('eventType', '==', 'watchlist_copied').where('createdAt', '>=', since);
+            return q.get();
+        },
+        mapRows: (docs) => {
+            return docs.map((doc) => {
+                const d = doc.data();
+                const createdAt = d.createdAt ? (d.createdAt.toDate ? d.createdAt.toDate() : d.createdAt) : new Date();
+                return {
+                    user_id: d.user_id ?? '',
+                    source_user_id: d.source_user_id ?? '',
+                    source_watchlist_id: d.source_watchlist_id ?? '',
+                    new_watchlist_id: d.new_watchlist_id ?? '',
+                    created_at: createdAt.toISOString().replace(/\.\d{3}Z$/, 'Z')
+                };
+            });
+        },
+        bqTable: 'watchlist_copied_events',
+        mergeKeys: null
+    },
+    {
+        sourceKey: 'alert_subscriptions',
+        ensureTable: (logger, datasetId) => ensureAlertSubscriptionsTable(logger, datasetId),
+        fetch: async (db, _since, fullSync) => {
+            return db.collectionGroup('alert_subscriptions').get();
+        },
+        mapRows: (docs) => {
+            return docs.map((doc) => {
+                const userId = doc.ref.parent.parent.id;
+                const data = doc.data();
+                const updatedAt = data.updatedAt ? (data.updatedAt.toDate ? data.updatedAt.toDate() : data.updatedAt) : new Date();
+                return {
+                    user_id: userId,
+                    pi_cid: doc.id,
+                    settings: JSON.stringify(data),
+                    updated_at: updatedAt.toISOString().replace(/\.\d{3}Z$/, 'Z')
+                };
+            });
+        },
+        bqTable: 'alert_subscriptions',
+        mergeKeys: ['user_id', 'pi_cid']
+    },
+    {
+        sourceKey: 'pi_addition_requests',
+        ensureTable: (logger, datasetId) => ensurePiAdditionRequestsTable(logger, datasetId),
+        fetch: async (db, since, fullSync) => {
+            const ref = db.collection('pi_addition_requests');
+            const q = fullSync ? ref : ref.where('requestedAt', '>=', since);
+            return q.get();
+        },
+        mapRows: (docs) => {
+            return docs.map((doc) => {
+                const d = doc.data();
+                const requestedAt = d.requestedAt ? (d.requestedAt.toDate ? d.requestedAt.toDate() : d.requestedAt) : new Date();
+                return {
+                    request_id: doc.id,
+                    username: d.username ?? null,
+                    requested_by: d.requestedBy ?? null,
+                    requested_at: requestedAt.toISOString().replace(/\.\d{3}Z$/, 'Z'),
+                    status: d.status ?? null
+                };
+            });
+        },
+        bqTable: 'pi_addition_requests',
+        mergeKeys: ['request_id']
+    }
+];
+
+/** Map sourceKey → config for quick lookup */
+const SOURCE_CONFIG_BY_KEY = Object.fromEntries(SOURCE_CONFIGS.map((c) => [c.sourceKey, c]));
+
+/** All source keys that have a config (for full sync fallback) */
+const ALL_SOURCE_KEYS = SOURCE_CONFIGS.map((c) => c.sourceKey);
+
+module.exports = {
+    SOURCE_CONFIGS,
+    SOURCE_CONFIG_BY_KEY,
+    ALL_SOURCE_KEYS
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.993",
+  "version": "1.0.995",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [