bulltrackers-module 1.0.658 → 1.0.659

package/functions/computation-system/services/SnapshotService.js

@@ -0,0 +1,129 @@
+/**
+ * @fileoverview Snapshot Service.
+ * Creates a complete "Frozen State" of the day's data in GCS.
+ * UPDATED: Now snapshots ALL data types (Ratings, Verification, Metadata, etc).
+ */
+const zlib = require('zlib');
+const { Storage } = require('@google-cloud/storage');
+const storage = new Storage();
+const dataLoader = require('../utils/data_loader');
+
+async function generateDailySnapshots(dateStr, config, deps) {
+    const { logger } = deps;
+    logger.log('INFO', `[SnapshotService] 📸 Starting Full System Snapshot for ${dateStr}`);
+
+    const bucketName = config.gcsBucketName || 'bulltrackers';
+    const bucket = storage.bucket(bucketName);
+
+    // parallelize independent fetches
+    await Promise.all([
+        snapshotPortfolios(dateStr, bucket, config, deps),  // Heavy
+        snapshotSocial(dateStr, bucket, config, deps),      // Heavy
+        snapshotHistory(dateStr, bucket, config, deps),     // Heavy (JSONL)
+        snapshotRatings(dateStr, bucket, config, deps),     // Sharded
+        snapshotVerification(dateStr, bucket, config, deps),// Collection Group
+        snapshotRankings(dateStr, bucket, config, deps),    // Single Doc
+        snapshotMetadata(dateStr, bucket, config, deps)     // Small Docs (Insights, Alerts, Watchlist)
+    ]);
+
+    logger.log('INFO', `[SnapshotService] ✅ Full System Snapshot Complete.`);
+    return { status: 'OK', date: dateStr };
+}
+
+// --- HEAVY DATA HANDLERS (As before) ---
+
+async function snapshotPortfolios(dateStr, bucket, config, deps) {
+    const file = bucket.file(`${dateStr}/snapshots/portfolios.json.gz`);
+    if ((await file.exists())[0] && !config.forceSnapshot) return;
+
+    // Fetch ALL types (PI + SignedIn) to ensure complete context
+    const refs = await dataLoader.getPortfolioPartRefs(config, deps, dateStr, ['ALL']);
+    const data = await dataLoader.loadDataByRefs(config, deps, refs);
+    if (Object.keys(data).length > 0) {
+        await file.save(JSON.stringify(data), { gzip: true });
+        deps.logger.log('INFO', `[Snapshot] Saved ${Object.keys(data).length} portfolios.`);
+    }
+}
+
+async function snapshotSocial(dateStr, bucket, config, deps) {
+    const file = bucket.file(`${dateStr}/snapshots/social.json.gz`);
+    if ((await file.exists())[0] && !config.forceSnapshot) return;
+
+    const data = await dataLoader.loadDailySocialPostInsights(config, deps, dateStr);
+    await file.save(JSON.stringify(data), { gzip: true });
+    deps.logger.log('INFO', `[Snapshot] Saved Social Data.`);
+}
+
+async function snapshotHistory(dateStr, bucket, config, deps) {
+    const file = bucket.file(`${dateStr}/snapshots/history.jsonl.gz`);
+    if ((await file.exists())[0] && !config.forceSnapshot) return;
+
+    const refs = await dataLoader.getHistoryPartRefs(config, deps, dateStr, ['ALL']);
+    const gcsStream = file.createWriteStream({ gzip: true });
+
+    // Stream line-by-line
+    const BATCH_SIZE = 10;
+    for (let i = 0; i < refs.length; i += BATCH_SIZE) {
+        const batchRefs = refs.slice(i, i + BATCH_SIZE);
+        const dataMap = await dataLoader.loadDataByRefs(config, deps, batchRefs);
+        let chunk = '';
+        for (const [uid, h] of Object.entries(dataMap)) chunk += JSON.stringify({ [uid]: h }) + '\n';
+        if (!gcsStream.write(chunk)) await new Promise(r => gcsStream.once('drain', r));
+    }
+    gcsStream.end();
+    await new Promise((resolve, reject) => { gcsStream.on('finish', resolve); gcsStream.on('error', reject); });
+    deps.logger.log('INFO', `[Snapshot] Saved History (JSONL).`);
+}
+
+// --- NEW HANDLERS (The "All Data" Expansion) ---
+
+async function snapshotRatings(dateStr, bucket, config, deps) {
+    const file = bucket.file(`${dateStr}/snapshots/ratings.json.gz`);
+    if ((await file.exists())[0] && !config.forceSnapshot) return;
+
+    // Load RAW ratings from Firestore shards
+    const data = await dataLoader.loadPIRatings(config, deps, dateStr);
+    await file.save(JSON.stringify(data), { gzip: true });
+    deps.logger.log('INFO', `[Snapshot] Saved Ratings for ${Object.keys(data).length} PIs.`);
+}
+
+async function snapshotVerification(dateStr, bucket, config, deps) {
+    const file = bucket.file(`${dateStr}/snapshots/verification.json.gz`);
+    if ((await file.exists())[0] && !config.forceSnapshot) return;
+
+    // Load global verification profiles (expensive scan)
+    const data = await dataLoader.loadVerificationProfiles(config, deps);
+    await file.save(JSON.stringify(data), { gzip: true });
+    deps.logger.log('INFO', `[Snapshot] Saved ${Object.keys(data).length} Verification Profiles.`);
+}
+
+async function snapshotRankings(dateStr, bucket, config, deps) {
+    const file = bucket.file(`${dateStr}/snapshots/rankings.json.gz`);
+    if ((await file.exists())[0] && !config.forceSnapshot) return;
+
+    const data = await dataLoader.loadPopularInvestorRankings(config, deps, dateStr);
+    if (data) await file.save(JSON.stringify(data), { gzip: true });
+    deps.logger.log('INFO', `[Snapshot] Saved Rankings.`);
+}
+
+async function snapshotMetadata(dateStr, bucket, config, deps) {
+    // Bundle small files into one "metadata.json" or keep separate. Separate is safer for loaders.
+    const ops = [
+        { name: 'insights', fn: () => dataLoader.loadDailyInsights(config, deps, dateStr) },
+        { name: 'page_views', fn: () => dataLoader.loadPIPageViews(config, deps, dateStr) },
+        { name: 'watchlist', fn: () => dataLoader.loadWatchlistMembership(config, deps, dateStr) },
+        { name: 'alerts', fn: () => dataLoader.loadPIAlertHistory(config, deps, dateStr) },
+        { name: 'master_list', fn: () => dataLoader.loadPopularInvestorMasterList(config, deps) } // Not date bound usually, but good to snapshot state
+    ];
+
+    for (const op of ops) {
+        const file = bucket.file(`${dateStr}/snapshots/${op.name}.json.gz`);
+        if ((await file.exists())[0] && !config.forceSnapshot) continue;
+        
+        const data = await op.fn();
+        if (data) await file.save(JSON.stringify(data), { gzip: true });
+    }
+    deps.logger.log('INFO', `[Snapshot] Saved Metadata files.`);
+}
+
+module.exports = { generateDailySnapshots };

package/functions/computation-system/tools/BuildReporter.js

@@ -14,6 +14,7 @@ const SYSTEM_EPOCH = require('../system_epoch');
 const REPORTER_EPOCH = require('../reporter_epoch');
 const pLimit = require('p-limit');
 const path = require('path');
+const fs = require('fs');
 const crypto = require('crypto');
 
 const BUILD_RECORDS_COLLECTION = 'computation_build_records';
@@ -27,20 +28,24 @@ function getPackageVersions() {
     let calcVersion = 'unknown';
 
     try {
-        const modulepkg = require('../../../../bulltrackers-module/package.json');
+        const modulePkgPath = path.join(__dirname, '../../../../bulltrackers-module/package.json');
+        const modulepkg = JSON.parse(fs.readFileSync(modulePkgPath, 'utf8'));
         moduleVersion = modulepkg.version;
 
-} catch (e) {
-    moduleVersion = "1"
-     console.warn('[BuildReporter] Could not resolve Module version', e.message); }
+    } catch (e) {
+        moduleVersion = "1"
+        console.warn('[BuildReporter] Could not resolve Module version', e.message); 
+    }
 
     try {
-        const calcpkg   = require('../../../../calculations/package.json')
+        const calcPkgPath = path.join(__dirname, '../../../../calculations/package.json');
+        const calcpkg = JSON.parse(fs.readFileSync(calcPkgPath, 'utf8'));
         calcVersion = calcpkg.version;
         
     } catch (e2) { 
         calcVersion = "1"
-        console.warn('[BuildReporter] Could not resolve Calculations version', e2.message); }
+        console.warn('[BuildReporter] Could not resolve Calculations version', e2.message); 
+    }
         
 
     return { moduleVersion, calcVersion };
@@ -257,7 +262,7 @@ async function generateBuildReport(config, dependencies, manifest) {
     const today = new Date();
     let dynamicDaysBack = 90; 
     if (absoluteEarliest) {
-        const diffTime = Math.abs(today - absoluteEarliest);
+        const diffTime = Math.abs(today.getTime() - absoluteEarliest.getTime());
         dynamicDaysBack = Math.ceil(diffTime / (1000 * 60 * 60 * 24)) + 2; 
     }
     const startDate = new Date();