bulltrackers-module 1.0.281 → 1.0.282

package/functions/computation-system/persistence/ResultCommitter.js

@@ -4,6 +4,7 @@
  * UPDATED: Implements Content-Based Hashing (ResultHash) for dependency short-circuiting.
  * UPDATED: Auto-enforces Weekend Mode validation.
  * UPDATED: Implements "Initial Write" logic to wipe stale data/shards on a fresh run.
+ * OPTIMIZED: Fetches pre-calculated 'simHash' from Registry (removes expensive simulation step).
  */
 const { commitBatchInChunks, generateDataHash } = require('../utils/utils');
 const { updateComputationStatus }          = require('./StatusRepository');
@@ -18,6 +19,8 @@ const NON_RETRYABLE_ERRORS = [
     'PERMISSION_DENIED',     'DATA_LOSS',             'FAILED_PRECONDITION'
 ];
 
+const SIMHASH_REGISTRY_COLLECTION = 'system_simhash_registry';
+
 /**
  * Commits results to Firestore.
  */
@@ -31,7 +34,7 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
     
     // Options defaults
     const flushMode        = options.flushMode || 'STANDARD';
-    const isInitialWrite   = options.isInitialWrite === true; // [NEW] Flag for clean run
+    const isInitialWrite   = options.isInitialWrite === true; 
     const shardIndexes     = options.shardIndexes || {};
     const nextShardIndexes = {};
 
@@ -80,6 +83,25 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
             const isEmpty = !result || (typeof result === 'object' && Object.keys(result).length === 0);
             const resultHash = isEmpty ? 'empty' : generateDataHash(result);
 
+            // [OPTIMIZATION] FETCH SimHash from Registry (Do NOT Calculate)
+            let simHash = null;
+            if (calc.manifest.hash && flushMode !== 'INTERMEDIATE') {
+                try {
+                    // Fast O(1) lookup using Code Hash
+                    // We simply check if the BuildReporter has already stamped this code version
+                    const regDoc = await db.collection(SIMHASH_REGISTRY_COLLECTION).doc(calc.manifest.hash).get();
+                    if (regDoc.exists) {
+                        simHash = regDoc.data().simHash;
+                    } else {
+                        // Fallback: This happens if BuildReporter didn't run or is out of sync.
+                        // We do NOT run SimRunner here to protect production performance.
+                        logger.log('WARN', `[ResultCommitter] SimHash not found in registry for ${name} (Hash: ${calc.manifest.hash}). Is BuildReporter skipped?`);
+                    }
+                } catch (regErr) {
+                    logger.log('WARN', `[ResultCommitter] Failed to read SimHash registry: ${regErr.message}`);
+                }
+            }
+
             if (isEmpty) { 
                 if (flushMode === 'INTERMEDIATE') {
                     nextShardIndexes[name] = currentShardIndex; 
@@ -88,6 +110,7 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
                 if (calc.manifest.hash) { 
                     successUpdates[name] = { 
                         hash:        calc.manifest.hash,
+                        simHash:     simHash, // [NEW] Populated from Registry
                         resultHash:  resultHash, 
                         dependencyResultHashes: calc.manifest.dependencyResultHashes || {},
                         category:    calc.manifest.category, 
@@ -115,8 +138,6 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
                         .collection(config.computationsSubcollection)
                         .doc(name);
 
-                    // Note: Multi-date fan-out rarely hits sharding, and tracking isInitialWrite per-date is complex.
-                    // We assume standard merging here.
                     await writeSingleResult(dailyData, historicalDocRef, name, historicalDate, logger, config, deps, 0, 'STANDARD', false);
                 }));
                 await Promise.all(datePromises);
@@ -124,6 +145,7 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
                 if (calc.manifest.hash) { 
                     successUpdates[name] = {  
                         hash:        calc.manifest.hash, 
+                        simHash:     simHash, // [NEW] Populated from Registry
                         resultHash:  resultHash, 
                         dependencyResultHashes: calc.manifest.dependencyResultHashes || {},
                         category:    calc.manifest.category, 
@@ -151,6 +173,7 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
                 if (calc.manifest.hash) { 
                     successUpdates[name] = {  
                         hash:        calc.manifest.hash, 
+                        simHash:     simHash, // [NEW] Populated from Registry
                         resultHash:  resultHash,
                         dependencyResultHashes: calc.manifest.dependencyResultHashes || {},
                         category:    calc.manifest.category,
@@ -188,10 +211,8 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
 
 async function writeSingleResult(result, docRef, name, dateContext, logger, config, deps, startShardIndex = 0, flushMode = 'STANDARD', isInitialWrite = false) {
     
-    // [NEW] Transition & Cleanup Logic
-    // If this is the initial write of a run, we verify the existing state to prevent "Ghost Data".
+    // Transition & Cleanup Logic
     let wasSharded = false;
-    let hadRootData = false;
     let shouldWipeShards = false;
 
     // Default: Merge updates. But if Initial Write, overwrite (merge: false) to clear stale fields.
@@ -203,11 +224,7 @@ async function writeSingleResult(result, docRef, name, dateContext, logger, conf
             if (currentSnap.exists) {
                 const d = currentSnap.data();
                 wasSharded = (d._sharded === true);
-                // If it was sharded, we MUST wipe the old shards because we are re-writing from scratch.
-                // Even if we write new shards, we want to ensure shard_10 doesn't persist if we only write up to shard_5.
                 if (wasSharded) shouldWipeShards = true;
-                
-                // If it wasn't sharded, it had root data. overwriting (merge: false) handles that automatically.
             }
         } catch (e) { /* ignore read error */ }
     }
@@ -276,7 +293,7 @@ async function writeSingleResult(result, docRef, name, dateContext, logger, conf
         try {
             const updates = await prepareAutoShardedWrites(result, docRef, logger, constraints.bytes, constraints.keys, startShardIndex, flushMode);
             
-            // [NEW] Inject Cleanup Ops
+            // Inject Cleanup Ops
             if (shouldWipeShards) {
                  logger.log('INFO', `[Cleanup] ${name}: Wiping old shards before Write (Initial).`);
                  const shardCol = docRef.collection('_shards');
@@ -410,4 +427,4 @@ function calculateFirestoreBytes(value) {
     if (typeof value === 'object') { let sum = 0; for (const k in value) { if (Object.prototype.hasOwnProperty.call(value, k)) { sum += (Buffer.byteLength(k, 'utf8') + 1) + calculateFirestoreBytes(value[k]); } } return sum; } return 0;
 }
 
-module.exports = { commitResults };
+module.exports = { commitResults };

package/functions/computation-system/simulation/Fabricator.js

@@ -0,0 +1,285 @@
+/**
+ * @fileoverview Generates deterministic mock contexts for Simulation Hashing.
+ * STRICTLY ALIGNED WITH SCHEMA.MD (Production V2 Schemas).
+ * UPGRADED: Supports Iteration (Seed Rotation) and Volume Scaling for Arrays.
+ */
+const SeededRandom = require('./SeededRandom');
+const { ContextFactory } = require('../context/ContextFactory');
+
+const FAKE_SECTORS = ['Technology', 'Healthcare', 'Financials', 'Energy', 'Crypto', 'Consumer Discretionary'];
+const FAKE_TICKERS = ['AAPL', 'GOOGL', 'MSFT', 'TSLA', 'AMZN', 'BTC', 'ETH', 'NVDA', 'META', 'AMD'];
+const FAKE_TOPICS  = ['AI', 'Earnings', 'Fed', 'Crypto', 'Macro'];
+
+class Fabricator {
+    constructor(calcName) {
+        this.baseSeed = calcName;
+        // Primary RNG for high-level structure
+        this.rng = new SeededRandom(calcName);
+    }
+
+    /**
+     * Generates a context for a specific user iteration.
+     * @param {number} iteration - The index of the user in the batch (0, 1, 2...).
+     */
+    async generateContext(calcManifest, dependenciesManifest, iteration = 0) {
+        // [CRITICAL] Rotate the RNG state based on iteration so User 1 != User 2
+        this.rng = new SeededRandom(`${this.baseSeed}_ITER_${iteration}`);
+
+        const FIXED_DATE = '2025-01-01'; // Fixed simulation date
+
+        // 1. Generate Root Data
+        const user = this._generateUser(calcManifest.userType, iteration);
+        const insights = this._generateInsights(FIXED_DATE);
+        
+        // 2. Generate Mock Dependencies (The "Schema Faking" Part)
+        const computed = {};
+        if (calcManifest.dependencies) {
+            for (const depName of calcManifest.dependencies) {
+                const depEntry = dependenciesManifest.get(depName);
+                if (depEntry && depEntry.class && depEntry.class.getSchema) {
+                    const schema = depEntry.class.getSchema();
+                    // [VOLUME UPGRADE] Dependencies usually represent aggregate data.
+                    computed[depName] = this._fakeFromSchema(schema, true); 
+                } else {
+                    computed[depName] = {};
+                }
+            }
+        }
+        
+        return ContextFactory.buildPerUserContext({
+            userId: user.id,
+            userType: user.type,
+            dateStr: FIXED_DATE,
+            todayPortfolio: user.portfolio.today,
+            yesterdayPortfolio: user.portfolio.yesterday,
+            todayHistory: user.history.today,
+            yesterdayHistory: user.history.yesterday,
+            metadata: calcManifest,
+            mappings: {
+                instrumentToTicker: this._generateMappings(),
+                instrumentToSector: this._generateSectorMappings()
+            },
+            insights: { today: insights },
+            socialData: { today: this._generateSocial(FIXED_DATE) },
+            computedDependencies: computed,
+            config: {},
+            deps: { logger: { log: () => {} } } 
+        });
+    }
+
+    // --- Schema Faker Logic (Unchanged) ---
+    _fakeFromSchema(schema, isHighVolume = false) {
+        if (!schema) return {};
+        if (schema.type === 'object') {
+            const obj = {};
+            if (schema.properties) {
+                for (const [key, propSchema] of Object.entries(schema.properties)) {
+                    obj[key] = this._fakeFromSchema(propSchema, isHighVolume);
+                }
+            }
+            if (schema.patternProperties) {
+                const count = isHighVolume ? this.rng.range(20, 50) : 3;
+                const propSchema = Object.values(schema.patternProperties)[0];
+                for (let i = 0; i < count; i++) {
+                    // Use deterministic ticker keys for stability
+                    const key = `${this.rng.choice(FAKE_TICKERS)}`; 
+                    // Note: In real scenarios tickers are unique, so we might need a suffix if count > tickers.length
+                    const safeKey = count > FAKE_TICKERS.length ? `${key}_${i}` : key;
+                    obj[safeKey] = this._fakeFromSchema(propSchema, isHighVolume);
+                }
+            }
+            return obj;
+        }
+        if (schema.type === 'array') {
+            const min = isHighVolume ? 50 : 1;
+            const max = isHighVolume ? 150 : 5;
+            const len = this.rng.range(min, max);
+            return Array.from({ length: len }, () => this._fakeFromSchema(schema.items, isHighVolume));
+        }
+        if (schema.type === 'number') return parseFloat(this.rng.next().toFixed(4)) * 100;
+        if (schema.type === 'string') return "SIMULATED_STRING";
+        if (schema.type === 'boolean') return this.rng.bool();
+        return null;
+    }
+
+    // --- PROD ALIGNED GENERATORS ---
+    
+    _generateUser(type, iteration) {
+        const userId = 1000000 + iteration; // Numeric ID to match Schema
+        const isSpeculator = (type === 'speculator');
+        
+        return {
+            id: String(userId),
+            type: type || 'all',
+            portfolio: {
+                today: isSpeculator ? this._genSpecPortfolio(userId) : this._genNormalPortfolio(userId),
+                yesterday: isSpeculator ? this._genSpecPortfolio(userId) : this._genNormalPortfolio(userId)
+            },
+            history: {
+                today: { PublicHistoryPositions: this._genHistoryTrades(userId) },
+                yesterday: { PublicHistoryPositions: this._genHistoryTrades(userId) }
+            }
+        };
+    }
+
+    // Schema 2: Speculator User Portfolio
+    _genSpecPortfolio(userId) {
+        const invested = this.rng.range(5000, 50000);
+        const netProfit = this.rng.range(-20, 30);
+        const equity = invested * (1 + (netProfit / 100));
+
+        return {
+            AverageOpen: this.rng.range(100, 3000),
+            Equity: parseFloat(equity.toFixed(4)),
+            Invested: parseFloat(invested.toFixed(4)),
+            NetProfit: parseFloat(netProfit.toFixed(4)),
+            PublicPositions: Array.from({ length: this.rng.range(2, 10) }, (_, i) => {
+                const openRate = this.rng.range(50, 500);
+                const isBuy = this.rng.bool();
+                return {
+                    Amount: parseFloat(this.rng.range(100, 1000).toFixed(4)),
+                    CID: userId,
+                    CurrentRate: parseFloat((openRate * (1 + (this.rng.next() - 0.5) * 0.1)).toFixed(2)),
+                    InstrumentID: 100 + (i % 20),
+                    IsBuy: isBuy,
+                    IsTslEnabled: this.rng.bool(0.1),
+                    Leverage: this.rng.choice([1, 2, 5, 10, 20]),
+                    MirrorID: 0,
+                    NetProfit: parseFloat(this.rng.range(-50, 50).toFixed(4)),
+                    OpenDateTime: '2024-12-01T10:00:00Z',
+                    OpenRate: parseFloat(openRate.toFixed(2)),
+                    ParentPositionID: 0,
+                    PipDifference: this.rng.range(-100, 100),
+                    PositionID: 3000000000 + i,
+                    StopLossRate: 0.01,
+                    TakeProfitRate: 0
+                };
+            })
+        };
+    }
+
+    // Schema 1: Normal User Portfolio
+    _genNormalPortfolio(userId) {
+        const positions = Array.from({ length: this.rng.range(3, 12) }, (_, i) => ({
+            Direction: "Buy",
+            InstrumentID: 100 + (i % 20),
+            Invested: parseFloat(this.rng.range(5, 20).toFixed(4)), // Percent
+            NetProfit: parseFloat(this.rng.range(-30, 40).toFixed(4)),
+            Value: parseFloat(this.rng.range(5, 25).toFixed(4)) // Percent (Invested + PnL approx)
+        }));
+
+        // [CRITICAL] DataExtractor.getPortfolioDailyPnl uses AggregatedPositionsByInstrumentTypeID
+        // We must generate this aggregation or PnL calcs return 0.
+        const aggByType = positions.map(p => ({
+            Direction: p.Direction,
+            InstrumentTypeID: 5, // Stock
+            Invested: p.Invested,
+            NetProfit: p.NetProfit,
+            Value: p.Value
+        }));
+
+        return {
+            AggregatedMirrors: [],
+            AggregatedPositions: positions,
+            AggregatedPositionsByInstrumentTypeID: aggByType, // Required for PnL
+            AggregatedPositionsByStockIndustryID: [],
+            CreditByRealizedEquity: 0,
+            CreditByUnrealizedEquity: 0
+        };
+    }
+
+    // Schema 3: Trade History
+    _genHistoryTrades(userId) {
+        return Array.from({ length: this.rng.range(5, 30) }, (_, i) => ({
+            PositionID: 4000000000 + i,
+            CID: userId,
+            OpenDateTime: '2024-12-01T10:00:00Z',
+            OpenRate: 100.50,
+            InstrumentID: 100 + (i % 20),
+            IsBuy: this.rng.bool(),
+            MirrorID: 0,
+            ParentPositionID: 0,
+            CloseDateTime: '2024-12-02T10:00:00Z',
+            CloseRate: 110.20,
+            CloseReason: this.rng.choice([1, 5, 0]), // 1=SL, 5=TP, 0=Manual
+            ParentCID: userId,
+            NetProfit: parseFloat(this.rng.range(-50, 50).toFixed(4)),
+            Leverage: this.rng.choice([1, 2, 5])
+        }));
+    }
+
+    // Schema 5: Insights
+    // [CRITICAL FIX] Must return object { fetchedAt, insights: [] }, not just array.
+    _generateInsights(dateStr) { 
+        const insightsArray = Array.from({ length: 50 }, (_, i) => ({
+            instrumentId: 100 + i,
+            total: this.rng.range(100, 50000), // Total owners
+            percentage: this.rng.next() * 0.05, // % of brokerage
+            growth: parseFloat((this.rng.next() * 10 - 5).toFixed(4)),
+            buy: this.rng.range(20, 95),
+            sell: 0, // Will calculate below
+            prevBuy: this.rng.range(20, 95),
+            prevSell: 0
+        }));
+
+        // Fix sell/prevSell math
+        insightsArray.forEach(i => {
+            i.sell = 100 - i.buy;
+            i.prevSell = 100 - i.prevBuy;
+        });
+
+        return {
+            fetchedAt: `${dateStr}T12:00:00Z`,
+            insights: insightsArray
+        };
+    }
+    
+    // Schema 4: Social Post Data
+    // Returns Map: { "postId": { ... } }
+    _generateSocial(dateStr) { 
+        const posts = {};
+        const count = this.rng.range(5, 20);
+        
+        for(let i=0; i<count; i++) {
+            const id = `post_${i}_${this.rng.next().toString(36).substring(7)}`;
+            const ticker = this.rng.choice(FAKE_TICKERS);
+            
+            posts[id] = {
+                commentCount: this.rng.range(0, 50),
+                createdAt: `${dateStr}T09:00:00Z`,
+                fetchedAt: `${dateStr}T10:00:00Z`,
+                fullText: `$${ticker} is looking bullish today!`,
+                language: 'en-gb',
+                likeCount: this.rng.range(0, 200),
+                postOwnerId: String(this.rng.range(100000, 999999)),
+                sentiment: {
+                    overallSentiment: this.rng.choice(['Bullish', 'Bearish', 'Neutral']),
+                    topics: [this.rng.choice(FAKE_TOPICS)]
+                },
+                textSnippet: `$${ticker} is looking...`,
+                tickers: [ticker]
+            };
+        }
+        return posts;
+    }
+    
+    _generateMappings() {
+        const map = {};
+        // Map ID 100-150 to FAKE_TICKERS deterministically
+        for(let i=0; i<50; i++) {
+            // cycle through tickers
+            map[100+i] = FAKE_TICKERS[i % FAKE_TICKERS.length];
+        }
+        return map;
+    }
+    
+    _generateSectorMappings() {
+        const map = {};
+        for(let i=0; i<50; i++) {
+            map[100+i] = FAKE_SECTORS[i % FAKE_SECTORS.length];
+        }
+        return map;
+    }
+}
+
+module.exports = Fabricator;

package/functions/computation-system/simulation/SeededRandom.js

@@ -0,0 +1,41 @@
+/**
+ * @fileoverview Deterministic Pseudo-Random Number Generator (LCG).
+ * Ensures that for a given seed, the sequence of numbers is identical across runs.
+ */
+class SeededRandom {
+    constructor(seedString) {
+        this.state = this._stringToSeed(seedString);
+    }
+
+    _stringToSeed(str) {
+        let h = 2166136261 >>> 0;
+        for (let i = 0; i < str.length; i++) {
+            h = Math.imul(h ^ str.charCodeAt(i), 16777619);
+        }
+        return h >>> 0;
+    }
+
+    /** Returns a float between 0 and 1 */
+    next() {
+        this.state = (Math.imul(48271, this.state) % 2147483647);
+        return (this.state - 1) / 2147483646;
+    }
+
+    /** Returns an integer between min and max (inclusive) */
+    range(min, max) {
+        return Math.floor(this.next() * (max - min + 1)) + min;
+    }
+
+    /** Returns a random element from an array */
+    choice(arr) {
+        if (!arr || arr.length === 0) return null;
+        return arr[this.range(0, arr.length - 1)];
+    }
+
+    /** Returns a boolean based on probability */
+    bool(probability = 0.5) {
+        return this.next() < probability;
+    }
+}
+
+module.exports = SeededRandom;

package/functions/computation-system/simulation/SimRunner.js

@@ -0,0 +1,51 @@
+/**
+ * @fileoverview Runner for Behavioral Hashing (SimHash).
+ * Executes a calculation against a fabricated, deterministic context.
+ */
+const Fabricator = require('./Fabricator');
+const { generateDataHash } = require('../utils/utils');
+
+class SimRunner {
+    /**
+     * Runs the simulation for a specific calculation.
+     * @param {Object} calcManifest - The manifest entry for the calculation.
+     * @param {Map} fullManifestMap - Map of all manifests (to look up dependencies).
+     * @returns {Promise<string>} The SimHash (SHA256 of the output).
+     */
+    static async run(calcManifest, fullManifestMap) {
+        try {
+            const fabricator = new Fabricator(calcManifest.name);
+            
+            // 1. Generate Deterministic Context
+            const context = await fabricator.generateContext(calcManifest, fullManifestMap);
+            
+            // 2. Instantiate
+            const instance = new calcManifest.class();
+            
+            // 3. Process
+            await instance.process(context);
+            
+            // 4. Get Result
+            // Note: If the calculation uses internal state buffering (like `results` property),
+            // getResult() usually returns that.
+            let result = null;
+            if (instance.getResult) {
+                result = await instance.getResult();
+            } else {
+                result = instance.result || instance.results || {};
+            }
+
+            // 5. Sanitize & Hash
+            // We strip any non-deterministic keys if they leak (like timestamps generated inside process)
+            // But ideally, the context mocking prevents this.
+            return generateDataHash(result);
+
+        } catch (e) {
+            console.error(`[SimRunner] Simulation failed for ${calcManifest.name}:`, e);
+            // If simulation crashes, we return a hash of the error to safely trigger a re-run
+            return generateDataHash({ error: e.message });
+        }
+    }
+}
+
+module.exports = SimRunner;