bulltrackers-module 1.0.732 → 1.0.733

package/functions/computation-system/utils/schema_capture.js

@@ -1,121 +0,0 @@
-/**
- * @fileoverview Schema capture utility for computation outputs
- * This module batches and stores pre-defined static schemas in Firestore.
- * UPDATED: Added schema validation to prevent silent batch failures.
- */
-
-/**
- * Recursively removes undefined values from an object.
- * Firestore doesn't allow undefined values, so we filter them out entirely.
- * @param {any} data - Data to sanitize
- * @returns {any} Sanitized data with undefined values removed
- */
-function removeUndefinedValues(data) {
-    if (data === undefined) return undefined; // Will be filtered out
-    if (data === null) return null;
-    if (data instanceof Date) return data;
-    
-    if (Array.isArray(data)) {
-        return data.map(item => removeUndefinedValues(item)).filter(item => item !== undefined);
-    }
-    
-    if (typeof data === 'object') {
-        const sanitized = {};
-        for (const [key, value] of Object.entries(data)) {
-            const sanitizedValue = removeUndefinedValues(value);
-            if (sanitizedValue !== undefined) {
-                sanitized[key] = sanitizedValue;
-            }
-        }
-        return sanitized;
-    }
-    
-    return data;
-}
-
-/**
- * Validates a schema object before storage.
- * Checks for circular references and size limits.
- * @param {object} schema 
- * @returns {object} { valid: boolean, reason: string }
- */
-function validateSchema(schema) {
-    try {
-        // 1. Detect circular references
-        const jsonStr = JSON.stringify(schema);
-        
-        // 2. Ensure it's not too large (Firestore limit: 1MB, reserve 100KB for metadata)
-        const size = Buffer.byteLength(jsonStr);
-        if (size > 900 * 1024) { return { valid: false, reason: `Schema exceeds 900KB limit (${(size/1024).toFixed(2)} KB)` }; }
-        return { valid: true };
-    } catch (e) { return { valid: false, reason: `Serialization failed: ${e.message}` }; }
-}
-
-/**
- * Batch store schemas for multiple computations.
- *
- * @param {object} dependencies - Contains db, logger
- * @param {object} config - Configuration object
- * @param {Array} schemas - Array of {name, category, schema, metadata} objects
- */
-async function batchStoreSchemas(dependencies, config, schemas) {
-    const { db, logger } = dependencies;
-    
-    if (config.captureSchemas === false) {
-        logger.log('INFO', '[SchemaCapture] Schema capture is disabled. Skipping.');
-        return;
-    }
-
-    const batch = db.batch();
-    const schemaCollection = config.schemaCollection || 'computation_schemas';
-    let validCount = 0;
-
-    for (const item of schemas) {
-        try {
-            if (!item.schema) {
-                logger.log('WARN', `[SchemaCapture] No schema provided for ${item.name}. Skipping.`);
-                continue;
-            }
-
-            // [IMPROVED] Validate before adding to batch
-            const validation = validateSchema(item.schema);
-            if (!validation.valid) {
-                logger.log('WARN', `[SchemaCapture] Invalid schema for ${item.name}: ${validation.reason}`);
-                continue;
-            }
-
-            const docRef = db.collection(schemaCollection).doc(item.name);
-            
-            // Critical: Always overwrite 'lastUpdated' to now
-            // Sanitize metadata to remove undefined values (Firestore doesn't allow undefined)
-            const sanitizedMetadata = item.metadata ? removeUndefinedValues(item.metadata) : {};
-            const docData = removeUndefinedValues({
-                computationName: item.name,
-                category:        item.category,
-                schema:          item.schema,
-                metadata:        sanitizedMetadata,
-                lastUpdated:     new Date() 
-            });
-            
-            batch.set(docRef, docData, { merge: true });
-            
-            validCount++;
-
-        } catch (error) {
-            logger.log('WARN', `[SchemaCapture] Failed to add schema to batch for ${item.name}`, { errorMessage: error.message });
-        }
-    }
-
-    if (validCount > 0) {
-        try {
-            await batch.commit();
-            logger.log('INFO', `[SchemaCapture] Batch stored ${validCount} computation schemas`);
-        } catch (error) {
-            logger.log('ERROR', '[SchemaCapture] Failed to commit schema batch', { errorMessage: error.message });
-        }
-    }
-}
-
-module.exports = {
-    batchStoreSchemas
-};

package/functions/computation-system/utils/utils.js

@@ -1,188 +0,0 @@
-/**
- * FILENAME: computation-system/utils/utils.js
- * UPDATED: Added undefined sanitization to prevent Firestore write errors.
- */
-
-const { FieldValue, FieldPath } = require('@google-cloud/firestore');
-const crypto                    = require('crypto');
-
-
-// [UPDATED] Registry for Data Availability. 
-// Populated dynamically by getEarliestDataDates().
-const DEFINITIVE_EARLIEST_DATES = {
-    portfolio: null,
-    history:   null,
-    social:    null,
-    insights:  null,
-    price:     null,
-    absoluteEarliest: null
-};
-
-/** Normalizes a calculation name to kebab-case */
-function normalizeName(name) { return name.replace(/_/g, '-'); }
-
-/** Generates a SHA-256 hash of a code string. */
-function generateCodeHash(codeString) {
-    if (!codeString) return 'unknown';
-    let clean = codeString.replace(/\/\/.*$/gm, '').replace(/\/\*[\s\S]*?\*\//g, '').replace(/\s+/g, '');
-    return crypto.createHash('sha256').update(clean).digest('hex');
-}
-
-/** Generates a stable SHA-256 hash of a data object. */
-function generateDataHash(data) {
-    if (data === undefined) return 'undefined';
-    const stableStringify = (obj) => {
-        if (typeof obj !== 'object' || obj === null) return JSON.stringify(obj);
-        if (Array.isArray(obj)) return '[' + obj.map(stableStringify).join(',') + ']';
-        return '{' + Object.keys(obj).sort().map(k => JSON.stringify(k) + ':' + stableStringify(obj[k])).join(',') + '}';
-    };
-    try {
-        return crypto.createHash('sha256').update(stableStringify(data)).digest('hex');
-    } catch (e) { return 'hash_error'; }
-}
-
-/** Exponential backoff retry logic. */
-async function withRetry(fn, operationName, maxRetries = 3) {
-    let attempt = 0;
-    while (attempt < maxRetries) {
-        try { return await fn(); } catch (error) {
-            attempt++;
-            if (attempt >= maxRetries) throw error;
-            await new Promise(resolve => setTimeout(resolve, 1000 * Math.pow(2, attempt - 1)));
-        }
-    }
-}
-
-/** * Recursively sanitizes data for Firestore compatibility.
- * Converts 'undefined' to 'null' to prevent validation errors.
- * Converts invalid Dates to null to prevent Timestamp serialization errors.
- * Converts NaN numbers to null to prevent serialization errors.
- */
-function sanitizeForFirestore(data) {
-    if (data === undefined) return null;
-    if (data === null) return null;
-    
-    // Handle Date objects - validate they're not Invalid Date (NaN timestamp)
-    if (data instanceof Date) {
-        if (isNaN(data.getTime())) {
-            // Invalid Date - return null instead of crashing Firestore
-            return null;
-        }
-        return data;
-    }
-    
-    // Handle NaN numbers (would cause serialization issues)
-    if (typeof data === 'number' && isNaN(data)) {
-        return null;
-    }
-    
-    // Handle Infinity (also not valid for Firestore)
-    if (data === Infinity || data === -Infinity) {
-        return null;
-    }
-    
-    // Handle Firestore Types if necessary (e.g. GeoPoint, Timestamp), passed through as objects usually.
-    
-    if (Array.isArray(data)) {
-        return data.map(item => sanitizeForFirestore(item));
-    }
-    
-    if (typeof data === 'object') {
-        // Check for specific Firestore types that shouldn't be traversed like plain objects
-        if (data.constructor && (data.constructor.name === 'DocumentReference' || data.constructor.name === 'Timestamp')) {
-            return data;
-        }
-
-        const sanitized = {};
-        for (const [key, value] of Object.entries(data)) {
-            sanitized[key] = sanitizeForFirestore(value);
-        }
-        return sanitized;
-    }
-    
-    return data;
-}
-
-/** Commit a batch of writes in chunks. */
-async function commitBatchInChunks(config, deps, writes, operationName) {
-    const { db, logger } = deps;
-    const retryFn = (deps.calculationUtils?.withRetry) || withRetry;
-    if (!writes?.length) return;
-
-    const MAX_BATCH_OPS = 300, MAX_BATCH_BYTES = 9 * 1024 * 1024; 
-    let currentBatch = db.batch(), currentOpsCount = 0, currentBytesEst = 0, batchIndex = 1;
-
-    const commitAndReset = async () => {
-        if (currentOpsCount > 0) {
-            await retryFn(() => currentBatch.commit(), `${operationName} (Chunk ${batchIndex})`);
-            batchIndex++;
-        }
-        currentBatch = db.batch(); currentOpsCount = 0; currentBytesEst = 0;
-    };
-
-    for (const write of writes) {
-        if (write.type === 'DELETE') {
-            if (currentOpsCount + 1 > MAX_BATCH_OPS) await commitAndReset();
-            currentBatch.delete(write.ref); currentOpsCount++; continue;
-        }
-        
-        // [FIX] Sanitize data to remove 'undefined' values before calculating size or setting
-        const safeData = sanitizeForFirestore(write.data);
-        
-        let docSize = 100;
-        try { if (safeData) docSize = JSON.stringify(safeData).length; } catch (e) { }
-        
-        if ((currentOpsCount + 1 > MAX_BATCH_OPS) || (currentBytesEst + docSize > MAX_BATCH_BYTES)) await commitAndReset();
-        
-        // Use safeData instead of write.data
-        currentBatch.set(write.ref, safeData, write.options || { merge: true });
-        currentOpsCount++; currentBytesEst += docSize;
-    }
-    await commitAndReset();
-}
-
-/** Generate array of date strings between two dates. */
-function getExpectedDateStrings(startDate, endDate) {
-    const dateStrings = [];
-    if (startDate && endDate && startDate <= endDate) {
-        const startUTC = new Date(Date.UTC(startDate.getUTCFullYear(), startDate.getUTCMonth(), startDate.getUTCDate()));
-        for (let d = startUTC; d <= endDate; d.setUTCDate(d.getUTCDate() + 1)) { dateStrings.push(new Date(d).toISOString().slice(0, 10)); }
-    }
-    return dateStrings;
-}
-
-/**
- * [UPDATED] Single Source of Truth for Data Availability.
- * Updates the global DEFINITIVE_EARLIEST_DATES object.
- */
-async function getEarliestDataDates(config, deps) {
-    const { db, logger } = deps;
-    const indexCollection = process.env.ROOT_DATA_AVAILABILITY_COLLECTION || 'system_root_data_index';
-    
-    const getEarliestForType = async (flagName) => {
-        try {
-            const snap = await db.collection(indexCollection).where(flagName, '==', true).orderBy(FieldPath.documentId(), 'asc').limit(1).get();
-            if (!snap.empty) return new Date(snap.docs[0].id + 'T00:00:00Z');
-        } catch (e) { logger.log('WARN', `[Utils] Index query failed for ${flagName}`); }
-        return null;
-    };
-
-    const [portfolio, history, social, insights, price] = await Promise.all([
-        getEarliestForType('hasPortfolio'), getEarliestForType('hasHistory'),
-        getEarliestForType('hasSocial'), getEarliestForType('hasInsights'), getEarliestForType('hasPrices')
-    ]);
-
-    const found = [portfolio, history, social, insights, price].filter(d => d !== null);
-    const absoluteEarliest = found.length > 0 ? new Date(Math.min(...found)) : new Date('2023-01-01T00:00:00Z');
-
-    // Sync the global registry
-    Object.assign(DEFINITIVE_EARLIEST_DATES, { portfolio, history, social, insights, price, absoluteEarliest });
-
-    return DEFINITIVE_EARLIEST_DATES;
-}
-
-module.exports = { 
-    FieldValue, FieldPath, normalizeName, commitBatchInChunks, 
-    getExpectedDateStrings, getEarliestDataDates, generateCodeHash,
-    generateDataHash, withRetry, DEFINITIVE_EARLIEST_DATES, sanitizeForFirestore
-};