bulltrackers-module 1.0.768 → 1.0.769

package/functions/computation-system-v2/devtools/builder/builder.js

@@ -0,0 +1,362 @@
+/**
+ * @fileoverview Declarative Computation Builder
+ * 
+ * Generates computation code from YAML specifications.
+ * 
+ * Usage:
+ *   node builder.js <spec.yaml> [--output <dir>]
+ *   node builder.js --interactive
+ * 
+ * YAML Schema:
+ *   name: ComputationName
+ *   description: What it does
+ *   type: per-entity | global
+ *   category: signed_in_user | popular_investor | global
+ *   isHistorical: true | false
+ *   requires:
+ *     table_name:
+ *       lookback: 30
+ *       mandatory: true
+ *       fields: [field1, field2]
+ *   dependencies: [OtherComputation]
+ *   storage:
+ *     bigquery: true
+ *     firestore:
+ *       enabled: true
+ *       path: 'collection/{entityId}'
+ *   logic:
+ *     extract:
+ *       - variable: positions
+ *         from: portfolio_snapshots
+ *         using: rules.portfolio.extractPositions
+ *     compute:
+ *       - variable: result
+ *         expression: positions.length
+ */
+
+const fs = require('fs');
+const path = require('path');
+const readline = require('readline');
+
+// Simple YAML parser (no external deps)
+function parseYAML(content) {
+    const lines = content.split('\n');
+    const result = {};
+    const stack = [{ indent: -2, obj: result }];
+    let currentKey = null;
+    let listContext = null;
+
+    for (const rawLine of lines) {
+        const line = rawLine.replace(/\r$/, '');
+        if (line.trim() === '' || line.trim().startsWith('#')) continue;
+
+        const indent = line.search(/\S/);
+        const trimmed = line.trim();
+
+        // Pop stack to find parent
+        while (stack.length > 1 && indent <= stack[stack.length - 1].indent) {
+            stack.pop();
+        }
+        const parent = stack[stack.length - 1].obj;
+
+        // Handle list item
+        if (trimmed.startsWith('- ')) {
+            const value = trimmed.slice(2).trim();
+            if (!Array.isArray(parent[currentKey])) {
+                parent[currentKey] = [];
+            }
+
+            if (value.includes(':')) {
+                // Object in list
+                const obj = {};
+                const [k, v] = value.split(':').map(s => s.trim());
+                obj[k] = parseValue(v);
+                parent[currentKey].push(obj);
+                stack.push({ indent, obj });
+            } else {
+                parent[currentKey].push(parseValue(value));
+            }
+            continue;
+        }
+
+        // Handle key: value
+        const colonIdx = trimmed.indexOf(':');
+        if (colonIdx > 0) {
+            const key = trimmed.slice(0, colonIdx).trim();
+            const value = trimmed.slice(colonIdx + 1).trim();
+
+            if (value === '') {
+                // Nested object
+                parent[key] = {};
+                currentKey = key;
+                stack.push({ indent, obj: parent[key] });
+            } else if (value.startsWith('[') && value.endsWith(']')) {
+                // Inline array
+                parent[key] = value.slice(1, -1).split(',').map(s => parseValue(s.trim()));
+                currentKey = key;
+            } else {
+                parent[key] = parseValue(value);
+                currentKey = key;
+            }
+        }
+    }
+
+    return result;
+}
+
+function parseValue(str) {
+    if (str === 'true') return true;
+    if (str === 'false') return false;
+    if (str === 'null') return null;
+    if (/^\d+$/.test(str)) return parseInt(str);
+    if (/^\d+\.\d+$/.test(str)) return parseFloat(str);
+    return str.replace(/^['"]|['"]$/g, '');
+}
+
+// Template engine
+function generateCode(spec) {
+    const { name, description, type, category, isHistorical, requires, dependencies, storage, logic } = spec;
+
+    // Build requires block
+    const requiresCode = Object.entries(requires || {}).map(([table, config]) => {
+        const fields = config.fields ? `fields: [${config.fields.map(f => `'${f}'`).join(', ')}]` : '';
+        return `            '${table}': {
+                lookback: ${config.lookback || 0},
+                mandatory: ${config.mandatory || false}${fields ? `,\n                ${fields}` : ''}
+            }`;
+    }).join(',\n');
+
+    // Build dependencies
+    const depsCode = dependencies && dependencies.length > 0
+        ? `\n            dependencies: [${dependencies.map(d => `'${d}'`).join(', ')}],`
+        : '';
+
+    // Build storage config
+    let storageCode = '';
+    if (storage) {
+        const firestoreConfig = storage.firestore?.enabled
+            ? `\n                firestore: {
+                    enabled: true,
+                    path: '${storage.firestore.path || 'collection/{entityId}'}',
+                    merge: ${storage.firestore.merge || true}
+                }`
+            : '\n                firestore: { enabled: false }';
+
+        storageCode = `
+            storage: {
+                bigquery: ${storage.bigquery !== false},${firestoreConfig}
+            }`;
+    }
+
+    // Build process method
+    const processCode = buildProcessCode(logic, requires, type);
+
+    return `const { Computation } = require('../framework');
+
+/**
+ * ${name}
+ * ${description || 'Auto-generated computation'}
+ * 
+ * Generated: ${new Date().toISOString()}
+ */
+class ${name} extends Computation {
+    static getConfig() {
+        return {
+            name: '${name}',
+            description: '${description || ''}',
+            type: '${type || 'per-entity'}',
+            category: '${category || 'signed_in_user'}',
+            isHistorical: ${isHistorical || false},${depsCode}
+            requires: {
+${requiresCode}
+            },${storageCode}
+        };
+    }
+
+${processCode}
+}
+
+module.exports = ${name};
+`;
+}
+
+function buildProcessCode(logic, requires, type) {
+    const lines = [];
+    const tables = Object.keys(requires || {});
+
+    if (type === 'global') {
+        lines.push('    async process(data, { targetDate, rules }) {');
+    } else {
+        lines.push('    async process(data, { entityId, targetDate, rules, getDependency }) {');
+    }
+
+    // Extract data from tables
+    for (const table of tables) {
+        if (type === 'global') {
+            lines.push(`        const ${toCamelCase(table)} = data.${table} || {};`);
+        } else {
+            lines.push(`        const ${toCamelCase(table)} = data.${table}[entityId] || [];`);
+        }
+    }
+
+    lines.push('');
+
+    // Generate extraction logic
+    if (logic?.extract) {
+        for (const ext of logic.extract) {
+            if (ext.using) {
+                lines.push(`        const ${ext.variable} = ${ext.using}(${toCamelCase(ext.from)});`);
+            } else {
+                lines.push(`        const ${ext.variable} = ${toCamelCase(ext.from)};`);
+            }
+        }
+        lines.push('');
+    }
+
+    // Generate computation logic
+    if (logic?.compute) {
+        for (const comp of logic.compute) {
+            lines.push(`        const ${comp.variable} = ${comp.expression};`);
+        }
+        lines.push('');
+    }
+
+    // Build return statement
+    if (logic?.compute) {
+        const resultVars = logic.compute.map(c => c.variable);
+        lines.push('        return {');
+        for (const v of resultVars) {
+            lines.push(`            ${v},`);
+        }
+        lines.push('        };');
+    } else {
+        lines.push('        // TODO: Implement computation logic');
+        lines.push('        return {};');
+    }
+
+    lines.push('    }');
+
+    return lines.join('\n');
+}
+
+function toCamelCase(str) {
+    return str.replace(/_([a-z])/g, (_, c) => c.toUpperCase());
+}
+
+// Interactive CLI
+async function runInteractive() {
+    const rl = readline.createInterface({
+        input: process.stdin,
+        output: process.stdout
+    });
+
+    const ask = (q) => new Promise(resolve => rl.question(q, resolve));
+
+    console.log('\n🔧 Computation Builder - Interactive Mode\n');
+
+    const spec = {
+        requires: {},
+        storage: { bigquery: true, firestore: { enabled: false } }
+    };
+
+    spec.name = await ask('Computation name (PascalCase): ');
+    spec.description = await ask('Description: ');
+
+    const typeChoice = await ask('Type (1=per-entity, 2=global): ');
+    spec.type = typeChoice === '2' ? 'global' : 'per-entity';
+
+    const catChoice = await ask('Category (1=signed_in_user, 2=popular_investor, 3=global): ');
+    spec.category = { '1': 'signed_in_user', '2': 'popular_investor', '3': 'global' }[catChoice] || 'signed_in_user';
+
+    const historical = await ask('Is historical? (y/N): ');
+    spec.isHistorical = historical.toLowerCase() === 'y';
+
+    // Add tables
+    console.log('\nAdd required tables (empty name to finish):');
+    while (true) {
+        const table = await ask('  Table name: ');
+        if (!table) break;
+
+        const lookback = await ask('  Lookback days (0): ');
+        const mandatory = await ask('  Mandatory? (Y/n): ');
+
+        spec.requires[table] = {
+            lookback: parseInt(lookback) || 0,
+            mandatory: mandatory.toLowerCase() !== 'n'
+        };
+    }
+
+    // Dependencies
+    const deps = await ask('\nDependencies (comma-separated, or empty): ');
+    if (deps) {
+        spec.dependencies = deps.split(',').map(d => d.trim());
+    }
+
+    // Firestore
+    const useFirestore = await ask('Enable Firestore? (y/N): ');
+    if (useFirestore.toLowerCase() === 'y') {
+        spec.storage.firestore.enabled = true;
+        spec.storage.firestore.path = await ask('  Firestore path: ');
+    }
+
+    rl.close();
+
+    const code = generateCode(spec);
+    const filename = `${spec.name}.js`;
+
+    console.log(`\n📝 Generated ${filename}\n`);
+    console.log(code);
+
+    const outputPath = path.join(process.cwd(), filename);
+    fs.writeFileSync(outputPath, code);
+    console.log(`\n✅ Saved to ${outputPath}`);
+}
+
+// CLI Entry
+async function main() {
+    const args = process.argv.slice(2);
+
+    if (args.includes('--interactive') || args.includes('-i')) {
+        await runInteractive();
+        return;
+    }
+
+    if (args.length === 0) {
+        console.log(`
+Computation Builder
+
+Usage:
+  node builder.js <spec.yaml>              Generate from YAML spec
+  node builder.js --interactive            Interactive wizard
+
+Options:
+  --output <dir>    Output directory (default: current)
+`);
+        return;
+    }
+
+    const specFile = args[0];
+    if (!fs.existsSync(specFile)) {
+        console.error(`File not found: ${specFile}`);
+        process.exit(1);
+    }
+
+    const content = fs.readFileSync(specFile, 'utf8');
+    const spec = parseYAML(content);
+    const code = generateCode(spec);
+
+    const outputIdx = args.indexOf('--output');
+    const outputDir = outputIdx >= 0 && args[outputIdx + 1]
+        ? args[outputIdx + 1]
+        : process.cwd();
+
+    const filename = `${spec.name}.js`;
+    const outputPath = path.join(outputDir, filename);
+
+    fs.writeFileSync(outputPath, code);
+    console.log(`✅ Generated ${outputPath}`);
+}
+
+main().catch(console.error);
+
+module.exports = { parseYAML, generateCode };

package/functions/computation-system-v2/devtools/builder/examples/user-metrics.yaml

@@ -0,0 +1,26 @@
+# Simple Example - No logic block (parser handles well)
+name: UserPortfolioMetrics
+description: Calculate portfolio metrics for signed-in users
+
+type: per-entity
+category: signed_in_user
+isHistorical: true
+
+requires:
+  portfolio_snapshots:
+    lookback: 30
+    mandatory: true
+    fields: [user_id, date, portfolio_data, cash, total_value]
+  trade_history_snapshots:
+    lookback: 30
+    mandatory: false
+    fields: [user_id, date, trade_history]
+
+dependencies: [SignedInUserList]
+
+storage:
+  bigquery: true
+  firestore:
+    enabled: true
+    path: users/{entityId}/metrics
+    merge: true

package/functions/computation-system-v2/devtools/index.js

@@ -0,0 +1,36 @@
+/**
+ * @fileoverview DevTools Main Index
+ */
+
+// Shared foundation
+const shared = require('./shared');
+const { SystemIntrospector, MockDataFactory, SchemaTemplates } = shared;
+
+// Simulation engine
+const simulation = require('./simulation');
+const { SimulationEngine, SimulationServer, MockDataFetcher, MockStorageManager, DAGAnalyzer } = simulation;
+
+// Builder
+const { parseYAML, generateCode } = require('./builder/builder');
+
+module.exports = {
+    // Foundation
+    SystemIntrospector,
+    MockDataFactory,
+    SchemaTemplates,
+
+    // Simulation
+    SimulationEngine,
+    SimulationServer,
+    MockDataFetcher,
+    MockStorageManager,
+    DAGAnalyzer,
+
+    // Builder
+    parseYAML,
+    generateCode,
+
+    // Re-export sub-modules
+    shared,
+    simulation
+};

package/functions/computation-system-v2/devtools/shared/MockDataFactory.js

@@ -0,0 +1,235 @@
+/**
+ * @fileoverview Mock Data Factory
+ * 
+ * Generates realistic mock data for the simulation engine.
+ * Uses SchemaTemplates to create data that matches actual BigQuery structures.
+ * 
+ * Features:
+ * - Schema-aware row generation
+ * - Entity-keyed data for per-entity computations
+ * - Date range support for lookback simulation
+ * - Reproducible output with seeding
+ */
+
+const { SchemaTemplates, helpers } = require('./SchemaTemplates');
+const { randomInt, randomFloat, randomChoice, randomPastDate, randomUUID, randomHex } = helpers;
+
+class MockDataFactory {
+    /**
+     * @param {Object} options
+     * @param {number} [options.seed] - Random seed for reproducibility
+     * @param {Object} [options.introspector] - SystemIntrospector for metadata
+     */
+    constructor(options = {}) {
+        this.introspector = options.introspector || null;
+        this.seed = options.seed || null;
+
+        // If seeded, we'd implement a seeded RNG here
+        // For now, using Math.random()
+    }
+
+    /**
+     * Generate mock data for a table.
+     * 
+     * @param {string} tableName - Table to generate data for
+     * @param {Object} options
+     * @param {number} [options.entityCount=10] - Number of entities
+     * @param {number} [options.daysBack=7] - Days of history to generate
+     * @param {string[]} [options.entityIds] - Specific entity IDs to use
+     * @param {Date|string} [options.asOfDate] - Reference date (defaults to today)
+     * @returns {Object} { rows: [], byEntity: Map<entityId, rows[]> }
+     */
+    generate(tableName, options = {}) {
+        const template = SchemaTemplates[tableName];
+        if (!template) {
+            throw new Error(`No schema template for table: ${tableName}`);
+        }
+
+        const entityCount = options.entityCount || 10;
+        const daysBack = options.daysBack || 7;
+        const asOfDate = options.asOfDate ? new Date(options.asOfDate) : new Date();
+
+        // Generate entity IDs
+        const entityIds = options.entityIds || this._generateEntityIds(tableName, entityCount);
+
+        // Get table metadata
+        const meta = this.introspector?.getTableMetadata(tableName) || this._inferMetadata(tableName, template);
+
+        const rows = [];
+        const byEntity = new Map();
+
+        // Generate rows
+        for (const entityId of entityIds) {
+            const entityRows = [];
+
+            if (meta.dateField) {
+                // Date-partitioned table: generate one row per day
+                for (let d = 0; d <= daysBack; d++) {
+                    const date = new Date(asOfDate);
+                    date.setDate(date.getDate() - d);
+
+                    const row = this._generateRow(tableName, template, entityId, date, meta);
+                    rows.push(row);
+                    entityRows.push(row);
+                }
+            } else {
+                // Non-partitioned table: single row per entity
+                const row = this._generateRow(tableName, template, entityId, null, meta);
+                rows.push(row);
+                entityRows.push(row);
+            }
+
+            byEntity.set(entityId, entityRows);
+        }
+
+        return { rows, byEntity };
+    }
+
+    /**
+     * Generate data mimicking the DataFetcher response format.
+     * This is what computations actually receive.
+     * 
+     * @param {Object} requires - The computation's requires config
+     * @param {Object} options
+     * @param {number} [options.entityCount=10]
+     * @param {string[]} [options.entityIds]
+     * @param {Date|string} [options.asOfDate]
+     * @returns {Object} Data object keyed by table name
+     */
+    generateForComputation(requires, options = {}) {
+        const data = {};
+        const entityCount = options.entityCount || 10;
+        const entityIds = options.entityIds || this._generateEntityIds('default', entityCount);
+
+        for (const [tableName, tableConfig] of Object.entries(requires)) {
+            const lookback = tableConfig.lookback || 0;
+
+            try {
+                const { byEntity } = this.generate(tableName, {
+                    entityCount,
+                    entityIds,
+                    daysBack: lookback,
+                    asOfDate: options.asOfDate
+                });
+
+                // DataFetcher returns data keyed by entityId (for per-entity fetches)
+                // or as flat array (for global fetches)
+                data[tableName] = Object.fromEntries(byEntity);
+            } catch (e) {
+                // Table not in templates, return empty
+                data[tableName] = {};
+            }
+        }
+
+        return data;
+    }
+
+    /**
+     * Generate computation results for dependency simulation.
+     * 
+     * @param {string} computationName
+     * @param {Object} results - Map of entityId -> result
+     * @param {Date|string} date
+     * @returns {Object[]} Rows in computation_results format
+     */
+    generateComputationResults(computationName, results, date) {
+        const rows = [];
+        const dateStr = (date instanceof Date ? date : new Date(date)).toISOString().split('T')[0];
+
+        for (const [entityId, result] of Object.entries(results)) {
+            rows.push({
+                date: dateStr,
+                computation_name: computationName,
+                category: 'mock',
+                entity_id: entityId,
+                result: result,
+                hash: randomHex(16),
+                created_at: new Date().toISOString()
+            });
+        }
+
+        return rows;
+    }
+
+    // =========================================================================
+    // PRIVATE METHODS
+    // =========================================================================
+
+    _generateRow(tableName, template, entityId, date, meta) {
+        const row = {};
+
+        for (const col of template.columns) {
+            const colName = col.name;
+
+            // Handle special fields
+            if (colName === meta.entityField) {
+                row[colName] = entityId;
+            } else if (colName === meta.dateField && date) {
+                row[colName] = date.toISOString().split('T')[0];
+            } else if (colName === 'fetched_at' || colName === 'last_updated') {
+                row[colName] = new Date().toISOString();
+            } else if (col.type === 'JSON') {
+                // Use JSON template
+                row[colName] = this._generateJsonField(tableName, colName, entityId, template);
+            } else if (template.generators && template.generators[colName]) {
+                // Use custom generator
+                row[colName] = template.generators[colName](entityId);
+            } else {
+                // Default by type
+                row[colName] = this._generateByType(col.type);
+            }
+        }
+
+        return row;
+    }
+
+    _generateJsonField(tableName, colName, entityId, template) {
+        const jsonTemplate = template.jsonTemplates?.[colName];
+        if (!jsonTemplate) return {};
+
+        const result = {};
+        for (const [key, generator] of Object.entries(jsonTemplate)) {
+            if (typeof generator === 'function') {
+                result[key] = generator(entityId);
+            } else {
+                result[key] = generator;
+            }
+        }
+
+        return result;
+    }
+
+    _generateByType(type) {
+        switch (type) {
+            case 'STRING': return randomHex(8);
+            case 'INTEGER': return randomInt(1, 1000);
+            case 'FLOAT': return randomFloat(0, 100);
+            case 'BOOLEAN': return randomChoice([true, false]);
+            case 'DATE': return new Date().toISOString().split('T')[0];
+            case 'TIMESTAMP': return new Date().toISOString();
+            case 'JSON': return {};
+            default: return null;
+        }
+    }
+
+    _generateEntityIds(tableName, count) {
+        // Generate realistic entity IDs based on table type
+        const ids = [];
+        for (let i = 0; i < count; i++) {
+            ids.push(String(randomInt(10000000, 50000000)));
+        }
+        return ids;
+    }
+
+    _inferMetadata(tableName, template) {
+        // Infer metadata from column names
+        const columns = template.columns.map(c => c.name);
+
+        return {
+            dateField: columns.includes('date') ? 'date' : null,
+            entityField: columns.find(c => ['user_id', 'pi_id', 'cid', 'entity_id', 'instrument_id'].includes(c)) || null
+        };
+    }
+}
+
+module.exports = { MockDataFactory };