bulltrackers-module 1.0.996 → 1.0.998

package/functions/api-v3/index.js

@@ -3,11 +3,22 @@
  * Exports the API factory for use in index.js
  */
 
+const http = require('http');
 const { createApiV3App, initialize, getServices, resetServices } = require('./core-api');
+const { attachVerificationWebSocketServer } = require('./websocket/verification');
+
+function createApiV3Server(apiConfig, dependencies) {
+    const app = createApiV3App(apiConfig, dependencies);
+    const server = http.createServer(app);
+    attachVerificationWebSocketServer(server, dependencies, apiConfig);
+    return { app, server };
+}
 
 module.exports = {
     createApiV3App,
+    createApiV3Server,
     initialize,
     getServices,
-    resetServices
+    resetServices,
+    attachVerificationWebSocketServer
 };

package/functions/api-v3/routes/computations.js

@@ -11,11 +11,13 @@ const { publishUploadStreamEvent, readUploadStream } = require('../middleware/up
 const { getRedis, checkVerificationUserLimit } = require('../middleware/upstashRateLimit');
 const { v4: uuidv4 } = require('uuid');
 const { writeComputationNotification } = require('../services/computationNotificationWriter');
-const { buildUserComputationContract } = require('../services/computationContract');
-const { buildSqlBlockFromRequires } = require('../../computation-system-v3/dev-tools/annotate-sql-core');
-
-const UPLOAD_SESSION_PREFIX = 'upload:session:';
-const UPLOAD_SESSION_TTL = 300;
+const { buildUserComputationContract } = require('../services/computationContract');
+const { buildSqlBlockFromRequires } = require('../../computation-system-v3/dev-tools/annotate-sql-core');
+
+const UPLOAD_SESSION_PREFIX = 'upload:session:';
+const UPLOAD_SESSION_TTL = 300;
+const VERIFY_TICKET_PREFIX = 'verify:ticket:';
+const VERIFY_TICKET_TTL_SECONDS = 30;
 
 // Helper to create router
 const createComputationsRouter = () => {
@@ -60,15 +62,79 @@ const createComputationsRouter = () => {
         }
     });
 
-    // GET /contract: canonical frontend contract for user computations
-    router.get('/contract', requireVerifiedUser, async (req, res, next) => {
-        try {
-            const contract = buildUserComputationContract();
-            res.json({ success: true, data: contract });
-        } catch (error) {
-            next(error);
-        }
-    });
+    // GET /contract: canonical frontend contract for user computations
+    router.head('/contract', requireVerifiedUser, async (req, res, next) => {
+        try {
+            const contract = buildUserComputationContract();
+            res.setHeader('x-bt-sdk-version', contract.sdkVersion || '');
+            res.setHeader('x-bt-contract-hash', contract.contractHash || '');
+            res.setHeader('x-bt-generated-at', contract.generatedAt || '');
+            res.status(200).end();
+        } catch (error) {
+            next(error);
+        }
+    });
+
+    router.get('/contract', requireVerifiedUser, async (req, res, next) => {
+        try {
+            const contract = buildUserComputationContract();
+            res.json({ success: true, data: contract });
+        } catch (error) {
+            next(error);
+        }
+    });
+
+    // POST /verify/start: issue a single-use verification ticket
+    router.post('/verify/start', requireVerifiedUser, async (req, res, next) => {
+        try {
+            const userId = req.targetUserId;
+            const { code, name } = req.body || {};
+            if (!code || typeof code !== 'string') {
+                return res.status(400).json({ error: 'Missing or invalid "code"' });
+            }
+            if (!name || typeof name !== 'string') {
+                return res.status(400).json({ error: 'Missing or invalid "name"' });
+            }
+            const limitCheck = await checkVerificationUserLimit(userId);
+            const { allowed, retryAfterSeconds, remaining, limit } = limitCheck;
+            if (!allowed) {
+                return res.status(429).json({
+                    success: false,
+                    error: 'Too many verifications',
+                    message: 'Verification rate limit exceeded. Please try again later.',
+                    retryAfterSeconds: retryAfterSeconds || 60,
+                    remainingVerifications: remaining ?? 0,
+                    limit
+                });
+            }
+            const redis = getRedis();
+            if (!redis) {
+                return res.status(503).json({ error: 'Verification ticket unavailable' });
+            }
+            const verificationId = uuidv4();
+            const ticket = uuidv4();
+            const payload = {
+                code,
+                name,
+                userId,
+                verificationId,
+                requestId: req.requestId || null,
+                createdAt: new Date().toISOString()
+            };
+            await redis.set(VERIFY_TICKET_PREFIX + ticket, JSON.stringify(payload), 'EX', VERIFY_TICKET_TTL_SECONDS);
+            const expiresAt = new Date(Date.now() + VERIFY_TICKET_TTL_SECONDS * 1000).toISOString();
+            res.json({
+                success: true,
+                ticket,
+                verificationId,
+                expiresAt,
+                remainingVerifications: typeof remaining === 'number' ? remaining : undefined,
+                limit
+            });
+        } catch (error) {
+            next(error);
+        }
+    });
 
     // GET /: list user's computations (metadata only). Includes status, schedule; nextRun/lastRun when available from scheduler.
     router.get('/', requireVerifiedUser, async (req, res, next) => {

package/functions/api-v3/services/computationContract.js

@@ -3,18 +3,110 @@
  * Backend verification remains the source of truth.
  */
 
+const fs = require('fs');
+const path = require('path');
 const systemConfig = require('../../computation-system-v3/config/bulltrackers.config');
 const { getForbiddenIdentifiers, getAllowedCtxKeys } = require('../../computation-system-v3/framework/core/manifest/AstValidator');
 const { getRegistry } = require('../../computation-system-v3/framework/metadata/FunctionRegistry');
+const { hashContract } = require('../../computation-system-v3/framework/metadata/ContractHash');
+const { ALL_VERIFICATION_CODES, ALL_PRODUCTION_CODES, ERROR_CODE_DOC } = require('../../computation-system-v3/verification/ErrorCodes');
+
+const SDK_VERSION = '2026-03-13';
+const ERROR_DOC_PATH = path.join(__dirname, '../../computation-system-v3/docs/16-ERROR-CODE-REFERENCE.md');
+
+function parseErrorCodeExamples() {
+    if (!fs.existsSync(ERROR_DOC_PATH)) return {};
+    try {
+        const content = fs.readFileSync(ERROR_DOC_PATH, 'utf8');
+        const lines = content.split('\n');
+        const examples = {};
+        for (const line of lines) {
+            if (!line.trim().startsWith('|')) continue;
+            const cells = line.split('|').map(c => c.trim()).filter(Boolean);
+            if (cells.length < 3) continue;
+            const code = cells[0];
+            const example = cells[2];
+            if (code && code !== 'Code' && !/^-+$/.test(code) && example) {
+                examples[code] = example;
+            }
+        }
+        return examples;
+    } catch {
+        return {};
+    }
+}
+
+function buildSdk(registry) {
+    const modules = {};
+    const functionsById = registry.functionsById || {};
+
+    const fnList = Object.values(functionsById).sort((a, b) => String(a.id).localeCompare(String(b.id)));
+    for (const fn of fnList) {
+        if (!fn || !fn.moduleName || !fn.functionName) continue;
+        if (!modules[fn.moduleName]) {
+            modules[fn.moduleName] = {
+                name: fn.moduleName,
+                moduleType: fn.moduleType || 'lib',
+                functions: []
+            };
+        }
+        modules[fn.moduleName].functions.push({
+            id: fn.id,
+            name: fn.functionName,
+            moduleType: fn.moduleType || 'lib',
+            description: fn.description || '',
+            example: fn.example || '',
+            scope: fn.scope,
+            tables: fn.tables || [],
+            fields: fn.fields || [],
+            lookback: fn.lookback || 0,
+            filterRequired: !!fn.filterRequired,
+            uses: fn.uses || '',
+            visibility: fn.visibility || 'public',
+            contract: fn.contract || null
+        });
+    }
+
+    for (const mod of Object.values(modules)) {
+        mod.functions.sort((a, b) => String(a.name).localeCompare(String(b.name)));
+    }
+
+    const wrapperFlat = Object.values(modules)
+        .filter(m => m.moduleType === 'wrapper')
+        .flatMap(m => (m.functions || []).map(f => f.name));
+
+    return {
+        modules,
+        wrapperFlat
+    };
+}
 
 function buildUserComputationContract() {
     const libModules = Array.isArray(systemConfig?.skills?.lib?.allowedModules)
         ? [...systemConfig.skills.lib.allowedModules]
         : [];
     const registry = getRegistry();
+    const errorExamples = parseErrorCodeExamples();
+    const sdk = buildSdk(registry);
 
-    return {
-        version: '2026-03-08',
+    const errorCodes = {
+        verification: [...ALL_VERIFICATION_CODES],
+        production: [...ALL_PRODUCTION_CODES],
+        docs: Object.fromEntries(
+            Object.entries(ERROR_CODE_DOC).map(([code, doc]) => [
+                code,
+                {
+                    description: doc.description || '',
+                    remediation: doc.remediation || '',
+                    example: errorExamples[code] || ''
+                }
+            ])
+        )
+    };
+
+    const baseContract = {
+        version: '2026-03-13',
+        sdkVersion: SDK_VERSION,
         exportShape: {
             root: 'module.exports',
             required: ['config', 'process']
@@ -45,9 +137,10 @@ function buildUserComputationContract() {
         },
         requiresTableKeys: Object.keys(systemConfig?.tables || {}),
         metadata: {
-            generatedAt: registry.generatedAt,
             modules: Object.keys(registry.modules || {}),
-            functions: Object.values(registry.functionsById || {}).map((f) => ({
+            functions: Object.values(registry.functionsById || {})
+                .sort((a, b) => String(a.id).localeCompare(String(b.id)))
+                .map((f) => ({
                 id: f.id,
                 moduleType: f.moduleType,
                 moduleName: f.moduleName,
@@ -58,9 +151,31 @@ function buildUserComputationContract() {
                 lookback: f.lookback,
                 filterRequired: f.filterRequired,
                 uses: f.uses,
+                description: f.description || '',
+                example: f.example || '',
                 visibility: f.visibility
             }))
-        }
+        },
+        sdk: {
+            modules: sdk.modules,
+            wrapperFlat: sdk.wrapperFlat,
+            ctx: { allowedKeys: Array.from(getAllowedCtxKeys()) },
+            security: { forbiddenIdentifiers: Array.from(getForbiddenIdentifiers()) },
+            tables: Object.keys(systemConfig?.tables || {})
+        },
+        errorCodes
+    };
+
+    const contractHash = hashContract(baseContract);
+
+    return {
+        ...baseContract,
+        metadata: {
+            ...baseContract.metadata,
+            generatedAt: registry.generatedAt
+        },
+        generatedAt: registry.generatedAt,
+        contractHash
     };
 }
 

package/functions/api-v3/websocket/verification.js

@@ -0,0 +1,320 @@
+'use strict';
+/**
+ * @fileoverview Verification WebSocket server for ticketed sandbox logs.
+ */
+
+const { WebSocketServer } = require('ws');
+const { getAuth } = require('firebase-admin/auth');
+const { z } = require('zod');
+const { Manifest } = require('../../computation-system-v3/framework/core/Manifest');
+const { runVerification } = require('../../computation-system-v3/verification/runVerification');
+const { getRedis } = require('../middleware/upstashRateLimit');
+const { isDevTenantRequest } = require('../middleware/identity');
+const { initialize } = require('../core-api');
+
+const VERIFY_TICKET_PREFIX = 'verify:ticket:';
+const VERIFY_WS_PATH = '/computations/verify/ws';
+const MAX_LOG_BYTES = 1024 * 1024;
+const MAX_MESSAGES_PER_SECOND = 50;
+
+const LogSchema = z.object({
+    level: z.enum(['info', 'warn', 'error', 'system']),
+    text: z.string().max(2000),
+    timestamp: z.string()
+});
+const StageSchema = z.object({
+    id: z.string(),
+    status: z.enum(['passed', 'failed', 'running', 'pending']),
+    message: z.string().optional(),
+    errorCode: z.string().optional()
+});
+const FinalSchema = z.object({
+    passed: z.boolean(),
+    errorCodes: z.array(z.string()).optional(),
+    errors: z.array(z.string()).optional()
+});
+
+async function resolveFirebaseUser(req) {
+    const authHeader = req.headers.authorization;
+    const testAuthHeader = req.headers['x-test-firebase-user'];
+    const allowTestAuth = process.env.ENABLE_TEST_AUTH === 'true' || process.env.NODE_ENV === 'test';
+
+    if (authHeader && authHeader.startsWith('Bearer ')) {
+        const token = authHeader.split('Bearer ')[1];
+        try {
+            const decodedToken = await getAuth().verifyIdToken(token);
+            return {
+                uid: decodedToken.uid,
+                email: decodedToken.email || null,
+                emailVerified: decodedToken.email_verified || false
+            };
+        } catch (_) {
+            if (!allowTestAuth) return null;
+        }
+    }
+
+    if (allowTestAuth && testAuthHeader) {
+        try {
+            let parsed;
+            try {
+                parsed = JSON.parse(testAuthHeader);
+            } catch {
+                parsed = testAuthHeader;
+            }
+            if (typeof parsed === 'object' && parsed !== null) {
+                return {
+                    uid: String(parsed.uid),
+                    email: parsed.email || null,
+                    emailVerified: true
+                };
+            }
+            const val = String(parsed);
+            return {
+                uid: val.includes('@') ? 'mock_uid' : val,
+                email: val.includes('@') ? val : `user${val}@test.com`,
+                emailVerified: true
+            };
+        } catch (_) {
+            return null;
+        }
+    }
+
+    return null;
+}
+
+function resolveTargetUserId(firebaseUser, userCid, req) {
+    const isDev = isDevTenantRequest(req);
+    if (isDev) {
+        if (userCid && String(userCid).startsWith('dev-')) {
+            return String(userCid);
+        }
+        const baseId = userCid || firebaseUser?.uid || 'anonymous';
+        return `dev_${baseId}`;
+    }
+    return userCid || firebaseUser?.uid || null;
+}
+
+function buildWsUrlBase(req) {
+    const host = req.headers.host || 'localhost';
+    return `http://${host}`;
+}
+
+function attachVerificationWebSocketServer(server, dependencies, apiConfig = {}) {
+    const wss = new WebSocketServer({ noServer: true });
+    let servicesPromise = null;
+
+    function getServices() {
+        if (!servicesPromise) {
+            servicesPromise = initialize({
+                bigquery: dependencies.bigquery,
+                firestore: dependencies.db,
+                storage: dependencies.storage,
+                pubsub: dependencies.pubsub,
+                logger: dependencies.logger
+            }, apiConfig);
+        }
+        return servicesPromise;
+    }
+
+    server.on('upgrade', async (req, socket, head) => {
+        try {
+            const url = new URL(req.url, buildWsUrlBase(req));
+            if (url.pathname !== VERIFY_WS_PATH) {
+                return;
+            }
+            wss.handleUpgrade(req, socket, head, (ws) => {
+                wss.emit('connection', ws, req);
+            });
+        } catch (e) {
+            socket.destroy();
+        }
+    });
+
+    wss.on('connection', async (ws, req) => {
+    let totalBytes = 0;
+        let msgWindowStart = Date.now();
+        let msgCount = 0;
+        const sendJson = (payload) => {
+            if (ws.readyState !== ws.OPEN) return;
+            const now = Date.now();
+            if (now - msgWindowStart >= 1000) {
+                msgWindowStart = now;
+                msgCount = 0;
+            }
+            msgCount += 1;
+            if (msgCount > MAX_MESSAGES_PER_SECOND) {
+                ws.close(1008, 'Rate limit exceeded');
+                return;
+            }
+            const json = JSON.stringify(payload);
+            totalBytes += Buffer.byteLength(json, 'utf8');
+            if (totalBytes > MAX_LOG_BYTES) {
+                try {
+                    ws.send(JSON.stringify({
+                        type: 'log',
+                        level: 'system',
+                        text: 'Log output exceeded allowed size. Closing connection.',
+                        timestamp: new Date().toISOString()
+                    }));
+                } catch (_) { }
+                ws.close(1008, 'Log quota exceeded');
+                return;
+            }
+            ws.send(json);
+        };
+
+        ws.on('message', () => {
+            ws.close(1008, 'Read-only verification stream');
+        });
+
+        try {
+            const url = new URL(req.url, buildWsUrlBase(req));
+            const ticket = url.searchParams.get('ticket');
+            if (!ticket) {
+                ws.close(1008, 'Missing ticket');
+                return;
+            }
+
+            const redis = getRedis();
+            if (!redis) {
+                ws.close(1011, 'Ticket store unavailable');
+                return;
+            }
+
+            const raw = await redis.get(VERIFY_TICKET_PREFIX + ticket);
+            if (!raw) {
+                ws.close(1008, 'Invalid or expired ticket');
+                return;
+            }
+            await redis.del(VERIFY_TICKET_PREFIX + ticket);
+
+            const payload = JSON.parse(raw);
+            const firebaseUser = await resolveFirebaseUser(req);
+            if (!firebaseUser) {
+                ws.close(1008, 'Authentication required');
+                return;
+            }
+
+            const services = await getServices();
+            let userCid = null;
+            if (firebaseUser.email) {
+                try {
+                    const lookup = await services.authService.lookupCidByEmail(firebaseUser.email, { isDevTenant: isDevTenantRequest(req) });
+                    if (lookup?.cid != null) userCid = String(lookup.cid);
+                } catch (_) { }
+            }
+            const targetUserId = resolveTargetUserId(firebaseUser, userCid, req);
+            if (!targetUserId || targetUserId !== payload.userId) {
+                ws.close(1008, 'Ticket user mismatch');
+                return;
+            }
+
+            const verificationId = payload.verificationId || '';
+            const startedAt = new Date().toISOString();
+            const auditRef = services.db
+                ? services.db.collection('users').doc(targetUserId)
+                    .collection('verification_sessions').doc(verificationId)
+                : null;
+
+            if (auditRef) {
+                await auditRef.set({
+                    verificationId,
+                    userId: targetUserId,
+                    name: payload.name || null,
+                    startedAt,
+                    requestId: payload.requestId || null,
+                    status: 'running'
+                }, { merge: true });
+            }
+
+            const stageEmitter = (stagePayload) => {
+                const parsed = StageSchema.safeParse(stagePayload);
+                if (!parsed.success) {
+                    sendJson({
+                        type: 'log',
+                        level: 'system',
+                        text: 'Verification emitted an invalid stage update.',
+                        timestamp: new Date().toISOString()
+                    });
+                    return;
+                }
+                sendJson({ type: 'stage', ...parsed.data });
+            };
+
+            const logEmitter = (entry) => {
+                let text = String(entry.message || '');
+                if (text.length > 2000) text = text.slice(0, 2000);
+                const logPayload = {
+                    level: entry.level === 'info' || entry.level === 'warn' || entry.level === 'error' ? entry.level : 'info',
+                    text,
+                    timestamp: new Date(entry.ts || Date.now()).toISOString()
+                };
+                const parsed = LogSchema.safeParse(logPayload);
+                if (!parsed.success) {
+                    sendJson({
+                        type: 'log',
+                        level: 'system',
+                        text: 'Verification emitted an invalid log entry.',
+                        timestamp: new Date().toISOString()
+                    });
+                    return;
+                }
+                sendJson({ type: 'log', ...parsed.data });
+            };
+
+            const userComputationNames = [];
+            try {
+                const ref = services.db.collection('users').doc(targetUserId).collection('computations');
+                const snap = await ref.get();
+                snap.docs.forEach(d => {
+                    const n = d.data()?.name || d.id;
+                    if (n) userComputationNames.push(Manifest.normalize(n));
+                });
+            } catch (_) { }
+
+            const result = await runVerification(payload.code, payload.name, targetUserId, {
+                userComputationNames,
+                logger: services.logger || console,
+                uploadSessionId: verificationId || 'verify-session',
+                onStageProgress: (_sid, stage) => stageEmitter(stage),
+                onLog: (entry) => logEmitter(entry)
+            });
+
+            const finalPayload = {
+                passed: result.passed === true,
+                errorCodes: result.errorCodes || undefined,
+                errors: result.errors || undefined
+            };
+            const finalParsed = FinalSchema.safeParse(finalPayload);
+            if (finalParsed.success) {
+                sendJson({ type: 'final', ...finalParsed.data });
+            }
+
+            if (auditRef) {
+                await auditRef.set({
+                    finishedAt: new Date().toISOString(),
+                    status: result.passed === true ? 'passed' : 'failed',
+                    errorCodes: result.errorCodes || null
+                }, { merge: true });
+            }
+
+            ws.close(1000, 'Verification complete');
+        } catch (err) {
+            try {
+                ws.send(JSON.stringify({
+                    type: 'log',
+                    level: 'system',
+                    text: 'Verification failed unexpectedly.',
+                    timestamp: new Date().toISOString()
+                }));
+            } catch (_) { }
+            ws.close(1011, 'Verification error');
+        }
+    });
+
+    return wss;
+}
+
+module.exports = {
+    attachVerificationWebSocketServer
+};

package/functions/computation-system-v3/docs/00-INDEX.md

@@ -20,3 +20,4 @@ One-line description and primary audience for each doc. Source of truth: codebas
 | [13-OBSERVABILITY-LOG-SINKS](13-OBSERVABILITY-LOG-SINKS.md) | Telemetry sink implementation, Logs Explorer and BigQuery query recipes, revision-based charting and dashboard-as-code workflow | Developer, ops |
 | [14-HOW-EXECUTION-WORKS](14-HOW-EXECUTION-WORKS.md) | Dependencies and execution order (DAG), what runs when, planner and dispatcher | Users, developer |
 | [15-CREDITS-AND-BILLING](15-CREDITS-AND-BILLING.md) | How credits are consumed, balance and negative balance, cost of 7-day overwrite | Users, developer |
+| [17-IDE-VERIFICATION-PLAN](17-IDE-VERIFICATION-PLAN.md) | Studio IDE DX plan: templates, IntelliSense, verification terminal, websocket security | Developer, frontend, ops |