bulltrackers-module 1.0.997 → 1.0.999

package/functions/api-v3/index.js

@@ -3,11 +3,22 @@
  * Exports the API factory for use in index.js
  */
 
+const http = require('http');
 const { createApiV3App, initialize, getServices, resetServices } = require('./core-api');
+const { attachVerificationWebSocketServer } = require('./websocket/verification');
+
+function createApiV3Server(apiConfig, dependencies) {
+    const app = createApiV3App(apiConfig, dependencies);
+    const server = http.createServer(app);
+    attachVerificationWebSocketServer(server, dependencies, apiConfig);
+    return { app, server };
+}
 
 module.exports = {
     createApiV3App,
+    createApiV3Server,
     initialize,
     getServices,
-    resetServices
+    resetServices,
+    attachVerificationWebSocketServer
 };

package/functions/api-v3/routes/computations.js

@@ -11,11 +11,13 @@ const { publishUploadStreamEvent, readUploadStream } = require('../middleware/up
 const { getRedis, checkVerificationUserLimit } = require('../middleware/upstashRateLimit');
 const { v4: uuidv4 } = require('uuid');
 const { writeComputationNotification } = require('../services/computationNotificationWriter');
-const { buildUserComputationContract } = require('../services/computationContract');
-const { buildSqlBlockFromRequires } = require('../../computation-system-v3/dev-tools/annotate-sql-core');
-
-const UPLOAD_SESSION_PREFIX = 'upload:session:';
-const UPLOAD_SESSION_TTL = 300;
+const { buildUserComputationContract } = require('../services/computationContract');
+const { buildSqlBlockFromRequires } = require('../../computation-system-v3/dev-tools/annotate-sql-core');
+
+const UPLOAD_SESSION_PREFIX = 'upload:session:';
+const UPLOAD_SESSION_TTL = 300;
+const VERIFY_TICKET_PREFIX = 'verify:ticket:';
+const VERIFY_TICKET_TTL_SECONDS = 30;
 
 // Helper to create router
 const createComputationsRouter = () => {
@@ -60,15 +62,79 @@ const createComputationsRouter = () => {
         }
     });
 
-    // GET /contract: canonical frontend contract for user computations
-    router.get('/contract', requireVerifiedUser, async (req, res, next) => {
-        try {
-            const contract = buildUserComputationContract();
-            res.json({ success: true, data: contract });
-        } catch (error) {
-            next(error);
-        }
-    });
+    // GET /contract: canonical frontend contract for user computations
+    router.head('/contract', requireVerifiedUser, async (req, res, next) => {
+        try {
+            const contract = buildUserComputationContract();
+            res.setHeader('x-bt-sdk-version', contract.sdkVersion || '');
+            res.setHeader('x-bt-contract-hash', contract.contractHash || '');
+            res.setHeader('x-bt-generated-at', contract.generatedAt || '');
+            res.status(200).end();
+        } catch (error) {
+            next(error);
+        }
+    });
+
+    router.get('/contract', requireVerifiedUser, async (req, res, next) => {
+        try {
+            const contract = buildUserComputationContract();
+            res.json({ success: true, data: contract });
+        } catch (error) {
+            next(error);
+        }
+    });
+
+    // POST /verify/start: issue a single-use verification ticket
+    router.post('/verify/start', requireVerifiedUser, async (req, res, next) => {
+        try {
+            const userId = req.targetUserId;
+            const { code, name } = req.body || {};
+            if (!code || typeof code !== 'string') {
+                return res.status(400).json({ error: 'Missing or invalid "code"' });
+            }
+            if (!name || typeof name !== 'string') {
+                return res.status(400).json({ error: 'Missing or invalid "name"' });
+            }
+            const limitCheck = await checkVerificationUserLimit(userId);
+            const { allowed, retryAfterSeconds, remaining, limit } = limitCheck;
+            if (!allowed) {
+                return res.status(429).json({
+                    success: false,
+                    error: 'Too many verifications',
+                    message: 'Verification rate limit exceeded. Please try again later.',
+                    retryAfterSeconds: retryAfterSeconds || 60,
+                    remainingVerifications: remaining ?? 0,
+                    limit
+                });
+            }
+            const redis = getRedis();
+            if (!redis) {
+                return res.status(503).json({ error: 'Verification ticket unavailable' });
+            }
+            const verificationId = uuidv4();
+            const ticket = uuidv4();
+            const payload = {
+                code,
+                name,
+                userId,
+                verificationId,
+                requestId: req.requestId || null,
+                createdAt: new Date().toISOString()
+            };
+            await redis.set(VERIFY_TICKET_PREFIX + ticket, JSON.stringify(payload), 'EX', VERIFY_TICKET_TTL_SECONDS);
+            const expiresAt = new Date(Date.now() + VERIFY_TICKET_TTL_SECONDS * 1000).toISOString();
+            res.json({
+                success: true,
+                ticket,
+                verificationId,
+                expiresAt,
+                remainingVerifications: typeof remaining === 'number' ? remaining : undefined,
+                limit
+            });
+        } catch (error) {
+            next(error);
+        }
+    });
 
     // GET /: list user's computations (metadata only). Includes status, schedule; nextRun/lastRun when available from scheduler.
     router.get('/', requireVerifiedUser, async (req, res, next) => {

package/functions/api-v3/services/computationContract.js

@@ -3,18 +3,110 @@
  * Backend verification remains the source of truth.
  */
 
+const fs = require('fs');
+const path = require('path');
 const systemConfig = require('../../computation-system-v3/config/bulltrackers.config');
 const { getForbiddenIdentifiers, getAllowedCtxKeys } = require('../../computation-system-v3/framework/core/manifest/AstValidator');
 const { getRegistry } = require('../../computation-system-v3/framework/metadata/FunctionRegistry');
+const { hashContract } = require('../../computation-system-v3/framework/metadata/ContractHash');
+const { ALL_VERIFICATION_CODES, ALL_PRODUCTION_CODES, ERROR_CODE_DOC } = require('../../computation-system-v3/verification/ErrorCodes');
+
+const SDK_VERSION = '2026-03-13';
+const ERROR_DOC_PATH = path.join(__dirname, '../../computation-system-v3/docs/16-ERROR-CODE-REFERENCE.md');
+
+function parseErrorCodeExamples() {
+    if (!fs.existsSync(ERROR_DOC_PATH)) return {};
+    try {
+        const content = fs.readFileSync(ERROR_DOC_PATH, 'utf8');
+        const lines = content.split('\n');
+        const examples = {};
+        for (const line of lines) {
+            if (!line.trim().startsWith('|')) continue;
+            const cells = line.split('|').map(c => c.trim()).filter(Boolean);
+            if (cells.length < 3) continue;
+            const code = cells[0];
+            const example = cells[2];
+            if (code && code !== 'Code' && !/^-+$/.test(code) && example) {
+                examples[code] = example;
+            }
+        }
+        return examples;
+    } catch {
+        return {};
+    }
+}
+
+function buildSdk(registry) {
+    const modules = {};
+    const functionsById = registry.functionsById || {};
+
+    const fnList = Object.values(functionsById).sort((a, b) => String(a.id).localeCompare(String(b.id)));
+    for (const fn of fnList) {
+        if (!fn || !fn.moduleName || !fn.functionName) continue;
+        if (!modules[fn.moduleName]) {
+            modules[fn.moduleName] = {
+                name: fn.moduleName,
+                moduleType: fn.moduleType || 'lib',
+                functions: []
+            };
+        }
+        modules[fn.moduleName].functions.push({
+            id: fn.id,
+            name: fn.functionName,
+            moduleType: fn.moduleType || 'lib',
+            description: fn.description || '',
+            example: fn.example || '',
+            scope: fn.scope,
+            tables: fn.tables || [],
+            fields: fn.fields || [],
+            lookback: fn.lookback || 0,
+            filterRequired: !!fn.filterRequired,
+            uses: fn.uses || '',
+            visibility: fn.visibility || 'public',
+            contract: fn.contract || null
+        });
+    }
+
+    for (const mod of Object.values(modules)) {
+        mod.functions.sort((a, b) => String(a.name).localeCompare(String(b.name)));
+    }
+
+    const wrapperFlat = Object.values(modules)
+        .filter(m => m.moduleType === 'wrapper')
+        .flatMap(m => (m.functions || []).map(f => f.name));
+
+    return {
+        modules,
+        wrapperFlat
+    };
+}
 
 function buildUserComputationContract() {
     const libModules = Array.isArray(systemConfig?.skills?.lib?.allowedModules)
         ? [...systemConfig.skills.lib.allowedModules]
         : [];
     const registry = getRegistry();
+    const errorExamples = parseErrorCodeExamples();
+    const sdk = buildSdk(registry);
 
-    return {
-        version: '2026-03-08',
+    const errorCodes = {
+        verification: [...ALL_VERIFICATION_CODES],
+        production: [...ALL_PRODUCTION_CODES],
+        docs: Object.fromEntries(
+            Object.entries(ERROR_CODE_DOC).map(([code, doc]) => [
+                code,
+                {
+                    description: doc.description || '',
+                    remediation: doc.remediation || '',
+                    example: errorExamples[code] || ''
+                }
+            ])
+        )
+    };
+
+    const baseContract = {
+        version: '2026-03-13',
+        sdkVersion: SDK_VERSION,
         exportShape: {
             root: 'module.exports',
             required: ['config', 'process']
@@ -45,9 +137,10 @@ function buildUserComputationContract() {
         },
         requiresTableKeys: Object.keys(systemConfig?.tables || {}),
         metadata: {
-            generatedAt: registry.generatedAt,
             modules: Object.keys(registry.modules || {}),
-            functions: Object.values(registry.functionsById || {}).map((f) => ({
+            functions: Object.values(registry.functionsById || {})
+                .sort((a, b) => String(a.id).localeCompare(String(b.id)))
+                .map((f) => ({
                 id: f.id,
                 moduleType: f.moduleType,
                 moduleName: f.moduleName,
@@ -58,9 +151,31 @@ function buildUserComputationContract() {
                 lookback: f.lookback,
                 filterRequired: f.filterRequired,
                 uses: f.uses,
+                description: f.description || '',
+                example: f.example || '',
                 visibility: f.visibility
             }))
-        }
+        },
+        sdk: {
+            modules: sdk.modules,
+            wrapperFlat: sdk.wrapperFlat,
+            ctx: { allowedKeys: Array.from(getAllowedCtxKeys()) },
+            security: { forbiddenIdentifiers: Array.from(getForbiddenIdentifiers()) },
+            tables: Object.keys(systemConfig?.tables || {})
+        },
+        errorCodes
+    };
+
+    const contractHash = hashContract(baseContract);
+
+    return {
+        ...baseContract,
+        metadata: {
+            ...baseContract.metadata,
+            generatedAt: registry.generatedAt
+        },
+        generatedAt: registry.generatedAt,
+        contractHash
     };
 }
 

package/functions/api-v3/websocket/verification.js

@@ -0,0 +1,320 @@
+'use strict';
+/**
+ * @fileoverview Verification WebSocket server for ticketed sandbox logs.
+ */
+
+const { WebSocketServer } = require('ws');
+const { getAuth } = require('firebase-admin/auth');
+const { z } = require('zod');
+const { Manifest } = require('../../computation-system-v3/framework/core/Manifest');
+const { runVerification } = require('../../computation-system-v3/verification/runVerification');
+const { getRedis } = require('../middleware/upstashRateLimit');
+const { isDevTenantRequest } = require('../middleware/identity');
+const { initialize } = require('../core-api');
+
+const VERIFY_TICKET_PREFIX = 'verify:ticket:';
+const VERIFY_WS_PATH = '/computations/verify/ws';
+const MAX_LOG_BYTES = 1024 * 1024;
+const MAX_MESSAGES_PER_SECOND = 50;
+
+const LogSchema = z.object({
+    level: z.enum(['info', 'warn', 'error', 'system']),
+    text: z.string().max(2000),
+    timestamp: z.string()
+});
+const StageSchema = z.object({
+    id: z.string(),
+    status: z.enum(['passed', 'failed', 'running', 'pending']),
+    message: z.string().optional(),
+    errorCode: z.string().optional()
+});
+const FinalSchema = z.object({
+    passed: z.boolean(),
+    errorCodes: z.array(z.string()).optional(),
+    errors: z.array(z.string()).optional()
+});
+
+async function resolveFirebaseUser(req) {
+    const authHeader = req.headers.authorization;
+    const testAuthHeader = req.headers['x-test-firebase-user'];
+    const allowTestAuth = process.env.ENABLE_TEST_AUTH === 'true' || process.env.NODE_ENV === 'test';
+
+    if (authHeader && authHeader.startsWith('Bearer ')) {
+        const token = authHeader.split('Bearer ')[1];
+        try {
+            const decodedToken = await getAuth().verifyIdToken(token);
+            return {
+                uid: decodedToken.uid,
+                email: decodedToken.email || null,
+                emailVerified: decodedToken.email_verified || false
+            };
+        } catch (_) {
+            if (!allowTestAuth) return null;
+        }
+    }
+
+    if (allowTestAuth && testAuthHeader) {
+        try {
+            let parsed;
+            try {
+                parsed = JSON.parse(testAuthHeader);
+            } catch {
+                parsed = testAuthHeader;
+            }
+            if (typeof parsed === 'object' && parsed !== null) {
+                return {
+                    uid: String(parsed.uid),
+                    email: parsed.email || null,
+                    emailVerified: true
+                };
+            }
+            const val = String(parsed);
+            return {
+                uid: val.includes('@') ? 'mock_uid' : val,
+                email: val.includes('@') ? val : `user${val}@test.com`,
+                emailVerified: true
+            };
+        } catch (_) {
+            return null;
+        }
+    }
+
+    return null;
+}
+
+function resolveTargetUserId(firebaseUser, userCid, req) {
+    const isDev = isDevTenantRequest(req);
+    if (isDev) {
+        if (userCid && String(userCid).startsWith('dev-')) {
+            return String(userCid);
+        }
+        const baseId = userCid || firebaseUser?.uid || 'anonymous';
+        return `dev_${baseId}`;
+    }
+    return userCid || firebaseUser?.uid || null;
+}
+
+function buildWsUrlBase(req) {
+    const host = req.headers.host || 'localhost';
+    return `http://${host}`;
+}
+
+function attachVerificationWebSocketServer(server, dependencies, apiConfig = {}) {
+    const wss = new WebSocketServer({ noServer: true });
+    let servicesPromise = null;
+
+    function getServices() {
+        if (!servicesPromise) {
+            servicesPromise = initialize({
+                bigquery: dependencies.bigquery,
+                firestore: dependencies.db,
+                storage: dependencies.storage,
+                pubsub: dependencies.pubsub,
+                logger: dependencies.logger
+            }, apiConfig);
+        }
+        return servicesPromise;
+    }
+
+    server.on('upgrade', async (req, socket, head) => {
+        try {
+            const url = new URL(req.url, buildWsUrlBase(req));
+            if (url.pathname !== VERIFY_WS_PATH) {
+                return;
+            }
+            wss.handleUpgrade(req, socket, head, (ws) => {
+                wss.emit('connection', ws, req);
+            });
+        } catch (e) {
+            socket.destroy();
+        }
+    });
+
+    wss.on('connection', async (ws, req) => {
+    let totalBytes = 0;
+        let msgWindowStart = Date.now();
+        let msgCount = 0;
+        const sendJson = (payload) => {
+            if (ws.readyState !== ws.OPEN) return;
+            const now = Date.now();
+            if (now - msgWindowStart >= 1000) {
+                msgWindowStart = now;
+                msgCount = 0;
+            }
+            msgCount += 1;
+            if (msgCount > MAX_MESSAGES_PER_SECOND) {
+                ws.close(1008, 'Rate limit exceeded');
+                return;
+            }
+            const json = JSON.stringify(payload);
+            totalBytes += Buffer.byteLength(json, 'utf8');
+            if (totalBytes > MAX_LOG_BYTES) {
+                try {
+                    ws.send(JSON.stringify({
+                        type: 'log',
+                        level: 'system',
+                        text: 'Log output exceeded allowed size. Closing connection.',
+                        timestamp: new Date().toISOString()
+                    }));
+                } catch (_) { }
+                ws.close(1008, 'Log quota exceeded');
+                return;
+            }
+            ws.send(json);
+        };
+
+        ws.on('message', () => {
+            ws.close(1008, 'Read-only verification stream');
+        });
+
+        try {
+            const url = new URL(req.url, buildWsUrlBase(req));
+            const ticket = url.searchParams.get('ticket');
+            if (!ticket) {
+                ws.close(1008, 'Missing ticket');
+                return;
+            }
+
+            const redis = getRedis();
+            if (!redis) {
+                ws.close(1011, 'Ticket store unavailable');
+                return;
+            }
+
+            const raw = await redis.get(VERIFY_TICKET_PREFIX + ticket);
+            if (!raw) {
+                ws.close(1008, 'Invalid or expired ticket');
+                return;
+            }
+            await redis.del(VERIFY_TICKET_PREFIX + ticket);
+
+            const payload = JSON.parse(raw);
+            const firebaseUser = await resolveFirebaseUser(req);
+            if (!firebaseUser) {
+                ws.close(1008, 'Authentication required');
+                return;
+            }
+
+            const services = await getServices();
+            let userCid = null;
+            if (firebaseUser.email) {
+                try {
+                    const lookup = await services.authService.lookupCidByEmail(firebaseUser.email, { isDevTenant: isDevTenantRequest(req) });
+                    if (lookup?.cid != null) userCid = String(lookup.cid);
+                } catch (_) { }
+            }
+            const targetUserId = resolveTargetUserId(firebaseUser, userCid, req);
+            if (!targetUserId || targetUserId !== payload.userId) {
+                ws.close(1008, 'Ticket user mismatch');
+                return;
+            }
+
+            const verificationId = payload.verificationId || '';
+            const startedAt = new Date().toISOString();
+            const auditRef = services.db
+                ? services.db.collection('users').doc(targetUserId)
+                    .collection('verification_sessions').doc(verificationId)
+                : null;
+
+            if (auditRef) {
+                await auditRef.set({
+                    verificationId,
+                    userId: targetUserId,
+                    name: payload.name || null,
+                    startedAt,
+                    requestId: payload.requestId || null,
+                    status: 'running'
+                }, { merge: true });
+            }
+
+            const stageEmitter = (stagePayload) => {
+                const parsed = StageSchema.safeParse(stagePayload);
+                if (!parsed.success) {
+                    sendJson({
+                        type: 'log',
+                        level: 'system',
+                        text: 'Verification emitted an invalid stage update.',
+                        timestamp: new Date().toISOString()
+                    });
+                    return;
+                }
+                sendJson({ type: 'stage', ...parsed.data });
+            };
+
+            const logEmitter = (entry) => {
+                let text = String(entry.message || '');
+                if (text.length > 2000) text = text.slice(0, 2000);
+                const logPayload = {
+                    level: entry.level === 'info' || entry.level === 'warn' || entry.level === 'error' ? entry.level : 'info',
+                    text,
+                    timestamp: new Date(entry.ts || Date.now()).toISOString()
+                };
+                const parsed = LogSchema.safeParse(logPayload);
+                if (!parsed.success) {
+                    sendJson({
+                        type: 'log',
+                        level: 'system',
+                        text: 'Verification emitted an invalid log entry.',
+                        timestamp: new Date().toISOString()
+                    });
+                    return;
+                }
+                sendJson({ type: 'log', ...parsed.data });
+            };
+
+            const userComputationNames = [];
+            try {
+                const ref = services.db.collection('users').doc(targetUserId).collection('computations');
+                const snap = await ref.get();
+                snap.docs.forEach(d => {
+                    const n = d.data()?.name || d.id;
+                    if (n) userComputationNames.push(Manifest.normalize(n));
+                });
+            } catch (_) { }
+
+            const result = await runVerification(payload.code, payload.name, targetUserId, {
+                userComputationNames,
+                logger: services.logger || console,
+                uploadSessionId: verificationId || 'verify-session',
+                onStageProgress: (_sid, stage) => stageEmitter(stage),
+                onLog: (entry) => logEmitter(entry)
+            });
+
+            const finalPayload = {
+                passed: result.passed === true,
+                errorCodes: result.errorCodes || undefined,
+                errors: result.errors || undefined
+            };
+            const finalParsed = FinalSchema.safeParse(finalPayload);
+            if (finalParsed.success) {
+                sendJson({ type: 'final', ...finalParsed.data });
+            }
+
+            if (auditRef) {
+                await auditRef.set({
+                    finishedAt: new Date().toISOString(),
+                    status: result.passed === true ? 'passed' : 'failed',
+                    errorCodes: result.errorCodes || null
+                }, { merge: true });
+            }
+
+            ws.close(1000, 'Verification complete');
+        } catch (err) {
+            try {
+                ws.send(JSON.stringify({
+                    type: 'log',
+                    level: 'system',
+                    text: 'Verification failed unexpectedly.',
+                    timestamp: new Date().toISOString()
+                }));
+            } catch (_) { }
+            ws.close(1011, 'Verification error');
+        }
+    });
+
+    return wss;
+}
+
+module.exports = {
+    attachVerificationWebSocketServer
+};

package/functions/computation-system-v3/docs/00-INDEX.md

@@ -20,3 +20,4 @@ One-line description and primary audience for each doc. Source of truth: codebas
 | [13-OBSERVABILITY-LOG-SINKS](13-OBSERVABILITY-LOG-SINKS.md) | Telemetry sink implementation, Logs Explorer and BigQuery query recipes, revision-based charting and dashboard-as-code workflow | Developer, ops |
 | [14-HOW-EXECUTION-WORKS](14-HOW-EXECUTION-WORKS.md) | Dependencies and execution order (DAG), what runs when, planner and dispatcher | Users, developer |
 | [15-CREDITS-AND-BILLING](15-CREDITS-AND-BILLING.md) | How credits are consumed, balance and negative balance, cost of 7-day overwrite | Users, developer |
+| [17-IDE-VERIFICATION-PLAN](17-IDE-VERIFICATION-PLAN.md) | Studio IDE DX plan: templates, IntelliSense, verification terminal, websocket security | Developer, frontend, ops |

package/functions/computation-system-v3/docs/17-IDE-VERIFICATION-PLAN.md

@@ -0,0 +1,248 @@
+# IDE verification plan (Studio DX)
+
+This document formalizes the plan to make computation authoring in the VS Code Web Studio fast, safe, and guided. It covers the IntelliSense experience, file templating, and the verification sandbox terminal and websocket security.
+
+## Goals
+
+- Make authoring computations feel effortless with rich IntelliSense, doc hovers, and safe diagnostics.
+- Provide a read-only verification terminal that streams sandbox logs without leaking internal data.
+- Enforce strict security controls for websocket ingress and log egress.
+- Keep frontend and backend contracts explicit and testable.
+
+## Non-goals
+
+- Replacing the computation-system-v3 execution engine.
+- Providing a general purpose shell inside the IDE.
+- Requiring users to install local extensions or desktop VS Code.
+
+## Scope by repository
+
+Computation system v3
+- Define the canonical SDK surface for libs and wrappers.
+- Provide a machine-readable SDK manifest for the frontend.
+- Enforce structured log egress from sandbox execution.
+- Add log validation and policy enforcement in verification websocket bridge.
+
+API v3
+- Provide ticketed verification start endpoint and auth enforcement.
+- Broker websocket connection to the verification sandbox.
+- Provide observable audit events for verification sessions.
+
+Frontend web2.0 (VS Code Web build)
+- Generate and inject `bulltrackers.d.ts` into the workspace.
+- Auto-template new computation files in `/computations`.
+- Provide custom completion and diagnostics based on libs, wrappers, and user code.
+- Open a read-only pseudoterminal on verification and persist logs to `/output`.
+
+## Current implementation (code references)
+
+Frontend web2.0
+- Soft checks are regex-based in `Bulltrackers-Frontend/web2.0/ide-extension/src/extension.ts` (`doSoftChecks`).
+- Verification runs via REST in `Bulltrackers-Frontend/web2.0/ide-extension/src/terminal.ts` using `verifyComputation` from `Bulltrackers-Frontend/web2.0/ide-extension/src/api.ts` (`POST /computations/verify`).
+- Terminal output is sanitized with `Bulltrackers-Frontend/web2.0/ide-extension/src/sanitize.ts`.
+- SDK placeholders live in `Bulltrackers-Frontend/web2.0/ide-extension/src/libsContent.ts` but are not wired into the filesystem provider.
+
+Computation system v3
+- Lib + wrapper injection occurs in `Bulltrackers-Backend/Core/bulltrackers-module/functions/computation-system-v3/framework/core/ContextBuilder.js` via `buildWrappers`.
+- Contracts are loaded from `__contracts` in `Bulltrackers-Backend/Core/bulltrackers-module/functions/computation-system-v3/framework/core/ContractRegistry.js`.
+- Contract metadata is flattened for frontend use in `Bulltrackers-Backend/Core/bulltrackers-module/functions/computation-system-v3/framework/metadata/FunctionRegistry.js`.
+- AST validation rules are in `Bulltrackers-Backend/Core/bulltrackers-module/functions/computation-system-v3/framework/core/manifest/AstValidator.js` and `.../framework/core/Manifest.js`.
+- Verification run output, log buffer quota, and error codes are in `Bulltrackers-Backend/Core/bulltrackers-module/functions/computation-system-v3/verification/runVerification.js` and `.../verification/ErrorCodes.js`.
+
+API v3
+- Canonical computation contract endpoint is `GET /computations/contract` in `Bulltrackers-Backend/Core/bulltrackers-module/functions/api-v3/routes/computations.js` using `.../services/computationContract.js`.
+
+## End-to-end flow (target)
+
+1. User creates a new `.js` file in `/computations`.
+2. Studio inserts the minimal computation template.
+3. TypeScript IntelliSense and custom completions guide authoring.
+4. User clicks Verify.
+5. Studio requests a single-use verification ticket from API v3.
+6. Studio opens a read-only terminal and connects to the verification websocket.
+7. Sandbox streams structured logs, validated and sanitized by the backend.
+8. Terminal remains open; logs are persisted to `/output/verification-<timestamp>.txt`.
+
+## Ambient intelligence (IntelliSense + templates)
+
+### SDK declaration file (`bulltrackers.d.ts`)
+
+Plan
+- On extension activation, write a generated `bulltrackers.d.ts` to the root workspace.
+- Use JSDoc comments to power hover docs and parameter hints.
+- Include `ComputationContext`, builtin types, and all libs/wrappers.
+
+Source of truth
+- Use the existing computation contract from API v3 (`GET /computations/contract`) and extend it as needed.
+- Back contract metadata by `ContractRegistry` + `FunctionRegistry` so wrappers and libs stay in sync with injection.
+- Backend is the canonical source of truth; frontend caches the manifest for performance and offline startup.
+- Include explicit SDK version metadata: `sdkVersion`, `generatedAt`, and `contractHash` (checksum).
+
+Cache invalidation policy
+- Cache the contract locally on first fetch, keyed by `sdkVersion` + `contractHash`.
+- On Studio startup, compare cached `sdkVersion`/`contractHash` with a lightweight `HEAD` or `GET /computations/contract` metadata fetch.
+- If mismatch, refresh full contract and regenerate `bulltrackers.d.ts`.
+- If network is unavailable, continue using cached contract and mark editor status as "offline contract".
+- Optional: revalidate every 24 hours in the background to pick up backend SDK changes.
+
+Required capabilities
+- Autocomplete for namespace paths, function names, and parameter lists.
+- Type errors when required params are missing or invalid types are passed.
+- Warnings when a lib or wrapper symbol does not exist.
+
+### Auto-templating new computations
+
+Trigger
+- When a `.js` file is created under `/computations`, insert a minimal template if empty.
+
+Template content
+- `export const config = { name, requires, schedule }`
+- `export function process(ctx) { /* user logic */ }`
+- A small placeholder comment for context
+
+Safety
+- Only apply to brand-new or empty files.
+- Do not overwrite user content.
+
+### Custom completion and diagnostics
+
+Baseline
+- TypeScript language service provides standard JS completions from `bulltrackers.d.ts`.
+
+Enhancements
+- A completion provider that prioritizes common libs/wrappers and recently used symbols.
+- Completion suggestions that include short docstrings from the manifest.
+- Diagnostics for invalid table/field names using bulltrackers config schema.
+
+Local personalization
+- Rank suggestions using symbols used in the current file and workspace.
+- Optionally use a lightweight static indexer over `/computations`.
+
+## Verification terminal (frontend)
+
+### Read-only pseudoterminal
+
+Requirements
+- The terminal is read-only and does not forward user input to any backend.
+- The terminal does not auto-close on websocket close.
+- The terminal can replay and display past logs.
+
+Implementation outline
+- Create a `vscode.Pseudoterminal` and ignore `handleInput`.
+- Connect to the verification websocket and stream logs into the terminal.
+- Buffer logs and write them to `workspace:/output/verification-<timestamp>.txt`.
+
+### Log persistence
+
+Behavior
+- Ensure `/output` exists in the workspace.
+- Write `verification-<timestamp>.txt` on websocket close or on terminal dispose (timestamp is UTC).
+- Keep per-run uniqueness by timestamp (no overwrite).
+- Enforce a 1 MB max log file size (truncate and append a final system line when exceeded).
+- Retain logs indefinitely unless the user deletes them.
+- Optional (if low effort): add a TTL cleanup command or scheduled cleanup to delete logs older than 7 days.
+
+TTL cleanup (optional, low effort)
+- Provide a manual command in the IDE extension (e.g., `bulltrackers.logs.cleanupVerification`).
+- Implementation location: `Bulltrackers-Frontend/web2.0/ide-extension/src/extension.ts` (command registration + invocation).
+- Implementation approach:
+  - List files in `workspace:/output/` and match `verification-*.txt`.
+  - Parse UTC timestamps from filenames; ignore files that do not match expected pattern.
+  - Delete files older than 7 days.
+  - Log a summary line to the verification terminal or show a toast with counts.
+- Optional background cleanup:
+  - Run once per Studio start (or every 24h) if a config flag is enabled.
+  - Track last-cleanup timestamp in extension global state.
+
+## Verification websocket security (backend)
+
+### Single-use ticket handshake
+
+Policy
+- Verification starts via `POST /api/verify/start`.
+- The response includes a single-use ticket bound to user ID and execution ID.
+- Ticket TTL is short and enforced (example: 30 seconds).
+- Websocket connects using `wss://.../verify?ticket=...`.
+- Ticket is consumed on successful connect and cannot be reused.
+
+### Strict egress schema (allowlist)
+
+Policy
+- Sandbox output must be JSON log records, not raw stdout.
+- Only structured log events pass through to the websocket.
+- All log messages are validated against an allowlisted schema.
+
+Recommended shape
+- `level`: enum `info | warn | error | system`
+- `text`: string, max length 2000
+- `timestamp`: ISO string
+- `code`: optional, enum from documented error code list (all codes allowed if docs are provided)
+
+Implementation notes
+- Use a schema validator like Zod for strict parsing.
+- Unknown fields are stripped and invalid payloads are dropped.
+- On invalid payload, send a safe generic system log line.
+
+### Ingress policy (read-only)
+
+Policy
+- Any inbound websocket message from the client is a violation.
+- Log the event and close with policy code 1008.
+
+### Sanitization
+
+Policy
+- Do not attempt to regex-strip secrets from arbitrary text.
+- Only structured logs make it to the client.
+- Any error emitted outside schema is replaced with a generic error and an internal trace ID.
+
+### Rate limits and abuse handling
+
+Policy
+- Rate limit websocket messages per connection.
+- Cap log line length and total bytes per run (1 MB).
+- Enforce connection timeout and idle timeout.
+
+## Interfaces and contracts
+
+Frontend to API v3
+- `POST /api/verify/start` returns `{ ticket, verificationId, expiresAt }`.
+- Errors return documented error codes only.
+
+API v3 to verification system
+- Start job with `verificationId` and user binding.
+- Bind websocket connection to a single job.
+
+Verification system to frontend
+- Only structured log schema.
+- No internal paths, raw stack traces, or environment info.
+
+## Implementation milestones (ordered, with code references)
+
+1. Upgrade backend contract metadata to match current injection model.
+   - Use `framework/core/ContractRegistry.js`, `framework/metadata/FunctionRegistry.js`, and `framework/core/ContextBuilder.js` as the source of truth.
+   - Ensure wrappers are exposed as `ctx.lib.<module>.<fn>` with docs/params for IDE usage.
+2. Extend API v3 computation contract to expose rich SDK metadata.
+   - Update `api-v3/services/computationContract.js` and `api-v3/routes/computations.js` (`GET /computations/contract`).
+   - Include error code references from `computation-system-v3/verification/ErrorCodes.js` and doc source `.../docs/16-ERROR-CODE-REFERENCE.md`.
+   - Add `sdkVersion`, `generatedAt`, and `contractHash` to the response for frontend cache validation.
+3. Replace the frontend SDK placeholders with contract-driven d.ts generation.
+   - Replace/extend `Bulltrackers-Frontend/web2.0/ide-extension/src/libsContent.ts`.
+   - Generate `bulltrackers.d.ts` in the workspace on activation in `.../src/extension.ts`.
+4. Upgrade frontend diagnostics to use contract metadata instead of regex.
+   - Replace `doSoftChecks` in `.../src/extension.ts` with AST + contract checks (forbidden identifiers, ctx.lib, ctx.data).
+   - Use table/field sets from the contract endpoint.
+5. Implement ticketed verification websocket and read-only terminal.
+   - Add start/ticket endpoint in API v3 (new route alongside `routes/computations.js`).
+   - Add websocket bridge in verification layer (new module under `computation-system-v3/verification`).
+   - Add read-only terminal in `Bulltrackers-Frontend/web2.0/ide-extension/src/extension.ts` (separate from `BulltrackersPty` in `.../src/terminal.ts`).
+6. Persist verification logs per run.
+   - Frontend: write `workspace:/output/verification-<timestamp>.txt` with a 1 MB cap.
+   - Backend: enforce log byte caps in `verification/runVerification.js` (increase from 8 KB to 1 MB for sandbox logs).
+7. Tests and hardening.
+   - Unit tests for contract generation, log egress schema, ticket reuse, websocket ingress policy.
+   - Regression tests for diagnostics and d.ts generation.
+
+## Open questions
+
+- None.

package/functions/computation-system-v3/framework/core/ContextBuilder.js

@@ -109,9 +109,10 @@ class ContextBuilder {
      * @param {UsageCollector} [params.usageCollector] - Optional collector for billing stats
      * @param {Array<{ts:number,level:string,message:string}>} [params.logBuffer] - Optional array to collect structured log entries (user computations). When used with logMaxBytes, quota is enforced.
      * @param {number} [params.logMaxBytes] - Max total bytes for logBuffer. When exceeded, logBuffer.quotaExceeded is set and a system line is appended.
+     * @param {(entry: { ts: number, level: string, message: string }) => void} [params.onLog] - Optional callback for streaming structured logs.
      * @returns {Object} The 'context' passed to process()
      */
-    build({ computationName, date, data, config, entityId, jobId = null, traceAlways = false, usageCollector = null, logBuffer = null, logMaxBytes = 0 }) {
+    build({ computationName, date, data, config, entityId, jobId = null, traceAlways = false, usageCollector = null, logBuffer = null, logMaxBytes = 0, onLog = null }) {
         let lib;
         if (config) {
             lib = buildLib(config);
@@ -199,12 +200,21 @@ class ContextBuilder {
                             const sysMsg = { ts: Date.now(), level: 'info', message: '[SYSTEM] Log quota exceeded. Subsequent logs suppressed.\n' };
                             logBuffer.push(sysMsg);
                             logBuffer._currentBytes += Buffer.byteLength(JSON.stringify(sysMsg), 'utf8');
+                            if (typeof onLog === 'function') {
+                                try { onLog(sysMsg); } catch (_) { }
+                            }
                             return;
                         }
                         logBuffer.push(entry);
                         logBuffer._currentBytes += entryBytes;
+                        if (typeof onLog === 'function') {
+                            try { onLog(entry); } catch (_) { }
+                        }
                     } else {
                         console.log(`[${computationName}] ${line.trim()}`);
+                        if (typeof onLog === 'function') {
+                            try { onLog({ ts: Date.now(), level, message: sanitized }); } catch (_) { }
+                        }
                     }
                 }
                 const logFn = (msg) => append('info', msg);
@@ -232,4 +242,4 @@ class ContextBuilder {
     }
 }
 
-module.exports = { ContextBuilder };
+module.exports = { ContextBuilder };

package/functions/computation-system-v3/framework/metadata/ContractHash.js

@@ -0,0 +1,31 @@
+'use strict';
+/**
+ * @fileoverview Stable JSON hashing for contract payloads.
+ */
+
+const crypto = require('crypto');
+
+function stableStringify(value) {
+    if (value === null || value === undefined) return JSON.stringify(value);
+    const t = typeof value;
+    if (t === 'string' || t === 'number' || t === 'boolean') return JSON.stringify(value);
+    if (Array.isArray(value)) {
+        return '[' + value.map(stableStringify).join(',') + ']';
+    }
+    if (t === 'object') {
+        const keys = Object.keys(value).sort();
+        const parts = keys.map((k) => JSON.stringify(k) + ':' + stableStringify(value[k]));
+        return '{' + parts.join(',') + '}';
+    }
+    return JSON.stringify(String(value));
+}
+
+function hashContract(payload) {
+    const json = stableStringify(payload);
+    return crypto.createHash('sha256').update(json).digest('hex');
+}
+
+module.exports = {
+    stableStringify,
+    hashContract
+};

package/functions/computation-system-v3/framework/metadata/FunctionRegistry.js

@@ -52,6 +52,9 @@ function getRegistry() {
             lookback: contract.lookback || 0,
             filterRequired: !!(contract.filter && Object.keys(contract.filter).length > 0),
             uses: contract.description || '',
+            description: contract.description || '',
+            example: contract.example || '',
+            contract,
             visibility: 'public'
         };
     }

package/functions/computation-system-v3/framework/tests/unit/LogQuota.test.js

@@ -11,12 +11,12 @@ const { VERIFICATION: VerifCodes } = require('../../../verification/ErrorCodes')
 describe('Log quota', () => {
     describe('verification: LOG_QUOTA_EXCEEDED', () => {
         it('fails dry_run when ctx.log() output exceeds limit', async () => {
-            // Log lines of ~100 bytes each; 100 lines = 10k bytes > 8192
+            // Log lines of ~120 bytes each; 12000 lines ~ 1.4MB > 1MB quota
             const code = `
                 module.exports = {
                     config: { name: 'LogFlood', skills: ['lib'] },
                     process: (ctx) => {
-                        for (let i = 0; i < 100; i++) {
+                        for (let i = 0; i < 12000; i++) {
                             ctx.log('line ' + i + ' xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx');
                         }
                         return { ok: true };
@@ -98,4 +98,4 @@ describe('Log quota', () => {
             assert.ok(logBuffer.length >= 2);
         });
     });
-});
+});

package/functions/computation-system-v3/verification/runVerification.js

@@ -48,7 +48,7 @@ function getStaticComputationNames() {
  * @param {string} code - User computation source code
  * @param {string} name - Computation name (for metadata)
  * @param {string} userId - Owner ID (for dependency resolution: allow own computations)
- * @param {object} options - { userComputationNames?: string[], fixtureData?: object, uploadSessionId?: string, onStageProgress?: (sessionId, payload) => void, forceMockBq?: boolean }
+ * @param {object} options - { userComputationNames?: string[], fixtureData?: object, uploadSessionId?: string, onStageProgress?: (sessionId, payload) => void, onLog?: (entry) => void, forceMockBq?: boolean }
  * @returns {Promise<{ stages: Array<{ id: string, status: string, message?: string, errorCode?: string }>, passed: boolean, errors?: string[], errorCodes?: string[], logOutput?: Array<{ ts: string|number, level: string, message: string }> }>}
  */
 async function runVerification(code, name, userId, options = {}) {
@@ -58,6 +58,7 @@ async function runVerification(code, name, userId, options = {}) {
     const logger = options.logger || console;
     const uploadSessionId = options.uploadSessionId || null;
     const onStageProgress = typeof options.onStageProgress === 'function' ? options.onStageProgress : null;
+    const onLog = typeof options.onLog === 'function' ? options.onLog : null;
 
     const userComputationNames = new Set(
         (options.userComputationNames || []).map(n => String(n).toLowerCase())
@@ -205,7 +206,7 @@ async function runVerification(code, name, userId, options = {}) {
             Object.assign(combinedData, fetched);
         }
 
-        const LOG_MAX_BYTES_VERIFY = 8192;
+        const LOG_MAX_BYTES_VERIFY = 1024 * 1024;
         const logBuffer = [];
         const ctx = contextBuilder.build({
             computationName: entry.originalName || entry.name,
@@ -216,7 +217,8 @@ async function runVerification(code, name, userId, options = {}) {
             jobId: 'verify-sandbox',
             usageCollector: null,
             logBuffer,
-            logMaxBytes: LOG_MAX_BYTES_VERIFY
+            logMaxBytes: LOG_MAX_BYTES_VERIFY,
+            onLog
         });
 
         // When not using WASM, recipe must have a callable process; dynamic entries only have processSource

package/index.js

@@ -12,7 +12,7 @@ const firestoreUtils = require('./functions/core/utils/firestore_utils');
 
 class FirestoreBatchManager { constructor(_db, _headerManager, _logger, _config) { } async flushBatches() { } } // No-op taskengineutils removed.
 const computationSystemV2 = require('./functions/computation-system-v3/index');
-const { createApiV3App } = require('./functions/api-v3/index');
+const { createApiV3App, createApiV3Server, attachVerificationWebSocketServer } = require('./functions/api-v3/index');
 const { createDataFetcherSystem } = require('./functions/data-fetcher-system-v1');
 const {
   runUpdateOrchestrator,
@@ -57,7 +57,7 @@ const taskEngine = {
     }, dependencies).handlers.taskEngine(message, context)
 };
 const computationSystem = computationSystemV2;
-const api = { createApiV3App, };
+const api = { createApiV3App, createApiV3Server, attachVerificationWebSocketServer };
 const dataFetcherSystem = { create: ({ config, dependencies, middlewares } = {}) => createDataFetcherSystem({ config, dependencies, middlewares }) };
 const maintenance = {
   runFetchInsights: (config, dependencies) => buildFetcherRuntime({ fetchInsights: config }, dependencies).handlers.insightsFetcher(),
@@ -76,4 +76,4 @@ const alertSystem = {
 };
 module.exports = {
   pipe: { core, orchestrator, dispatcher, taskEngine, computationSystem, api, maintenance, proxy, alertSystem, dataFetcherSystem },
-};
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.997",
+  "version": "1.0.999",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [
@@ -39,7 +39,8 @@
     "sharedsetup": "latest",
     "stripe": "^17.5.0",
     "supertest": "^7.2.2",
-    "zod": "^4.3.6"
+    "zod": "^4.3.6",
+    "ws": "^8.18.1"
   },
   "devDependencies": {
     "bulltracker-deployer": "file:../bulltracker-deployer",