bulltrackers-module 1.0.892 → 1.0.894

package/functions/api-v3/testing/routes.integration.test.js

@@ -16,6 +16,11 @@ process.env.COMPUTATION_VERIFICATION_URL = '';
 // Mock Redis so sync route uses Firestore rate-limit path (path relative to Core where Jest runs)
 jest.mock('bulltrackers-module/functions/shared/upstashRedis', () => ({ getRedis: () => null }));
 
+// Force in-process verification in tests (no sandbox HTTP). Use same path as route require() so Jest applies the mock.
+jest.mock('../services/verificationSandboxClient', () => ({
+    callVerificationSandbox: () => Promise.resolve(null),
+}));
+
 const request = require('supertest');
 const { createTestApp } = require('./test-app-factory');
 
@@ -983,9 +988,10 @@ describe('Cross-Cutting', () => {
         });
 
         it('returns 200 and writes to GCS when code passes verification', async () => {
+            // Use process: (ctx) => ... so extracted process source re-parses when wrapped in parens (method shorthand process(ctx){ } is invalid as a standalone expression).
             const validCode = `module.exports = {
                 config: { name: 'SafeComp', skills: ['lib'], requires: {} },
-                process(ctx) { return { ok: true, date: ctx.runDate }; }
+                process: (ctx) => ({ ok: true, date: ctx.date })
             };`;
             const res = await request(app)
                 .post('/computations/upload')

package/functions/api-v3/testing/test-app-factory.js

@@ -169,6 +169,13 @@ function createMockFirestore() {
                 commit: async () => { for (const op of ops) await op(); },
             };
         },
+        runTransaction(callback) {
+            const t = {
+                get: async (ref) => ref.get(),
+                set: async (ref, data, opts) => ref.set(data, opts),
+            };
+            return callback(t);
+        },
         // ─── Test helper ─────────────────────────────────────────
         // Seed a document at a Firestore-like path
         seed(path, data) {

package/functions/computation-system-v3/verification/runVerification.js

@@ -116,6 +116,8 @@ async function runVerification(code, name, userId, options = {}) {
     addStage(STAGE_GCS_RULES, 'passed', 'GCS rules (no select-all) satisfied');
 
     // Stage 3: Dry run with mock BQ and short timeout
+    // In test env, use in-process run (no WASM) so integration tests don't require quickjs-emscripten
+    const isTestEnv = process.env.NODE_ENV === 'test';
     const sandboxExecutionConfig = {
         ...config.execution,
         timeoutMs: SANDBOX_TIMEOUT_MS,
@@ -124,8 +126,8 @@ async function runVerification(code, name, userId, options = {}) {
         memoryLimitBytes: config.execution?.memoryLimitBytes ?? 64 * 1024 * 1024,
         maxStackSizeBytes: config.execution?.maxStackSizeBytes ?? 327680,
         runtime: config.execution?.runtime ?? 'default',
-        sandbox: true,
-        useQuickJS: config.execution?.useQuickJS !== false
+        sandbox: !isTestEnv,
+        useQuickJS: isTestEnv ? false : (config.execution?.useQuickJS !== false)
     };
 
     const mockBq = new MockBQAdapter(options.fixtureData || {}, config);
@@ -154,6 +156,15 @@ async function runVerification(code, name, userId, options = {}) {
         usageCollector: null
     });
 
+    // In test, dynamic recipe only has processSource; provide a callable process so in-process run works without WASM
+    if (isTestEnv && entry.recipe.processSource && entry.recipe.process === null) {
+        const processSource = entry.recipe.processSource;
+        entry.recipe.process = function (context) {
+            const fn = new Function('ctx', `return (${processSource})(ctx);`);
+            return Promise.resolve(fn(context));
+        };
+    }
+
     try {
         const result = await runner.run(entry.recipe, ctx);
         if (result && result.status === 'completed') {

package/functions/core/utils/intelligent_proxy_manager.js

@@ -62,14 +62,15 @@ class IntelligentProxyManager {
     /**
      * Direct fetch to target URL (no proxy). Used when circuit is open or as fallback after proxy exhaustion.
      * @private
+     * @param {object} fetchMeta - { usedDirect: true, attemptCount: number } to attach to response.
      */
-    async _directFetch(targetUrl, options = {}) {
+    async _directFetch(targetUrl, options = {}, fetchMeta = { usedDirect: true, attemptCount: 1 }) {
         const method = options.method || 'GET';
         const headers = options.headers || {};
         const fetchOptions = { method, headers };
         if (options.signal) fetchOptions.signal = options.signal;
         const res = await fetch(targetUrl, fetchOptions);
-        return {
+        const out = {
             ok: res.ok,
             status: res.status,
             headers: res.headers,
@@ -78,6 +79,8 @@ class IntelligentProxyManager {
             isUrlFetchError: false,
             isRateLimitError: false
         };
+        out._fetchMeta = fetchMeta;
+        return out;
     }
 
     /**
@@ -177,11 +180,17 @@ class IntelligentProxyManager {
             } catch (_) { /* use in-memory */ }
         }
         if (circuitOpenUntil && Date.now() < circuitOpenUntil) {
-            this.logger.log('INFO', '[ProxyManager] Circuit open — using direct fetch (no proxy).', { targetUrl, cooldownRemainingMs: circuitOpenUntil - Date.now(), shared: !!redis });
-            return this._directFetch(targetUrl, options);
+            this.logger.log('INFO', '[ProxyManager] Circuit breaker open: using direct fetch only (1 hour TTL). 3 consecutive request failures.', {
+                targetUrl,
+                cooldownRemainingMs: circuitOpenUntil - Date.now(),
+                ttlMinutes: Math.ceil(this.CIRCUIT_COOLDOWN_MS / 60000),
+                shared: !!redis
+            });
+            return this._directFetch(targetUrl, options, { usedDirect: true, attemptCount: 1 });
         }
 
         let lastResponse = null;
+        let proxyAttemptCount = 0;
 
         for (let attempt = 1; attempt <= this.MAX_RETRIES; attempt++) {
             let proxy;
@@ -189,15 +198,20 @@ class IntelligentProxyManager {
                 proxy = await this._selectProxy();
             } catch (error) {
                 this.logger.log('ERROR', '[ProxyManager] fetch failed: All proxies are locked.', { url: targetUrl });
-                return { ok: false, status: 503, error: { message: error.message }, headers: new Headers() };
+                const errResp = { ok: false, status: 503, error: { message: error.message }, headers: new Headers() };
+                errResp._fetchMeta = { usedDirect: false, attemptCount: 0 };
+                return errResp;
             }
 
+            proxyAttemptCount = attempt;
             const response = await this._fetchViaAppsScript(proxy.url, targetUrl, options);
             lastResponse = response;
 
             if (response.ok) {
                 this.consecutiveFailures = 0;
                 if (redis) { try { await redis.set(CIRCUIT_FAILURES_KEY, '0', { ex: CIRCUIT_FAILURES_TTL_SEC }); } catch (_) {} }
+                if (!response._fetchMeta) response._fetchMeta = { usedDirect: false, attemptCount: proxyAttemptCount };
+                else response._fetchMeta.attemptCount = proxyAttemptCount;
                 return response;
             }
 
@@ -216,7 +230,8 @@ class IntelligentProxyManager {
 
             this.consecutiveFailures = 0;
             if (redis) { try { await redis.set(CIRCUIT_FAILURES_KEY, '0', { ex: CIRCUIT_FAILURES_TTL_SEC }); } catch (_) {} }
-            return response;
+            if (!lastResponse._fetchMeta) lastResponse._fetchMeta = { usedDirect: false, attemptCount: proxyAttemptCount };
+            return lastResponse;
         }
 
         let newCount = this.consecutiveFailures + 1;
@@ -236,10 +251,15 @@ class IntelligentProxyManager {
                     await redis.del(CIRCUIT_FAILURES_KEY);
                 } catch (_) {}
             }
-            this.logger.log('WARN', '[ProxyManager] Circuit breaker open: 3 consecutive proxy failures. Bypassing proxy for 60 minutes.', { cooldownMs: this.CIRCUIT_COOLDOWN_MS, shared: !!redis });
-            return this._directFetch(targetUrl, options);
+            this.logger.log('WARN', '[ProxyManager] Circuit breaker open: 3 consecutive request failures overall. Using direct fetch only for 1 hour TTL.', {
+                cooldownMs: this.CIRCUIT_COOLDOWN_MS,
+                ttlMinutes: Math.ceil(this.CIRCUIT_COOLDOWN_MS / 60000),
+                shared: !!redis
+            });
+            return this._directFetch(targetUrl, options, { usedDirect: true, attemptCount: proxyAttemptCount + 1 });
         }
         this.logger.log('ERROR', `[ProxyManager] Request failed after ${this.MAX_RETRIES} proxy attempts.`, { url: targetUrl, lastStatus: lastResponse?.status });
+        if (lastResponse && !lastResponse._fetchMeta) lastResponse._fetchMeta = { usedDirect: false, attemptCount: proxyAttemptCount };
         return lastResponse;
     }
 

package/functions/data-fetcher-system-v1/services/taskEngine/taskEngineService.js

@@ -9,6 +9,10 @@ function parseIncomingPayload(message) {
     return (typeof rawData === 'string') ? JSON.parse(rawData) : rawData;
 }
 
+function defaultFetchMeta() {
+    return { usedDirect: true, attemptCount: 1 };
+}
+
 async function fetchJsonWithHeader({ url, dependencies }) {
     const { headerManager, proxyManager, adapters, logger } = dependencies;
     const selectedHeader = await headerManager.selectHeader();
@@ -20,8 +24,12 @@ async function fetchJsonWithHeader({ url, dependencies }) {
         logger
     });
 
+    const fetchMeta = response?._fetchMeta || defaultFetchMeta();
+
     if (!response?.ok) {
-        throw new Error(`[TaskEngineV1] Fetch failed: ${response?.status || 'unknown'} for ${url}`);
+        const err = new Error(`[TaskEngineV1] Fetch failed: ${response?.status || 'unknown'} for ${url}`);
+        err.fetchMeta = fetchMeta;
+        throw err;
     }
 
     const contentType = (response.headers?.get?.('content-type') || '').toLowerCase();
@@ -29,20 +37,15 @@ async function fetchJsonWithHeader({ url, dependencies }) {
     const looksLikeHtml = contentType.includes('text/html') || /^\s*</.test(rawBody);
 
     if (looksLikeHtml) {
-        const snippet = rawBody.length > 4000 ? rawBody.slice(0, 4000) + '\n... [truncated]' : rawBody;
-        logger.log('WARN', '[TaskEngineV1] Received HTML instead of JSON', {
-            url,
-            contentType,
-            bodyLength: rawBody.length,
-            bodySnippet: snippet
-        });
-        throw new Error(`[TaskEngineV1] Expected JSON but received HTML (${rawBody.length} chars) for ${url}`);
+        const err = new Error(`[TaskEngineV1] Expected JSON but received HTML (${rawBody.length} chars) for ${url}`);
+        err.fetchMeta = fetchMeta;
+        throw err; // End-result failure will be logged by caller; no per-attempt log here.
     }
 
     const data = JSON.parse(rawBody);
     if (url.includes('/portfolios')) dependencies.contracts?.assertPortfolioResponse(data);
     if (url.includes('/history/public/credit/flat')) dependencies.contracts?.assertHistoryResponse(data);
-    return data;
+    return { data, fetchMeta };
 }
 
 async function storeUserSnapshot({ dependencies, collection, cid, date, payload, type }) {
@@ -74,38 +77,55 @@ async function handleUserUpdate(taskData, configObj, dependencies, isPopularInve
     const historyUrl = `${historyBase}?StartTime=${encodeURIComponent(startTime)}&PageNumber=1&ItemsPerPage=30000&PublicHistoryPortfolioFilter=&CID=${cid}&client_request_id=${clientRequestId}`;
 
     const userType = isPopularInvestor ? 'popular_investor' : 'signed_in_user';
-    // Fetch independently so one failure (e.g. bad history URL) does not discard portfolio or block social
-    let portfolioData = null;
-    let historyData = null;
-    try {
-        portfolioData = await fetchJsonWithHeader({ url: portfolioUrl, dependencies });
-        if (portfolioData) {
-            logger.log('INFO', '[TaskEngineV1] Fetched portfolio', { cid, username, userType, dataType: 'portfolio' });
-        }
-    } catch (err) {
-        logger.log('WARN', '[TaskEngineV1] Portfolio fetch failed', { cid, error: err.message, url: portfolioUrl });
+    const collections = isPopularInvestor
+        ? { portfolio: config.FIRESTORE_COLLECTION_PI_PORTFOLIOS_OVERALL || 'pi_portfolios_overall', history: config.FIRESTORE_COLLECTION_PI_HISTORY || 'pi_trade_history' }
+        : { portfolio: config.FIRESTORE_COLLECTION_SIGNED_IN_USER_PORTFOLIOS || 'signed_in_users', history: config.FIRESTORE_COLLECTION_SIGNED_IN_HISTORY || 'signed_in_user_history' };
+
+    function formatFetchMeta(meta) {
+        const n = (meta && meta.attemptCount) != null ? meta.attemptCount : 1;
+        const mode = (meta && meta.usedDirect) ? 'direct' : 'proxy';
+        return { n, mode };
     }
+
+    // Portfolio: one end-result log per task type / user type / user ID
+    let portfolioData = null;
     try {
-        historyData = await fetchJsonWithHeader({ url: historyUrl, dependencies });
-        if (historyData) {
-            logger.log('INFO', '[TaskEngineV1] Fetched trade history', { cid, username, userType, dataType: 'history' });
-        }
+        const result = await fetchJsonWithHeader({ url: portfolioUrl, dependencies });
+        portfolioData = result.data;
+        const { n, mode } = formatFetchMeta(result.fetchMeta);
+        logger.log('INFO', `[TaskEngineV1] Success to fetch portfolio for ${userType} for ID ${cid} after ${n} attempts with ${mode}`, {
+            taskType: 'portfolio', userType, userId: cid, attemptCount: n, mode
+        });
     } catch (err) {
-        logger.log('WARN', '[TaskEngineV1] History fetch failed', { cid, error: err.message, url: historyUrl });
+        const { n, mode } = formatFetchMeta(err.fetchMeta);
+        logger.log('WARN', `[TaskEngineV1] Failure to fetch portfolio for ${userType} for ID ${cid} after ${n} attempts with ${mode}`, {
+            taskType: 'portfolio', userType, userId: cid, attemptCount: n, mode, error: err.message
+        });
     }
-
-    const collections = isPopularInvestor
-        ? { portfolio: config.FIRESTORE_COLLECTION_PI_PORTFOLIOS_OVERALL || 'pi_portfolios_overall', history: config.FIRESTORE_COLLECTION_PI_HISTORY || 'pi_trade_history' }
-        : { portfolio: config.FIRESTORE_COLLECTION_SIGNED_IN_USER_PORTFOLIOS || 'signed_in_users', history: config.FIRESTORE_COLLECTION_SIGNED_IN_HISTORY || 'signed_in_user_history' };
     if (portfolioData) {
         await storeUserSnapshot({ dependencies, collection: collections.portfolio, cid, date, payload: portfolioData, type: 'portfolio' });
-        logger.log('INFO', '[TaskEngineV1] Stored portfolio', { cid, username, userType, dataType: 'portfolio', collection: collections.portfolio, date });
+    }
+
+    // History: one end-result log per task type / user type / user ID
+    let historyData = null;
+    try {
+        const result = await fetchJsonWithHeader({ url: historyUrl, dependencies });
+        historyData = result.data;
+        const { n, mode } = formatFetchMeta(result.fetchMeta);
+        logger.log('INFO', `[TaskEngineV1] Success to fetch history for ${userType} for ID ${cid} after ${n} attempts with ${mode}`, {
+            taskType: 'history', userType, userId: cid, attemptCount: n, mode
+        });
+    } catch (err) {
+        const { n, mode } = formatFetchMeta(err.fetchMeta);
+        logger.log('WARN', `[TaskEngineV1] Failure to fetch history for ${userType} for ID ${cid} after ${n} attempts with ${mode}`, {
+            taskType: 'history', userType, userId: cid, attemptCount: n, mode, error: err.message
+        });
     }
     if (historyData) {
         await storeUserSnapshot({ dependencies, collection: collections.history, cid, date, payload: historyData, type: 'history' });
-        logger.log('INFO', '[TaskEngineV1] Stored trade history', { cid, username, userType, dataType: 'history', collection: collections.history, date });
     }
 
+    // Social: one end-result log per task type / user type / user ID (no fetchMeta from inner call; use defaults)
     if (taskData.data?.includeSocial !== false) {
         try {
             await handleSocialTaskPayload({
@@ -113,10 +133,13 @@ async function handleUserUpdate(taskData, configObj, dependencies, isPopularInve
                 id: String(cid),
                 username
             }, configObj.social || configObj.taskEngine?.social || {}, dependencies);
-            logger.log('INFO', '[TaskEngineV1] Fetched social', { cid, username, userType, dataType: 'social' });
+            logger.log('INFO', `[TaskEngineV1] Success to fetch social for ${userType} for ID ${cid} after 1 attempts with proxy`, {
+                taskType: 'social', userType, userId: cid, attemptCount: 1, mode: 'proxy'
+            });
         } catch (socialErr) {
-            // 404 = eToro has no user for this username (e.g. placeholder "ondemand-{cid}" or private). Portfolio + history already saved.
-            logger.log('WARN', '[TaskEngineV1] Social fetch skipped', { cid, username, error: socialErr.message });
+            logger.log('WARN', `[TaskEngineV1] Failure to fetch social for ${userType} for ID ${cid} after 1 attempts with proxy`, {
+                taskType: 'social', userType, userId: cid, attemptCount: 1, mode: 'proxy', error: socialErr.message
+            });
         }
     }
 }

package/functions/data-fetcher-system-v1/shared/appscriptClient.js

@@ -29,6 +29,7 @@ async function fetchWithProxyFallback({
         throw new Error('[AppscriptClient] No fetch implementation available (pass fetchImpl or use Node 18+)');
     }
 
+    let proxyMeta = null;
     if (proxyManager && typeof proxyManager.fetch === 'function') {
         const timeoutMs = options.timeout ?? DEFAULT_PROXY_TIMEOUT_MS;
         try {
@@ -39,23 +40,23 @@ async function fetchWithProxyFallback({
             if (response?.ok) {
                 return response;
             }
-            if (logger && typeof logger.log === 'function') {
-                logger.log('INFO', '[AppscriptClient] Proxy returned non-ok, using direct fetch fallback', {
-                    status: response?.status,
-                    url
-                });
-            }
+            proxyMeta = response?._fetchMeta || null;
+            // No per-attempt log: caller logs end-result only (e.g. task engine).
         } catch (proxyErr) {
-            if (logger && typeof logger.log === 'function') {
-                logger.log('WARN', '[AppscriptClient] Proxy fetch failed, using direct fetch fallback', {
-                    error: proxyErr?.message,
-                    url
-                });
-            }
+            // No per-attempt log: caller logs end-result only (e.g. task engine).
         }
     }
 
-    return impl(url, options);
+    const directResponse = await impl(url, options);
+    if (directResponse && typeof directResponse === 'object' && !directResponse._fetchMeta) {
+        // If we had proxy attempts then fell back to direct, proxyMeta may have attemptCount; add 1 for the direct attempt.
+        const usedDirect = true;
+        const attemptCount = proxyMeta
+            ? (proxyMeta.attemptCount || 0) + 1
+            : 1;
+        directResponse._fetchMeta = { usedDirect, attemptCount };
+    }
+    return directResponse;
 }
 
 async function fetchJsonWithProxyFallback(params) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.892",
+  "version": "1.0.894",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [