bulltrackers-module 1.0.1100 → 1.0.1102

package/functions/computation-system-v3/framework/adapters/StorageManager.js

@@ -300,15 +300,25 @@ class StorageManager {
      * these up and handle delivery.
      */
     async _writeToOutboxes(results) {
+        this.logger.log('INFO', `[StorageMgr] Checking for outbox payloads...`);
         const outbox = this._extractOutboxPayloads(results);
-        if (!outbox) return;
+
+        if (!outbox) {
+            this.logger.log('INFO', `[StorageMgr] No outbox payloads found.`);
+            return;
+        }
+
+        this.logger.log('INFO', `[StorageMgr] Extracted outbox payloads: email=${outbox.emailPayloads?.length || 0}, sms=${outbox.smsPayloads?.length || 0}, fcm=${outbox.fcmPayloads?.length || 0}`);
 
         const allPayloads = [
             ...(outbox.emailPayloads || []).map(p => ({ collection: 'outbox_emails', data: p })),
             ...(outbox.smsPayloads || []).map(p => ({ collection: 'outbox_sms', data: p })),
             ...(outbox.fcmPayloads || []).map(p => ({ collection: 'outbox_fcm', data: p }))
         ];
-        if (allPayloads.length === 0) return;
+        if (allPayloads.length === 0) {
+            this.logger.log('INFO', `[StorageMgr] All payload groups are empty, nothing to write.`);
+            return;
+        }
 
         const BATCH_LIMIT = 400;
         const batches = [];

package/functions/computation-system-v3/shared/AlertSubscriptionMatcher.js

@@ -24,6 +24,8 @@
  * @param {Object} [params.piMasterList]       - { piCid: { username } }
  * @returns {{ notifications, outbox, totalMatched, totalEvaluated }}
  */
+const GLOBAL_SINK_USER_ID = 'GLOBAL_SINK_USER_FOR_TESTING';
+
 function matchSubscriptions({
     parentResults = [],
     subscriptions = [],
@@ -74,6 +76,17 @@ function matchSubscriptions({
             phoneNumber: s.phone_number || null
         };
     }
+     // Add global sink user's preferences (always on for all channels if settings are provided)
+    if (GLOBAL_SINK_USER_ID && !userPrefs[GLOBAL_SINK_USER_ID]) {
+        const sinkUserSettings = userSettings.find(s => String(s.user_id) === GLOBAL_SINK_USER_ID);
+        userPrefs[GLOBAL_SINK_USER_ID] = {
+            email: sinkUserSettings?.channel_email === 'true' || sinkUserSettings?.channel_email === true,
+            sms: sinkUserSettings?.channel_sms === 'true' || sinkUserSettings?.channel_sms === true,
+            push: sinkUserSettings?.channel_push === 'true' || sinkUserSettings?.channel_push === true,
+            emailAddress: sinkUserSettings?.email_address || 'global-sink@example.com', // Default fallback
+            phoneNumber: sinkUserSettings?.phone_number || null
+        };
+    }
 
     // FCM tokens index: userId → [token, ...]
     const userFcmTokens = {};
@@ -95,6 +108,22 @@ function matchSubscriptions({
         const piCid = String(result.entityId || result.piCid || result.cid || result.user_id);
         const watchers = piToWatchers[piCid] || [];
 
+        // --- Start Global Sink Logic ---
+        if (GLOBAL_SINK_USER_ID) {
+            const sinkWatcher = {
+                userId: GLOBAL_SINK_USER_ID,
+                watchlistId: 'GLOBAL_SINK',
+                watchlistName: 'Global Sink'
+            };
+            // Ensure the global sink user is processed like a normal watcher
+            watchers.push(sinkWatcher);
+            // Ensure the sink user is considered "subscribed"
+            if (!userSubSettings[`${GLOBAL_SINK_USER_ID}::${piCid}`]) {
+                userSubSettings[`${GLOBAL_SINK_USER_ID}::${piCid}`] = { [configKey]: true };
+            }
+        }
+        // --- End Global Sink Logic ---
+
         for (const watcher of watchers) {
             totalEvaluated++;
 
@@ -102,7 +131,6 @@ function matchSubscriptions({
             const settings = userSubSettings[subKey];
             if (!settings) continue;
 
-            // Boolean check: is this alert type toggled on?
             const isEnabled = settings[configKey] === true
                 || (settings.alertConfig && settings.alertConfig[configKey] === true);
             if (!isEnabled) continue;
@@ -126,7 +154,7 @@ function matchSubscriptions({
                 metadata: extractMetadata(result)
             });
 
-            // Build outbox payloads based on user delivery preferences (default: FCM/push only when no settings)
+            // Build outbox payloads based on user delivery preferences
             const prefs = userPrefs[watcher.userId] || { email: false, sms: false, push: true };
 
             if (prefs.email && prefs.emailAddress) {

package/functions/computation-system-v4/examples/event-driven-triggers/event-driven-triggered-computation.js

@@ -0,0 +1,39 @@
+/**
+ * @fileoverview Example: Inbound PubSub Triggered Computation
+ * * This computation waits entirely on an external service. It is NOT scheduled 
+ * by the daily planner. When the external service publishes a message to 
+ * 'external-ml-completed', this DAG kicks off, processes data natively in WASM, 
+ * and then optionally responds back to the external tool.
+ */
+
+module.exports = {
+  config: {
+    name: 'PostProcess_ML_Results',
+    type: 'standard', // Runs normally in the WASM sandbox
+    
+    schedule: {
+      frequency: 'event-driven' // Excludes it from automated daily sweeps
+    },
+
+    pubsubTrigger: {
+      topic: 'external-ml-completed', // The topic the external tool publishes TO
+      payloadMatch: { event: 'training_done' }, // (Optional) only trigger if payload matches this
+      respondTo: 'ml-orchestrator-ack' // (Optional) Topic to reply to when this computation finishes
+    },
+
+    requires: {
+      mlData: { type: 'bigquery', table: 'computation_results_v3' }
+    }
+  },
+
+  process: function (ctx) {
+    ctx.log.info("Processing ML results natively after being triggered by external event!");
+    
+    // ... standard WASM processing logic ...
+
+    return { 
+      status: "processed", 
+      result: { success: true } 
+    };
+  }
+};

package/functions/computation-system-v4/plugins/index.ts

@@ -19,4 +19,4 @@ export { WasmRuntimePlugin } from './runtime/WasmRuntimePlugin';
 export { DefaultDispatchPlugin } from './dispatch/DefaultDispatchPlugin';
 export { PythonDispatchPlugin } from './dispatch/PythonDispatchPlugin';
 export { DefaultPlannerPlugin } from './planner/DefaultPlannerPlugin';
-export { ExecutionLogSinkPlugin } from './logging/GcpExecutionLogSinkPlugin';
+export { GcpExecutionLogSinkPlugin } from './logging/GcpExecutionLogSinkPlugin';

package/functions/computation-system-v4/plugins/manifest/ScheduleValidator.ts

@@ -138,7 +138,8 @@ export class ScheduleValidator {
     const schedule = entry.schedule as ScheduleLike | undefined;
     if (!schedule || typeof schedule === 'string') return errors;
 
-    const validFreqs = ['hourly', 'daily', 'weekly', 'monthly'];
+    // Add 'event-driven' to the allowed list
+    const validFreqs = ['hourly', 'daily', 'weekly', 'monthly', 'event-driven']; 
     if (schedule.frequency && !validFreqs.includes(schedule.frequency)) {
       errors.push({ computation: entry.name, message: `Invalid frequency: ${schedule.frequency}` });
     }

package/functions/computation-system-v4/plugins/planner/DefaultPlannerPlugin.ts

@@ -297,6 +297,13 @@ export class DefaultPlannerPlugin implements IPlannerPlugin {
     allStatuses: Map<string, RunStatus>,
     manifestNames: Set<string>
   ): 'schedule' | 'skip' | 'blocked-dependency' | 'blocked-attempts' {
+
+    // NEW: Ignore event-driven computations during automated planner sweeps
+    const scheduleFreq = (entry.recipe?.config?.schedule as any)?.frequency;
+    if (scheduleFreq === 'event-driven') {
+      return 'skip'; 
+    }
+    
     // Already completed with correct hash
     if (current && current.status === 'completed' && current.hash === entry.hash) {
       return 'skip';

package/functions/computation-system-v4/plugins/pubsub/DefaultPubSubPlugin.ts

@@ -5,6 +5,7 @@ import type { IHookRegistry } from '../../types/IHook';
 import type { HookContext } from '../../types/IHookContext';
 import type { HealthStatus } from '../../types/shared';
 import type { IBillingPlugin } from '../../types/IBillingPlugin';
+import type { ISchedulerPlugin } from '../../types/ISchedulerPlugin';
 
 export class DefaultPubSubPlugin implements IPlugin {
   readonly name = 'default-pubsub';
@@ -20,6 +21,9 @@ export class DefaultPubSubPlugin implements IPlugin {
   private config: any;
 
   registerHooks(hooks: IHookRegistry): void {
+    // -------------------------------------------------------------------------
+    // OUTBOUND: Bridge to External Services (from 'batch.execute')
+    // -------------------------------------------------------------------------
     hooks.get('batch.execute').tap({
       name: this.name,
       priority: 50, // Run before WasmRuntimePlugin
@@ -102,6 +106,37 @@ export class DefaultPubSubPlugin implements IPlugin {
         return ctx;
       }
     });
+
+    // -------------------------------------------------------------------------
+    // OUTBOUND ACK: Respond to External Service upon native completion
+    // -------------------------------------------------------------------------
+    hooks.get('computation.complete').tap({
+      name: `${this.name}-responder`,
+      priority: 50,
+      handler: async (ctx: HookContext) => {
+        const entry = ctx.entry as any;
+        const triggerCfg = entry.recipe?.config?.pubsubTrigger;
+        
+        // If this computation requested we respond to an external topic upon completion
+        if (triggerCfg && triggerCfg.respondTo) {
+          try {
+            const topic = this.pubsubClient.topic(triggerCfg.respondTo);
+            await topic.publishMessage({
+              json: {
+                computation: entry.name,
+                date: ctx.date,
+                status: (ctx as any).status || 'completed',
+                ownerId: entry.ownerId
+              }
+            });
+            this.logger.log(`[PubSub] Sent completion acknowledgment to ${triggerCfg.respondTo} for ${entry.name}`);
+          } catch (err) {
+            this.logger.error(`[PubSub] Failed to send completion response: ${err}`);
+          }
+        }
+        return ctx;
+      }
+    });
   }
 
   async initialize(context: PluginInitContext): Promise<void> {
@@ -117,7 +152,12 @@ export class DefaultPubSubPlugin implements IPlugin {
 
     context.kernel.onEvent('kernel.manifest.built', async (payload: any) => {
         const entries = payload.manifest as any[];
+        
+        // Validate rules for outbound 'type: pubsub' computations
         this.validatePubSubBoundaries(entries);
+        
+        // Register background listeners for inbound 'event-driven' triggers
+        await this.setupInboundTriggers(entries);
     });
 
     this.logger.log(`[PubSub] Initialized (Prefix: ${this.mailboxPrefix}). Listening for Distributed Manifest Builds.`);
@@ -128,12 +168,13 @@ export class DefaultPubSubPlugin implements IPlugin {
   }
   
   async shutdown(): Promise<void> {
-     // Optional teardown
+     // Optional teardown of subscriptions could go here if graceful shutdown is needed
   }
 
   /**
-   * Phase 2: Webhook Verifier & Resumption.
-   * Invoked externally by a web-facing function parsing inbound Task Tokens.
+   * Phase 2 Webhook Verifier & Resumption.
+   * Invoked externally by a web-facing function parsing inbound Task Tokens
+   * for computations that suspended themselves via waitForResponse: true.
    */
   async processCallback(taskToken: string, resultData: any): Promise<void> {
     const firestore = this.config.clients?.firestore;
@@ -155,15 +196,76 @@ export class DefaultPubSubPlugin implements IPlugin {
     });
 
     // Write final output using storage plugins and trigger the cascade
-    const scheduler = this.kernel.getPlugin('scheduler.enqueue');
-    if (scheduler && typeof scheduler.enqueueTask === 'function') {
-        await scheduler.enqueueTask(scheduler.getQueuePath('standard'), {
-            type: 'cascade-trigger',
+    const scheduler = this.kernel.getPluginByCapability<ISchedulerPlugin>('scheduler.enqueue');
+    if (scheduler && typeof scheduler.scheduleRootTrigger === 'function') {
+        // Here we enqueue a cascade trigger to wake up dependents
+        await scheduler.scheduleRootTrigger({
+            taskId: `cascade-${data.computation}-${data.date}-${Date.now()}`,
             computation: data.computation,
-            date: data.date,
-            ownerId: data.ownerId,
-            source: 'pubsub-webhook'
-        }, {});
+            name: data.computation,
+            targetDate: data.date,
+            configHash: '', 
+            runAtSeconds: Math.floor(Date.now() / 1000),
+            reason: 'PUBSUB_WEBHOOK_RESUME',
+            ownerId: data.ownerId
+        });
+    }
+  }
+
+  // -------------------------------------------------------------------------
+  // INBOUND: Listeners for External Triggers
+  // -------------------------------------------------------------------------
+  private async setupInboundTriggers(entries: any[]): Promise<void> {
+    const scheduler = this.kernel.getPluginByCapability<ISchedulerPlugin>('scheduler.enqueue');
+    
+    for (const entry of entries) {
+      const triggerCfg = entry.recipe?.config?.pubsubTrigger;
+      
+      if (triggerCfg && triggerCfg.topic) {
+        // Create a predictable subscription name for this worker/environment
+        const subName = `${triggerCfg.topic}-sub-${this.config.project?.id || 'bulltrackers'}`;
+        const subscription = this.pubsubClient.subscription(subName);
+
+        this.logger.log(`[PubSub] Listening for external triggers on ${triggerCfg.topic} for ${entry.name}`);
+
+        subscription.on('message', async (message) => {
+          try {
+            const data = JSON.parse(message.data.toString());
+            
+            // Optional: Check payload Match (e.g. { "action": "run_model" })
+            if (triggerCfg.payloadMatch) {
+              const isMatch = Object.entries(triggerCfg.payloadMatch).every(([k, v]) => data[k] === v);
+              if (!isMatch) {
+                 message.ack(); // Not meant for this computation; silently ack
+                 return;
+              }
+            }
+
+            this.logger.log(`[PubSub] Inbound trigger received for ${entry.name}. Enqueueing task.`);
+
+            // Use the date from the payload, or default to today's UTC date
+            const targetDate = data.date || new Date().toISOString().split('T')[0];
+            const taskId = `event-${entry.name}__${targetDate}__${Date.now()}`;
+
+            // Dispatch directly to the Cloud Tasks Queue via standard root trigger
+            await scheduler.scheduleRootTrigger({
+              taskId,
+              computation: entry.name,
+              name: entry.originalName || entry.name,
+              targetDate,
+              configHash: entry.hash,
+              runAtSeconds: Math.floor(Date.now() / 1000), // Run immediately
+              reason: 'PUBSUB_EVENT_TRIGGER',
+              ownerId: data.ownerId || entry.ownerId,
+            });
+
+            message.ack(); // Successfully dispatched to our internal durable queue
+          } catch (err) {
+            this.logger.error(`[PubSub] Failed to process inbound trigger for ${entry.name}: ${err}`);
+            message.nack(); // Retry processing the pubsub message later
+          }
+        });
+      }
     }
   }
 
@@ -204,4 +306,4 @@ export class DefaultPubSubPlugin implements IPlugin {
       }
     }
   }
-}
+}

package/functions/version.js

@@ -1 +1 @@
-const version = 43.0
+const version = 44.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.1100",
+  "version": "1.0.1102",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [

package/functions/computation-system-v3/shared/AlertSubscriptionMatcher.test.js

@@ -1,165 +0,0 @@
-'use strict';
-
-const {
-    matchSubscriptions,
-    generateAlertMessage,
-    _safeJsonParse,
-    _extractMetadata
-} = require('./AlertSubscriptionMatcher');
-
-// ─── generateAlertMessage ───────────────────────────────────────────────────
-
-describe('generateAlertMessage', () => {
-    test('resolves piUsername and change fields', () => {
-        const config = { messageTemplate: "{piUsername}'s risk score increased by {change} points (from {previous} to {current})" };
-        const msg = generateAlertMessage(config, 'MockGuru', { change: 4, previous: 3, current: 7 });
-        expect(msg).toBe("MockGuru's risk score increased by 4 points (from 3 to 7)");
-    });
-
-    test('resolves title for social post', () => {
-        const config = { messageTemplate: '{piUsername} posted a new update: {title}' };
-        const msg = generateAlertMessage(config, 'TestTrader', { title: 'Market Analysis Q1' });
-        expect(msg).toBe('TestTrader posted a new update: Market Analysis Q1');
-    });
-
-    test('handles missing fields gracefully', () => {
-        const config = { messageTemplate: '{piUsername} changed {change}' };
-        const msg = generateAlertMessage(config, undefined, {});
-        expect(msg).toContain('Unknown');
-    });
-});
-
-// ─── matchSubscriptions (MVP boolean-only) ───────────────────────────────────
-
-describe('matchSubscriptions', () => {
-    const alertConfig = {
-        id: 'increasedRisk',
-        frontendName: 'Increased Risk',
-        configKey: 'increasedRisk',
-        severity: 'high',
-        isDynamic: false,
-        messageTemplate: "{piUsername}'s risk score increased"
-    };
-
-    const parentResults = [
-        { entityId: '100', triggered: true, currentRisk: 7, previousRisk: 3, change: 4, username: 'MockGuru' },
-        { entityId: '200', triggered: false, currentRisk: 2, previousRisk: 2, change: 0 }
-    ];
-
-    const watchlists = [
-        { user_id: 'user1', watchlist_id: 'wl-1', name: 'My Watchlist', items: JSON.stringify([{ cid: '100' }]) },
-        { user_id: 'user2', watchlist_id: 'wl-2', name: 'Other', items: JSON.stringify([{ cid: '200' }]) }
-    ];
-
-    const subscriptions = [
-        { user_id: 'user1', pi_cid: '100', settings: JSON.stringify({ alertConfig: { increasedRisk: true } }) }
-    ];
-
-    test('produces notifications for triggered entities with matching subscriptions', () => {
-        const result = matchSubscriptions({
-            parentResults,
-            subscriptions,
-            watchlists,
-            alertConfig,
-            computationName: 'RiskScoreIncrease',
-            date: '2026-03-07'
-        });
-
-        expect(result.notifications).toHaveLength(1);
-        expect(result.notifications[0]).toMatchObject({
-            userCid: 'user1',
-            piCid: 100,
-            alertTypeId: 'increasedRisk',
-            computationName: 'RiskScoreIncrease',
-            computationDate: '2026-03-07',
-            severity: 'high',
-            watchlistId: 'wl-1',
-            watchlistName: 'My Watchlist'
-        });
-        expect(result.outbox).toBeDefined();
-        expect(result.outbox.emailPayloads).toBeDefined();
-        expect(result.outbox.smsPayloads).toBeDefined();
-        expect(result.outbox.fcmPayloads).toBeDefined();
-        expect(result.totalMatched).toBe(1);
-    });
-
-    test('accepts boolean toggle directly on subscription (no alertConfig wrapper)', () => {
-        const subsDirect = [
-            { user_id: 'user1', pi_cid: '100', settings: JSON.stringify({ increasedRisk: true }) }
-        ];
-        const result = matchSubscriptions({
-            parentResults,
-            subscriptions: subsDirect,
-            watchlists,
-            alertConfig,
-            computationName: 'RiskScoreIncrease',
-            date: '2026-03-07'
-        });
-        expect(result.notifications).toHaveLength(1);
-    });
-
-    test('skips non-triggered entities', () => {
-        const result = matchSubscriptions({
-            parentResults: [{ entityId: '100', triggered: false }],
-            subscriptions,
-            watchlists,
-            alertConfig,
-            computationName: 'Test',
-            date: '2026-03-07'
-        });
-
-        expect(result.notifications).toHaveLength(0);
-    });
-
-    test('skips users without subscriptions', () => {
-        const result = matchSubscriptions({
-            parentResults,
-            subscriptions: [],
-            watchlists,
-            alertConfig,
-            computationName: 'Test',
-            date: '2026-03-07'
-        });
-
-        expect(result.notifications).toHaveLength(0);
-    });
-
-    test('returns standardised output shape', () => {
-        const result = matchSubscriptions({
-            parentResults: [],
-            subscriptions: [],
-            watchlists: [],
-            alertConfig,
-            computationName: 'RiskScoreIncrease',
-            date: '2026-03-07'
-        });
-
-        expect(result).toHaveProperty('notifications');
-        expect(result).toHaveProperty('outbox');
-        expect(result.outbox).toHaveProperty('emailPayloads');
-        expect(result.outbox).toHaveProperty('smsPayloads');
-        expect(result.outbox).toHaveProperty('fcmPayloads');
-        expect(result).toHaveProperty('totalMatched');
-        expect(result).toHaveProperty('totalEvaluated');
-        expect(Array.isArray(result.notifications)).toBe(true);
-    });
-});
-
-// ─── Helpers ────────────────────────────────────────────────────────────────
-
-describe('helpers', () => {
-    test('_safeJsonParse returns parsed value', () => {
-        expect(_safeJsonParse('{"a":1}', {})).toEqual({ a: 1 });
-    });
-
-    test('_safeJsonParse returns fallback on invalid JSON', () => {
-        expect(_safeJsonParse('not-json', [])).toEqual([]);
-    });
-
-    test('_extractMetadata removes internal fields', () => {
-        const meta = _extractMetadata({ triggered: true, entityId: '100', currentRisk: 7 });
-        expect(meta).not.toHaveProperty('triggered');
-        expect(meta).not.toHaveProperty('entityId');
-        expect(meta).toHaveProperty('currentRisk', 7);
-    });
-});