bulltrackers-module 1.0.658 → 1.0.659

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.658",
+  "version": "1.0.659",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [
@@ -45,7 +45,8 @@
     "p-limit": "^3.1.0",
     "require-all": "^3.0.0",
     "sharedsetup": "latest",
-    "zod": "^4.3.5"
+    "zod": "^4.3.5",
+    "@google-cloud/storage": "^7.18.0"
   },
   "devDependencies": {
     "bulltracker-deployer": "file:../bulltracker-deployer"

package/functions/computation-system/workflows/bulltrackers_pipeline.yaml

@@ -1,163 +0,0 @@
-# Cloud Workflows: Precision Cursor-Based Orchestrator with Manual Override
-main:
-  params: [input]
-  steps:
-    - init:
-        assign:
-          - project: '${sys.get_env("GOOGLE_CLOUD_PROJECT_ID")}'
-          - passes: ["1", "2", "3", "4", "5"]
-          - current_date: '${text.split(time.format(sys.now()), "T")[0]}'
-          - date_to_run: '${default(map.get(input, "date"), current_date)}'
-
-    # --- 🔀 NEW: CHECK FOR MANUAL OVERRIDE ---
-    - check_manual_override:
-        switch:
-          - condition: '${map.get(input, "action") == "FORCE_RUN"}'
-            next: execute_manual_force_run
-
-    # --- PHASE 1: EXECUTION (Standard + High Mem Retry) ---
-    - run_sequential_passes:
-        for:
-          value: pass_id
-          in: ${passes}
-          steps:
-            - init_cursor:
-                assign:
-                  - n_cursor: 1
-                  - pass_complete: false
-
-            - sequential_date_loop:
-                switch:
-                  - condition: ${not pass_complete}
-                    steps:
-                      - call_dispatcher:
-                          call: http.post
-                          args:
-                            url: '${"https://europe-west1-" + project + ".cloudfunctions.net/computation-dispatcher"}'
-                            body:
-                              pass: '${pass_id}'
-                              cursorIndex: '${n_cursor}'
-                              date: '${date_to_run}'
-                              action: 'DISPATCH'
-                            auth: { type: OIDC }
-                          result: dispatch_res
-
-                      - evaluate_dispatch:
-                          switch:
-                            # 1. End of Session
-                            - condition: '${dispatch_res.body.status == "MOVE_TO_NEXT_PASS"}'
-                              assign:
-                                - pass_complete: true
-                            
-                            # 2. Satiation Check
-                            - condition: '${dispatch_res.body.status == "CONTINUE_PASS" and dispatch_res.body.remainingDates == 0 and dispatch_res.body.dispatched == 0}'
-                              steps:
-                                - log_satiation:
-                                    call: sys.log
-                                    args:
-                                      text: '${"Pass " + pass_id + " - ✅ Pass satiated. Starting Verification."}'
-                                - mark_complete:
-                                    assign:
-                                      - pass_complete: true
-
-                            # 3. Work Dispatched
-                            - condition: '${dispatch_res.body.dispatched > 0}'
-                              steps:
-                                - wait_for_completion:
-                                    call: sys.sleep
-                                    args:
-                                      seconds: '${int(dispatch_res.body.etaSeconds)}'
-                                - update_cursor:
-                                    assign:
-                                      - n_cursor: '${default(dispatch_res.body.nextCursor, n_cursor + 1)}'
-                                - next_loop_work:
-                                    next: sequential_date_loop
-
-                            # 4. No Work (Fast-Forward)
-                            - condition: '${dispatch_res.body.dispatched == 0}'
-                              steps:
-                                - wait_short:
-                                    call: sys.sleep
-                                    args:
-                                      seconds: 2
-                                - update_cursor_retry:
-                                    assign:
-                                        - n_cursor: '${default(dispatch_res.body.nextCursor, n_cursor + 1)}'
-                                - next_loop_retry:
-                                    next: sequential_date_loop
-            
-            # --- VERIFICATION & SWEEP (CRITICAL LOGIC PRESERVED) ---
-            - verify_pass_completion:
-                call: http.post
-                args:
-                  url: '${"https://europe-west1-" + project + ".cloudfunctions.net/computation-dispatcher"}'
-                  body:
-                    action: 'VERIFY'
-                    pass: '${pass_id}'
-                    date: '${date_to_run}'
-                  auth: { type: OIDC }
-                result: verify_res
-
-            - process_sweep_tasks:
-                for:
-                  value: sweep_task
-                  in: '${verify_res.body.missingTasks}'
-                  steps:
-                    - log_sweep:
-                        call: sys.log
-                        args:
-                          text: '${"🧹 SWEEP: Disposing " + sweep_task.taskCount + " high-mem tasks for " + sweep_task.date}'
-                    - dispatch_force_sweep:
-                        call: http.post
-                        args:
-                          url: '${"https://europe-west1-" + project + ".cloudfunctions.net/computation-dispatcher"}'
-                          body:
-                            action: 'SWEEP'
-                            pass: '${pass_id}'
-                            date: '${sweep_task.date}'
-                          auth: { type: OIDC }
-                    - wait_sweep_completion:
-                        call: sys.sleep
-                        args:
-                          seconds: '${int(sweep_task.eta)}'
-
-    # --- PHASE 2: FINAL FORENSIC REPORTING ---
-    - run_final_forensics:
-        for:
-          value: pass_id
-          in: ${passes}
-          steps:
-            - generate_final_report:
-                call: http.post
-                args:
-                  url: '${"https://europe-west1-" + project + ".cloudfunctions.net/computation-dispatcher"}'
-                  body:
-                    action: 'REPORT'
-                    pass: '${pass_id}'
-                    date: '${date_to_run}'
-                  auth: { type: OIDC }
-                result: report_res
-            
-            - log_forensics:
-                call: sys.log
-                args:
-                   text: '${"📝 FINAL REPORT: Pass " + pass_id + " -> " + report_res.body.issuesFound + " detailed forensic documents created."}'
-
-    - finish_standard:
-        return: "Pipeline Complete with Forensic Analysis"
-
-    # --- 🚨 MANUAL OVERRIDE EXECUTION PATH ---
-    - execute_manual_force_run:
-        call: http.post
-        args:
-            url: '${"https://europe-west1-" + project + ".cloudfunctions.net/computation-dispatcher"}'
-            body:
-                action: 'FORCE_RUN'
-                computation: '${input.computation}'
-                date: '${map.get(input, "date")}' # Can be null for ALL_DATES
-                resources: '${map.get(input, "resources")}'
-            auth: { type: OIDC }
-        result: force_res
-
-    - finish_manual:
-        return: '${force_res.body}'

package/functions/computation-system/workflows/data_feeder_pipeline.yaml

@@ -1,115 +0,0 @@
-# Data Feeder Pipeline (V4.0 - Lean Market Close)
-# Schedule: 22:00 UTC
-# Objective: Capture Market Close data & perform Global Indexing at midnight.
-
-main:
-  params: [input]
-  steps:
-    - init:
-        assign:
-          - project: '${sys.get_env("GOOGLE_CLOUD_PROJECT_ID")}'
-          - location: "europe-west1"
-
-    # ==========================================
-    # TEST MODE ROUTING (Preserved for Debugging)
-    # ==========================================
-    - check_test_mode:
-        switch:
-          - condition: '${input != null and "target_step" in input}'
-            steps:
-              - route_test:
-                  switch:
-                    - condition: '${input.target_step == "test_price"}'
-                      steps:
-                        - call_price_iso:
-                            call: http.post
-                            args:
-                              url: '${"https://" + location + "-" + project + ".cloudfunctions.net/price-fetcher"}'
-                              auth: { type: OIDC }
-                              timeout: 300
-                        - return_price:
-                            return: "Test Complete: Price Fetcher executed."
-
-                    - condition: '${input.target_step == "test_insights"}'
-                      steps:
-                        - call_insights_iso:
-                            call: http.post
-                            args:
-                              url: '${"https://" + location + "-" + project + ".cloudfunctions.net/insights-fetcher"}'
-                              auth: { type: OIDC }
-                              timeout: 300
-                        - return_insights:
-                            return: "Test Complete: Insights Fetcher executed."
-                            
-                    - condition: '${input.target_step == "test_indexer"}'
-                      steps:
-                        - call_indexer_iso:
-                            call: http.post
-                            args:
-                              url: '${"https://" + location + "-" + project + ".cloudfunctions.net/root-data-indexer"}'
-                              auth: { type: OIDC }
-                              timeout: 300
-                        - return_indexer:
-                            return: "Test Complete: Root Data Indexer executed."
-
-    # ==========================================
-    # PHASE 1: MARKET CLOSE (Starts 22:00 UTC)
-    # ==========================================
-
-    # 1. Price Fetcher
-    - phase_2200_price:
-        try:
-          call: http.post
-          args:
-            url: '${"https://" + location + "-" + project + ".cloudfunctions.net/price-fetcher"}'
-            auth: { type: OIDC }
-            timeout: 300
-        except:
-          as: e
-          steps:
-            - log_price_error:
-                call: sys.log
-                args: { severity: "WARNING", text: "Price fetch timed out/failed. Proceeding." }
-
-    # 2. Insights Fetcher (Can run immediately after or parallel)
-    - phase_2200_insights:
-        try:
-          call: http.post
-          args:
-            url: '${"https://" + location + "-" + project + ".cloudfunctions.net/insights-fetcher"}'
-            auth: { type: OIDC }
-            timeout: 300
-        except:
-          as: e
-          steps:
-            - log_insights_error:
-                call: sys.log
-                args: { severity: "WARNING", text: "Insights fetch timed out/failed. Proceeding." }
-
-    # ==========================================
-    # PHASE 2: WAIT FOR MIDNIGHT (Indexing)
-    # ==========================================
-
-    - align_to_midnight:
-        assign:
-          - now_sec: '${int(sys.now())}'
-          - day_sec: 86400
-          # Calculates seconds remaining until the next 00:00 UTC
-          - sleep_midnight: '${day_sec - (now_sec % day_sec)}'
-    
-    - wait_for_midnight:
-        call: sys.sleep
-        args: { seconds: '${sleep_midnight}' }
-
-    # ==========================================
-    # PHASE 3: GLOBAL INDEXING (00:00 UTC)
-    # ==========================================
-
-    - global_index_midnight:
-        call: http.post
-        args:
-          url: '${"https://" + location + "-" + project + ".cloudfunctions.net/root-data-indexer"}'
-          auth: { type: OIDC }
-
-    - finish:
-        return: "Daily Close Cycle Finished (Price, Insights, Indexing)."

package/functions/computation-system/workflows/datafeederpipelineinstructions.md

@@ -1,30 +0,0 @@
-# 🧪 Workflow Testing Guide
-
-Below is a quick-reference guide for manually triggering the **Data Feeder Pipeline** using the Google Cloud Console UI.
-
-## How to Run a Test
-
-1. Navigate to **Workflows** in your GCP Console.
-2. Select `data-feeder-pipeline`.
-3. Click the **Execute** button at the top.
-4. Paste the specific **JSON Input** from the table below into the input box to bypass schedules and target specific phases.
-
-### Test Commands
-
-Cloud Function,Input JSON to Use
-Price Fetcher,"{"target_step": "test_price"}"
-Insights Fetcher,"{"target_step": "test_insights"}"
-Popular Investors,"{"target_step": "test_rankings"}"
-Social Orchestrator,"{"target_step": "test_social"}"
-Root Indexer,"{"target_step": "test_indexer"}"
-
----
-
-### 💡 Testing Notes & Pro-Tips
-
-* **Automatic Indexing:** As of V3.2, you will not see explicit "Index" steps in the workflow visualization for Price, Insights, or Rankings. These functions now trigger the `root-data-indexer` automatically upon completion. The only visible index step is the **Global Verification** in the Midnight Phase.
-* **Verification:** To confirm data was indexed during a test:
-    * Check the logs of the individual Cloud Functions (`price-fetcher`, etc.).
-    * Or, run the **Midnight Phase** test, which ends with the explicit `global_index_midnight` step.
-* **Variable Checking:** After execution, check the **Variables** tab to view `sleep_midnight` calculations to ensure UTC alignment is functioning correctly.
-* **Permissions:** Ensure the executor has `roles/workflows.invoker`.

package/functions/computation-system/workflows/morning_prep_pipeline.yaml

@@ -1,55 +0,0 @@
-# Morning Prep Pipeline
-# Schedule: 04:00 UTC
-# Objective: Refresh Rankings -> Trigger Daily Update (Task Engine)
-
-main:
-  params: [input]
-  steps:
-    - init:
-        assign:
-          - project: '${sys.get_env("GOOGLE_CLOUD_PROJECT_ID")}'
-          - location: "europe-west1"
-          # Define the Task Engine workflow ID to trigger later
-          - task_engine_workflow_id: "daily-update-pipeline"
-
-    # ==========================================
-    # STEP 1: REFRESH RANKINGS (The Master List)
-    # ==========================================
-    - fetch_rankings:
-        try:
-          call: http.post
-          args:
-            url: '${"https://" + location + "-" + project + ".cloudfunctions.net/fetch-popular-investors"}'
-            auth: { type: OIDC }
-            timeout: 540
-        except:
-          as: e
-          steps:
-            - log_rankings_fail:
-                call: sys.log
-                args: { severity: "ERROR", text: "Rankings Fetch Failed. Stopping pipeline to prevent stale Task Engine run." }
-            - raise_error:
-                raise: ${e}
-
-    # ==========================================
-    # STEP 2: TRIGGER TASK ENGINE
-    # ==========================================
-    # We trigger the workflow execution directly. 
-    # This replaces the need for a separate 05:00 UTC Scheduler.
-    
-    - trigger_daily_update:
-        call: googleapis.workflowexecutions.v1.projects.locations.workflows.executions.create
-        args:
-          parent: '${"projects/" + project + "/locations/" + location + "/workflows/" + task_engine_workflow_id}'
-          body:
-            # Pass explicit arguments to the Task Engine
-            argument: '{"userTypes": ["normal", "speculator"], "source": "morning_prep_trigger"}'
-        result: execution_result
-
-    - log_trigger:
-        call: sys.log
-        args:
-          text: '${"✅ Rankings Complete. Triggered Task Engine: " + execution_result.name}'
-
-    - finish:
-        return: "Morning Prep Complete. Task Engine launched."