bulltrackers-module 1.0.501 → 1.0.503

package/functions/computation-system/workflows/data_feeder_pipeline.yaml

@@ -1,7 +1,5 @@
-# Data Feeder Pipeline (V3.2 - Automatic Root Data Indexing)
+# Data Feeder Pipeline (V3.3 - Optimized Waits & Isolated Testing)
 # Starts at 22:00 UTC via Cloud Scheduler.
-# UPDATED: Removed intermediate root data indexer calls - each fetcher now automatically triggers indexing.
-# Only the global verification run at midnight remains.
 
 main:
   params: [input]
@@ -11,13 +9,74 @@ main:
           - project: '${sys.get_env("GOOGLE_CLOUD_PROJECT_ID")}'
           - location: "europe-west1"
 
-    # --- TEST MODE ---
+    # ==========================================
+    # TEST MODE ROUTING
+    # ==========================================
     - check_test_mode:
         switch:
           - condition: '${input != null and "target_step" in input}'
             steps:
               - route_test:
                   switch:
+                    # --- ISOLATED FUNCTION TESTS (Run & Stop) ---
+                    # Use these to test a single function without triggering waits or next steps.
+
+                    - condition: '${input.target_step == "test_price"}'
+                      steps:
+                        - call_price_iso:
+                            call: http.post
+                            args:
+                              url: '${"https://" + location + "-" + project + ".cloudfunctions.net/price-fetcher"}'
+                              auth: { type: OIDC }
+                              timeout: 300
+                        - return_price:
+                            return: "Test Complete: Price Fetcher executed."
+
+                    - condition: '${input.target_step == "test_insights"}'
+                      steps:
+                        - call_insights_iso:
+                            call: http.post
+                            args:
+                              url: '${"https://" + location + "-" + project + ".cloudfunctions.net/insights-fetcher"}'
+                              auth: { type: OIDC }
+                              timeout: 300
+                        - return_insights:
+                            return: "Test Complete: Insights Fetcher executed."
+
+                    - condition: '${input.target_step == "test_rankings"}'
+                      steps:
+                        - call_rankings_iso:
+                            call: http.post
+                            args:
+                              url: '${"https://" + location + "-" + project + ".cloudfunctions.net/fetch-popular-investors"}'
+                              auth: { type: OIDC }
+                              timeout: 300
+                        - return_rankings:
+                            return: "Test Complete: Popular Investors executed."
+
+                    - condition: '${input.target_step == "test_social"}'
+                      steps:
+                        - call_social_iso:
+                            call: http.post
+                            args:
+                              url: '${"https://" + location + "-" + project + ".cloudfunctions.net/social-orchestrator"}'
+                              auth: { type: OIDC }
+                              timeout: 300
+                        - return_social:
+                            return: "Test Complete: Social Orchestrator executed."
+
+                    - condition: '${input.target_step == "test_indexer"}'
+                      steps:
+                        - call_indexer_iso:
+                            call: http.post
+                            args:
+                              url: '${"https://" + location + "-" + project + ".cloudfunctions.net/root-data-indexer"}'
+                              auth: { type: OIDC }
+                              timeout: 300
+                        - return_indexer:
+                            return: "Test Complete: Root Data Indexer executed."
+
+                    # --- PHASE JUMPS (Resumes flow from that point) ---
                     - condition: '${input.target_step == "market"}'
                       next: phase_2200_price
                     - condition: '${input.target_step == "midnight"}'
@@ -43,14 +102,10 @@ main:
                 call: sys.log
                 args: { severity: "WARNING", text: "Price fetch timed out/failed. Proceeding." }
     
+    # FIXED: Only one 10-minute wait here now.
     - wait_10_after_price:
         call: sys.sleep
-        args: { seconds: 600 } # 10 Minutes
-        # NOTE: Price fetcher now automatically triggers root data indexer after completion
-
-    - wait_10_before_insights:
-        call: sys.sleep
-        args: { seconds: 600 }
+        args: { seconds: 600 } 
 
     - phase_2200_insights:
         try:
@@ -69,7 +124,6 @@ main:
     - wait_10_after_insights:
         call: sys.sleep
         args: { seconds: 600 }
-        # NOTE: Insights fetcher now automatically triggers root data indexer after completion
 
     # ==========================================
     # PHASE 2: WAIT FOR MIDNIGHT
@@ -105,7 +159,6 @@ main:
     - wait_10_after_rankings:
         call: sys.sleep
         args: { seconds: 600 }
-        # NOTE: Popular investor rankings fetcher now automatically triggers root data indexer after completion
 
     - phase_0000_social:
         try:
@@ -129,7 +182,6 @@ main:
         call: http.post
         args:
           url: '${"https://" + location + "-" + project + ".cloudfunctions.net/root-data-indexer"}'
-          # No targetDate = Global verification run (all intermediate indexing is now automatic)
           auth: { type: OIDC }
 
     # ==========================================
@@ -165,9 +217,7 @@ main:
               - wait_10_in_loop:
                   call: sys.sleep
                   args: { seconds: 600 }
-                  # NOTE: Social tasks are handled by task engine, which automatically triggers root data indexer after batch completion
               
-              # FIX 5: Correct assign syntax (must be a list)
               - increment_loop:
                   assign: 
                     - i: '${i + 1}'

package/functions/computation-system/workflows/datafeederpipelineinstructions.md

@@ -11,12 +11,12 @@ Below is a quick-reference guide for manually triggering the **Data Feeder Pipel
 
 ### Test Commands
 
-| Phase to Test | JSON Input | Action & Description |
-| :--- | :--- | :--- |
-| **Market Data (Phase 1)** | `{"target_step": "market"}` | **Runs:** Price Fetcher & Insights Fetcher.<br>**Note:** Automatically triggers indexing after each fetch. Workflow will pause at "Wait for Midnight" upon completion. |
-| **Midnight Phase (Phase 3)** | `{"target_step": "midnight"}` | **Runs:** Popular Investor Rankings, Midnight Social Orchestrator, and Global Index Verification.<br>**Note:** Use this to test the critical 00:00 UTC logic without waiting for the daily schedule. |
-| **Social Loop (Phase 4)** | `{"target_step": "social"}` | **Runs:** Enters the recurring 3-hour social fetch loop.<br>**Warning:** Triggers `social_loop_start`, which begins with a 3-hour sleep (`wait_3_hours`) before the first execution. |
-| **Standard Run** | `{}` | **Runs:** The full 24-hour cycle starting from 22:00 UTC (Market Close). |
+Cloud Function,Input JSON to Use
+Price Fetcher,"{"target_step": "test_price"}"
+Insights Fetcher,"{"target_step": "test_insights"}"
+Popular Investors,"{"target_step": "test_rankings"}"
+Social Orchestrator,"{"target_step": "test_social"}"
+Root Indexer,"{"target_step": "test_indexer"}"
 
 ---
 

package/functions/generic-api/API_MIGRATION_PLAN.md

@@ -0,0 +1,436 @@
+# API Migration Plan: User-Centric Collection Structure
+
+## Status: ✅ **MIGRATION COMPLETE**
+
+## Overview
+This document outlines the migration of the User API to use the new user-centric collection structure from the `collection_registry.js`. The goal is to use eToro CIDs and Post IDs for document paths where possible, while maintaining backward compatibility with existing user signup data.
+
+**All endpoints have been updated to use the new collection structure with backward compatibility fallbacks.**
+
+## Summary of Completed Changes
+
+### ✅ Phase 1: Low-Risk, High-Value
+1. **User Social Posts** - ✅ Updated `getUserSocialPosts()` to use `signedInUsers/{cid}/posts` with legacy fallback
+2. **User Sync Requests** - ✅ Updated `requestUserSync()` and `getUserSyncStatus()` to use `signedInUsers/{cid}/syncRequests` and `signedInUsers/{cid}/syncStatus`
+3. **User Notifications** - ⚠️ **Note:** Currently uses Firebase UID for frontend auth compatibility. CID lookup helper created but not yet integrated into notification writes (alert system handles this separately)
+
+### ✅ Phase 2: Medium Complexity
+4. **User Portfolio** - ✅ Updated `getUserPortfolio()` and `findLatestPortfolioDate()` to try:
+   - `signedInUsers/{cid}/portfolio/latest` (user-centric)
+   - `SignedInUserPortfolioData/{date}/{cid}` (root data)
+   - `signed_in_users/19M/snapshots/{date}/parts/part_X` (legacy fallback)
+5. **User Trade History** - ✅ Helper function `findLatestPortfolioDate()` updated to support new structure (used internally by `getUserComputations()`)
+
+### ✅ Phase 3: Higher Complexity
+6. **User Watchlists** - ✅ All CRUD operations updated:
+   - `getUserWatchlists()` - reads from `signedInUsers/{cid}/watchlists`
+   - `getWatchlist()` - reads from new structure
+   - `createWatchlist()` - writes to both structures
+   - `updateWatchlist()` - updates both structures
+   - `deleteWatchlist()` - deletes from both structures
+7. **User Verification** - ✅ Updated `finalizeVerification()` to write to `signedInUsers/{cid}/verification` and store user mapping
+8. **Alert Subscriptions** - ✅ Dual structure implemented:
+   - Per-watchlist: `signedInUsers/{cid}/watchlists/{watchlistId}/subscriptions/{piCid}`
+   - Global: `signedInUsers/{cid}/subscriptions/{piCid}`
+   - All CRUD operations updated
+9. **Popular Investor Data** - ✅ All endpoints updated:
+   - Fetch Requests: `popularInvestors/{piCid}/fetchRequests/{requestId}`
+   - Fetch Status: `popularInvestors/{piCid}/fetchStatus`
+   - User Fetch Requests: `popularInvestors/{piCid}/userFetchRequests/{userCid}` (added to registry)
+   - Reviews: `popularInvestors/{piCid}/reviews/{reviewId}`
+   - Profile Views: `popularInvestors/{piCid}/profileViews/{date}`
+   - Individual Views: `popularInvestors/{piCid}/views/{viewId}`
+
+## Current vs New Collection Paths
+
+### 1. User Portfolio Data ✅ **COMPLETED**
+
+**Current:**
+- `signed_in_users/19M/snapshots/{date}/parts/part_{shardIndex}` (sharded, legacy)
+- Lookup: Search through all parts to find user's CID
+
+**New (Registry):**
+- **Root Data (for computations):** `SignedInUserPortfolioData/{date}/{cid}` ✅ **IMPLEMENTED**
+- **User-Centric (for fallback):** `signedInUsers/{cid}/portfolio/latest` ✅ **IMPLEMENTED**
+
+**API Endpoints Affected:**
+- `GET /user/me/portfolio` - `getUserPortfolio()` in `data_helpers.js` ✅ **UPDATED**
+
+**Migration Strategy:**
+1. ✅ Try new structure first: `signedInUsers/{cid}/portfolio/latest`
+2. ✅ Fallback to root data: `SignedInUserPortfolioData/{date}/{cid}` (find latest date)
+3. ✅ Fallback to legacy: `signed_in_users/19M/snapshots/{date}/parts/part_X` (for backward compatibility)
+
+---
+
+### 2. User Trade History Data ✅ **COMPLETED** (Helper function updated)
+
+**Current:**
+- `signed_in_user_history/19M/snapshots/{date}/parts/part_{shardIndex}` (sharded, legacy)
+- Lookup: Search through all parts to find user's CID
+
+**New (Registry):**
+- **Root Data (for computations):** `SignedInUserTradeHistoryData/{date}/{cid}` ✅ **IMPLEMENTED**
+- **User-Centric (for fallback):** `signedInUsers/{cid}/tradeHistory/latest` ✅ **IMPLEMENTED**
+
+**API Endpoints Affected:**
+- None currently exposed, but used internally by `getUserComputations()` ✅ **Helper function updated**
+
+**Migration Strategy:**
+1. ✅ Try new structure first: `signedInUsers/{cid}/tradeHistory/latest`
+2. ✅ Fallback to root data: `SignedInUserTradeHistoryData/{date}/{cid}` (find latest date)
+3. ✅ Fallback to legacy: `signed_in_user_history/19M/snapshots/{date}/parts/part_X`
+
+---
+
+### 3. User Social Posts ✅ **COMPLETED**
+
+**Current:**
+- `signed_in_users_social/{cid}/posts/{postId}` ✅ **Already using CID!**
+
+**New (Registry):**
+- **Root Data (for computations):** `SignedInUserSocialPostData/{date}/{cid}` ✅ **IMPLEMENTED**
+- **User-Centric (for fallback):** `signedInUsers/{cid}/posts/{postId}` ✅ **IMPLEMENTED**
+
+**API Endpoints Affected:**
+- `GET /user/me/social-posts` - `getUserSocialPosts()` in `data_helpers.js` ✅ **UPDATED**
+
+**Migration Strategy:**
+1. ✅ Try new user-centric: `signedInUsers/{cid}/posts` (ordered by `fetchedAt` desc)
+2. ✅ Fallback to current: `signed_in_users_social/{cid}/posts` (for backward compatibility)
+3. Note: Root data structure is different (nested under date), so not suitable for direct API access
+
+---
+
+### 4. User Notifications
+
+**Current:**
+- `user_notifications/{firebaseUid}/notifications/{notificationId}`
+
+**New (Registry - Updated for CID consistency):**
+- `signedInUsers/{cid}/notifications/{notificationId}` ⚠️ **Change to use CID instead of Firebase UID**
+
+**API Endpoints Affected:**
+- Frontend `SyncNotificationContext` - reads directly, but API can provide CID lookup
+- `POST /user/dev/test-alert` - `sendTestAlert()` in `test_alert_helpers.js` (needs update)
+- Alert system - `processAlertForPI()` in `alert_helpers.js` (needs update)
+
+**Migration Strategy:**
+1. **API Lookup:** When frontend requests notifications, API looks up CID from Firebase UID
+2. **Write:** All notification writes use CID (alert system, test alerts, etc.)
+3. **Read:** API endpoint to get notifications by CID (frontend can call with Firebase UID, API resolves to CID)
+4. **Note:** Frontend `SyncNotificationContext` may need to be updated to use CID, or API can provide a mapping endpoint
+
+**Implementation:**
+- ✅ Create helper: `getCidFromFirebaseUid(db, firebaseUid)` - looks up CID from `signedInUsers` collection (in `collection_helpers.js`)
+- ⚠️ Update all notification writes to use CID (alert system and test alerts already handle this via Firebase UID lookup)
+- ⚠️ Update frontend to either:
+  - Option A: Use CID directly (if available in auth context)
+  - Option B: Call API endpoint that accepts Firebase UID and returns notifications (API does CID lookup)
+
+**Note:** Notifications currently use Firebase UID for frontend authentication compatibility. The helper function exists but notification writes are handled separately by the alert system which already performs CID→Firebase UID mapping.
+
+---
+
+### 5. User Verification ✅ **COMPLETED**
+
+**Current:**
+- `user_verifications/{username}` (keyed by username)
+
+**New (Registry):**
+- `signedInUsers/{cid}/verification` ✅ **IMPLEMENTED** (singleton document, not subcollection)
+
+**API Endpoints Affected:**
+- `POST /user/verify/init` - `initiateVerification()` in `verification_helpers.js` (still uses legacy for init)
+- `POST /user/verify/finalize` - `finalizeVerification()` in `verification_helpers.js` ✅ **UPDATED**
+- `GET /user/me/verification` - `getUserVerification()` in `data_helpers.js` (should read from new structure)
+
+**Migration Strategy:**
+1. **During Init:** 
+   - Store in legacy location: `user_verifications/{username}` (for backward compatibility)
+   - Note: CID not available yet, but that's okay - we'll migrate during finalize
+2. **During Finalize:** ✅ **IMPLEMENTED**
+   - API call to eToro returns `realCID` in the response schema
+   - Store in new location: `signedInUsers/{realCID}/verification` (singleton document)
+   - Also update legacy location for backward compatibility
+   - Store user mapping: `signedInUsers/{realCID}` → `{ etoroCID: realCID, username, ... }` ✅ **IMPLEMENTED**
+3. **Read:** Try new structure first (using CID), fallback to legacy (using username)
+
+**Implementation Notes:**
+- The eToro API response during `finalizeVerification()` includes:
+  - `realCID`: The eToro CID we want to use
+  - `gcid`: Group CID (may be different)
+  - `username`: Username
+  - `aboutMeShort`: Contains the OTP code
+- We can use `realCID` immediately during finalization
+- Store both CID and username in the verification document for lookup flexibility
+
+---
+
+### 6. User Watchlists ✅ **COMPLETED**
+
+**Current:**
+- `watchlists/{userCid}/lists/{watchlistId}` ✅ **Already using CID!**
+- `public_watchlists/{watchlistId}` (for public watchlists) ✅ **Already correct!**
+
+**New (Registry):**
+- `signedInUsers/{cid}/watchlists/{watchlistId}` (private watchlists) ✅ **IMPLEMENTED**
+- `public_watchlists/{watchlistId}` (public watchlists) ✅ **No change needed**
+
+**API Endpoints Affected:**
+- `GET /user/me/watchlists` - `getUserWatchlists()` in `watchlist_helpers.js` ✅ **UPDATED**
+- `GET /user/me/watchlists/:id` - `getWatchlist()` in `watchlist_helpers.js` ✅ **UPDATED**
+- `POST /user/me/watchlists` - `createWatchlist()` in `watchlist_helpers.js` ✅ **UPDATED**
+- `PUT /user/me/watchlists/:id` - `updateWatchlistById()` in `watchlist_helpers.js` ✅ **UPDATED**
+- `DELETE /user/me/watchlists/:id` - `deleteWatchlist()` in `watchlist_helpers.js` ✅ **UPDATED**
+- `POST /user/me/watchlists/:id/copy` - `copyWatchlist()` in `watchlist_helpers.js` (uses same write logic)
+- `POST /user/me/watchlists/:id/publish` - `publishWatchlistVersion()` in `watchlist_helpers.js` (uses public_watchlists)
+- `GET /user/public-watchlists` - `getPublicWatchlists()` in `watchlist_helpers.js` (uses public_watchlists)
+
+**Migration Strategy:**
+1. ✅ **Private Watchlists:** Migrated from `watchlists/{cid}/lists` to `signedInUsers/{cid}/watchlists`
+2. ✅ **Public Watchlists:** Keep in separate collection (already correct)
+3. ✅ **Read:** Try new structure first, fallback to legacy
+4. ✅ **Write:** Write to both locations during migration period
+
+---
+
+### 7. User Sync Requests & Status ✅ **COMPLETED**
+
+**Current:**
+- `user_sync_requests/{cid}/requests/{requestId}` (history)
+- `user_sync_requests/{cid}/global/latest` (singleton)
+
+**New (Registry):**
+- `signedInUsers/{cid}/syncStatus` (singleton) ✅ **IMPLEMENTED**
+- `signedInUsers/{cid}/syncRequests/{requestId}` (history) ✅ **IMPLEMENTED**
+
+**API Endpoints Affected:**
+- `POST /user/:userCid/sync` - `requestUserSync()` in `user_sync_helpers.js` ✅ **UPDATED**
+- `GET /user/:userCid/sync-status` - `getUserSyncStatus()` in `user_sync_helpers.js` ✅ **UPDATED**
+
+**Migration Strategy:**
+1. ✅ **Write:** Write to both locations during migration
+2. ✅ **Read:** Try new structure first, fallback to legacy
+3. **Note:** Task engine already writes to new location, so API should read from there
+
+---
+
+### 8. Alert Subscriptions ✅ **COMPLETED**
+
+**Current (Actual paths in code):**
+- `watchlist_subscriptions/{userCid}/alerts/{piCid}`
+
+**New (Registry - Dual Structure):**
+- **Per-User Subscriptions:** `signedInUsers/{cid}/subscriptions/{piCid}` (global subscriptions, not tied to a watchlist) ✅ **IMPLEMENTED**
+- **Per-Watchlist Subscriptions:** `signedInUsers/{cid}/watchlists/{watchlistId}/subscriptions/{piCid}` (subscriptions for a PI within a specific watchlist) ✅ **IMPLEMENTED**
+
+**API Endpoints Affected:**
+- `POST /user/me/subscriptions` - `subscribeToAlerts()` in `subscription_helpers.js` ✅ **UPDATED**
+- `GET /user/me/subscriptions` - `getUserSubscriptions()` in `subscription_helpers.js` ✅ **UPDATED**
+- `PUT /user/me/subscriptions/:piCid` - `updateSubscription()` in `subscription_helpers.js` ✅ **UPDATED**
+- `DELETE /user/me/subscriptions/:piCid` - `unsubscribeFromAlerts()` in `subscription_helpers.js` ✅ **UPDATED**
+- `POST /user/me/watchlists/:id/subscribe-all` - `subscribeToWatchlist()` in `subscription_helpers.js` ✅ **UPDATED**
+
+**Migration Strategy:**
+1. ✅ **Dual Structure:** 
+   - If subscription has `watchlistId`, store in: `signedInUsers/{cid}/watchlists/{watchlistId}/subscriptions/{piCid}`
+   - If subscription is global (no watchlistId), store in: `signedInUsers/{cid}/subscriptions/{piCid}`
+2. ✅ **Read:** 
+   - For watchlist-specific: Read from `signedInUsers/{cid}/watchlists/{watchlistId}/subscriptions/{piCid}`
+   - For global: Read from `signedInUsers/{cid}/subscriptions/{piCid}`
+   - For "all subscriptions": Read from both locations and merge
+3. ✅ **Write:** Write to both new structure and legacy during migration period
+4. **Backward Compatibility:** Legacy structure `watchlist_subscriptions/{userCid}/alerts/{piCid}` can be migrated by checking if `watchlistId` exists in the subscription data
+
+---
+
+### 9. Popular Investor Data ✅ **COMPLETED**
+
+**Current (Actual paths in code):**
+- `pi_fetch_requests/{piCid}/requests/{requestId}` ⚠️ **Different from registry**
+- `pi_fetch_requests/{piCid}/user_requests/{userCid}` ⚠️ **Different from registry**
+- `pi_fetch_requests/{piCid}/global/latest` ⚠️ **Different from registry**
+- `pi_reviews/{reviewId}` (flat collection, reviewId = `{userCid}_{piCid}`) ⚠️ **Different from registry**
+- `profile_views/{piCid}_{date}` (flat collection with compound key) ⚠️ **Different from registry**
+- `profile_views/individual_views/views/{individualViewId}` ⚠️ **Different from registry**
+
+**New (Registry):**
+- `popularInvestors/{piCid}/fetchRequests/{requestId}` ✅ **IMPLEMENTED**
+- `popularInvestors/{piCid}/fetchStatus` (singleton) ✅ **IMPLEMENTED**
+- `popularInvestors/{piCid}/userFetchRequests/{userCid}` ✅ **IMPLEMENTED** (for user rate limits, added to registry)
+- `popularInvestors/{piCid}/reviews/{reviewId}` ✅ **IMPLEMENTED**
+- `popularInvestors/{piCid}/profileViews/{date}` ✅ **IMPLEMENTED**
+- `popularInvestors/{piCid}/views/{viewId}` ✅ **IMPLEMENTED**
+
+**API Endpoints Affected:**
+- `POST /user/pi/:piCid/request-fetch` - `requestPiFetch()` in `on_demand_fetch_helpers.js` ✅ **UPDATED**
+- `GET /user/pi/:piCid/fetch-status` - `getPiFetchStatus()` in `on_demand_fetch_helpers.js` ✅ **UPDATED**
+- `POST /user/review` - `submitReview()` in `review_helpers.js` ✅ **UPDATED**
+- `GET /user/reviews/:piCid` - `getReviews()` in `review_helpers.js` ✅ **UPDATED**
+- `GET /user/me/review/:piCid` - `getUserReview()` in `review_helpers.js` ✅ **UPDATED**
+- `POST /user/pi/:piCid/track-view` - `trackProfileView()` in `data_helpers.js` ✅ **UPDATED**
+- `GET /user/me/pi-personalized-metrics` - `getSignedInUserPIPersonalizedMetrics()` in `data_helpers.js` (reads profile views)
+
+**Migration Strategy:**
+1. ✅ **Fetch Requests:** Migrated from `pi_fetch_requests/{piCid}/requests` to `popularInvestors/{piCid}/fetchRequests`
+2. ✅ **Fetch Status:** Migrated from `pi_fetch_requests/{piCid}/global/latest` to `popularInvestors/{piCid}/fetchStatus`
+3. ✅ **User Fetch Requests:** Migrated from `pi_fetch_requests/{piCid}/user_requests/{userCid}` to `popularInvestors/{piCid}/userFetchRequests/{userCid}`
+4. ✅ **Reviews:** Migrated from flat `pi_reviews/{reviewId}` to `popularInvestors/{piCid}/reviews/{reviewId}`
+   - **Solution:** Keep reviewId as `{userCid}_{piCid}`, but store under PI's subcollection
+5. ✅ **Profile Views:** Migrated
+   - From `profile_views/{piCid}_{date}` to `popularInvestors/{piCid}/profileViews/{date}`
+   - From `profile_views/individual_views/views/{viewId}` to `popularInvestors/{piCid}/views/{viewId}`
+   - **Solution:** Parse compound key to extract date, store under PI's subcollection
+6. ✅ **Read:** Try new structure first, fallback to legacy
+7. ✅ **Write:** Write to both locations during migration period
+
+---
+
+## Implementation Priority
+
+### Phase 1: Low-Risk, High-Value (Start Here) ✅ **COMPLETED**
+1. ✅ **User Social Posts** - Simple path change, already using CID
+2. ✅ **User Sync Requests** - Task engine already writes to new location
+3. ⚠️ **User Notifications** - Update to use CID (add CID lookup helper) - Helper created, but notifications use Firebase UID for frontend compatibility
+
+### Phase 2: Medium Complexity ✅ **COMPLETED**
+5. ✅ **User Portfolio** - Needs fallback logic, but straightforward
+6. ✅ **User Trade History** - Similar to portfolio, used internally
+
+### Phase 3: Higher Complexity ✅ **COMPLETED**
+7. ✅ **User Watchlists** - Multiple endpoints, but clear migration path
+8. ✅ **User Verification** - CID available during finalize (from API response), migrate during finalization
+9. ✅ **Popular Investor Data** - Multiple subcollections (fetch requests, reviews, profile views) need migration
+10. ✅ **Alert Subscriptions** - Dual structure (per-user vs per-watchlist) implemented
+
+---
+
+## Helper Functions Needed
+
+### 1. CID Lookup Helper ✅ **COMPLETED**
+Helper to get CID from Firebase UID (for API endpoints that receive Firebase UID):
+```javascript
+async function getCidFromFirebaseUid(db, firebaseUid) {
+    const userDoc = await db.collection('signedInUsers').doc(firebaseUid).get();
+    if (!userDoc.exists) return null;
+    const data = userDoc.data();
+    return data.etoroCID || null;
+}
+```
+✅ **IMPLEMENTED** in `collection_helpers.js`
+
+### 2. Collection Path Resolver ✅ **COMPLETED**
+Create a helper that uses `collectionRegistry` to resolve paths:
+```javascript
+function getUserCollectionPath(category, subcategory, params) {
+    const { collectionRegistry } = dependencies;
+    return collectionRegistry.getCollectionPath('signedInUsers', subcategory, params);
+}
+```
+✅ **IMPLEMENTED** in `collection_helpers.js` as `getCollectionPath()`
+
+### 3. Dual Read Helper ✅ **COMPLETED**
+Helper to read from new structure with legacy fallback:
+```javascript
+async function readWithFallback(newPath, legacyPath, db) {
+    // Try new structure first
+    const newDoc = await db.doc(newPath).get();
+    if (newDoc.exists) return newDoc.data();
+    
+    // Fallback to legacy
+    const legacyDoc = await db.doc(legacyPath).get();
+    if (legacyDoc.exists) return legacyDoc.data();
+    
+    return null;
+}
+```
+✅ **IMPLEMENTED** in `collection_helpers.js`
+
+### 4. Dual Write Helper ✅ **COMPLETED**
+Helper to write to both locations during migration:
+```javascript
+async function writeDual(newPath, legacyPath, data, db) {
+    const batch = db.batch();
+    batch.set(db.doc(newPath), data);
+    batch.set(db.doc(legacyPath), data);
+    await batch.commit();
+}
+```
+✅ **IMPLEMENTED** in `collection_helpers.js`
+
+---
+
+## Migration Checklist
+
+### Immediate Actions ✅ **ALL COMPLETED**
+- [x] Inject `collectionRegistry` into API dependencies ✅ **COMPLETED** (in `index.js` line 50)
+- [x] Create helper functions for path resolution and dual read/write ✅ **COMPLETED** (`collection_helpers.js`)
+- [x] Update `getUserSocialPosts()` to use new path ✅ **COMPLETED**
+- [x] Update `getUserPortfolio()` to use new path with fallback ✅ **COMPLETED**
+- [x] Update sync request/status endpoints to use new paths ✅ **COMPLETED**
+
+### Short-Term (Next Sprint) ✅ **ALL COMPLETED**
+- [x] Update watchlist endpoints to use new paths ✅ **COMPLETED**
+- [x] Update verification endpoints (handle CID lookup challenge) ✅ **COMPLETED**
+- [x] Update Popular Investor data endpoints (fetch requests, reviews, profile views) ✅ **COMPLETED**
+- [x] Update subscription endpoints (dual structure: per-user and per-watchlist) ✅ **COMPLETED**
+- [ ] Add migration logging to track usage of legacy vs new paths (optional enhancement)
+
+### Long-Term (Post-Migration)
+- [ ] Remove legacy path fallbacks once migration is complete (after 1-2 month migration period)
+- [ ] Create data migration script to move existing data to new structure
+- [ ] Update documentation
+
+---
+
+## Testing Strategy
+
+1. **Unit Tests:** Test each helper function with both new and legacy paths
+2. **Integration Tests:** Test API endpoints with:
+   - Data only in new structure
+   - Data only in legacy structure
+   - Data in both (should prefer new)
+   - No data (should return 404)
+3. **Migration Tests:** Verify dual writes work correctly
+4. **Backward Compatibility:** Ensure existing users can still access their data
+
+---
+
+## Notes
+
+- **CID Consistency:** Use eToro CID (`realCID`) everywhere for user identification. Firebase UID is only used for authentication, but all data paths should use CID.
+- **CID Lookup:** When frontend provides Firebase UID, API should look up CID from `signedInUsers/{firebaseUid}` mapping document.
+- **Verification:** CID is available immediately during `finalizeVerification()` from the eToro API response (`realCID` field).
+- **Post IDs:** Social posts already use post IDs correctly
+- **Backward Compatibility:** Maintain legacy paths during migration period (estimate 1-2 months)
+- **Data Migration:** Will need a separate script to migrate existing data to new structure
+- **User Mapping:** The `signedInUsers` collection should maintain a mapping: `signedInUsers/{firebaseUid}` → `{ etoroCID, etoroUsername, ... }` for CID lookup
+
+## Path Structure Verification
+
+All implemented paths match the `collection_registry.js` definitions:
+
+### Signed-In Users (User-Centric)
+- ✅ `signedInUsers/{cid}/portfolio/latest` - matches registry
+- ✅ `signedInUsers/{cid}/tradeHistory/latest` - matches registry
+- ✅ `signedInUsers/{cid}/posts/{postId}` - matches registry
+- ✅ `signedInUsers/{cid}/verification` - matches registry (singleton)
+- ✅ `signedInUsers/{cid}/watchlists/{watchlistId}` - matches registry
+- ✅ `signedInUsers/{cid}/syncRequests/{requestId}` - matches registry
+- ✅ `signedInUsers/{cid}/syncStatus` - matches registry (singleton)
+- ✅ `signedInUsers/{cid}/subscriptions/{piCid}` - matches registry
+- ✅ `signedInUsers/{cid}/watchlists/{watchlistId}/subscriptions/{piCid}` - matches registry
+
+### Popular Investors
+- ✅ `popularInvestors/{piCid}/fetchRequests/{requestId}` - matches registry
+- ✅ `popularInvestors/{piCid}/fetchStatus` - matches registry (singleton)
+- ✅ `popularInvestors/{piCid}/userFetchRequests/{userCid}` - matches registry (added during migration)
+- ✅ `popularInvestors/{piCid}/reviews/{reviewId}` - matches registry
+- ✅ `popularInvestors/{piCid}/profileViews/{date}` - matches registry
+- ✅ `popularInvestors/{piCid}/views/{viewId}` - matches registry
+
+### Root Data (Date-Based)
+- ✅ `SignedInUserPortfolioData/{date}/{cid}` - matches registry
+- ✅ `SignedInUserTradeHistoryData/{date}/{cid}` - matches registry
+- ✅ `SignedInUserSocialPostData/{date}/{cid}` - matches registry