coffeeinabit 0.0.47 → 0.0.49

package/tools/get_new_messages.js

@@ -1,7 +1,10 @@
 import axios from 'axios';
 import { safeGoto } from './navigation.js';
 
-export async function executeGetNewMessages(page, action, accessToken) {
+/**
+ * Parse lastChecked timestamp from action parameters
+ */
+function parseLastCheckedTimestamp(action) {
   let lastChecked = action.parameters?.last_checked;
   if (!lastChecked) {
     lastChecked = 0;
@@ -12,14 +15,112 @@ export async function executeGetNewMessages(page, action, accessToken) {
     lastCheckedTimestamp = lastCheckedTimestamp * 1000;
   }
 
-  const allMessageEntities = [];
-  const processedMessages = new Set();
-  const conversationParticipantsMap = new Map();
+  return lastCheckedTimestamp;
+}
+
+/**
+ * Extract participant encoded URL from conversation data
+ */
+function extractParticipantUrl(conversation) {
+  const convUrn = conversation.backendUrn;
+  const participants = conversation.conversationParticipants || [];
+  const otherPerson = participants.find(p => {
+    const member = p.participantType?.member;
+    return member && member.distance !== 'SELF';
+  });
+  
+  if (otherPerson && otherPerson.hostIdentityUrn) {
+    const encodedUrl = otherPerson.hostIdentityUrn.includes('urn:li:fsd_profile:')
+      ? otherPerson.hostIdentityUrn.split('urn:li:fsd_profile:')[1]
+      : '';
+    return { convUrn, encodedUrl };
+  }
+  
+  return { convUrn, encodedUrl: null };
+}
+
+/**
+ * Parse conversation participants from API response
+ */
+function parseConversationParticipants(data, conversationParticipantsMap) {
+  if (data?.data?.messengerConversationsBySyncToken?.elements) {
+    const conversations = data.data.messengerConversationsBySyncToken.elements;
+    conversations.forEach(conv => {
+      const { convUrn, encodedUrl } = extractParticipantUrl(conv);
+      if (encodedUrl) {
+        conversationParticipantsMap.set(convUrn, encodedUrl);
+      }
+    });
+  }
+}
+
+/**
+ * Extract message entity from API response message object
+ */
+function extractMessageEntity(msg) {
+  const senderInfo = msg.sender || msg.actor;
+  const senderMember = senderInfo?.participantType?.member;
+  const authorId = senderInfo?.backendUrn || 'unknown';
+  const authorName = senderMember 
+    ? `${senderMember.firstName?.text || ''} ${senderMember.lastName?.text || ''}`.trim()
+    : 'Unknown';
+  
+  const hostIdentityUrn = senderInfo?.hostIdentityUrn || '';
+  const authorUrl = hostIdentityUrn.includes('urn:li:fsd_profile:') 
+    ? hostIdentityUrn.split('urn:li:fsd_profile:')[1] 
+    : '';
+  
+  const conversationId = msg.backendConversationUrn || '';
+  const isSelf = senderMember?.distance === 'SELF';
+  const messageText = msg.body?.text || '';
+  const deliveredAt = msg.deliveredAt;
+
+  return {
+    authorId,
+    authorName,
+    authorUrl,
+    conversationId,
+    isSelf,
+    messageText,
+    deliveredAt
+  };
+}
 
-  const routeHandler = async (route) => {
+/**
+ * Parse messages from API response
+ */
+function parseMessagesFromResponse(data, allMessageEntities, processedMessages) {
+  if (data?.data?.messengerMessagesBySyncToken?.elements) {
+    const elements = data.data.messengerMessagesBySyncToken.elements;
+    console.log(`[GetNewMessages] Found messengerMessagesBySyncToken with ${elements.length} messages`);
+    
+    let addedCount = 0;
+    let duplicateCount = 0;
+    
+    elements.forEach(msg => {
+      const messageEntity = extractMessageEntity(msg);
+      const messageKey = `${messageEntity.conversationId}-${messageEntity.deliveredAt}-${messageEntity.messageText}`;
+      
+      if (messageEntity.messageText && messageEntity.deliveredAt && !processedMessages.has(messageKey)) {
+        processedMessages.add(messageKey);
+        allMessageEntities.push(messageEntity);
+        addedCount++;
+      } else if (processedMessages.has(messageKey)) {
+        duplicateCount++;
+      }
+    });
+    
+    console.log(`[GetNewMessages] Added ${addedCount} new messages, ${duplicateCount} duplicates, total now: ${allMessageEntities.length}`);
+  }
+}
+
+/**
+ * Create route handler for intercepting LinkedIn API calls
+ */
+function createRouteHandler(allMessageEntities, processedMessages, conversationParticipantsMap) {
+  return async (route) => {
     const url = route.request().url();
     
-    /////
     if (url.includes('voyagerMessagingGraphQL')) {
       const requestBody = route.request().postData();
       if (requestBody) {
@@ -37,27 +138,7 @@ export async function executeGetNewMessages(page, action, accessToken) {
       try {
         const responseBody = await response.text();
         const data = JSON.parse(responseBody);
-        
-        if (data?.data?.messengerConversationsBySyncToken?.elements) {
-          const conversations = data.data.messengerConversationsBySyncToken.elements;
-          conversations.forEach(conv => {
-            const convUrn = conv.backendUrn;
-            const participants = conv.conversationParticipants || [];
-            const otherPerson = participants.find(p => {
-              const member = p.participantType?.member;
-              return member && member.distance !== 'SELF';
-            });
-            
-            if (otherPerson && otherPerson.hostIdentityUrn) {
-              const encodedUrl = otherPerson.hostIdentityUrn.includes('urn:li:fsd_profile:')
-                ? otherPerson.hostIdentityUrn.split('urn:li:fsd_profile:')[1]
-                : '';
-              if (encodedUrl) {
-                conversationParticipantsMap.set(convUrn, encodedUrl);
-              }
-            }
-          });
-        }
+        parseConversationParticipants(data, conversationParticipantsMap);
       } catch (error) {
         console.error('[GetNewMessages] Error parsing conversations:', error);
       }
@@ -75,56 +156,8 @@ export async function executeGetNewMessages(page, action, accessToken) {
       try {
         const responseBody = await response.text();
         const data = JSON.parse(responseBody);
-        
         console.log(`[GetNewMessages] Intercepted messengerMessages response`);
-        
-        if (data?.data?.messengerMessagesBySyncToken?.elements) {
-          const elements = data.data.messengerMessagesBySyncToken.elements;
-          console.log(`[GetNewMessages] Found messengerMessagesBySyncToken with ${elements.length} messages`);
-          
-          let addedCount = 0;
-          let duplicateCount = 0;
-          
-          elements.forEach(msg => {
-            const senderInfo = msg.sender || msg.actor;
-            const senderMember = senderInfo?.participantType?.member;
-            const authorId = senderInfo?.backendUrn || 'unknown';
-            const authorName = senderMember 
-              ? `${senderMember.firstName?.text || ''} ${senderMember.lastName?.text || ''}`.trim()
-              : 'Unknown';
-            
-            const hostIdentityUrn = senderInfo?.hostIdentityUrn || '';
-            const authorUrl = hostIdentityUrn.includes('urn:li:fsd_profile:') 
-              ? hostIdentityUrn.split('urn:li:fsd_profile:')[1] 
-              : '';
-            
-            const conversationId = msg.backendConversationUrn || '';
-            const isSelf = senderMember?.distance === 'SELF';
-            
-            const messageText = msg.body?.text || '';
-            const deliveredAt = msg.deliveredAt;
-
-            const messageKey = `${conversationId}-${deliveredAt}-${messageText}`;
-            
-            if (messageText && deliveredAt && !processedMessages.has(messageKey)) {
-              processedMessages.add(messageKey);
-              allMessageEntities.push({
-                authorId: authorId,
-                authorName: authorName,
-                authorUrl: authorUrl,
-                conversationId: conversationId,
-                isSelf: isSelf,
-                messageText: messageText,
-                deliveredAt: deliveredAt
-              });
-              addedCount++;
-            } else if (processedMessages.has(messageKey)) {
-              duplicateCount++;
-            }
-          });
-          
-          console.log(`[GetNewMessages] Added ${addedCount} new messages, ${duplicateCount} duplicates, total now: ${allMessageEntities.length}`);
-        }
+        parseMessagesFromResponse(data, allMessageEntities, processedMessages);
       } catch (error) {
         console.error('[GetNewMessages] Error parsing response:', error);
       }
@@ -138,73 +171,382 @@ export async function executeGetNewMessages(page, action, accessToken) {
     
     await route.continue();
   };
+}
+
+/**
+ * Navigate to messaging page and wait for UI to load
+ */
+async function navigateToMessagingPage(page) {
+  await safeGoto(page, 'https://www.linkedin.com/messaging/', {
+    waitUntil: 'domcontentloaded',
+    timeout: 60000
+  });
+
+  await page.waitForLoadState('domcontentloaded');
+  await new Promise(resolve => setTimeout(resolve, 2000));
+}
 
+/**
+ * Wait for conversations list to appear
+ * Returns early with empty result if not found (STOPPING POINT #1)
+ */
+async function waitForConversationsList(page, routeHandler) {
+  console.log('[GetNewMessages] Waiting for conversations list to load...');
+  
   try {
-    await page.route('**/voyagerMessagingGraphQL/**', routeHandler);
+    await page.waitForSelector('.msg-conversations-container__conversations-list', {
+      timeout: 30000,
+      state: 'visible'
+    });
+  } catch (error) {
+    console.log('[GetNewMessages] Timeout waiting for conversations list');
+    try {
+      await page.unroute('**/voyagerMessagingGraphQL/**', routeHandler);
+    } catch (unrouteError) {
+      console.log('[GetNewMessages] Error unrouting (non-critical):', unrouteError.message);
+    }
+    // ⚠️ STOPPING POINT #1: Early return when UI not available
+    return { success: false, items: null };
+  }
+  
+  await new Promise(resolve => setTimeout(resolve, 2000));
+  return { success: true, items: null };
+}
 
-    await safeGoto(page, 'https://www.linkedin.com/messaging/', {
-      waitUntil: 'domcontentloaded',
-      timeout: 60000
+/**
+ * Get conversation items from the list
+ * Returns early with empty result if none found (STOPPING POINT #2 & #3)
+ */
+async function getConversationItems(page, routeHandler) {
+  console.log('[GetNewMessages] Looking for conversations list...');
+  const conversationsList = await page.locator('.msg-conversations-container__conversations-list').first();
+  const listCount = await conversationsList.count();
+  console.log('[GetNewMessages] Conversations list count:', listCount);
+  
+  if (listCount === 0) {
+    console.log('[GetNewMessages] No conversations list found, exiting');
+    try {
+      await page.unroute('**/voyagerMessagingGraphQL/**', routeHandler);
+    } catch (unrouteError) {
+      console.log('[GetNewMessages] Error unrouting (non-critical):', unrouteError.message);
+    }
+    // ⚠️ STOPPING POINT #2: No conversations found
+    return { success: false, items: null };
+  }
+
+  console.log('[GetNewMessages] Waiting for conversation items to load...');
+  try {
+    await page.waitForSelector('li.msg-conversation-listitem .msg-conversation-listitem__link', {
+      timeout: 15000,
+      state: 'visible'
     });
+  } catch (error) {
+    console.log('[GetNewMessages] No conversation items found');
+    try {
+      await page.unroute('**/voyagerMessagingGraphQL/**', routeHandler);
+    } catch (unrouteError) {
+      console.log('[GetNewMessages] Error unrouting (non-critical):', unrouteError.message);
+    }
+    // ⚠️ STOPPING POINT #3: No conversation items found
+    return { success: false, items: null };
+  }
 
-    await page.waitForLoadState('domcontentloaded');
-    await new Promise(resolve => setTimeout(resolve, 2000));
+  await new Promise(resolve => setTimeout(resolve, 1000));
+  const conversationItems = await conversationsList.locator('li.msg-conversation-listitem .msg-conversation-listitem__link').all();
+  console.log('[GetNewMessages] Found', conversationItems.length, 'conversation items');
+
+  await new Promise(resolve => setTimeout(resolve, 2000));
+  return { success: true, items: conversationItems };
+}
 
-    console.log('[GetNewMessages] Waiting for conversations list to load...');
+/**
+ * Scroll up in message container to load older messages
+ * Returns true if reached old messages (STOPPING POINT #4)
+ */
+async function scrollToLoadOldMessages(page, messageContainer, allMessageEntities, lastCheckedTimestamp) {
+  let reachedOldMessages = false;
+  let scrollAttempts = 0;
+  const maxScrollAttempts = 20;
+  
+  await messageContainer.hover();
+  await new Promise(resolve => setTimeout(resolve, 500));
+  
+  let noNewMessagesRetries = 0;
+  
+  while (!reachedOldMessages && scrollAttempts < maxScrollAttempts) {
+    const beforeScrollCount = allMessageEntities.length;
+    console.log(`[GetNewMessages] Scroll cycle ${scrollAttempts + 1}, current messages: ${beforeScrollCount}`);
     
-    try {
-      await page.waitForSelector('.msg-conversations-container__conversations-list', {
-        timeout: 30000,
-        state: 'visible'
+
+    for (let i = 0; i < 5; i++) {
+      await messageContainer.evaluate(function(el) {
+        el.scrollBy({ top: -2000, behavior: 'smooth' });
       });
-    } catch (error) {
-      console.log('[GetNewMessages] Timeout waiting for conversations list');
-      try {
-        await page.unroute('**/voyagerMessagingGraphQL/**', routeHandler);
-      } catch (unrouteError) {
-        console.log('[GetNewMessages] Error unrouting (non-critical):', unrouteError.message);
+      console.log(`[GetNewMessages] Scrolled up -2000px (${i + 1}/5)`);
+      await new Promise(resolve => setTimeout(resolve, 1000));
+    }
+    
+    console.log(`[GetNewMessages] Waiting for LinkedIn to load older messages...`);
+    await new Promise(resolve => setTimeout(resolve, 3000));
+    
+    const afterScrollCount = allMessageEntities.length;
+    const newMessagesLoaded = afterScrollCount - beforeScrollCount;
+    console.log(`[GetNewMessages] After 5 scrolls: ${afterScrollCount} messages (${newMessagesLoaded} new)`);
+    
+    if (newMessagesLoaded === 0) {
+      noNewMessagesRetries++;
+      console.log(`[GetNewMessages] No new messages loaded, retry ${noNewMessagesRetries}/2`);
+      
+      if (noNewMessagesRetries >= 2) {
+        console.log('[GetNewMessages] No new messages after 2 retries, reached the top');
+        break;
       }
-      return { newMessagesCount: 0, messages: [] };
+    } else {
+      noNewMessagesRetries = 0;
+      console.log(`[GetNewMessages] Loaded ${newMessagesLoaded} new messages, continuing...`);
     }
     
-    await new Promise(resolve => setTimeout(resolve, 2000));
+    const oldestMessage = allMessageEntities
+      .filter(msg => msg.deliveredAt)
+      .sort((a, b) => a.deliveredAt - b.deliveredAt)[0];
+    
+    // ⚠️ STOPPING POINT #4: Check if reached messages older than lastChecked
+    if (oldestMessage && oldestMessage.deliveredAt < lastCheckedTimestamp) {
+      reachedOldMessages = true;
+      console.log('[GetNewMessages] Reached messages older than lastChecked in this conversation');
+    }
+    
+    scrollAttempts++;
+  }
+  
+  console.log(`[GetNewMessages] Finished scrolling after ${scrollAttempts} attempts`);
+  return reachedOldMessages;
+}
 
-    console.log('[GetNewMessages] Looking for conversations list...');
-    const conversationsList = await page.locator('.msg-conversations-container__conversations-list').first();
-    const listCount = await conversationsList.count();
-    console.log('[GetNewMessages] Conversations list count:', listCount);
+/**
+ * Process a single conversation: click, scroll, and collect messages
+ * Returns shouldContinue flag (STOPPING POINT #5)
+ */
+async function processConversation(page, item, index, allMessageEntities, lastCheckedTimestamp) {
+  const messageCountBefore = allMessageEntities.length;
+  
+  await item.scrollIntoViewIfNeeded();
+  await new Promise(resolve => setTimeout(resolve, 300));
+  await item.click({ timeout: 10000, force: false });
+  await new Promise(resolve => setTimeout(resolve, 2500));
+  
+  const messageContainer = await page.locator('.msg-s-message-list.scrollable').first();
+  const containerExists = await messageContainer.count();
+  
+  if (containerExists > 0) {
+    await scrollToLoadOldMessages(page, messageContainer, allMessageEntities, lastCheckedTimestamp);
+  }
+  
+  const messageCountAfter = allMessageEntities.length;
+  const newMessages = messageCountAfter - messageCountBefore;
+  console.log(`[GetNewMessages] Total messages from conversation ${index + 1}: ${newMessages}`);
+  
+  let shouldContinue = true;
+  if (newMessages > 0) {
+    const conversationMessages = allMessageEntities.slice(messageCountBefore);
+    const latestInThisConversation = conversationMessages
+      .filter(msg => msg.deliveredAt)
+      .sort((a, b) => b.deliveredAt - a.deliveredAt)[0];
     
-    if (listCount === 0) {
-      console.log('[GetNewMessages] No conversations list found, exiting');
-      try {
-        await page.unroute('**/voyagerMessagingGraphQL/**', routeHandler);
-      } catch (unrouteError) {
-        console.log('[GetNewMessages] Error unrouting (non-critical):', unrouteError.message);
-      }
-      return { newMessagesCount: 0, messages: [] };
+    // ⚠️ STOPPING POINT #5: Stop processing if latest message is older than lastChecked
+    if (latestInThisConversation && latestInThisConversation.deliveredAt < lastCheckedTimestamp) {
+      console.log('[GetNewMessages] Latest message in conversation is older than lastChecked, stopping conversation loop');
+      shouldContinue = false;
+    }
+  }
+  
+  return shouldContinue;
+}
+
+/**
+ * Filter and organize new messages by conversation
+ */
+function filterAndOrganizeNewMessages(allMessageEntities, lastCheckedTimestamp) {
+  const newMessages = allMessageEntities.filter(msg => msg.deliveredAt > lastCheckedTimestamp);
+  newMessages.sort((a, b) => a.deliveredAt - b.deliveredAt);
+  console.log('[GetNewMessages] New messages count:', newMessages.length);
+
+  const newMessagesByConversation = {};
+  newMessages.forEach(msg => {
+    if (!newMessagesByConversation[msg.conversationId]) {
+      newMessagesByConversation[msg.conversationId] = [];
+    }
+    newMessagesByConversation[msg.conversationId].push(msg);
+  });
+
+  console.log('[GetNewMessages] Conversations with new messages:', Object.keys(newMessagesByConversation).length);
+  return { newMessages, newMessagesByConversation };
+}
+
+/**
+ * Build map of encoded URLs that need to be decoded
+ */
+function buildEncodedUrlMap(newMessagesByConversation, conversationParticipantsMap) {
+  const encodedUrlMap = new Map();
+  for (const conversationId in newMessagesByConversation) {
+    const encodedUrl = conversationParticipantsMap.get(conversationId);
+    if (encodedUrl) {
+      encodedUrlMap.set(encodedUrl, null);
+    } else {
+      console.log(`[GetNewMessages] No participant found for conversation:`, conversationId);
     }
+  }
+  console.log('[GetNewMessages] Unique profiles to decode:', encodedUrlMap.size);
+  return encodedUrlMap;
+}
 
-    console.log('[GetNewMessages] Waiting for conversation items to load...');
+/**
+ * Decode encoded LinkedIn profile URLs by visiting them
+ */
+async function decodeProfileUrls(page, encodedUrlMap) {
+  const decodedUrlMap = new Map();
+  const maxDecodeAttempts = Math.min(encodedUrlMap.size, 50);
+  let decodeCount = 0;
+  
+  for (const encodedUrl of encodedUrlMap.keys()) {
+    if (decodeCount >= maxDecodeAttempts) {
+      console.log(`[GetNewMessages] Reached max decode attempts (${maxDecodeAttempts}), skipping remaining profiles`);
+      break;
+    }
+    
     try {
-      await page.waitForSelector('li.msg-conversation-listitem .msg-conversation-listitem__link', {
-        timeout: 15000,
-        state: 'visible'
+      console.log(`[GetNewMessages] Decoding profile ${decodeCount + 1}/${encodedUrlMap.size}: ${encodedUrl}`);
+      
+      const decodePromise = safeGoto(page, `https://www.linkedin.com/in/${encodedUrl}`, {
+        waitUntil: 'domcontentloaded',
+        timeout: 30000
       });
+      
+      const timeoutPromise = new Promise((_, reject) => 
+        setTimeout(() => reject(new Error('Decode timeout')), 35000)
+      );
+      
+      await Promise.race([decodePromise, timeoutPromise]);
+      await page.waitForLoadState('domcontentloaded', { timeout: 10000 });
+      await new Promise(resolve => setTimeout(resolve, 1000));
+      
+      const currentUrl = page.url();
+      const match = currentUrl.match(/linkedin\.com\/in\/([^\/\?]+)/);
+      if (match && match[1]) {
+        decodedUrlMap.set(encodedUrl, match[1]);
+        console.log(`[GetNewMessages] Successfully decoded: ${encodedUrl} -> ${match[1]}`);
+      } else {
+        console.log(`[GetNewMessages] Could not extract URL from: ${currentUrl}`);
+      }
+      decodeCount++;
     } catch (error) {
-      console.log('[GetNewMessages] No conversation items found');
+      console.error(`[GetNewMessages] Failed to decode URL ${encodedUrl}:`, error.message);
+      decodeCount++;
+      continue;
+    }
+  }
+  
+  return decodedUrlMap;
+}
+
+/**
+ * ⚠️ SENDING FUNCTIONALITY: Send messages to API endpoint
+ * This is the core functionality for sending collected messages to the backend
+ */
+async function sendMessagesToApi(newMessagesByConversation, conversationParticipantsMap, decodedUrlMap, accessToken) {
+  let sentCount = 0;
+  let failedCount = 0;
+  let totalToSend = 0;
+
+  for (const conversationId in newMessagesByConversation) {
+    const messages = newMessagesByConversation[conversationId];
+    const encodedUrl = conversationParticipantsMap.get(conversationId);
+    const linkedinUrl = encodedUrl ? decodedUrlMap.get(encodedUrl) : null;
+
+    if (!linkedinUrl) {
+      console.log(`[GetNewMessages] Skipping conversation ${conversationId} - could not find or decode other person's URL`);
+      continue;
+    }
+
+    console.log(`[GetNewMessages] Sending ${messages.length} messages to ${linkedinUrl}`);
+    totalToSend += messages.length;
+
+    for (const msg of messages) {
       try {
-        await page.unroute('**/voyagerMessagingGraphQL/**', routeHandler);
-      } catch (unrouteError) {
-        console.log('[GetNewMessages] Error unrouting (non-critical):', unrouteError.message);
+        const payload = {
+          linkedin_url: linkedinUrl,
+          message: msg.messageText,
+          is_receipent: !msg.isSelf,
+          timestamp: msg.deliveredAt
+        };
+
+        console.log(`[GetNewMessages] Posting message: "${msg.messageText.substring(0, 50)}..." (${msg.deliveredAt})`);
+
+        await axios.post('https://api.coffeeinabit.com/messages', payload, {
+          headers: {
+            'Authorization': `Bearer ${accessToken}`,
+            'Content-Type': 'application/json'
+          }
+        });
+
+        sentCount++;
+      } catch (error) {
+        failedCount++;
+        console.error(`[GetNewMessages] Failed to send message to ${linkedinUrl}: "${msg.messageText.substring(0, 50)}..."`);
+        console.error(`[GetNewMessages] Error:`, error.message);
+        if (error.response) {
+          console.error(`[GetNewMessages] Response status:`, error.response.status);
+          console.error(`[GetNewMessages] Response data:`, error.response.data);
+        }
       }
+    }
+  }
+
+  console.log(`[GetNewMessages] Successfully sent ${sentCount}/${totalToSend} messages${failedCount > 0 ? ` (${failedCount} failed)` : ''}`);
+  return { sentCount, failedCount, totalToSend };
+}
+
+/**
+ * Cleanup route handlers
+ */
+async function cleanupRouteHandlers(page, routeHandler) {
+  console.log('[GetNewMessages] Unrouting message handlers...');
+  try {
+    await page.unroute('**/voyagerMessagingGraphQL/**', routeHandler);
+    console.log('[GetNewMessages] Successfully unrouted message handlers');
+  } catch (error) {
+    console.log('[GetNewMessages] Error unrouting (non-critical):', error.message);
+  }
+}
+
+/**
+ * Main function to get new messages from LinkedIn
+ */
+export async function executeGetNewMessages(page, action, accessToken) {
+  const lastCheckedTimestamp = parseLastCheckedTimestamp(action);
+
+  const allMessageEntities = [];
+  const processedMessages = new Set();
+  const conversationParticipantsMap = new Map();
+
+  const routeHandler = createRouteHandler(allMessageEntities, processedMessages, conversationParticipantsMap);
+
+  try {
+    await page.route('**/voyagerMessagingGraphQL/**', routeHandler);
+
+    await navigateToMessagingPage(page);
+
+    const conversationsResult = await waitForConversationsList(page, routeHandler);
+    if (!conversationsResult.success) {
       return { newMessagesCount: 0, messages: [] };
     }
 
-    await new Promise(resolve => setTimeout(resolve, 1000));
-    const conversationItems = await conversationsList.locator('li.msg-conversation-listitem .msg-conversation-listitem__link').all();
-    console.log('[GetNewMessages] Found', conversationItems.length, 'conversation items');
+    const itemsResult = await getConversationItems(page, routeHandler);
+    if (!itemsResult.success) {
+      return { newMessagesCount: 0, messages: [] };
+    }
 
-    await new Promise(resolve => setTimeout(resolve, 2000));
+    const conversationItems = itemsResult.items;
     const initialMessageCount = allMessageEntities.length;
     console.log(`[GetNewMessages] Messages from initially opened conversation: ${initialMessageCount}`);
 
@@ -213,89 +555,7 @@ export async function executeGetNewMessages(page, action, accessToken) {
       const item = conversationItems[i];
       
       try {
-        const messageCountBefore = allMessageEntities.length;
-        
-        await item.scrollIntoViewIfNeeded();
-        await new Promise(resolve => setTimeout(resolve, 300));
-        await item.click({ timeout: 10000, force: false });
-        await new Promise(resolve => setTimeout(resolve, 2500));
-        
-        const messageContainer = await page.locator('.msg-s-message-list.scrollable').first();
-        const containerExists = await messageContainer.count();
-        
-        if (containerExists > 0) {
-          let reachedOldMessages = false;
-          let scrollAttempts = 0;
-          const maxScrollAttempts = 20;
-          
-          await messageContainer.hover();
-          await new Promise(resolve => setTimeout(resolve, 500));
-          
-          let noNewMessagesRetries = 0;
-          
-          while (!reachedOldMessages && scrollAttempts < maxScrollAttempts) {
-            const beforeScrollCount = allMessageEntities.length;
-            console.log(`[GetNewMessages] Scroll cycle ${scrollAttempts + 1}, current messages: ${beforeScrollCount}`);
-            
-
-            for (let i = 0; i < 5; i++) {
-              await messageContainer.evaluate((el) => {
-                el.scrollBy({ top: -2000, behavior: 'smooth' });
-              });
-              console.log(`[GetNewMessages] Scrolled up -2000px (${i + 1}/5)`);
-              await new Promise(resolve => setTimeout(resolve, 1000));
-            }
-            
-            console.log(`[GetNewMessages] Waiting for LinkedIn to load older messages...`);
-            await new Promise(resolve => setTimeout(resolve, 3000));
-            
-            const afterScrollCount = allMessageEntities.length;
-            const newMessagesLoaded = afterScrollCount - beforeScrollCount;
-            console.log(`[GetNewMessages] After 5 scrolls: ${afterScrollCount} messages (${newMessagesLoaded} new)`);
-            
-            if (newMessagesLoaded === 0) {
-              noNewMessagesRetries++;
-              console.log(`[GetNewMessages] No new messages loaded, retry ${noNewMessagesRetries}/2`);
-              
-              if (noNewMessagesRetries >= 2) {
-                console.log('[GetNewMessages] No new messages after 2 retries, reached the top');
-                break;
-              }
-            } else {
-              noNewMessagesRetries = 0;
-              console.log(`[GetNewMessages] Loaded ${newMessagesLoaded} new messages, continuing...`);
-            }
-            
-            const oldestMessage = allMessageEntities
-              .filter(msg => msg.deliveredAt)
-              .sort((a, b) => a.deliveredAt - b.deliveredAt)[0];
-            
-            if (oldestMessage && oldestMessage.deliveredAt < lastCheckedTimestamp) {
-              reachedOldMessages = true;
-              console.log('[GetNewMessages] Reached messages older than lastChecked in this conversation');
-            }
-            
-            scrollAttempts++;
-          }
-          
-          console.log(`[GetNewMessages] Finished scrolling after ${scrollAttempts} attempts`);
-        }
-        
-        const messageCountAfter = allMessageEntities.length;
-        const newMessages = messageCountAfter - messageCountBefore;
-        console.log(`[GetNewMessages] Total messages from conversation ${i + 1}: ${newMessages}`);
-        
-        if (newMessages > 0) {
-          const conversationMessages = allMessageEntities.slice(messageCountBefore);
-          const latestInThisConversation = conversationMessages
-            .filter(msg => msg.deliveredAt)
-            .sort((a, b) => b.deliveredAt - a.deliveredAt)[0];
-          
-          if (latestInThisConversation && latestInThisConversation.deliveredAt < lastCheckedTimestamp) {
-            console.log('[GetNewMessages] Latest message in conversation is older than lastChecked, stopping conversation loop');
-            shouldContinue = false;
-          }
-        }
+        shouldContinue = await processConversation(page, item, i, allMessageEntities, lastCheckedTimestamp);
       } catch (error) {
         console.error('[GetNewMessages] Error with conversation:', error.message);
       }
@@ -304,133 +564,18 @@ export async function executeGetNewMessages(page, action, accessToken) {
     console.log('[GetNewMessages] Finished processing all conversations, waiting before cleanup...');
     await new Promise(resolve => setTimeout(resolve, 2000));
 
-    console.log('[GetNewMessages] Unrouting message handlers...');
-    try {
-      await page.unroute('**/voyagerMessagingGraphQL/**', routeHandler);
-      console.log('[GetNewMessages] Successfully unrouted message handlers');
-    } catch (error) {
-      console.log('[GetNewMessages] Error unrouting (non-critical):', error.message);
-    }
+    await cleanupRouteHandlers(page, routeHandler);
 
     console.log('[GetNewMessages] Total messages collected:', allMessageEntities.length);
     console.log('[GetNewMessages] Conversations with participants mapped:', conversationParticipantsMap.size);
 
-    const newMessages = allMessageEntities.filter(msg => msg.deliveredAt > lastCheckedTimestamp);
-    newMessages.sort((a, b) => a.deliveredAt - b.deliveredAt);
-    console.log('[GetNewMessages] New messages count:', newMessages.length);
-
-    const newMessagesByConversation = {};
-    newMessages.forEach(msg => {
-      if (!newMessagesByConversation[msg.conversationId]) {
-        newMessagesByConversation[msg.conversationId] = [];
-      }
-      newMessagesByConversation[msg.conversationId].push(msg);
-    });
-
-    console.log('[GetNewMessages] Conversations with new messages:', Object.keys(newMessagesByConversation).length);
-
-    const encodedUrlMap = new Map();
-    for (const conversationId in newMessagesByConversation) {
-      const encodedUrl = conversationParticipantsMap.get(conversationId);
-      if (encodedUrl) {
-        encodedUrlMap.set(encodedUrl, null);
-      } else {
-        console.log(`[GetNewMessages] No participant found for conversation:`, conversationId);
-      }
-    }
-    console.log('[GetNewMessages] Unique profiles to decode:', encodedUrlMap.size);
+    const { newMessages, newMessagesByConversation } = filterAndOrganizeNewMessages(allMessageEntities, lastCheckedTimestamp);
 
-    const decodedUrlMap = new Map();
-    const maxDecodeAttempts = Math.min(encodedUrlMap.size, 50);
-    let decodeCount = 0;
+    const encodedUrlMap = buildEncodedUrlMap(newMessagesByConversation, conversationParticipantsMap);
+    const decodedUrlMap = await decodeProfileUrls(page, encodedUrlMap);
     
-    for (const encodedUrl of encodedUrlMap.keys()) {
-      if (decodeCount >= maxDecodeAttempts) {
-        console.log(`[GetNewMessages] Reached max decode attempts (${maxDecodeAttempts}), skipping remaining profiles`);
-        break;
-      }
-      
-      try {
-        console.log(`[GetNewMessages] Decoding profile ${decodeCount + 1}/${encodedUrlMap.size}: ${encodedUrl}`);
-        
-        const decodePromise = safeGoto(page, `https://www.linkedin.com/in/${encodedUrl}`, {
-          waitUntil: 'domcontentloaded',
-          timeout: 30000
-        });
-        
-        const timeoutPromise = new Promise((_, reject) => 
-          setTimeout(() => reject(new Error('Decode timeout')), 35000)
-        );
-        
-        await Promise.race([decodePromise, timeoutPromise]);
-        await page.waitForLoadState('domcontentloaded', { timeout: 10000 });
-        await new Promise(resolve => setTimeout(resolve, 1000));
-        
-        const currentUrl = page.url();
-        const match = currentUrl.match(/linkedin\.com\/in\/([^\/\?]+)/);
-        if (match && match[1]) {
-          decodedUrlMap.set(encodedUrl, match[1]);
-          console.log(`[GetNewMessages] Successfully decoded: ${encodedUrl} -> ${match[1]}`);
-        } else {
-          console.log(`[GetNewMessages] Could not extract URL from: ${currentUrl}`);
-        }
-        decodeCount++;
-      } catch (error) {
-        console.error(`[GetNewMessages] Failed to decode URL ${encodedUrl}:`, error.message);
-        decodeCount++;
-        continue;
-      }
-    }
-    
-    let sentCount = 0;
-    let failedCount = 0;
-    let totalToSend = 0;
-
-    for (const conversationId in newMessagesByConversation) {
-      const messages = newMessagesByConversation[conversationId];
-      const encodedUrl = conversationParticipantsMap.get(conversationId);
-      const linkedinUrl = encodedUrl ? decodedUrlMap.get(encodedUrl) : null;
-
-      if (!linkedinUrl) {
-        console.log(`[GetNewMessages] Skipping conversation ${conversationId} - could not find or decode other person's URL`);
-        continue;
-      }
-
-      console.log(`[GetNewMessages] Sending ${messages.length} messages to ${linkedinUrl}`);
-      totalToSend += messages.length;
-
-      for (const msg of messages) {
-        try {
-          const payload = {
-            linkedin_url: linkedinUrl,
-            message: msg.messageText,
-            is_receipent: !msg.isSelf,
-            timestamp: msg.deliveredAt
-          };
-
-          console.log(`[GetNewMessages] Posting message: "${msg.messageText.substring(0, 50)}..." (${msg.deliveredAt})`);
-
-          await axios.post('https://api.coffeeinabit.com/messages', payload, {
-            headers: {
-              'Authorization': `Bearer ${accessToken}`,
-              'Content-Type': 'application/json'
-            }
-          });
-
-          sentCount++;
-        } catch (error) {
-          failedCount++;
-          console.error(`[GetNewMessages] Failed to send message to ${linkedinUrl}: "${msg.messageText.substring(0, 50)}..."`);
-          console.error(`[GetNewMessages] Error:`, error.message);
-          if (error.response) {
-            console.error(`[GetNewMessages] Response status:`, error.response.status);
-            console.error(`[GetNewMessages] Response data:`, error.response.data);
-          }
-        }
-      }
-    }
-
-    console.log(`[GetNewMessages] Successfully sent ${sentCount}/${totalToSend} messages${failedCount > 0 ? ` (${failedCount} failed)` : ''}`);
+    // ⚠️ SENDING FUNCTIONALITY: Send all collected messages to API
+    await sendMessagesToApi(newMessagesByConversation, conversationParticipantsMap, decodedUrlMap, accessToken);
 
     return {
       newMessagesCount: newMessages.length,