@lhremote/core 0.8.0 → 0.10.0

package/dist/operations/get-feed.js

@@ -1,164 +1,355 @@
 // SPDX-License-Identifier: AGPL-3.0-only
 // Copyright (C) 2026 Oleksii PELYKH
+import { resolveInstancePort } from "../cdp/index.js";
 import { CDPClient } from "../cdp/client.js";
 import { discoverTargets } from "../cdp/discovery.js";
-import { VoyagerInterceptor } from "../voyager/interceptor.js";
-import { DEFAULT_CDP_PORT } from "../constants.js";
+import { humanizedScrollY, humanizedScrollToByIndex, retryInteraction } from "../linkedin/dom-automation.js";
+import { delay as utilsDelay, gaussianDelay, gaussianBetween, maybeHesitate, maybeBreak, simulateReadingTime } from "../utils/delay.js";
+import { navigateAwayIf } from "./navigate-away.js";
 // ---------------------------------------------------------------------------
-// Parsing helpers
+// In-page DOM scraping script
 // ---------------------------------------------------------------------------
 /**
- * Resolve a text value that may be a string or `{ text: string }`.
+ * JavaScript source evaluated inside the LinkedIn page context via
+ * `Runtime.evaluate`.  Returns an array of {@link RawDomPost} objects
+ * (without URNs — those are extracted separately via the three-dot menu).
+ *
+ * ## Discovery strategy (2026-03 onwards)
+ *
+ * LinkedIn's SSR feed uses `div[data-testid="mainFeed"]` as the feed
+ * list (`role="list"`) and `div[role="listitem"]` for each post.
+ * CSS class names are obfuscated hashes (CSS Modules), so the script
+ * relies on semantic attributes and structural heuristics.
+ *
+ * Post URNs are NOT available in the DOM.  They are extracted in a
+ * separate phase by opening each post's three-dot menu, clicking
+ * "Copy link to post", and deriving the URN from the captured URL.
  */
-function resolveText(value) {
-    if (value === undefined || value === null)
-        return null;
-    if (typeof value === "string")
-        return value;
-    return value.text ?? null;
+const SCRAPE_FEED_POSTS_SCRIPT = `(() => {
+  const posts = [];
+
+  // --- Step 1: Find the feed list via data-testid ---
+  const feedList = document.querySelector('[data-testid="mainFeed"]');
+  if (!feedList) return posts;
+
+  // --- Step 2: Iterate listitem children ---
+  const items = feedList.querySelectorAll('div[role="listitem"]');
+  for (const wrapper of items) {
+    // The listitem wraps the actual post content in nested divs.
+    // Some listitems may be zero-height (virtualized/hidden) or
+    // non-post items (composer, suggestions).
+    const item = wrapper;
+    if (item.offsetHeight < 100) continue;
+
+    // Detect real posts: must have a three-dot menu button
+    const menuBtn = item.querySelector('button[aria-label^="Open control menu for post"]');
+    if (!menuBtn) continue;
+
+    // --- Author info ---
+    let authorName = null;
+    let authorHeadline = null;
+    let authorProfileUrl = null;
+
+    const authorLink = item.querySelector('a[href*="/in/"], a[href*="/company/"]');
+    if (authorLink) {
+      authorProfileUrl = authorLink.href.split('?')[0] || null;
+      const nameEl = authorLink.querySelector('span[dir="ltr"], span[aria-hidden="true"]')
+        || authorLink;
+      const rawName = (nameEl.textContent || '').trim();
+      authorName = rawName || null;
+    }
+
+    // Author headline: look for a short descriptive text near the author.
+    const allSpans = item.querySelectorAll('span');
+    for (const span of allSpans) {
+      const txt = (span.textContent || '').trim();
+      if (
+        txt &&
+        txt.length > 5 &&
+        txt.length < 200 &&
+        txt !== authorName &&
+        !txt.match(/^\\d+[smhdw]$/) &&
+        !txt.match(/^\\d[\\d,]*\\s+(reactions?|comments?|reposts?|likes?)$/i) &&
+        !txt.match(/^Follow$|^Promoted$/i)
+      ) {
+        authorHeadline = txt;
+        break;
+      }
+    }
+
+    // --- Post text ---
+    let text = null;
+    const ltrSpans = item.querySelectorAll('span[dir="ltr"]');
+    let longestText = '';
+    for (const span of ltrSpans) {
+      const txt = (span.textContent || '').trim();
+      if (txt.length > longestText.length && txt !== authorName && txt !== authorHeadline) {
+        longestText = txt;
+      }
+    }
+    if (longestText.length > 20) {
+      text = longestText;
+    }
+
+    // --- Media type ---
+    let mediaType = null;
+    if (item.querySelector('video')) {
+      mediaType = 'video';
+    } else if (item.querySelector('img[src*="media.licdn.com"]')) {
+      const imgs = item.querySelectorAll('img[src*="media.licdn.com"]');
+      for (const img of imgs) {
+        if (img.offsetHeight > 100) { mediaType = 'image'; break; }
+      }
+    }
+
+    // --- Engagement counts ---
+    const itemText = item.textContent || '';
+
+    function parseCount(pattern) {
+      const m = itemText.match(pattern);
+      if (!m) return 0;
+      const raw = m[1].replace(/,/g, '');
+      const num = parseInt(raw, 10);
+      return isNaN(num) ? 0 : num;
+    }
+
+    const reactionCount = parseCount(/(\\d[\\d,]*)\\s+reactions?/i);
+    const commentCount = parseCount(/(\\d[\\d,]*)\\s+comments?/i);
+    const shareCount = parseCount(/(\\d[\\d,]*)\\s+reposts?/i);
+
+    // --- Timestamp ---
+    let timestamp = null;
+    const timeEl = item.querySelector('time');
+    if (timeEl) {
+      const dt = timeEl.getAttribute('datetime');
+      if (dt) timestamp = dt;
+    }
+    if (!timestamp) {
+      const timeMatch = itemText.match(/(?:^|\\s)(\\d+[smhdw])(?:\\s|$|\\u00B7|\\xB7)/);
+      if (timeMatch) timestamp = timeMatch[1];
+    }
+
+    posts.push({
+      url: null,
+      authorName: authorName,
+      authorHeadline: authorHeadline,
+      authorProfileUrl: authorProfileUrl,
+      text: text,
+      mediaType: mediaType,
+      reactionCount: reactionCount,
+      commentCount: commentCount,
+      shareCount: shareCount,
+      timestamp: timestamp,
+    });
+  }
+
+  return posts;
+})()`;
+/**
+ * Legacy scraping script using structural heuristics to find the feed
+ * container.  Used by search-posts which navigates to search result
+ * pages where `data-testid="mainFeed"` is not present.
+ *
+ * @internal Exported for reuse by search-posts.
+ */
+export { SCRAPE_FEED_POSTS_SCRIPT as SCRAPE_FEED_SCRIPT };
+// ---------------------------------------------------------------------------
+// URL capture via three-dot menu → "Copy link to post"
+// ---------------------------------------------------------------------------
+/** CSS selector for feed post menu buttons. */
+const FEED_MENU_BUTTON_SELECTOR = '[data-testid="mainFeed"] div[role="listitem"] button[aria-label^="Open control menu for post"]';
+/**
+ * Capture the post URL for a single feed item by opening its three-dot
+ * menu and clicking "Copy link to post".
+ *
+ * Requires the clipboard interceptor to be installed beforehand via
+ * {@link installClipboardInterceptor}.
+ *
+ * @returns The post URL (query params stripped) or `null` if capture failed.
+ */
+async function capturePostUrl(client, postIndex, mouse) {
+    const MAX_ATTEMPTS = 3;
+    for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
+        await maybeHesitate(); // Probabilistic pause before menu interaction
+        // Reset clipboard capture
+        await client.evaluate(`window.__capturedClipboard = null;`);
+        // Scroll the menu button into view (humanized when mouse available)
+        await humanizedScrollToByIndex(client, FEED_MENU_BUTTON_SELECTOR, postIndex, mouse);
+        // Click the menu button
+        const clicked = await client.evaluate(`(() => {
+      const btns = document.querySelectorAll(
+        ${JSON.stringify(FEED_MENU_BUTTON_SELECTOR)}
+      );
+      const btn = btns[${postIndex}];
+      if (!btn) return false;
+      btn.click();
+      return true;
+    })()`);
+        if (!clicked)
+            return null; // No menu button — structural, retrying won't help
+        await gaussianDelay(700, 100, 500, 900);
+        // Click "Copy link to post" menu item
+        await client.evaluate(`(() => {
+      for (const el of document.querySelectorAll('[role="menuitem"]')) {
+        if (el.textContent.trim() === 'Copy link to post') {
+          el.click();
+          return;
+        }
+      }
+    })()`);
+        await gaussianDelay(550, 75, 400, 700);
+        // Read captured URL
+        const postUrl = await client.evaluate(`window.__capturedClipboard`);
+        if (postUrl) {
+            // Strip query parameters
+            return postUrl.split("?")[0] ?? postUrl;
+        }
+        // Dismiss any open menu before retrying
+        await client.evaluate(`(() => {
+      document.dispatchEvent(new KeyboardEvent('keydown', { key: 'Escape', bubbles: true }));
+    })()`);
+        // Escalating retry delays: longer waits on later attempts
+        const retryDelays = [
+            { mean: 700, stdDev: 200 },
+            { mean: 1_200, stdDev: 400 },
+            { mean: 2_500, stdDev: 800 },
+        ];
+        const rd = retryDelays[attempt] ?? retryDelays[2];
+        await gaussianDelay(rd.mean, rd.stdDev, rd.mean * 0.5, rd.mean * 1.5);
+        // 50% chance of a small "confusion" scroll to reset visual state
+        if (Math.random() < 0.5) {
+            const scrollDist = Math.round(gaussianBetween(75, 15, 50, 100));
+            const dir = Math.random() < 0.5 ? -1 : 1;
+            await humanizedScrollY(client, scrollDist * dir, 300, 400, mouse);
+            await gaussianDelay(300, 100, 150, 500);
+        }
+    }
+    return null;
+}
+/**
+ * Install a clipboard interceptor that captures `navigator.clipboard.writeText`
+ * calls into `window.__capturedClipboard`.  Required because Electron's
+ * clipboard API is broken (readText returns `{}`).
+ */
+async function installClipboardInterceptor(client) {
+    await client.evaluate(`navigator.clipboard.writeText = function(text) {
+      window.__capturedClipboard = text;
+      return Promise.resolve();
+    };`);
 }
+// ---------------------------------------------------------------------------
+// Parsing helpers
+// ---------------------------------------------------------------------------
 /**
  * Extract hashtags from post text.
  */
-function extractHashtags(text) {
+export function extractHashtags(text) {
     if (!text)
         return [];
     const matches = text.match(/#[\w\u00C0-\u024F]+/g);
     return matches ? [...new Set(matches.map((t) => t.slice(1)))] : [];
 }
 /**
- * Infer media type from Voyager content metadata.
+ * Parse a relative timestamp string (e.g. "52m", "16h", "2d", "1w") or an
+ * ISO date into epoch milliseconds.  Returns null for unrecognised formats.
  */
-function inferMediaType(content) {
-    if (!content)
+export function parseTimestamp(raw) {
+    if (!raw)
         return null;
-    if (content.mediaCategory) {
-        const cat = content.mediaCategory.toLowerCase();
-        if (cat.includes("image"))
-            return "image";
-        if (cat.includes("video"))
-            return "video";
-        if (cat.includes("article"))
-            return "article";
-        if (cat.includes("document"))
-            return "document";
-        return cat;
-    }
-    const type = content.$type ?? "";
-    if (type.includes("Image"))
-        return "image";
-    if (type.includes("Video"))
-        return "video";
-    if (type.includes("Article") || content.navigationUrl)
-        return "article";
-    if (type.includes("Document"))
-        return "document";
-    if (type)
-        return type.split(".").pop()?.toLowerCase() ?? null;
-    return null;
+    // ISO datetime
+    const asDate = Date.parse(raw);
+    if (!isNaN(asDate))
+        return asDate;
+    // Relative time: Ns, Nm, Nh, Nd, Nw
+    const match = raw.match(/^(\d+)([smhdw])$/);
+    if (!match)
+        return null;
+    const value = parseInt(match[1] ?? "0", 10);
+    const unit = match[2] ?? "";
+    const now = Date.now();
+    const multipliers = {
+        s: 1_000,
+        m: 60_000,
+        h: 3_600_000,
+        d: 86_400_000,
+        w: 604_800_000,
+    };
+    return now - value * (multipliers[unit] ?? 0);
 }
 /**
- * Build a LinkedIn post URL from an update URN.
+ * Build a LinkedIn post URL from an activity URN.
  */
-function buildPostUrl(urn) {
+/** @internal Exported for reuse by search-posts. */
+export function buildPostUrl(urn) {
     return `https://www.linkedin.com/feed/update/${urn}/`;
 }
 /**
- * Parse the Voyager feed response into normalised FeedPost entries.
+ * Convert raw DOM-scraped posts into normalised FeedPost entries.
  */
-function parseFeedResponse(raw) {
-    const elements = raw.data?.elements ?? raw.elements ?? [];
-    const metadata = raw.data?.metadata ?? raw.metadata;
-    const included = raw.included ?? [];
-    // Build lookup for included entities (actors, social details)
-    const entitiesByUrn = new Map();
-    for (const entity of included) {
-        if (entity.entityUrn) {
-            entitiesByUrn.set(entity.entityUrn, entity);
-        }
-    }
-    const posts = [];
-    for (const el of elements) {
-        const urn = el.updateUrn ?? el.urn;
-        if (!urn)
-            continue;
-        // Resolve actor — inline or via included entities
-        let authorName = "";
-        let authorHeadline = null;
-        let authorProfileUrl = null;
-        if (el.actor) {
-            authorName = resolveText(el.actor.name) ?? "";
-            authorHeadline = resolveText(el.actor.description);
-            authorProfileUrl = el.actor.navigationUrl ?? null;
-        }
-        else if (el["*actor"]) {
-            const actorEntity = entitiesByUrn.get(el["*actor"]);
-            if (actorEntity) {
-                // Person actor
-                if (actorEntity.firstName || actorEntity.lastName) {
-                    authorName = [actorEntity.firstName, actorEntity.lastName]
-                        .filter(Boolean)
-                        .join(" ");
-                    authorHeadline =
-                        resolveText(actorEntity.headline) ??
-                            actorEntity.occupation ??
-                            null;
-                    if (actorEntity.publicIdentifier) {
-                        authorProfileUrl = `https://www.linkedin.com/in/${actorEntity.publicIdentifier}/`;
-                    }
-                }
-                else {
-                    // Company / page actor
-                    authorName = resolveText(actorEntity.name) ?? "";
-                    authorHeadline = resolveText(actorEntity.description);
-                    authorProfileUrl = actorEntity.navigationUrl ?? null;
-                }
-            }
-        }
-        // Post text
-        const text = resolveText(el.commentary?.text) ?? null;
-        // Media type
-        const mediaType = inferMediaType(el.content);
-        // Engagement counts — inline or via included social detail
-        let reactionCount = 0;
-        let commentCount = 0;
-        let shareCount = 0;
-        const socialCounts = el.socialDetail?.totalSocialActivityCounts;
-        if (socialCounts) {
-            reactionCount = socialCounts.numLikes ?? 0;
-            commentCount = socialCounts.numComments ?? 0;
-            shareCount = socialCounts.numShares ?? 0;
-        }
-        else if (el["*socialDetail"]) {
-            const socialEntity = entitiesByUrn.get(el["*socialDetail"]);
-            if (socialEntity?.totalSocialActivityCounts) {
-                reactionCount = socialEntity.totalSocialActivityCounts.numLikes ?? 0;
-                commentCount = socialEntity.totalSocialActivityCounts.numComments ?? 0;
-                shareCount = socialEntity.totalSocialActivityCounts.numShares ?? 0;
-            }
+/** @internal Exported for reuse by search-posts. */
+export function mapRawPosts(raw) {
+    return raw.map((r) => ({
+        url: r.url ?? null,
+        authorName: r.authorName,
+        authorHeadline: r.authorHeadline,
+        authorProfileUrl: r.authorProfileUrl,
+        authorPublicId: null,
+        text: r.text,
+        mediaType: r.mediaType,
+        reactionCount: r.reactionCount,
+        commentCount: r.commentCount,
+        shareCount: r.shareCount,
+        timestamp: parseTimestamp(r.timestamp),
+        hashtags: extractHashtags(r.text),
+    }));
+}
+// ---------------------------------------------------------------------------
+// Scroll helper
+// ---------------------------------------------------------------------------
+/** @internal Exported for reuse by other operations. */
+export const delay = utilsDelay;
+/**
+ * Scroll the feed down by a randomised viewport-like distance.
+ *
+ * The distance varies between 600–1000 px per scroll to avoid the
+ * detection signal of a perfectly uniform scroll cadence.
+ *
+ * When a {@link HumanizedMouse} is provided, scrolling uses incremental
+ * mouse-wheel strokes (150 px / 25 ms) that mimic a physical scroll
+ * wheel.  Falls back to a single CDP `mouseWheel` event otherwise.
+ *
+ * @internal Exported for reuse by search-posts.
+ */
+export async function scrollFeed(client, mouse) {
+    const distance = Math.round(gaussianBetween(800, 100, 600, 1_000));
+    const x = Math.round(gaussianBetween(350, 100, 150, 550));
+    const y = Math.round(gaussianBetween(400, 80, 250, 550));
+    await humanizedScrollY(client, distance, x, y, mouse);
+}
+// ---------------------------------------------------------------------------
+// Wait for feed to load
+// ---------------------------------------------------------------------------
+/** @internal Exported for reuse by search-posts. */
+export async function waitForFeedLoad(client, timeoutMs = 15_000) {
+    const deadline = Date.now() + timeoutMs;
+    while (Date.now() < deadline) {
+        const ready = await client.evaluate(`(() => {
+      const feed = document.querySelector('[data-testid="mainFeed"]');
+      if (!feed) return false;
+      const items = feed.querySelectorAll('div[role="listitem"]');
+      // Ready when at least one listitem has a post menu button
+      for (const item of items) {
+        if (item.querySelector('button[aria-label^="Open control menu for post"]')) {
+          return true;
         }
-        // Timestamp
-        const timestamp = el.createdAt ?? el.publishedAt ?? null;
-        // Hashtags
-        const hashtags = extractHashtags(text);
-        posts.push({
-            urn,
-            url: buildPostUrl(urn),
-            authorName,
-            authorHeadline,
-            authorProfileUrl,
-            authorPublicId: null,
-            text,
-            mediaType,
-            reactionCount,
-            commentCount,
-            shareCount,
-            timestamp,
-            hashtags,
-        });
+      }
+      return false;
+    })()`);
+        if (ready)
+            return;
+        await delay(500);
     }
-    const nextCursor = metadata?.paginationToken ?? metadata?.nextPageToken ?? null;
-    return { posts, nextCursor };
+    throw new Error("Timed out waiting for feed posts to appear in the DOM");
 }
 // ---------------------------------------------------------------------------
 // Public API
@@ -166,19 +357,20 @@ function parseFeedResponse(raw) {
 /**
  * Read the LinkedIn home feed and return structured post data.
  *
- * Connects to the LinkedIn webview in LinkedHelper and calls the
- * Voyager feed updates API. Supports cursor-based pagination: the
- * first call returns the first page; pass the returned `nextCursor`
- * in subsequent calls to retrieve additional pages.
+ * Navigates to the feed page and extracts posts from the rendered DOM.
+ * Supports cursor-based pagination: the first call returns the first page;
+ * pass the returned `nextCursor` in subsequent calls to retrieve additional
+ * pages via scroll + re-scrape.
  *
  * @param input - Pagination parameters and CDP connection options.
  * @returns Feed posts with a cursor for the next page.
  */
 export async function getFeed(input) {
-    const cdpPort = input.cdpPort ?? DEFAULT_CDP_PORT;
+    const cdpPort = await resolveInstancePort(input.cdpPort, input.cdpHost);
     const cdpHost = input.cdpHost ?? "127.0.0.1";
     const allowRemote = input.allowRemote ?? false;
     const count = input.count ?? 10;
+    const cursor = input.cursor ?? null;
     // Enforce loopback guard
     if (!allowRemote && cdpHost !== "127.0.0.1" && cdpHost !== "localhost") {
         throw new Error(`Non-loopback CDP host "${cdpHost}" requires --allow-remote. ` +
@@ -193,25 +385,80 @@ export async function getFeed(input) {
     const client = new CDPClient(cdpPort, { host: cdpHost, allowRemote });
     await client.connect(linkedInTarget.id);
     try {
-        const voyager = new VoyagerInterceptor(client);
-        let path = `/voyager/api/feed/dash/feedUpdates` +
-            `?count=${String(count)}&q=feedByType&moduleKey=feed`;
-        if (input.cursor) {
-            path += `&paginationToken=${encodeURIComponent(input.cursor)}`;
+        const mouse = input.mouse ?? null;
+        // Navigate away if already on the feed page to force a fresh load
+        await navigateAwayIf(client, "/feed");
+        await client.navigate("https://www.linkedin.com/feed/");
+        // Wait for the initial feed content to render
+        await waitForFeedLoad(client);
+        // Collect posts — scroll to load more if needed
+        const maxScrollAttempts = 10;
+        let allPosts = [];
+        let previousCount = 0;
+        // If resuming with a cursor, we need to scroll past already-seen posts
+        const cursorUrl = cursor;
+        for (let scroll = 0; scroll <= maxScrollAttempts; scroll++) {
+            const countBeforeScroll = previousCount;
+            const scraped = await client.evaluate(SCRAPE_FEED_POSTS_SCRIPT);
+            allPosts = scraped ?? [];
+            // Determine which posts to return
+            const available = allPosts.length;
+            if (available >= count && !cursorUrl)
+                break;
+            // No new posts appeared after scroll — stop
+            if (allPosts.length === previousCount && scroll > 0)
+                break;
+            previousCount = allPosts.length;
+            // Scroll to load more
+            if (scroll < maxScrollAttempts) {
+                await scrollFeed(client, mouse);
+                // Progressive session fatigue: delays increase with each scroll
+                const fatigueMultiplier = 1 + scroll * 0.1;
+                // Scale delay by newly visible content volume
+                const newPostCount = allPosts.length - countBeforeScroll;
+                const contentBonus = Math.min(newPostCount * gaussianBetween(350, 75, 200, 500), 3_000);
+                await gaussianDelay(1_500 * fatigueMultiplier + contentBonus, 150 * fatigueMultiplier, 1_200 * fatigueMultiplier + contentBonus, 1_800 * fatigueMultiplier + contentBonus);
+                // Reading simulation: pause proportional to visible content volume.
+                // Estimate ~300 chars per newly visible post (headline + snippet).
+                if (newPostCount > 0) {
+                    await simulateReadingTime(newPostCount * 300);
+                }
+                await maybeBreak();
+            }
         }
-        const response = await voyager.fetch(path);
-        if (response.status !== 200) {
-            throw new Error(`Voyager API returned HTTP ${String(response.status)} for feed`);
+        // --- URL extraction phase ---
+        // Open each post's three-dot menu, click "Copy link to post", and
+        // derive the URN from the captured URL.  This populates the urn/url
+        // fields that the scrape script left null.
+        await installClipboardInterceptor(client);
+        for (let i = 0; i < allPosts.length; i++) {
+            const post = allPosts[i];
+            if (!post)
+                continue;
+            if (i > 0)
+                await gaussianDelay(550, 125, 300, 800); // Inter-post delay
+            await maybeBreak();
+            const url = await retryInteraction(() => capturePostUrl(client, i, mouse));
+            if (url) {
+                post.url = url;
+            }
         }
-        const body = response.body;
-        if (body === null || typeof body !== "object") {
-            throw new Error("Voyager API returned an unexpected response format for feed");
+        // Slice the result window
+        let startIdx = 0;
+        if (cursorUrl) {
+            const cursorIdx = allPosts.findIndex((p) => p.url === cursorUrl);
+            if (cursorIdx >= 0) {
+                startIdx = cursorIdx + 1;
+            }
         }
-        const parsed = parseFeedResponse(body);
-        return {
-            posts: parsed.posts,
-            nextCursor: parsed.nextCursor,
-        };
+        const window = allPosts.slice(startIdx, startIdx + count);
+        const posts = mapRawPosts(window);
+        // Determine next cursor
+        const hasMore = startIdx + count < allPosts.length;
+        const lastPost = window[window.length - 1];
+        const nextCursor = hasMore && lastPost ? lastPost.url : null;
+        await gaussianDelay(800, 300, 300, 1_800); // Post-action dwell
+        return { posts, nextCursor };
     }
     finally {
         client.disconnect();