@lhremote/core 0.8.0 → 0.10.0

package/dist/operations/get-post.js

@@ -1,201 +1,311 @@
 // SPDX-License-Identifier: AGPL-3.0-only
 // Copyright (C) 2026 Oleksii PELYKH
+import { resolveInstancePort } from "../cdp/index.js";
 import { CDPClient } from "../cdp/client.js";
 import { discoverTargets } from "../cdp/discovery.js";
-import { VoyagerInterceptor } from "../voyager/interceptor.js";
-import { DEFAULT_CDP_PORT } from "../constants.js";
-import { extractPostUrn } from "./get-post-stats.js";
+import { gaussianDelay } from "../utils/delay.js";
+import { extractPostUrn, resolvePostDetailUrl } from "./get-post-stats.js";
+import { delay, parseTimestamp } from "./get-feed.js";
+import { navigateAwayIf } from "./navigate-away.js";
 // ---------------------------------------------------------------------------
-// Parsing helpers
+// In-page DOM scraping scripts
 // ---------------------------------------------------------------------------
 /**
- * Resolve a text value that may be a string or an object with a `text` field.
- */
-export function resolveTextValue(value) {
-    if (value === undefined || value === null)
-        return "";
-    if (typeof value === "string")
-        return value;
-    return value.text ?? "";
-}
-/**
- * Extract public identifier from a LinkedIn navigation URL.
+ * JavaScript source evaluated inside the LinkedIn post detail page to
+ * extract post metadata from the rendered DOM.
+ *
+ * The post detail page (`/feed/update/{urn}/`) renders a single post
+ * using the same SSR feed structure as the home feed.  The script looks
+ * for `[data-testid="mainFeed"]` → `div[role="listitem"]` first, then
+ * falls back to the full document.
  */
-function extractPublicId(url) {
-    if (!url)
-        return null;
-    const match = /linkedin\.com\/in\/([^/?]+)/.exec(url);
-    return match?.[1] ?? null;
-}
+const SCRAPE_POST_DETAIL_SCRIPT = `(() => {
+  let authorName = null;
+  let authorHeadline = null;
+  let authorProfileUrl = null;
+  let text = null;
+  let reactionCount = 0;
+  let commentCount = 0;
+  let shareCount = 0;
+  let timestamp = null;
+
+  // Narrow scope: try mainFeed listitem, then <main>, then document
+  let scope = document.querySelector('main') || document;
+  const feedList = document.querySelector('[data-testid="mainFeed"]');
+  if (feedList) {
+    const items = feedList.querySelectorAll('div[role="listitem"]');
+    for (const item of items) {
+      if (item.offsetHeight < 100) continue;
+      const menuBtn = item.querySelector('button[aria-label^="Open control menu for post"]');
+      if (menuBtn) {
+        scope = item;
+        break;
+      }
+    }
+  }
+
+  // --- Author info ---
+  const authorLink = scope.querySelector('a[href*="/in/"], a[href*="/company/"]');
+  if (authorLink) {
+    authorProfileUrl = authorLink.href.split('?')[0] || null;
+
+    // Try name from span inside the link first
+    const nameSpan = authorLink.querySelector('span[dir="ltr"], span[aria-hidden="true"]');
+    let rawName = nameSpan ? (nameSpan.textContent || '').trim() : '';
+
+    // Fallback: link's own textContent (trimmed, first line only)
+    if (!rawName) {
+      rawName = (authorLink.textContent || '').trim().split('\\n')[0].trim();
+    }
+
+    // Fallback: look for a nearby heading or span that contains the name
+    // (LinkedIn sometimes renders the name outside the <a> tag)
+    if (!rawName) {
+      const parent = authorLink.closest('div');
+      if (parent) {
+        const nearby = parent.querySelector('span[dir="ltr"], span[aria-hidden="true"]');
+        if (nearby) rawName = (nearby.textContent || '').trim();
+      }
+    }
+
+    authorName = rawName || null;
+  }
+
+  // --- Author headline ---
+  // Search within <main> scope, skip navigation text and the author name
+  const allSpans = scope.querySelectorAll('span');
+  for (const span of allSpans) {
+    const txt = (span.textContent || '').trim();
+    if (
+      txt &&
+      txt.length > 5 &&
+      txt.length < 200 &&
+      txt !== authorName &&
+      !txt.match(/^\\d+[smhdw]$/) &&
+      !txt.match(/^\\d[\\d,]*\\s+(reactions?|comments?|reposts?|likes?)$/i) &&
+      !txt.match(/^Follow$|^Promoted$/i) &&
+      !txt.match(/^Skip to|^Keyboard shortcuts$|^Close jump menu$/i) &&
+      !txt.match(/^Feed detail update$|^Feed post$/i)
+    ) {
+      authorHeadline = txt;
+      break;
+    }
+  }
+
+  // --- Post text ---
+  const ltrSpans = scope.querySelectorAll('span[dir="ltr"]');
+  let longestText = '';
+  for (const span of ltrSpans) {
+    const txt = (span.textContent || '').trim();
+    if (txt.length > longestText.length && txt !== authorName && txt !== authorHeadline) {
+      longestText = txt;
+    }
+  }
+  if (longestText.length > 20) {
+    text = longestText;
+  }
+
+  // --- Engagement counts ---
+  const countText = document.body.textContent || '';
+
+  function parseCount(pattern) {
+    const m = countText.match(pattern);
+    if (!m) return 0;
+    const raw = m[1].replace(/,/g, '');
+    const num = parseInt(raw, 10);
+    return isNaN(num) ? 0 : num;
+  }
+
+  reactionCount = parseCount(/(\\d[\\d,]*)\\s+reactions?/i);
+  commentCount = parseCount(/(\\d[\\d,]*)\\s+comments?/i);
+  shareCount = parseCount(/(\\d[\\d,]*)\\s+reposts?/i);
+
+  // --- Timestamp ---
+  const timeEl = scope.querySelector('time');
+  if (timeEl) {
+    const dt = timeEl.getAttribute('datetime');
+    if (dt) timestamp = dt;
+  }
+  if (!timestamp) {
+    const scopeText = scope.textContent || '';
+    const timeMatch = scopeText.match(/(?:^|\\s)(\\d+[smhdw])(?:\\s|$|\\u00B7|\\xB7)/);
+    if (timeMatch) timestamp = timeMatch[1];
+  }
+
+  return {
+    authorName,
+    authorHeadline,
+    authorProfileUrl,
+    text,
+    reactionCount,
+    commentCount,
+    shareCount,
+    timestamp,
+  };
+})()`;
 /**
- * Parse the Voyager feed update response into a normalised PostDetail.
+ * JavaScript source evaluated inside the LinkedIn post detail page to
+ * extract visible comments from the DOM.
+ *
+ * Comments are rendered as `article` elements containing an author link
+ * and text content.  Only first-page (visible) comments are extracted;
+ * "Load more" is not clicked.
  */
-export function parseFeedUpdateResponse(raw, postUrn, included) {
-    // Resolve the feed update element — try nested data, data.elements, or flat
-    const element = raw.data?.elements?.[0] ?? raw.data ?? raw;
-    // Build included entity lookup
-    const profilesByUrn = new Map();
-    for (const entity of included) {
-        if (entity.entityUrn) {
-            profilesByUrn.set(entity.entityUrn, entity);
-        }
-    }
-    // Resolve actor — may be inline object, URN reference, or in included
-    let authorName = "";
+const SCRAPE_COMMENTS_SCRIPT = `(() => {
+  const comments = [];
+  const articles = document.querySelectorAll('article');
+
+  for (const article of articles) {
+    if (article.offsetHeight < 30) continue;
+
+    // --- Author ---
+    let authorName = '';
     let authorHeadline = null;
     let authorPublicId = null;
-    const actorValue = element.actor;
-    if (typeof actorValue === "string") {
-        // URN reference — look up in included
-        const profile = profilesByUrn.get(actorValue);
-        if (profile) {
-            authorName = resolveActorName(profile);
-            authorHeadline = resolveTextValue(profile.description ?? profile.headline) || null;
-            authorPublicId = profile.publicIdentifier ?? extractPublicId(profile.navigationUrl ?? undefined);
+    const authorLink = article.querySelector('a[href*="/in/"]');
+
+    if (authorLink) {
+      const nameSpan = authorLink.querySelector('span[dir="ltr"], span[aria-hidden="true"]');
+      authorName = nameSpan ? (nameSpan.textContent || '').trim() : '';
+      if (!authorName) {
+        authorName = (authorLink.textContent || '').trim().split('\\n')[0].trim();
+      }
+      if (!authorName) {
+        const parent = authorLink.closest('div');
+        if (parent) {
+          const nearby = parent.querySelector('span[dir="ltr"], span[aria-hidden="true"]');
+          if (nearby) authorName = (nearby.textContent || '').trim();
         }
+      }
+
+      const href = authorLink.href.split('?')[0] || '';
+      const idMatch = href.match(/\\/in\\/([^/?]+)/);
+      if (idMatch) authorPublicId = idMatch[1];
     }
-    else if (actorValue) {
-        authorName = resolveActorName(actorValue);
-        authorHeadline = resolveTextValue(actorValue.description ?? actorValue.headline) || null;
-        authorPublicId = actorValue.publicIdentifier ?? extractPublicId(actorValue.navigationUrl);
+
+    // --- Author headline ---
+    const spans = article.querySelectorAll('span');
+    for (const span of spans) {
+      const txt = (span.textContent || '').trim();
+      if (
+        txt &&
+        txt.length > 5 &&
+        txt.length < 200 &&
+        txt !== authorName &&
+        !txt.match(/^\\d+[smhdw]$/) &&
+        !txt.match(/^\\d[\\d,]*\\s+(reactions?|comments?|reposts?|likes?)$/i) &&
+        !txt.match(/^Reply$|^Like$/i)
+      ) {
+        authorHeadline = txt;
+        break;
+      }
     }
-    else {
-        // Try *actor URN reference
-        const actorUrn = element["*actor"] ?? raw.data?.["*actor"];
-        if (actorUrn) {
-            const profile = profilesByUrn.get(actorUrn);
-            if (profile) {
-                authorName = resolveActorName(profile);
-                authorHeadline = resolveTextValue(profile.description ?? profile.headline) || null;
-                authorPublicId = profile.publicIdentifier ?? extractPublicId(profile.navigationUrl ?? undefined);
-            }
-        }
+
+    // --- Comment text ---
+    let text = '';
+    const ltrSpans = article.querySelectorAll('span[dir="ltr"]');
+    for (const span of ltrSpans) {
+      const txt = (span.textContent || '').trim();
+      if (txt.length > text.length && txt !== authorName && txt !== authorHeadline) {
+        text = txt;
+      }
     }
-    // Resolve commentary text
-    const commentary = element.commentary ?? raw.data?.commentary;
-    const text = resolveTextValue(commentary?.text);
-    // Resolve timestamps
-    const publishedAt = element.publishedAt ?? raw.data?.publishedAt ?? null;
-    // Resolve social counts
-    const social = element.socialDetail ?? raw.data?.socialDetail ?? raw.socialDetail;
-    const counts = social?.totalSocialActivityCounts;
-    return {
-        postUrn,
-        authorName,
-        authorHeadline,
-        authorPublicId,
-        text,
-        publishedAt,
-        reactionCount: counts?.numLikes ?? 0,
-        commentCount: counts?.numComments ?? 0,
-        shareCount: counts?.numShares ?? 0,
-    };
-}
+
+    // Skip if no meaningful content
+    if (!text && !authorName) continue;
+
+    // --- Timestamp ---
+    let createdAt = null;
+    const timeEl = article.querySelector('time');
+    if (timeEl) {
+      const dt = timeEl.getAttribute('datetime');
+      if (dt) createdAt = dt;
+    }
+
+    // --- Reaction count ---
+    let reactionCount = 0;
+    const articleText = article.textContent || '';
+    const likesMatch = articleText.match(/(\\d[\\d,]*)\\s+reactions?/i);
+    if (likesMatch) {
+      reactionCount = parseInt(likesMatch[1].replace(/,/g, ''), 10) || 0;
+    }
+
+    comments.push({
+      authorName,
+      authorHeadline,
+      authorPublicId,
+      text,
+      createdAt,
+      reactionCount,
+    });
+  }
+
+  return comments;
+})()`;
 /**
- * Resolve actor display name from various response shapes.
+ * JavaScript source that finds and clicks the "Load more comments" button.
+ * Returns `true` if a button was clicked, `false` otherwise.
+ *
+ * LinkedIn renders the load-more trigger as a `button` or `span` whose
+ * text content includes "Load more comments" (or locale equivalents).
+ * The script also recognises "Load previous replies" for nested threads.
  */
-function resolveActorName(actor) {
-    // Try name.text pattern (actor object from feed updates)
-    const nameText = resolveTextValue(actor.name);
-    if (nameText)
-        return nameText;
-    // Try firstName + lastName pattern (mini-profile)
-    if (actor.firstName || actor.lastName) {
-        return [actor.firstName, actor.lastName].filter(Boolean).join(" ");
+const CLICK_LOAD_MORE_COMMENTS_SCRIPT = `(() => {
+  const loadMoreTexts = [
+    'load more comments', 'show more comments', 'show previous replies',
+    'load previous replies', 'view more comments',
+  ];
+
+  // Try buttons first, then spans and anchors
+  const candidates = [
+    ...document.querySelectorAll('button'),
+    ...document.querySelectorAll('span[role="button"]'),
+  ];
+
+  for (const el of candidates) {
+    const txt = (el.textContent || '').trim().toLowerCase();
+    if (loadMoreTexts.some(t => txt.includes(t))) {
+      el.scrollIntoView({ block: 'center' });
+      el.click();
+      return true;
     }
-    return "";
-}
+  }
+  return false;
+})()`;
+// ---------------------------------------------------------------------------
+// Wait for post detail to load
+// ---------------------------------------------------------------------------
 /**
- * Parse the Voyager comments response into normalised PostComment entries.
+ * Poll the DOM until the post detail page has rendered.  The page is
+ * considered ready when an author link and at least one `span[dir="ltr"]`
+ * are present.
  */
-export function parseCommentsResponse(raw) {
-    const elements = raw.data?.elements ?? raw.elements ?? [];
-    const paging = raw.data?.paging ?? raw.paging;
-    const included = raw.included ?? [];
-    // Build a lookup for included mini-profile entities
-    const profilesByUrn = new Map();
-    for (const entity of included) {
-        if (entity.entityUrn) {
-            profilesByUrn.set(entity.entityUrn, entity);
-        }
+async function waitForPostLoad(client, timeoutMs = 15_000) {
+    const deadline = Date.now() + timeoutMs;
+    while (Date.now() < deadline) {
+        const ready = await client.evaluate(`(() => {
+      const authorLink = document.querySelector('a[href*="/in/"], a[href*="/company/"]');
+      if (!authorLink) return false;
+      const ltrSpans = document.querySelectorAll('span[dir="ltr"]');
+      return ltrSpans.length > 0;
+    })()`);
+        if (ready)
+            return;
+        await delay(500);
     }
-    const comments = [];
-    for (const el of elements) {
-        const commentUrn = el.urn ?? el.entityUrn ?? null;
-        // Resolve commenter
-        let authorName = "";
-        let authorHeadline = null;
-        let authorPublicId = null;
-        if (el.commenter) {
-            authorName = [el.commenter.firstName, el.commenter.lastName]
-                .filter(Boolean)
-                .join(" ");
-            authorHeadline =
-                resolveTextValue(el.commenter.headline) ||
-                    el.commenter.occupation ||
-                    null;
-            authorPublicId = el.commenter.publicIdentifier ?? null;
-        }
-        else {
-            // Try URN reference lookup
-            const commenterUrn = el.commenterUrn ?? el["*commenter"];
-            if (commenterUrn) {
-                const profile = profilesByUrn.get(commenterUrn);
-                if (profile) {
-                    authorName = [profile.firstName, profile.lastName]
-                        .filter(Boolean)
-                        .join(" ");
-                    authorHeadline =
-                        resolveTextValue(profile.headline) ||
-                            profile.occupation ||
-                            null;
-                    authorPublicId = profile.publicIdentifier ?? null;
-                }
-            }
-        }
-        // Resolve comment text — multiple possible shapes
-        let text = "";
-        if (el.commentV2) {
-            text = resolveTextValue(el.commentV2.text);
-        }
-        else if (el.commentary) {
-            text = resolveTextValue(el.commentary.text);
-        }
-        else if (el.comment) {
-            if (typeof el.comment === "string") {
-                text = el.comment;
-            }
-            else if ("text" in el.comment) {
-                text = resolveTextValue(el.comment.text);
-            }
-            else if ("values" in el.comment) {
-                const vals = el.comment
-                    .values;
-                text = vals?.map((v) => v.value ?? "").join("") ?? "";
-            }
-        }
-        // Resolve timestamp
-        const createdAt = el.createdTime ?? el.created?.time ?? null;
-        // Resolve reaction count
-        const reactionCount = el.socialDetail?.totalSocialActivityCounts?.numLikes ?? 0;
-        comments.push({
-            commentUrn,
-            authorName,
-            authorHeadline,
-            authorPublicId,
-            text,
-            createdAt,
-            reactionCount,
-        });
-    }
-    return {
-        comments,
-        paging: {
-            start: paging?.start ?? 0,
-            count: paging?.count ?? comments.length,
-            total: paging?.total ?? comments.length,
-        },
-    };
+    throw new Error("Timed out waiting for post detail to appear in the DOM");
+}
+// ---------------------------------------------------------------------------
+// Parsing helpers
+// ---------------------------------------------------------------------------
+/**
+ * Extract public identifier from a LinkedIn profile URL.
+ */
+function extractPublicId(url) {
+    if (!url)
+        return null;
+    const match = /\/in\/([^/?]+)/.exec(url);
+    return match?.[1] ?? null;
 }
 // ---------------------------------------------------------------------------
 // Main operation
@@ -203,19 +313,27 @@ export function parseCommentsResponse(raw) {
 /**
  * Retrieve detailed data for a single LinkedIn post with its comment thread.
  *
- * Connects to the LinkedIn webview in LinkedHelper and calls the
- * Voyager API to fetch the post entity and its comments.
+ * Connects to the LinkedIn webview in LinkedHelper, navigates to the
+ * post detail page, and extracts post data and comments from the
+ * rendered DOM.
  *
- * @param input - Post URL or URN, comment pagination parameters, and CDP connection options.
+ * @param input - Post URL or URN, and CDP connection options.
  * @returns Post detail with comments and pagination metadata.
  */
 export async function getPost(input) {
-    const cdpPort = input.cdpPort ?? DEFAULT_CDP_PORT;
+    const cdpPort = await resolveInstancePort(input.cdpPort, input.cdpHost);
     const cdpHost = input.cdpHost ?? "127.0.0.1";
     const allowRemote = input.allowRemote ?? false;
-    const commentCount = input.commentCount ?? 10;
-    const commentStart = input.commentStart ?? 0;
-    const postUrn = extractPostUrn(input.postUrl);
+    const maxComments = input.commentCount ?? 100;
+    const postDetailUrl = resolvePostDetailUrl(input.postUrl);
+    // Try to extract URN for the output postUrn field
+    let postUrn;
+    try {
+        postUrn = extractPostUrn(input.postUrl);
+    }
+    catch {
+        postUrn = input.postUrl;
+    }
     // Enforce loopback guard
     if (!allowRemote && cdpHost !== "127.0.0.1" && cdpHost !== "localhost") {
         throw new Error(`Non-loopback CDP host "${cdpHost}" requires --allow-remote. ` +
@@ -230,37 +348,65 @@ export async function getPost(input) {
     const client = new CDPClient(cdpPort, { host: cdpHost, allowRemote });
     await client.connect(linkedInTarget.id);
     try {
-        const voyager = new VoyagerInterceptor(client);
-        // Fetch post detail
-        const encodedUrn = encodeURIComponent(postUrn);
-        const postPath = `/voyager/api/feed/updates/${encodedUrn}`;
-        const postResponse = await voyager.fetch(postPath);
-        if (postResponse.status !== 200) {
-            throw new Error(`Voyager API returned HTTP ${String(postResponse.status)} for post detail`);
-        }
-        const postBody = postResponse.body;
-        if (postBody === null || typeof postBody !== "object") {
-            throw new Error("Voyager API returned an unexpected response format for post detail");
+        // Navigate away if already on the post detail page to force a fresh load
+        await navigateAwayIf(client, "/feed/update/");
+        // Navigate to the post detail page
+        await client.navigate(postDetailUrl);
+        // Wait for the post content to render
+        await waitForPostLoad(client);
+        // Extract post metadata from the DOM
+        const rawPost = await client.evaluate(SCRAPE_POST_DETAIL_SCRIPT);
+        if (!rawPost) {
+            throw new Error("Failed to extract post detail from the DOM");
         }
-        const rawPost = postBody;
-        const post = parseFeedUpdateResponse(rawPost, postUrn, rawPost.included ?? []);
-        // Fetch comments
-        const commentsPath = `/voyager/api/feed/dash/feedComments` +
-            `?q=commentsUnderFeedUpdate&updateUrn=${encodedUrn}` +
-            `&start=${String(commentStart)}&count=${String(commentCount)}`;
-        const commentsResponse = await voyager.fetch(commentsPath);
-        if (commentsResponse.status !== 200) {
-            throw new Error(`Voyager API returned HTTP ${String(commentsResponse.status)} for post comments`);
-        }
-        const commentsBody = commentsResponse.body;
-        if (commentsBody === null || typeof commentsBody !== "object") {
-            throw new Error("Voyager API returned an unexpected response format for post comments");
+        const post = {
+            postUrn,
+            authorName: rawPost.authorName ?? "",
+            authorHeadline: rawPost.authorHeadline ?? null,
+            authorPublicId: extractPublicId(rawPost.authorProfileUrl),
+            text: rawPost.text ?? "",
+            publishedAt: parseTimestamp(rawPost.timestamp),
+            reactionCount: rawPost.reactionCount,
+            commentCount: rawPost.commentCount,
+            shareCount: rawPost.shareCount,
+        };
+        // --- Comment loading ---
+        // Click "Load more comments" repeatedly until we have enough or no more
+        // are available.  Each click loads an additional batch of comments.
+        const maxLoadMoreAttempts = 20;
+        if (maxComments > 0) {
+            for (let attempt = 0; attempt < maxLoadMoreAttempts; attempt++) {
+                const currentCount = await client.evaluate(`document.querySelectorAll('article').length`);
+                if (currentCount >= maxComments)
+                    break;
+                const clicked = await client.evaluate(CLICK_LOAD_MORE_COMMENTS_SCRIPT);
+                if (!clicked)
+                    break;
+                await delay(1500);
+            }
         }
-        const parsed = parseCommentsResponse(commentsBody);
+        // Extract all visible comments from the DOM
+        const rawComments = await client.evaluate(SCRAPE_COMMENTS_SCRIPT);
+        const allRaw = rawComments ?? [];
+        const limited = maxComments > 0 ? allRaw.slice(0, maxComments) : [];
+        const comments = limited.map((c) => ({
+            commentUrn: null,
+            authorName: c.authorName,
+            authorHeadline: c.authorHeadline,
+            authorPublicId: c.authorPublicId,
+            text: c.text,
+            createdAt: parseTimestamp(c.createdAt),
+            reactionCount: c.reactionCount,
+        }));
+        await gaussianDelay(800, 300, 300, 1_800); // Post-action dwell
         return {
             post,
-            comments: parsed.comments,
-            commentsPaging: parsed.paging,
+            comments,
+            commentsPaging: {
+                start: 0,
+                count: comments.length,
+                total: comments.length,
+            },
         };
     }
     finally {