@jackwener/opencli 1.3.2 → 1.4.0

package/src/clis/linkedin/timeline.ts

@@ -0,0 +1,532 @@
+import { cli, Strategy } from '../../registry.js';
+import type { IPage } from '../../types.js';
+import { AuthRequiredError, EmptyResultError } from '../../errors.js';
+
+interface TimelinePost {
+  rank?: number;
+  id: string;
+  author: string;
+  author_url: string;
+  headline: string;
+  text: string;
+  posted_at: string;
+  reactions: number;
+  comments: number;
+  url: string;
+}
+
+interface ExtractedBatch {
+  loginRequired?: boolean;
+  posts?: TimelinePost[];
+}
+
+function normalizeWhitespace(value: unknown): string {
+  return String(value ?? '').replace(/\s+/g, ' ').trim();
+}
+
+function parseMetric(value: unknown): number {
+  const raw = normalizeWhitespace(value).toLowerCase();
+  if (!raw) return 0;
+
+  const compact = raw.replace(/,/g, '');
+  const match = compact.match(/(\d+(?:\.\d+)?)(k|m)?/i);
+  if (!match) return 0;
+
+  const base = Number(match[1]);
+  const suffix = (match[2] || '').toLowerCase();
+  if (suffix === 'k') return Math.round(base * 1000);
+  if (suffix === 'm') return Math.round(base * 1000000);
+  return Math.round(base);
+}
+
+function buildPostId(post: Partial<TimelinePost>): string {
+  const url = normalizeWhitespace(post.url);
+  if (url) return url;
+
+  const author = normalizeWhitespace(post.author);
+  const text = normalizeWhitespace(post.text);
+  const postedAt = normalizeWhitespace(post.posted_at);
+  return `${author}::${postedAt}::${text.slice(0, 120)}`;
+}
+
+function mergeTimelinePosts(existing: TimelinePost[], batch: TimelinePost[]): TimelinePost[] {
+  const seen = new Set(existing.map(post => post.id));
+  const merged = [...existing];
+
+  for (const rawPost of batch) {
+    const post: TimelinePost = {
+      id: buildPostId(rawPost),
+      author: normalizeWhitespace(rawPost.author),
+      author_url: normalizeWhitespace(rawPost.author_url),
+      headline: normalizeWhitespace(rawPost.headline),
+      text: normalizeWhitespace(rawPost.text),
+      posted_at: normalizeWhitespace(rawPost.posted_at),
+      reactions: Number(rawPost.reactions) || 0,
+      comments: Number(rawPost.comments) || 0,
+      url: normalizeWhitespace(rawPost.url),
+    };
+
+    if (!post.author || !post.text) continue;
+    if (seen.has(post.id)) continue;
+    seen.add(post.id);
+    merged.push(post);
+  }
+
+  return merged;
+}
+
+async function extractVisiblePosts(page: IPage): Promise<ExtractedBatch> {
+  return page.evaluate(`(function () {
+    function normalize(value) {
+      return String(value || '').replace(/\\s+/g, ' ').trim();
+    }
+    function textOf(root, selector) {
+      var el = root.querySelector(selector);
+      return el ? el.textContent : '';
+    }
+    function hrefOf(root, selector) {
+      var el = root.querySelector(selector);
+      return el && el.href ? el.href : '';
+    }
+    function attrOf(root, selector, attr) {
+      var el = root.querySelector(selector);
+      return el ? el.getAttribute(attr) : '';
+    }
+    function cleanTimestamp(value) {
+      return normalize(String(value || '').replace(/[•.]/g, ' '));
+    }
+    function parseMetric(value) {
+      var raw = normalize(value).toLowerCase();
+      var match;
+      var base;
+      var suffix;
+      if (!raw) return 0;
+      match = raw.replace(/,/g, '').match(/(\\d+(?:\\.\\d+)?)(k|m)?/i);
+      if (!match) return 0;
+      base = Number(match[1]);
+      suffix = (match[2] || '').toLowerCase();
+      if (suffix === 'k') return Math.round(base * 1000);
+      if (suffix === 'm') return Math.round(base * 1000000);
+      return Math.round(base);
+    }
+    function splitBlocks(text) {
+      var lines = String(text || '').split('\\n');
+      var blocks = [];
+      var current = [];
+      var i;
+      var line;
+      for (i = 0; i < lines.length; i += 1) {
+        line = normalize(lines[i]);
+        if (!line) {
+          if (current.length) {
+            blocks.push(normalize(current.join(' ')));
+            current = [];
+          }
+          continue;
+        }
+        current.push(line);
+      }
+      if (current.length) blocks.push(normalize(current.join(' ')));
+      return blocks;
+    }
+    function looksLikeTimestamp(value) {
+      var lower = String(value || '').toLowerCase();
+      return /^\\d+\\s*(s|m|h|d|w|mo|yr|min)(\\s*[•.])?$/i.test(lower);
+    }
+    function looksLikeBadge(value) {
+      var lower = String(value || '').toLowerCase();
+      return String(value || '').indexOf('•') === 0
+        || lower === '1st'
+        || lower === '2nd'
+        || lower === '3rd'
+        || lower === 'degree connection';
+    }
+    function looksLikeAction(value) {
+      return /^(follow|send message|connect|visit my website|view my newsletter|subscribe)$/i.test((value || '').toLowerCase());
+    }
+    function looksLikeCta(value) {
+      return /^(book an appointment|view my services|visit my website|view my newsletter|subscribe|learn more|contact us)$/i.test((value || '').toLowerCase());
+    }
+    function looksLikeEngagement(value) {
+      return /(reactions?|comments?|reposts?)/i.test(String(value || ''));
+    }
+    function looksLikeFooterAction(value) {
+      return /^(like|comment|repost|send|reply|load more comments)$/i.test((value || '').toLowerCase());
+    }
+    function findActivityUrn(root) {
+      var elements = [root].concat(Array.from(root.querySelectorAll('*')));
+      var i;
+      var j;
+      var attrs;
+      var value;
+      var match;
+      for (i = 0; i < elements.length; i += 1) {
+        attrs = Array.from(elements[i].attributes || []);
+        for (j = 0; j < attrs.length; j += 1) {
+          value = String(attrs[j].value || '');
+          match = value.match(/urn:li:activity:\\d+/);
+          if (match) return match[0];
+        }
+      }
+      return '';
+    }
+    function parseReactionCount(root, blocks) {
+      var direct = textOf(root, '.social-details-social-counts__reactions-count');
+      var rootText = String(root.innerText || '');
+      var i;
+      var value;
+      value = rootText.match(/and\\s+(\\d[\\d,]*)\\s+others\\s+reacted/i);
+      if (value) return parseMetric(value[1]) + 1;
+      value = rootText.match(/and\\s+(\\d[\\d,]*)\\s+others(?!\\s+comments?)(?!\\s+reposts?)/i);
+      if (value) return parseMetric(value[1]) + 1;
+      value = rootText.match(/(\\d[\\d,]*)\\s+reactions?/i);
+      if (value) return parseMetric(value[0]);
+      if (direct) return parseMetric(direct);
+      for (i = 0; i < blocks.length; i += 1) {
+        value = blocks[i];
+        if (/and\\s+\\d[\\d,]*\\s+others(?!\\s+comments?)(?!\\s+reposts?)/i.test(value)) {
+          return parseMetric(value) + 1;
+        }
+        if (/reactions?/i.test(value)) return parseMetric(value);
+        if (/and\\s+\\d+[\\d,]*\\s+others\\s+reacted/i.test(value)) return parseMetric(value) + 1;
+      }
+      return 0;
+    }
+    function parseCommentCount(blocks) {
+      var i;
+      var text = blocks.join(' ');
+      var match = text.match(/(\\d[\\d,]*)\\s+comments?/i);
+      if (match) return parseMetric(match[0]);
+      for (i = 0; i < blocks.length; i += 1) {
+        if (/comments?/i.test(blocks[i])) return parseMetric(blocks[i]);
+      }
+      return 0;
+    }
+    function selectProfileLink(root, author) {
+      var links = Array.from(root.querySelectorAll('a[href*="/in/"], a[href*="/company/"]'));
+      var normalizedAuthor = normalize(author).toLowerCase();
+      var i;
+      var label;
+      for (i = 0; i < links.length; i += 1) {
+        label = normalize(links[i].textContent || links[i].getAttribute('aria-label')).toLowerCase();
+        if (!links[i].href) continue;
+        if (normalizedAuthor && label.indexOf(normalizedAuthor) >= 0) return links[i];
+      }
+      return links[0] || null;
+    }
+    function selectProfileUrl(root, author) {
+      var link = selectProfileLink(root, author);
+      return link && link.href ? link.href : '';
+    }
+    function parseActorLinkMeta(root, author) {
+      var link = selectProfileLink(root, author);
+      var text = normalize(link ? link.textContent : '');
+      var normalizedAuthor = normalize(author);
+      var match;
+      var rest;
+      var headline = '';
+      var postedAt = '';
+      if (!text || !normalizedAuthor) return { headline: '', postedAt: '' };
+      if (text.indexOf(normalizedAuthor) === 0) {
+        rest = normalize(text.slice(normalizedAuthor.length));
+      } else {
+        rest = text;
+      }
+      rest = normalize(rest.replace(/^[•·]\\s*(1st|2nd|3rd\\+?|3rd|degree connection)/i, ''));
+      match = rest.match(/(\\d+\\s*(?:s|m|h|d|w|mo|yr|min))\\s*[•·]?$/i);
+      if (match) {
+        postedAt = cleanTimestamp(match[1]);
+        headline = normalize(rest.slice(0, rest.length - match[0].length));
+      } else {
+        headline = rest;
+      }
+      headline = normalize(headline.replace(/^(book an appointment|view my services|visit my website|view my newsletter)\\s*/i, ''));
+      return { headline: headline, postedAt: postedAt };
+    }
+    function stripBodyTail(value) {
+      return normalize(String(value || '')
+        .replace(/\\s+\\d[\\d,]*\\s+reactions?[\\s\\S]*$/i, '')
+        .replace(/\\s+\\d[\\d,]*\\s+comments?[\\s\\S]*$/i, '')
+        .replace(/\\s+[A-Z][^\\n]+\\s+and\\s+\\d[\\d,]*\\s+others\\s+reacted[\\s\\S]*$/i, '')
+        .replace(/\\s+Like\\s+Comment\\s+Repost\\s+Send[\\s\\S]*$/i, '')
+        .replace(/\\s+Reaction button state:[\\s\\S]*$/i, '')
+        .replace(/^\\d+\\s*(?:s|m|h|d|w|mo|yr|min)\\s*[•.]?\\s*Follow\\s+/i, '')
+      );
+    }
+    function parseActorMeta(root) {
+      var actorLink = root.querySelector('a[href*="/in/"], a[href*="/company/"]');
+      var actorText = normalize(actorLink ? actorLink.textContent : '');
+      var author = '';
+      var headline = '';
+      var postedAt = '';
+      var match;
+      if (actorText) {
+        match = actorText.match(/^(.+?)\\s+[•·]\\s+(1st|2nd|3rd\\+?|3rd|degree connection)(.*)$/i);
+        if (match) {
+          author = normalize(match[1]);
+          actorText = normalize(match[3]);
+        }
+      }
+      match = actorText.match(/(.+?)\\s+(\\d+\\s*(?:s|m|h|d|w|mo|yr|min))\\s*[•·]?$/i);
+      if (match) {
+        headline = normalize(match[1]);
+        postedAt = cleanTimestamp(match[2]);
+      } else if (actorText) {
+        headline = actorText;
+      }
+      return {
+        author: author,
+        headline: headline,
+        postedAt: postedAt,
+        authorUrl: actorLink && actorLink.href ? actorLink.href : '',
+      };
+    }
+    function extractFromListItem(root) {
+      var blocks = splitBlocks(root.innerText || '');
+      var filtered = [];
+      var i;
+      var value;
+      var author = '';
+      var authorUrl = '';
+      var headline = '';
+      var postedAt = '';
+      var text = '';
+      var bodyStart = -1;
+      var permalink;
+      var url;
+      var reactions;
+      var comments;
+      var endIndex = -1;
+      var urn;
+
+      if (blocks.length < 5) return null;
+      if (blocks[0] !== 'Feed post') return null;
+
+      for (i = 1; i < blocks.length; i += 1) {
+        value = blocks[i];
+        if (!value) continue;
+        if (/commented on this|reposted this|liked this|suggested/i.test(value)) continue;
+        filtered.push(value);
+      }
+      if (filtered.length < 4) return null;
+
+      for (i = 0; i < filtered.length; i += 1) {
+        value = filtered[i];
+        if (!author && !looksLikeBadge(value) && !looksLikeAction(value) && !looksLikeTimestamp(value)) {
+          author = value;
+          continue;
+        }
+        if (author && !headline && !looksLikeBadge(value) && !looksLikeAction(value) && !looksLikeTimestamp(value) && !looksLikeCta(value)) {
+          headline = value;
+          continue;
+        }
+        if (!postedAt && looksLikeTimestamp(value)) {
+          postedAt = cleanTimestamp(value);
+          continue;
+        }
+      }
+
+      if (!author) return null;
+      authorUrl = selectProfileUrl(root, author);
+      if (!headline || !postedAt) {
+        var actorMeta = parseActorLinkMeta(root, author);
+        if (!headline && actorMeta.headline) headline = actorMeta.headline;
+        if (!postedAt && actorMeta.postedAt) postedAt = actorMeta.postedAt;
+      }
+
+      for (i = 0; i < filtered.length; i += 1) {
+        value = filtered[i];
+        if (looksLikeAction(value)) {
+          bodyStart = i + 1;
+          break;
+        }
+      }
+      if (bodyStart < 0 && postedAt) {
+        bodyStart = filtered.indexOf(postedAt) + 1;
+      }
+      if (bodyStart < 0) bodyStart = Math.min(filtered.length, headline ? 2 : 1);
+
+      for (i = bodyStart; i < filtered.length; i += 1) {
+        value = filtered[i];
+        if (looksLikeEngagement(value) || looksLikeFooterAction(value)) {
+          endIndex = i;
+          break;
+        }
+      }
+      if (endIndex < 0) endIndex = filtered.length;
+
+      text = stripBodyTail(filtered.slice(bodyStart, endIndex).join('\\n\\n'));
+      if (!text) return null;
+
+      permalink = root.querySelector('a[href*="/feed/update/"], a[href*="/posts/"], a[href*="/pulse/"]');
+      url = permalink ? permalink.href : '';
+      urn = findActivityUrn(root);
+      if (!url && urn) url = 'https://www.linkedin.com/feed/update/' + urn + '/';
+      reactions = parseReactionCount(root, filtered);
+      comments = parseCommentCount(filtered);
+
+      return {
+        id: url || (author + '::' + postedAt + '::' + text.slice(0, 120)),
+        author: author,
+        author_url: authorUrl,
+        headline: headline,
+        text: text,
+        posted_at: postedAt,
+        reactions: reactions,
+        comments: comments,
+        url: url,
+      };
+    }
+    function commentMetric(root) {
+      var links = Array.from(root.querySelectorAll('button, a'));
+      var i;
+      var label;
+      for (i = 0; i < links.length; i += 1) {
+        label = normalize(links[i].textContent || links[i].getAttribute('aria-label'));
+        if (/comment/i.test(label)) return parseMetric(label);
+      }
+      return 0;
+    }
+
+    var currentUrl = window.location.href;
+    var path = String(window.location.pathname || '');
+    var loginRequired = path.indexOf('/login') >= 0
+      || path.indexOf('/checkpoint/') >= 0
+      || Boolean(document.querySelector('input[name="session_key"], form.login__form'));
+    var moreButtons = Array.from(document.querySelectorAll('button, a[role="button"]'))
+      .filter(function (el) {
+        return /see more|more/i.test(normalize(el.textContent))
+          || /see more|more/i.test(normalize(el.getAttribute('aria-label')));
+      })
+      .slice(0, 8);
+    var cards = Array.from(document.querySelectorAll('article, .feed-shared-update-v2, .occludable-update, [role="listitem"]'));
+    var seen = new Set();
+    var posts = [];
+    var i;
+    var card;
+    var root;
+    var author;
+    var headline;
+    var text;
+    var postedAt;
+    var permalink;
+    var url;
+    var reactions;
+    var comments;
+
+    for (i = 0; i < moreButtons.length; i += 1) {
+      try { moreButtons[i].click(); } catch (err) {}
+    }
+
+    for (i = 0; i < cards.length; i += 1) {
+      card = cards[i];
+      root = card.closest('article, .feed-shared-update-v2, .occludable-update, [role="listitem"]') || card;
+      if (!root || seen.has(root)) continue;
+      seen.add(root);
+
+      if (String(root.getAttribute('role') || '') === 'listitem') {
+        var extracted = extractFromListItem(root);
+        if (extracted) posts.push(extracted);
+        continue;
+      }
+
+      author = normalize(
+        textOf(root, '.update-components-actor__title span[dir="ltr"]')
+        || textOf(root, '.update-components-actor__title')
+        || textOf(root, '[data-control-name="actor"] span[dir="ltr"]')
+        || textOf(root, '[data-control-name="actor"]')
+      );
+      headline = normalize(
+        textOf(root, '.update-components-actor__description')
+        || textOf(root, '.update-components-actor__sub-description')
+      );
+      text = normalize(
+        textOf(root, '.update-components-text span[dir="ltr"]')
+        || textOf(root, '.update-components-text')
+        || textOf(root, '.feed-shared-inline-show-more-text span[dir="ltr"]')
+        || textOf(root, '.feed-shared-inline-show-more-text')
+        || textOf(root, '[data-test-id="main-feed-activity-card"] .break-words')
+      );
+      postedAt = normalize(
+        textOf(root, '.update-components-actor__sub-description a')
+        || textOf(root, '.update-components-actor__sub-description span[aria-hidden="true"]')
+        || textOf(root, 'time')
+      );
+      permalink = root.querySelector('a[href*="/feed/update/"], a[href*="/posts/"], a[href*="/pulse/"]');
+      url = permalink ? permalink.href : '';
+      if (url && url.indexOf('/') === 0) url = new URL(url, currentUrl).toString();
+      reactions = parseMetric(
+        textOf(root, '.social-details-social-counts__reactions-count')
+        || attrOf(root, '[aria-label*="reaction"]', 'aria-label')
+        || attrOf(root, '[aria-label*="like"]', 'aria-label')
+      );
+      comments = commentMetric(root);
+
+      if (!author || !text) continue;
+
+      posts.push({
+        id: url || (author + '::' + postedAt + '::' + text.slice(0, 120)),
+        author: author,
+        author_url: hrefOf(root, 'a[href*="/in/"], a[href*="/company/"]'),
+        headline: headline,
+        text: text,
+        posted_at: postedAt,
+        reactions: reactions,
+        comments: comments,
+        url: url,
+      });
+    }
+
+    return { loginRequired: loginRequired, posts: posts };
+  })()`);
+}
+
+cli({
+  site: 'linkedin',
+  name: 'timeline',
+  description: 'Read LinkedIn home timeline posts',
+  domain: 'www.linkedin.com',
+  strategy: Strategy.COOKIE,
+  browser: true,
+  args: [
+    { name: 'limit', type: 'int', default: 20, help: 'Number of posts to return (max 100)' },
+  ],
+  columns: ['rank', 'author', 'author_url', 'headline', 'text', 'posted_at', 'reactions', 'comments', 'url'],
+  func: async (page, kwargs) => {
+    const limit = Math.max(1, Math.min(kwargs.limit ?? 20, 100));
+
+    await page.goto('https://www.linkedin.com/feed/');
+    await page.wait(4);
+
+    let posts: TimelinePost[] = [];
+    let sawLoginWall = false;
+
+    for (let i = 0; i < 6 && posts.length < limit; i++) {
+      const batch = await extractVisiblePosts(page);
+      if (batch?.loginRequired) sawLoginWall = true;
+      posts = mergeTimelinePosts(posts, Array.isArray(batch?.posts) ? batch.posts : []);
+      if (posts.length >= limit) break;
+      await page.autoScroll({ times: 1, delayMs: 1200 });
+      await page.wait(1);
+    }
+
+    if (sawLoginWall && posts.length === 0) {
+      throw new AuthRequiredError('linkedin.com', 'LinkedIn timeline requires an active signed-in browser session');
+    }
+
+    if (posts.length === 0) {
+      throw new EmptyResultError('linkedin timeline', 'Make sure your LinkedIn home feed is visible in the browser.');
+    }
+
+    return posts.slice(0, limit).map((post, index) => ({
+      rank: index + 1,
+      ...post,
+    }));
+  },
+});
+
+export const __test__ = {
+  parseMetric,
+  buildPostId,
+  mergeTimelinePosts,
+};

package/src/clis/medium/feed.ts

@@ -1,5 +1,5 @@
 import { cli, Strategy } from '../../registry.js';
-import { buildMediumTagUrl, loadMediumPosts } from './shared.js';
+import { buildMediumTagUrl, loadMediumPosts } from './utils.js';
 
 cli({
   site: 'medium',

package/src/clis/medium/search.ts

@@ -1,5 +1,5 @@
 import { cli, Strategy } from '../../registry.js';
-import { buildMediumSearchUrl, loadMediumPosts } from './shared.js';
+import { buildMediumSearchUrl, loadMediumPosts } from './utils.js';
 
 cli({
   site: 'medium',

package/src/clis/medium/user.ts

@@ -1,5 +1,5 @@
 import { cli, Strategy } from '../../registry.js';
-import { buildMediumUserUrl, loadMediumPosts } from './shared.js';
+import { buildMediumUserUrl, loadMediumPosts } from './utils.js';
 
 cli({
   site: 'medium',

package/src/clis/medium/{shared.ts → utils.ts}

@@ -1,3 +1,4 @@
+import { CommandExecutionError } from '../../errors.js';
 import type { IPage } from '../../types.js';
 
 export function buildMediumTagUrl(topic?: string): string {
@@ -13,7 +14,7 @@ export function buildMediumUserUrl(username: string): string {
 }
 
 export async function loadMediumPosts(page: IPage, url: string, limit: number): Promise<any[]> {
-  if (!page) throw new Error('Requires browser session');
+  if (!page) throw new CommandExecutionError('Browser session required for medium posts');
   await page.goto(url);
   await page.wait(5);
   const data = await page.evaluate(`

package/src/clis/pixiv/detail.yaml

@@ -0,0 +1,49 @@
+site: pixiv
+name: detail
+description: View illustration details (tags, stats, URLs)
+domain: www.pixiv.net
+strategy: cookie
+browser: true
+
+args:
+  id:
+    type: str
+    required: true
+    positional: true
+    description: Illustration ID
+
+pipeline:
+  - navigate: https://www.pixiv.net
+
+  - evaluate: |
+      (async () => {
+        const id = ${{ args.id | json }};
+        const res = await fetch(
+          'https://www.pixiv.net/ajax/illust/' + id,
+          { credentials: 'include' }
+        );
+        if (!res.ok) {
+          if (res.status === 401 || res.status === 403) throw new Error('Authentication required — please log in to Pixiv in Chrome');
+          if (res.status === 404) throw new Error('Illustration not found: ' + id);
+          throw new Error('Pixiv request failed (HTTP ' + res.status + ')');
+        }
+        const data = await res.json();
+        const b = data?.body;
+        if (!b) throw new Error('Illustration not found');
+        return [{
+          illust_id: b.illustId,
+          title: b.illustTitle,
+          author: b.userName,
+          user_id: b.userId,
+          type: b.illustType === 0 ? 'illust' : b.illustType === 1 ? 'manga' : b.illustType === 2 ? 'ugoira' : String(b.illustType),
+          pages: b.pageCount,
+          bookmarks: b.bookmarkCount,
+          likes: b.likeCount,
+          views: b.viewCount,
+          tags: (b.tags?.tags || []).map(t => t.tag).join(', '),
+          created: b.createDate?.split('T')[0] || '',
+          url: 'https://www.pixiv.net/artworks/' + b.illustId
+        }];
+      })()
+
+columns: [illust_id, title, author, type, pages, bookmarks, likes, views, tags, created, url]