@jackwener/opencli 0.7.2 → 0.7.4

package/dist/clis/linkedin/search.js

@@ -0,0 +1,366 @@
+import { cli, Strategy } from '../../registry.js';
+// ── Filter value mappings ──────────────────────────────────────────────
+const EXPERIENCE_LEVELS = {
+    internship: '1',
+    entry: '2',
+    'entry-level': '2',
+    associate: '3',
+    mid: '4',
+    senior: '4',
+    'mid-senior': '4',
+    'mid-senior-level': '4',
+    director: '5',
+    executive: '6',
+};
+const JOB_TYPES = {
+    'full-time': 'F',
+    fulltime: 'F',
+    full: 'F',
+    'part-time': 'P',
+    parttime: 'P',
+    part: 'P',
+    contract: 'C',
+    temporary: 'T',
+    temp: 'T',
+    volunteer: 'V',
+    internship: 'I',
+    other: 'O',
+};
+const DATE_POSTED = {
+    any: 'on',
+    month: 'r2592000',
+    'past-month': 'r2592000',
+    week: 'r604800',
+    'past-week': 'r604800',
+    day: 'r86400',
+    '24h': 'r86400',
+    'past-24h': 'r86400',
+};
+const REMOTE_TYPES = {
+    onsite: '1',
+    'on-site': '1',
+    hybrid: '3',
+    remote: '2',
+};
+// ── Helpers ────────────────────────────────────────────────────────────
+function parseCsvArg(value) {
+    if (value === undefined || value === null || value === '')
+        return [];
+    return String(value)
+        .split(',')
+        .map(item => item.trim())
+        .filter(Boolean);
+}
+function mapFilterValues(input, mapping, label) {
+    const values = parseCsvArg(input);
+    const resolved = values.map(value => {
+        const key = value.toLowerCase();
+        const mapped = mapping[key];
+        if (!mapped)
+            throw new Error(`Unsupported ${label}: ${value}`);
+        return mapped;
+    });
+    return [...new Set(resolved)];
+}
+function normalizeWhitespace(value) {
+    return String(value ?? '').replace(/\s+/g, ' ').trim();
+}
+function decodeLinkedinRedirect(url) {
+    if (!url)
+        return '';
+    try {
+        const parsed = new URL(url);
+        if (parsed.pathname === '/redir/redirect/') {
+            return parsed.searchParams.get('url') || url;
+        }
+    }
+    catch { }
+    return url;
+}
+function buildVoyagerSearchQuery(input) {
+    const hasFilters = input.companyIds.length ||
+        input.experienceLevels.length ||
+        input.jobTypes.length ||
+        input.datePostedValues.length ||
+        input.remoteTypes.length;
+    const parts = [
+        'origin:' + (hasFilters ? 'JOB_SEARCH_PAGE_JOB_FILTER' : 'JOB_SEARCH_PAGE_OTHER_ENTRY'),
+        'keywords:' + input.keywords,
+    ];
+    if (input.location) {
+        parts.push('locationUnion:(seoLocation:(location:' + input.location + '))');
+    }
+    const filters = [];
+    if (input.companyIds.length)
+        filters.push('company:List(' + input.companyIds.join(',') + ')');
+    if (input.experienceLevels.length)
+        filters.push('experience:List(' + input.experienceLevels.join(',') + ')');
+    if (input.jobTypes.length)
+        filters.push('jobType:List(' + input.jobTypes.join(',') + ')');
+    if (input.datePostedValues.length)
+        filters.push('timePostedRange:List(' + input.datePostedValues.join(',') + ')');
+    if (input.remoteTypes.length)
+        filters.push('workplaceType:List(' + input.remoteTypes.join(',') + ')');
+    if (filters.length)
+        parts.push('selectedFilters:(' + filters.join(',') + ')');
+    parts.push('spellCorrectionEnabled:true');
+    return '(' + parts.join(',') + ')';
+}
+function buildVoyagerUrl(input, offset, count) {
+    const params = new URLSearchParams({
+        decorationId: 'com.linkedin.voyager.dash.deco.jobs.search.JobSearchCardsCollection-220',
+        count: String(count),
+        q: 'jobSearch',
+    });
+    const query = encodeURIComponent(buildVoyagerSearchQuery(input))
+        .replace(/%3A/gi, ':')
+        .replace(/%2C/gi, ',')
+        .replace(/%28/gi, '(')
+        .replace(/%29/gi, ')');
+    return '/voyager/api/voyagerJobsDashJobCards?' + params.toString() + '&query=' + query + '&start=' + offset;
+}
+// ── Company ID resolution (requires DOM interaction) ──────────────────
+async function resolveCompanyIds(page, input) {
+    const rawValues = parseCsvArg(input);
+    const ids = new Set();
+    const names = [];
+    for (const value of rawValues) {
+        if (/^\d+$/.test(value))
+            ids.add(value);
+        else
+            names.push(value);
+    }
+    if (!names.length)
+        return [...ids];
+    const resolved = await page.evaluate(`(async () => {
+    const targets = ${JSON.stringify(names)};
+    const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));
+    const normalize = (v) => (v || '').toLowerCase().replace(/\\s+/g, ' ').trim();
+
+    // Open "All filters" panel to expose company filter inputs
+    const allBtn = [...document.querySelectorAll('button')]
+      .find(b => ((b.innerText || '').trim().replace(/\\s+/g, ' ')) === 'All filters');
+    if (allBtn) { allBtn.click(); await sleep(300); }
+
+    const getCompanyMap = () => {
+      const map = {};
+      for (const el of document.querySelectorAll('input[name="company-filter-value"]')) {
+        const text = (el.parentElement?.innerText || el.closest('label')?.innerText || '')
+          .replace(/\\s+/g, ' ').trim().replace(/\\s*Filter by.*$/i, '').trim();
+        if (text) map[normalize(text)] = el.value;
+      }
+      return map;
+    };
+
+    const match = (map, name) => {
+      const n = normalize(name);
+      if (map[n]) return map[n];
+      const k = Object.keys(map).find(e => e === n || e.includes(n) || n.includes(e));
+      return k ? map[k] : null;
+    };
+
+    const results = {};
+    let map = getCompanyMap();
+
+    for (const name of targets) {
+      let found = match(map, name);
+      if (!found) {
+        const inp = [...document.querySelectorAll('input')]
+          .find(el => el.getAttribute('aria-label') === 'Add a company');
+        if (inp) {
+          inp.focus();
+          inp.value = name;
+          inp.dispatchEvent(new Event('input', { bubbles: true }));
+          inp.dispatchEvent(new KeyboardEvent('keyup', { key: 'Enter', bubbles: true }));
+          await sleep(1200);
+          map = getCompanyMap();
+          found = match(map, name);
+          inp.value = '';
+          inp.dispatchEvent(new Event('input', { bubbles: true }));
+          await sleep(100);
+        }
+      }
+      results[name] = found || null;
+    }
+    return results;
+  })()`);
+    const unresolved = [];
+    for (const name of names) {
+        const id = resolved?.[name];
+        if (id)
+            ids.add(id);
+        else
+            unresolved.push(name);
+    }
+    if (unresolved.length) {
+        throw new Error(`Could not resolve LinkedIn company filter: ${unresolved.join(', ')}`);
+    }
+    return [...ids];
+}
+// ── Voyager API fetch (runs inside page context for cookie access) ────
+async function fetchJobCards(page, input) {
+    const MAX_BATCH = 25;
+    const allJobs = [];
+    let offset = input.start;
+    while (allJobs.length < input.limit) {
+        const count = Math.min(MAX_BATCH, input.limit - allJobs.length);
+        const apiPath = buildVoyagerUrl(input, offset, count);
+        const batch = await page.evaluate(`(async () => {
+      const jsession = document.cookie.split(';').map(p => p.trim())
+        .find(p => p.startsWith('JSESSIONID='))?.slice('JSESSIONID='.length);
+      if (!jsession) return { error: 'LinkedIn JSESSIONID cookie not found. Please sign in to LinkedIn in the browser.' };
+
+      const csrf = jsession.replace(/^"|"$/g, '');
+      const res = await fetch(${JSON.stringify(apiPath)}, {
+        credentials: 'include',
+        headers: { 'csrf-token': csrf, 'x-restli-protocol-version': '2.0.0' },
+      });
+      if (!res.ok) {
+        const text = await res.text();
+        return { error: 'LinkedIn API error: HTTP ' + res.status + ' ' + text.slice(0, 200) };
+      }
+      return res.json();
+    })()`);
+        if (!batch || batch.error) {
+            throw new Error(batch?.error || 'LinkedIn search returned an unexpected response');
+        }
+        const elements = Array.isArray(batch?.elements) ? batch.elements : [];
+        if (elements.length === 0)
+            break;
+        for (const element of elements) {
+            const card = element?.jobCardUnion?.jobPostingCard;
+            if (!card)
+                continue;
+            // Extract job ID from URN fields
+            const jobId = [card.jobPostingUrn, card.jobPosting?.entityUrn, card.entityUrn]
+                .filter(Boolean)
+                .map(s => String(s).match(/(\d+)/)?.[1])
+                .find(Boolean) ?? '';
+            // Extract listed date
+            const listedItem = (card.footerItems || []).find((i) => i?.type === 'LISTED_DATE' && i?.timeAt);
+            const listed = listedItem?.timeAt ? new Date(listedItem.timeAt).toISOString().slice(0, 10) : '';
+            allJobs.push({
+                title: card.jobPostingTitle || card.title?.text || '',
+                company: card.primaryDescription?.text || '',
+                location: card.secondaryDescription?.text || '',
+                listed,
+                salary: card.tertiaryDescription?.text || '',
+                url: jobId ? 'https://www.linkedin.com/jobs/view/' + jobId : '',
+            });
+        }
+        if (elements.length < count)
+            break;
+        offset += elements.length;
+    }
+    return allJobs.slice(0, input.limit).map((item, index) => ({
+        rank: input.start + index + 1,
+        ...item,
+    }));
+}
+// ── Job detail enrichment (--details flag) ────────────────────────────
+async function enrichJobDetails(page, jobs) {
+    const enriched = [];
+    for (let i = 0; i < jobs.length; i++) {
+        const job = jobs[i];
+        console.error(`[opencli:linkedin] Fetching details ${i + 1}/${jobs.length}: ${job.title}`);
+        if (!job.url) {
+            enriched.push({ ...job, description: '', apply_url: '' });
+            continue;
+        }
+        try {
+            await page.goto(job.url);
+            await page.wait({ text: 'About the job', timeout: 8 });
+            // Expand "Show more" button if present
+            await page.evaluate(`(() => {
+        const norm = (v) => (v || '').replace(/\\s+/g, ' ').trim().toLowerCase();
+        const section = [...document.querySelectorAll('div, section, article')]
+          .find(el => norm(el.querySelector('h1,h2,h3,h4')?.textContent || '') === 'about the job');
+        const btn = [...(section?.querySelectorAll('button, a[role="button"]') || [])]
+          .find(el => /more/.test(norm(el.textContent || '')) || /more/.test(norm(el.getAttribute('aria-label') || '')));
+        if (btn) btn.click();
+      })()`);
+            await page.wait(1);
+            // Extract description and apply URL
+            const detail = await page.evaluate(`(() => {
+        const norm = (v) => (v || '').replace(/\\s+/g, ' ').trim();
+        // Find the most specific (shortest) container with "About the job" heading
+        // Shortest = most specific DOM node, avoiding outer wrappers that include unrelated text
+        const candidates = [...document.querySelectorAll('div, section, article')]
+          .map(el => ({
+            heading: norm(el.querySelector('h1,h2,h3,h4')?.textContent || ''),
+            text: norm(el.innerText || ''),
+          }))
+          .filter(c => c.text && c.heading.toLowerCase() === 'about the job' && c.text.length > 'About the job'.length)
+          .sort((a, b) => a.text.length - b.text.length);
+
+        const description = candidates[0]?.text.replace(/^About the job\\s*/i, '') || '';
+        const applyLink = [...document.querySelectorAll('a[href]')]
+          .map(a => ({ href: a.href || '', text: norm(a.textContent || ''), aria: norm(a.getAttribute('aria-label') || '') }))
+          .find(a => /apply/i.test(a.text) || /apply/i.test(a.aria));
+
+        return { description, applyUrl: applyLink?.href || '' };
+      })()`);
+            enriched.push({
+                ...job,
+                description: normalizeWhitespace(detail?.description),
+                apply_url: decodeLinkedinRedirect(String(detail?.applyUrl ?? '')),
+            });
+        }
+        catch {
+            enriched.push({ ...job, description: '', apply_url: '' });
+        }
+    }
+    return enriched;
+}
+// ── CLI registration ──────────────────────────────────────────────────
+cli({
+    site: 'linkedin',
+    name: 'search',
+    description: 'Search LinkedIn jobs',
+    domain: 'www.linkedin.com',
+    strategy: Strategy.HEADER,
+    browser: true,
+    args: [
+        { name: 'query', type: 'string', required: true, help: 'Job search keywords' },
+        { name: 'location', type: 'string', required: false, help: 'Location text such as San Francisco Bay Area' },
+        { name: 'limit', type: 'int', default: 10, help: 'Number of jobs to return (max 100)' },
+        { name: 'start', type: 'int', default: 0, help: 'Result offset for pagination' },
+        { name: 'details', type: 'bool', default: false, help: 'Include full job description and apply URL (slower)' },
+        { name: 'company', type: 'string', required: false, help: 'Comma-separated company names or LinkedIn company IDs' },
+        { name: 'experience_level', type: 'string', required: false, help: 'Comma-separated: internship, entry, associate, mid-senior, director, executive' },
+        { name: 'job_type', type: 'string', required: false, help: 'Comma-separated: full-time, part-time, contract, temporary, volunteer, internship, other' },
+        { name: 'date_posted', type: 'string', required: false, help: 'One of: any, month, week, 24h' },
+        { name: 'remote', type: 'string', required: false, help: 'Comma-separated: on-site, hybrid, remote' },
+    ],
+    columns: ['rank', 'title', 'company', 'location', 'listed', 'salary', 'url'],
+    func: async (page, kwargs) => {
+        const limit = Math.max(1, Math.min(kwargs.limit ?? 10, 100));
+        const start = Math.max(0, kwargs.start ?? 0);
+        const includeDetails = Boolean(kwargs.details);
+        const location = (kwargs.location ?? '').trim();
+        const keywords = String(kwargs.query ?? '').trim();
+        if (!keywords)
+            throw new Error('query is required');
+        const searchParams = new URLSearchParams({ keywords });
+        if (location)
+            searchParams.set('location', location);
+        await page.goto(`https://www.linkedin.com/jobs/search/?${searchParams.toString()}`);
+        await page.wait({ text: 'Jobs', timeout: 10 });
+        const companyIds = await resolveCompanyIds(page, kwargs.company);
+        const input = {
+            keywords,
+            location,
+            limit,
+            start,
+            companyIds,
+            experienceLevels: mapFilterValues(kwargs.experience_level, EXPERIENCE_LEVELS, 'experience_level'),
+            jobTypes: mapFilterValues(kwargs.job_type, JOB_TYPES, 'job_type'),
+            datePostedValues: mapFilterValues(kwargs.date_posted, DATE_POSTED, 'date_posted'),
+            remoteTypes: mapFilterValues(kwargs.remote, REMOTE_TYPES, 'remote'),
+        };
+        const data = await fetchJobCards(page, input);
+        if (!includeDetails)
+            return data;
+        return enrichJobDetails(page, data);
+    },
+});

package/dist/clis/reddit/read.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/clis/reddit/read.js

@@ -0,0 +1,184 @@
+/**
+ * Reddit post reader with threaded comment tree.
+ *
+ * Replaces the original flat read.yaml with recursive comment traversal:
+ * - Top-K comments by score at each level
+ * - Configurable depth and replies-per-level
+ * - Indented output showing conversation threads
+ */
+import { cli, Strategy } from '../../registry.js';
+cli({
+    site: 'reddit',
+    name: 'read',
+    description: 'Read a Reddit post and its comments',
+    domain: 'reddit.com',
+    strategy: Strategy.COOKIE,
+    args: [
+        { name: 'post_id', required: true, help: 'Post ID (e.g. 1abc123) or full URL' },
+        { name: 'sort', default: 'best', help: 'Comment sort: best, top, new, controversial, old, qa' },
+        { name: 'limit', type: 'int', default: 25, help: 'Number of top-level comments' },
+        { name: 'depth', type: 'int', default: 2, help: 'Max reply depth (1=no replies, 2=one level of replies, etc.)' },
+        { name: 'replies', type: 'int', default: 5, help: 'Max replies shown per comment at each level (sorted by score)' },
+        { name: 'max_length', type: 'int', default: 2000, help: 'Max characters per comment body (min 100)' },
+    ],
+    columns: ['type', 'author', 'score', 'text'],
+    func: async (page, kwargs) => {
+        const sort = kwargs.sort ?? 'best';
+        const limit = Math.max(1, kwargs.limit ?? 25);
+        const maxDepth = Math.max(1, kwargs.depth ?? 2);
+        const maxReplies = Math.max(1, kwargs.replies ?? 5);
+        const maxLength = Math.max(100, kwargs.max_length ?? 2000);
+        await page.goto('https://www.reddit.com');
+        await page.wait(2);
+        const data = await page.evaluate(`
+      (async function() {
+        var postId = ${JSON.stringify(kwargs.post_id)};
+        var urlMatch = postId.match(/comments\\/([a-z0-9]+)/);
+        if (urlMatch) postId = urlMatch[1];
+
+        var sort = ${JSON.stringify(sort)};
+        var limit = ${limit};
+        var maxDepth = ${maxDepth};
+        var maxReplies = ${maxReplies};
+        var maxLength = ${maxLength};
+
+        // Request more from API than top-level limit to get inline replies
+        // depth param tells Reddit how deep to inline replies vs "more" stubs
+        var apiLimit = Math.max(limit * 3, 100);
+        var res = await fetch(
+          '/comments/' + postId + '.json?sort=' + sort + '&limit=' + apiLimit + '&depth=' + (maxDepth + 1) + '&raw_json=1',
+          { credentials: 'include' }
+        );
+        if (!res.ok) return { error: 'Reddit API returned HTTP ' + res.status };
+
+        var data;
+        try { data = await res.json(); } catch(e) { return { error: 'Failed to parse response' }; }
+        if (!Array.isArray(data) || data.length < 2) return { error: 'Unexpected response format' };
+
+        var results = [];
+
+        // Post
+        var post = data[0] && data[0].data && data[0].data.children && data[0].data.children[0] && data[0].data.children[0].data;
+        if (post) {
+          var body = post.selftext || '';
+          if (body.length > maxLength) body = body.slice(0, maxLength) + '\\n... [truncated]';
+          results.push({
+            type: 'POST',
+            author: post.author || '[deleted]',
+            score: post.score || 0,
+            text: post.title + (body ? '\\n\\n' + body : '') + (post.url && !post.is_self ? '\\n' + post.url : ''),
+          });
+        }
+
+        // Recursive comment walker
+        // depth 0 = top-level comments; maxDepth is exclusive,
+        // so --depth 1 means top-level only, --depth 2 means one reply level, etc.
+        function walkComment(node, depth) {
+          if (!node || node.kind !== 't1') return;
+          var d = node.data;
+          var body = d.body || '';
+          if (body.length > maxLength) body = body.slice(0, maxLength) + '...';
+
+          // Indent prefix: apply to every line so multiline bodies stay aligned
+          var indent = '';
+          for (var i = 0; i < depth; i++) indent += '  ';
+          var prefix = depth === 0 ? '' : indent + '> ';
+          var indentedBody = depth === 0
+            ? body
+            : body.split('\\n').map(function(line) { return prefix + line; }).join('\\n');
+
+          results.push({
+            type: depth === 0 ? 'L0' : 'L' + depth,
+            author: d.author || '[deleted]',
+            score: d.score || 0,
+            text: indentedBody,
+          });
+
+          // Count all available replies (for accurate "more" count)
+          var t1Children = [];
+          var moreCount = 0;
+          if (d.replies && d.replies.data && d.replies.data.children) {
+            var children = d.replies.data.children;
+            for (var i = 0; i < children.length; i++) {
+              if (children[i].kind === 't1') {
+                t1Children.push(children[i]);
+              } else if (children[i].kind === 'more') {
+                moreCount += children[i].data.count || 0;
+              }
+            }
+          }
+
+          // At depth cutoff: don't recurse, but show all replies as hidden
+          if (depth + 1 >= maxDepth) {
+            var totalHidden = t1Children.length + moreCount;
+            if (totalHidden > 0) {
+              var cutoffIndent = '';
+              for (var j = 0; j <= depth; j++) cutoffIndent += '  ';
+              results.push({
+                type: 'L' + (depth + 1),
+                author: '',
+                score: '',
+                text: cutoffIndent + '[+' + totalHidden + ' more replies]',
+              });
+            }
+            return;
+          }
+
+          // Sort by score descending, take top N
+          t1Children.sort(function(a, b) { return (b.data.score || 0) - (a.data.score || 0); });
+          var toProcess = Math.min(t1Children.length, maxReplies);
+          for (var i = 0; i < toProcess; i++) {
+            walkComment(t1Children[i], depth + 1);
+          }
+
+          // Show hidden count (skipped replies + "more" stubs)
+          var hidden = t1Children.length - toProcess + moreCount;
+          if (hidden > 0) {
+            var moreIndent = '';
+            for (var j = 0; j <= depth; j++) moreIndent += '  ';
+            results.push({
+              type: 'L' + (depth + 1),
+              author: '',
+              score: '',
+              text: moreIndent + '[+' + hidden + ' more replies]',
+            });
+          }
+        }
+
+        // Walk top-level comments
+        var topLevel = data[1].data.children || [];
+        var t1TopLevel = [];
+        for (var i = 0; i < topLevel.length; i++) {
+          if (topLevel[i].kind === 't1') t1TopLevel.push(topLevel[i]);
+        }
+
+        // Top-level are already sorted by Reddit (sort param), take top N
+        for (var i = 0; i < Math.min(t1TopLevel.length, limit); i++) {
+          walkComment(t1TopLevel[i], 0);
+        }
+
+        // Count remaining
+        var moreTopLevel = topLevel.filter(function(c) { return c.kind === 'more'; })
+          .reduce(function(sum, c) { return sum + (c.data.count || 0); }, 0);
+        var hiddenTopLevel = Math.max(0, t1TopLevel.length - limit) + moreTopLevel;
+        if (hiddenTopLevel > 0) {
+          results.push({
+            type: '',
+            author: '',
+            score: '',
+            text: '[+' + hiddenTopLevel + ' more top-level comments]',
+          });
+        }
+
+        return results;
+      })()
+    `);
+        if (!data || typeof data !== 'object')
+            throw new Error('Failed to fetch post data');
+        if (!Array.isArray(data) && data.error)
+            throw new Error(data.error);
+        if (!Array.isArray(data))
+            throw new Error('Unexpected response');
+        return data;
+    },
+});

package/dist/clis/youtube/transcript-group.d.ts

@@ -0,0 +1,44 @@
+/**
+ * Transcript grouping: sentence merging, speaker detection, and chapter support.
+ * Ported and simplified from Defuddle's YouTube extractor.
+ *
+ * Raw segments (2-3 second fragments) are grouped into readable paragraphs:
+ * - Sentence boundaries: merge until sentence-ending punctuation (.!?)
+ * - Speaker turns: detect ">>" markers from YouTube auto-captions
+ * - Chapters: optional chapter headings inserted at appropriate timestamps
+ */
+export interface RawSegment {
+    start: number;
+    end: number;
+    text: string;
+}
+export interface GroupedSegment {
+    start: number;
+    text: string;
+    speakerChange: boolean;
+    speaker?: number;
+}
+export interface Chapter {
+    title: string;
+    start: number;
+}
+/**
+ * Group raw transcript segments into readable blocks.
+ * If speaker markers (>>) are present, groups by speaker turn.
+ * Otherwise, groups by sentence boundaries.
+ */
+export declare function groupTranscriptSegments(segments: {
+    start: number;
+    text: string;
+}[]): GroupedSegment[];
+/**
+ * Format grouped segments + chapters into a final text output.
+ */
+export declare function formatGroupedTranscript(segments: GroupedSegment[], chapters?: Chapter[]): {
+    rows: Array<{
+        timestamp: string;
+        speaker: string;
+        text: string;
+    }>;
+    plainText: string;
+};