@jackwener/opencli 0.7.2 → 0.7.3

package/dist/clis/reddit/read.js

@@ -0,0 +1,184 @@
+/**
+ * Reddit post reader with threaded comment tree.
+ *
+ * Replaces the original flat read.yaml with recursive comment traversal:
+ * - Top-K comments by score at each level
+ * - Configurable depth and replies-per-level
+ * - Indented output showing conversation threads
+ */
+import { cli, Strategy } from '../../registry.js';
+cli({
+    site: 'reddit',
+    name: 'read',
+    description: 'Read a Reddit post and its comments',
+    domain: 'reddit.com',
+    strategy: Strategy.COOKIE,
+    args: [
+        { name: 'post_id', required: true, help: 'Post ID (e.g. 1abc123) or full URL' },
+        { name: 'sort', default: 'best', help: 'Comment sort: best, top, new, controversial, old, qa' },
+        { name: 'limit', type: 'int', default: 25, help: 'Number of top-level comments' },
+        { name: 'depth', type: 'int', default: 2, help: 'Max reply depth (1=no replies, 2=one level of replies, etc.)' },
+        { name: 'replies', type: 'int', default: 5, help: 'Max replies shown per comment at each level (sorted by score)' },
+        { name: 'max_length', type: 'int', default: 2000, help: 'Max characters per comment body (min 100)' },
+    ],
+    columns: ['type', 'author', 'score', 'text'],
+    func: async (page, kwargs) => {
+        const sort = kwargs.sort ?? 'best';
+        const limit = Math.max(1, kwargs.limit ?? 25);
+        const maxDepth = Math.max(1, kwargs.depth ?? 2);
+        const maxReplies = Math.max(1, kwargs.replies ?? 5);
+        const maxLength = Math.max(100, kwargs.max_length ?? 2000);
+        await page.goto('https://www.reddit.com');
+        await page.wait(2);
+        const data = await page.evaluate(`
+      (async function() {
+        var postId = ${JSON.stringify(kwargs.post_id)};
+        var urlMatch = postId.match(/comments\\/([a-z0-9]+)/);
+        if (urlMatch) postId = urlMatch[1];
+
+        var sort = ${JSON.stringify(sort)};
+        var limit = ${limit};
+        var maxDepth = ${maxDepth};
+        var maxReplies = ${maxReplies};
+        var maxLength = ${maxLength};
+
+        // Request more from API than top-level limit to get inline replies
+        // depth param tells Reddit how deep to inline replies vs "more" stubs
+        var apiLimit = Math.max(limit * 3, 100);
+        var res = await fetch(
+          '/comments/' + postId + '.json?sort=' + sort + '&limit=' + apiLimit + '&depth=' + (maxDepth + 1) + '&raw_json=1',
+          { credentials: 'include' }
+        );
+        if (!res.ok) return { error: 'Reddit API returned HTTP ' + res.status };
+
+        var data;
+        try { data = await res.json(); } catch(e) { return { error: 'Failed to parse response' }; }
+        if (!Array.isArray(data) || data.length < 2) return { error: 'Unexpected response format' };
+
+        var results = [];
+
+        // Post
+        var post = data[0] && data[0].data && data[0].data.children && data[0].data.children[0] && data[0].data.children[0].data;
+        if (post) {
+          var body = post.selftext || '';
+          if (body.length > maxLength) body = body.slice(0, maxLength) + '\\n... [truncated]';
+          results.push({
+            type: 'POST',
+            author: post.author || '[deleted]',
+            score: post.score || 0,
+            text: post.title + (body ? '\\n\\n' + body : '') + (post.url && !post.is_self ? '\\n' + post.url : ''),
+          });
+        }
+
+        // Recursive comment walker
+        // depth 0 = top-level comments; maxDepth is exclusive,
+        // so --depth 1 means top-level only, --depth 2 means one reply level, etc.
+        function walkComment(node, depth) {
+          if (!node || node.kind !== 't1') return;
+          var d = node.data;
+          var body = d.body || '';
+          if (body.length > maxLength) body = body.slice(0, maxLength) + '...';
+
+          // Indent prefix: apply to every line so multiline bodies stay aligned
+          var indent = '';
+          for (var i = 0; i < depth; i++) indent += '  ';
+          var prefix = depth === 0 ? '' : indent + '> ';
+          var indentedBody = depth === 0
+            ? body
+            : body.split('\\n').map(function(line) { return prefix + line; }).join('\\n');
+
+          results.push({
+            type: depth === 0 ? 'L0' : 'L' + depth,
+            author: d.author || '[deleted]',
+            score: d.score || 0,
+            text: indentedBody,
+          });
+
+          // Count all available replies (for accurate "more" count)
+          var t1Children = [];
+          var moreCount = 0;
+          if (d.replies && d.replies.data && d.replies.data.children) {
+            var children = d.replies.data.children;
+            for (var i = 0; i < children.length; i++) {
+              if (children[i].kind === 't1') {
+                t1Children.push(children[i]);
+              } else if (children[i].kind === 'more') {
+                moreCount += children[i].data.count || 0;
+              }
+            }
+          }
+
+          // At depth cutoff: don't recurse, but show all replies as hidden
+          if (depth + 1 >= maxDepth) {
+            var totalHidden = t1Children.length + moreCount;
+            if (totalHidden > 0) {
+              var cutoffIndent = '';
+              for (var j = 0; j <= depth; j++) cutoffIndent += '  ';
+              results.push({
+                type: 'L' + (depth + 1),
+                author: '',
+                score: '',
+                text: cutoffIndent + '[+' + totalHidden + ' more replies]',
+              });
+            }
+            return;
+          }
+
+          // Sort by score descending, take top N
+          t1Children.sort(function(a, b) { return (b.data.score || 0) - (a.data.score || 0); });
+          var toProcess = Math.min(t1Children.length, maxReplies);
+          for (var i = 0; i < toProcess; i++) {
+            walkComment(t1Children[i], depth + 1);
+          }
+
+          // Show hidden count (skipped replies + "more" stubs)
+          var hidden = t1Children.length - toProcess + moreCount;
+          if (hidden > 0) {
+            var moreIndent = '';
+            for (var j = 0; j <= depth; j++) moreIndent += '  ';
+            results.push({
+              type: 'L' + (depth + 1),
+              author: '',
+              score: '',
+              text: moreIndent + '[+' + hidden + ' more replies]',
+            });
+          }
+        }
+
+        // Walk top-level comments
+        var topLevel = data[1].data.children || [];
+        var t1TopLevel = [];
+        for (var i = 0; i < topLevel.length; i++) {
+          if (topLevel[i].kind === 't1') t1TopLevel.push(topLevel[i]);
+        }
+
+        // Top-level are already sorted by Reddit (sort param), take top N
+        for (var i = 0; i < Math.min(t1TopLevel.length, limit); i++) {
+          walkComment(t1TopLevel[i], 0);
+        }
+
+        // Count remaining
+        var moreTopLevel = topLevel.filter(function(c) { return c.kind === 'more'; })
+          .reduce(function(sum, c) { return sum + (c.data.count || 0); }, 0);
+        var hiddenTopLevel = Math.max(0, t1TopLevel.length - limit) + moreTopLevel;
+        if (hiddenTopLevel > 0) {
+          results.push({
+            type: '',
+            author: '',
+            score: '',
+            text: '[+' + hiddenTopLevel + ' more top-level comments]',
+          });
+        }
+
+        return results;
+      })()
+    `);
+        if (!data || typeof data !== 'object')
+            throw new Error('Failed to fetch post data');
+        if (!Array.isArray(data) && data.error)
+            throw new Error(data.error);
+        if (!Array.isArray(data))
+            throw new Error('Unexpected response');
+        return data;
+    },
+});

package/dist/clis/youtube/transcript-group.d.ts

@@ -0,0 +1,44 @@
+/**
+ * Transcript grouping: sentence merging, speaker detection, and chapter support.
+ * Ported and simplified from Defuddle's YouTube extractor.
+ *
+ * Raw segments (2-3 second fragments) are grouped into readable paragraphs:
+ * - Sentence boundaries: merge until sentence-ending punctuation (.!?)
+ * - Speaker turns: detect ">>" markers from YouTube auto-captions
+ * - Chapters: optional chapter headings inserted at appropriate timestamps
+ */
+export interface RawSegment {
+    start: number;
+    end: number;
+    text: string;
+}
+export interface GroupedSegment {
+    start: number;
+    text: string;
+    speakerChange: boolean;
+    speaker?: number;
+}
+export interface Chapter {
+    title: string;
+    start: number;
+}
+/**
+ * Group raw transcript segments into readable blocks.
+ * If speaker markers (>>) are present, groups by speaker turn.
+ * Otherwise, groups by sentence boundaries.
+ */
+export declare function groupTranscriptSegments(segments: {
+    start: number;
+    text: string;
+}[]): GroupedSegment[];
+/**
+ * Format grouped segments + chapters into a final text output.
+ */
+export declare function formatGroupedTranscript(segments: GroupedSegment[], chapters?: Chapter[]): {
+    rows: Array<{
+        timestamp: string;
+        speaker: string;
+        text: string;
+    }>;
+    plainText: string;
+};

package/dist/clis/youtube/transcript-group.js

@@ -0,0 +1,226 @@
+/**
+ * Transcript grouping: sentence merging, speaker detection, and chapter support.
+ * Ported and simplified from Defuddle's YouTube extractor.
+ *
+ * Raw segments (2-3 second fragments) are grouped into readable paragraphs:
+ * - Sentence boundaries: merge until sentence-ending punctuation (.!?)
+ * - Speaker turns: detect ">>" markers from YouTube auto-captions
+ * - Chapters: optional chapter headings inserted at appropriate timestamps
+ */
+// Include CJK sentence-ending punctuation: 。！？ (fullwidth: ．！？)
+const SENTENCE_END = /[.!?\u3002\uFF01\uFF1F\uFF0E]["'\u2019\u201D)]*\s*$/;
+const QUESTION_END = /[?\uFF1F]["'\u2019\u201D)]*\s*$/;
+const TRANSCRIPT_GROUP_GAP_SECONDS = 20;
+const TURN_MERGE_MAX_WORDS = 80;
+const TURN_MERGE_MAX_SPAN_SECONDS = 45;
+const SHORT_UTTERANCE_MAX_WORDS = 3;
+const FIRST_GROUP_MERGE_MIN_WORDS = 8;
+function countWords(text) {
+    return text.split(/\s+/).filter(Boolean).length;
+}
+/**
+ * Group raw transcript segments into readable blocks.
+ * If speaker markers (>>) are present, groups by speaker turn.
+ * Otherwise, groups by sentence boundaries.
+ */
+export function groupTranscriptSegments(segments) {
+    if (segments.length === 0)
+        return [];
+    const hasSpeakerMarkers = segments.some(s => /^>>/.test(s.text));
+    return hasSpeakerMarkers ? groupBySpeaker(segments) : groupBySentence(segments);
+}
+/**
+ * Format grouped segments + chapters into a final text output.
+ */
+export function formatGroupedTranscript(segments, chapters = []) {
+    const sortedChapters = [...chapters].sort((a, b) => a.start - b.start);
+    let chapterIdx = 0;
+    const rows = [];
+    const textParts = [];
+    for (const segment of segments) {
+        // Insert chapter headings
+        while (chapterIdx < sortedChapters.length && sortedChapters[chapterIdx].start <= segment.start) {
+            const title = sortedChapters[chapterIdx].title;
+            rows.push({ timestamp: fmtTime(sortedChapters[chapterIdx].start), speaker: '', text: `[Chapter] ${title}` });
+            if (textParts.length > 0)
+                textParts.push('');
+            textParts.push(`### ${title}`);
+            textParts.push('');
+            chapterIdx++;
+        }
+        const timestamp = fmtTime(segment.start);
+        const speaker = segment.speaker !== undefined ? `Speaker ${segment.speaker + 1}` : '';
+        rows.push({ timestamp, speaker, text: segment.text });
+        if (segment.speakerChange && textParts.length > 0) {
+            textParts.push('');
+        }
+        textParts.push(`${timestamp} ${segment.text}`);
+    }
+    return { rows, plainText: textParts.join('\n') };
+}
+function fmtTime(sec) {
+    const h = Math.floor(sec / 3600);
+    const m = Math.floor((sec % 3600) / 60);
+    const s = Math.floor(sec % 60);
+    if (h > 0) {
+        return `${h}:${String(m).padStart(2, '0')}:${String(s).padStart(2, '0')}`;
+    }
+    return `${m}:${String(s).padStart(2, '0')}`;
+}
+// ── Sentence grouping ─────────────────────────────────────────────────────
+// Max time span (seconds) for a single group when no sentence boundaries are found.
+// Prevents unbounded merging for languages without punctuation (Chinese, etc.).
+const MAX_GROUP_SPAN_SECONDS = 30;
+function groupBySentence(segments) {
+    const groups = [];
+    let buffer = '';
+    let bufferStart = 0;
+    let lastStart = 0;
+    const flush = () => {
+        if (buffer.trim()) {
+            groups.push({ start: bufferStart, text: buffer.trim(), speakerChange: false });
+            buffer = '';
+        }
+    };
+    for (const seg of segments) {
+        // Large gap between segments — always flush
+        if (buffer && seg.start - lastStart > TRANSCRIPT_GROUP_GAP_SECONDS) {
+            flush();
+        }
+        // Time-based flush: prevent unbounded groups for unpunctuated languages
+        if (buffer && seg.start - bufferStart > MAX_GROUP_SPAN_SECONDS) {
+            flush();
+        }
+        if (!buffer)
+            bufferStart = seg.start;
+        buffer += (buffer ? ' ' : '') + seg.text;
+        lastStart = seg.start;
+        if (SENTENCE_END.test(seg.text))
+            flush();
+    }
+    flush();
+    return groups;
+}
+// ── Speaker grouping ──────────────────────────────────────────────────────
+function groupBySpeaker(segments) {
+    const turns = [];
+    let currentTurn = null;
+    let speakerIndex = -1;
+    let prevSegText = '';
+    for (const seg of segments) {
+        const isSpeakerChange = /^>>/.test(seg.text);
+        const cleanText = seg.text.replace(/^>>\s*/, '').replace(/^-\s+/, '');
+        const prevEndsWithComma = /,\s*$/.test(prevSegText);
+        const prevEndedSentence = (SENTENCE_END.test(prevSegText) || !prevSegText) && !prevEndsWithComma;
+        const isRealSpeakerChange = isSpeakerChange && prevEndedSentence;
+        if (isRealSpeakerChange) {
+            if (currentTurn)
+                turns.push(currentTurn);
+            speakerIndex = (speakerIndex + 1) % 2;
+            currentTurn = {
+                start: seg.start,
+                segments: [{ start: seg.start, text: cleanText }],
+                speakerChange: true,
+                speaker: speakerIndex,
+            };
+        }
+        else {
+            if (!currentTurn) {
+                currentTurn = { start: seg.start, segments: [], speakerChange: false };
+            }
+            currentTurn.segments.push({ start: seg.start, text: cleanText });
+        }
+        prevSegText = cleanText;
+    }
+    if (currentTurn)
+        turns.push(currentTurn);
+    splitAffirmativeTurns(turns);
+    const groups = [];
+    for (const turn of turns) {
+        const sentenceGroups = turn.speaker === undefined
+            ? groupBySentence(turn.segments)
+            : mergeSentenceGroupsWithinTurn(groupBySentence(turn.segments));
+        for (let i = 0; i < sentenceGroups.length; i++) {
+            groups.push({
+                ...sentenceGroups[i],
+                speakerChange: i === 0 && turn.speakerChange,
+                speaker: turn.speaker,
+            });
+        }
+    }
+    return groups;
+}
+function splitAffirmativeTurns(turns) {
+    const affirmativePattern = /^(mhm|yeah|yes|yep|right|okay|ok|absolutely|sure|exactly|uh-huh|mm-hmm)[.!,]?\s+/i;
+    for (let i = 0; i < turns.length; i++) {
+        const turn = turns[i];
+        if (turn.speaker === undefined || turn.segments.length === 0)
+            continue;
+        const firstSeg = turn.segments[0];
+        const match = affirmativePattern.exec(firstSeg.text);
+        if (!match)
+            continue;
+        if (/,\s*$/.test(match[0]))
+            continue;
+        const remainder = firstSeg.text.slice(match[0].length).trim();
+        const restSegments = turn.segments.slice(1);
+        const restWords = countWords(remainder) + restSegments.reduce((sum, s) => sum + countWords(s.text), 0);
+        if (restWords < 30)
+            continue;
+        const affirmativeText = match[0].trimEnd();
+        const newRestSegments = remainder
+            ? [{ start: firstSeg.start, text: remainder }, ...restSegments]
+            : restSegments;
+        turns.splice(i, 1, {
+            start: turn.start,
+            segments: [{ start: firstSeg.start, text: affirmativeText }],
+            speakerChange: turn.speakerChange,
+            speaker: turn.speaker,
+        }, {
+            start: newRestSegments[0].start,
+            segments: newRestSegments,
+            speakerChange: true,
+            speaker: turn.speaker === 0 ? 1 : 0,
+        });
+        i++;
+    }
+}
+function mergeSentenceGroupsWithinTurn(groups) {
+    if (groups.length <= 1)
+        return groups;
+    const merged = [];
+    let current = { ...groups[0] };
+    let currentIsFirstInTurn = true;
+    for (let i = 1; i < groups.length; i++) {
+        const next = groups[i];
+        if (shouldMergeSentenceGroups(current, next, currentIsFirstInTurn)) {
+            current.text = `${current.text} ${next.text}`;
+            continue;
+        }
+        merged.push(current);
+        current = { ...next };
+        currentIsFirstInTurn = false;
+    }
+    merged.push(current);
+    return merged;
+}
+function shouldMergeSentenceGroups(current, next, currentIsFirstInTurn) {
+    const currentWords = countWords(current.text);
+    const nextWords = countWords(next.text);
+    if (isShortStandaloneUtterance(current.text, currentWords)
+        || isShortStandaloneUtterance(next.text, nextWords))
+        return false;
+    if (currentIsFirstInTurn && currentWords < FIRST_GROUP_MERGE_MIN_WORDS)
+        return false;
+    if (QUESTION_END.test(current.text) || QUESTION_END.test(next.text))
+        return false;
+    if (currentWords + nextWords > TURN_MERGE_MAX_WORDS)
+        return false;
+    if (next.start - current.start > TURN_MERGE_MAX_SPAN_SECONDS)
+        return false;
+    return true;
+}
+function isShortStandaloneUtterance(text, words) {
+    const w = words ?? countWords(text);
+    return w > 0 && w <= SHORT_UTTERANCE_MAX_WORDS && SENTENCE_END.test(text);
+}

package/dist/clis/youtube/transcript-group.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/clis/youtube/transcript-group.test.js

@@ -0,0 +1,99 @@
+import { describe, it, expect } from 'vitest';
+import { groupTranscriptSegments, formatGroupedTranscript } from './transcript-group.js';
+describe('groupTranscriptSegments', () => {
+    it('groups segments by sentence boundaries', () => {
+        const segments = [
+            { start: 0, text: 'Hello there.' },
+            { start: 2, text: 'How are you doing today?' },
+            { start: 5, text: 'I am' },
+            { start: 6, text: 'doing well.' },
+        ];
+        const result = groupTranscriptSegments(segments);
+        expect(result).toHaveLength(3);
+        expect(result[0].text).toBe('Hello there.');
+        expect(result[1].text).toBe('How are you doing today?');
+        expect(result[2].text).toBe('I am doing well.');
+    });
+    it('flushes on large time gaps', () => {
+        const segments = [
+            { start: 0, text: 'First part' },
+            { start: 2, text: 'still first' },
+            { start: 25, text: 'second part after gap' },
+        ];
+        const result = groupTranscriptSegments(segments);
+        expect(result).toHaveLength(2);
+        expect(result[0].text).toBe('First part still first');
+        expect(result[1].text).toBe('second part after gap');
+    });
+    it('respects 30s max group span for unpunctuated text', () => {
+        // Simulate CJK captions without punctuation
+        const segments = Array.from({ length: 20 }, (_, i) => ({
+            start: i * 2,
+            text: `segment${i}`,
+        }));
+        const result = groupTranscriptSegments(segments);
+        // 20 segments * 2s = 40s total, should be split into at least 2 groups
+        expect(result.length).toBeGreaterThanOrEqual(2);
+        // No single group should span more than ~30s
+        for (const g of result) {
+            const words = g.text.split(' ');
+            // With 2s per segment and 30s max, each group should have at most ~16 segments
+            expect(words.length).toBeLessThanOrEqual(16);
+        }
+    });
+    it('detects speaker changes via >> markers', () => {
+        const segments = [
+            { start: 0, text: '>> How are you?' },
+            { start: 3, text: '>> I am fine.' },
+        ];
+        const result = groupTranscriptSegments(segments);
+        expect(result.some(g => g.speakerChange)).toBe(true);
+        expect(result.some(g => g.speaker !== undefined)).toBe(true);
+    });
+    it('recognizes CJK sentence-ending punctuation', () => {
+        const segments = [
+            { start: 0, text: '你好世界。' },
+            { start: 2, text: '这是测试' },
+            { start: 4, text: '内容。' },
+        ];
+        const result = groupTranscriptSegments(segments);
+        expect(result).toHaveLength(2);
+        expect(result[0].text).toBe('你好世界。');
+        expect(result[1].text).toBe('这是测试 内容。');
+    });
+    it('returns empty array for empty input', () => {
+        expect(groupTranscriptSegments([])).toEqual([]);
+    });
+});
+describe('formatGroupedTranscript', () => {
+    it('formats timestamps correctly', () => {
+        const segments = [
+            { start: 65, text: 'One minute five.', speakerChange: false },
+            { start: 3661, text: 'One hour one minute.', speakerChange: false },
+        ];
+        const { rows } = formatGroupedTranscript(segments);
+        expect(rows[0].timestamp).toBe('1:05');
+        expect(rows[1].timestamp).toBe('1:01:01');
+    });
+    it('inserts chapter headings at correct positions', () => {
+        const segments = [
+            { start: 0, text: 'Intro text.', speakerChange: false },
+            { start: 60, text: 'Chapter content.', speakerChange: false },
+        ];
+        const chapters = [{ title: 'Introduction', start: 0 }, { title: 'Main', start: 50 }];
+        const { rows } = formatGroupedTranscript(segments, chapters);
+        expect(rows[0].text).toBe('[Chapter] Introduction');
+        expect(rows[1].text).toBe('Intro text.');
+        expect(rows[2].text).toBe('[Chapter] Main');
+        expect(rows[3].text).toBe('Chapter content.');
+    });
+    it('labels speakers', () => {
+        const segments = [
+            { start: 0, text: 'Hello.', speakerChange: true, speaker: 0 },
+            { start: 5, text: 'Hi there.', speakerChange: true, speaker: 1 },
+        ];
+        const { rows } = formatGroupedTranscript(segments);
+        expect(rows[0].speaker).toBe('Speaker 1');
+        expect(rows[1].speaker).toBe('Speaker 2');
+    });
+});

package/dist/clis/youtube/transcript.d.ts

@@ -0,0 +1 @@
+export {};