@huyuan-ai/cli 1.0.0 → 1.0.2

package/vendor/opencli/dist/clis/youtube/comments.js

@@ -0,0 +1,95 @@
+/**
+ * YouTube comments — get video comments via InnerTube API.
+ */
+import { cli, Strategy } from '../../registry.js';
+import { CommandExecutionError } from '../../errors.js';
+import { parseVideoId } from './utils.js';
+cli({
+    site: 'youtube',
+    name: 'comments',
+    description: 'Get YouTube video comments',
+    domain: 'www.youtube.com',
+    strategy: Strategy.COOKIE,
+    args: [
+        { name: 'url', required: true, positional: true, help: 'YouTube video URL or video ID' },
+        { name: 'limit', type: 'int', default: 20, help: 'Max comments (max 100)' },
+    ],
+    columns: ['rank', 'author', 'text', 'likes', 'replies', 'time'],
+    func: async (page, kwargs) => {
+        const videoId = parseVideoId(kwargs.url);
+        const limit = Math.min(kwargs.limit || 20, 100);
+        await page.goto(`https://www.youtube.com/watch?v=${videoId}`);
+        await page.wait(3);
+        const data = await page.evaluate(`
+      (async () => {
+        const videoId = ${JSON.stringify(videoId)};
+        const limit = ${limit};
+        const cfg = window.ytcfg?.data_ || {};
+        const apiKey = cfg.INNERTUBE_API_KEY;
+        const context = cfg.INNERTUBE_CONTEXT;
+        if (!apiKey || !context) return {error: 'YouTube config not found'};
+
+        // Step 1: Get comment continuation token
+        let continuationToken = null;
+
+        // Try from current page ytInitialData
+        if (window.ytInitialData) {
+          const results = window.ytInitialData.contents?.twoColumnWatchNextResults?.results?.results?.contents || [];
+          const commentSection = results.find(i => i.itemSectionRenderer?.targetId === 'comments-section');
+          continuationToken = commentSection?.itemSectionRenderer?.contents?.[0]?.continuationItemRenderer?.continuationEndpoint?.continuationCommand?.token;
+        }
+
+        // Fallback: fetch via next API
+        if (!continuationToken) {
+          const nextResp = await fetch('/youtubei/v1/next?key=' + apiKey + '&prettyPrint=false', {
+            method: 'POST', credentials: 'include',
+            headers: {'Content-Type': 'application/json'},
+            body: JSON.stringify({context, videoId})
+          });
+          if (!nextResp.ok) return {error: 'Failed to get video data: HTTP ' + nextResp.status};
+          const nextData = await nextResp.json();
+          const results = nextData.contents?.twoColumnWatchNextResults?.results?.results?.contents || [];
+          const commentSection = results.find(i => i.itemSectionRenderer?.targetId === 'comments-section');
+          continuationToken = commentSection?.itemSectionRenderer?.contents?.[0]?.continuationItemRenderer?.continuationEndpoint?.continuationCommand?.token;
+        }
+
+        if (!continuationToken) return {error: 'No comment section found — comments may be disabled'};
+
+        // Step 2: Fetch comments
+        const commentResp = await fetch('/youtubei/v1/next?key=' + apiKey + '&prettyPrint=false', {
+          method: 'POST', credentials: 'include',
+          headers: {'Content-Type': 'application/json'},
+          body: JSON.stringify({context, continuation: continuationToken})
+        });
+        if (!commentResp.ok) return {error: 'Failed to fetch comments: HTTP ' + commentResp.status};
+        const commentData = await commentResp.json();
+
+        // Parse from frameworkUpdates (new ViewModel format)
+        const mutations = commentData.frameworkUpdates?.entityBatchUpdate?.mutations || [];
+        const commentEntities = mutations.filter(m => m.payload?.commentEntityPayload);
+
+        return commentEntities.slice(0, limit).map((m, i) => {
+          const p = m.payload.commentEntityPayload;
+          const props = p.properties || {};
+          const author = p.author || {};
+          const toolbar = p.toolbar || {};
+          return {
+            rank: i + 1,
+            author: author.displayName || '',
+            text: (props.content?.content || '').substring(0, 300),
+            likes: toolbar.likeCountNotliked || '0',
+            replies: toolbar.replyCount || '0',
+            time: props.publishedTime || '',
+          };
+        });
+      })()
+    `);
+        if (!Array.isArray(data)) {
+            const errMsg = data && typeof data === 'object' ? String(data.error || '') : '';
+            if (errMsg)
+                throw new CommandExecutionError(errMsg);
+            return [];
+        }
+        return data;
+    },
+});

package/vendor/opencli/dist/clis/youtube/search.d.ts

@@ -0,0 +1 @@
+export {};

package/vendor/opencli/dist/clis/youtube/search.js

@@ -0,0 +1,99 @@
+/**
+ * YouTube search — innertube API via browser session.
+ */
+import { cli, Strategy } from '../../registry.js';
+cli({
+    site: 'youtube',
+    name: 'search',
+    description: 'Search YouTube videos',
+    domain: 'www.youtube.com',
+    strategy: Strategy.COOKIE,
+    args: [
+        { name: 'query', required: true, positional: true, help: 'Search query' },
+        { name: 'limit', type: 'int', default: 20, help: 'Max results (max 50)' },
+        { name: 'type', default: '', help: 'Filter type: shorts, video, channel, playlist' },
+        { name: 'upload', default: '', help: 'Upload date: hour, today, week, month, year' },
+        { name: 'sort', default: '', help: 'Sort by: relevance, date, views, rating' },
+    ],
+    columns: ['rank', 'title', 'channel', 'views', 'duration', 'published', 'url'],
+    func: async (page, kwargs) => {
+        const limit = Math.min(kwargs.limit || 20, 50);
+        const query = encodeURIComponent(kwargs.query);
+        // Build search URL with filter params
+        // YouTube uses sp= parameter for filters — we use the URL approach for reliability
+        const spMap = {
+            // type filters
+            'shorts': 'EgIQCQ%3D%3D', // Shorts (type=9)
+            'video': 'EgIQAQ%3D%3D',
+            'channel': 'EgIQAg%3D%3D',
+            'playlist': 'EgIQAw%3D%3D',
+            // upload date filters (can be combined with type via URL)
+            'hour': 'EgIIAQ%3D%3D',
+            'today': 'EgIIAg%3D%3D',
+            'week': 'EgIIAw%3D%3D',
+            'month': 'EgIIBA%3D%3D',
+            'year': 'EgIIBQ%3D%3D',
+        };
+        const sortMap = {
+            'date': 'CAI%3D',
+            'views': 'CAM%3D',
+            'rating': 'CAE%3D',
+        };
+        // YouTube only supports a single sp= parameter — pick the most specific filter.
+        // Priority: type > upload > sort (type is the most common use case)
+        let sp = '';
+        if (kwargs.type && spMap[kwargs.type])
+            sp = spMap[kwargs.type];
+        else if (kwargs.upload && spMap[kwargs.upload])
+            sp = spMap[kwargs.upload];
+        else if (kwargs.sort && sortMap[kwargs.sort])
+            sp = sortMap[kwargs.sort];
+        let url = `https://www.youtube.com/results?search_query=${query}`;
+        if (sp)
+            url += `&sp=${sp}`;
+        await page.goto(url);
+        await page.wait(3);
+        const data = await page.evaluate(`
+      (async () => {
+        const data = window.ytInitialData;
+        if (!data) return {error: 'YouTube data not found'};
+
+        const contents = data.contents?.twoColumnSearchResultsRenderer?.primaryContents?.sectionListRenderer?.contents || [];
+        const videos = [];
+        for (const section of contents) {
+          const items = section.itemSectionRenderer?.contents || section.reelShelfRenderer?.items || [];
+          for (const item of items) {
+            if (videos.length >= ${limit}) break;
+            if (item.videoRenderer) {
+              const v = item.videoRenderer;
+              videos.push({
+                rank: videos.length + 1,
+                title: v.title?.runs?.[0]?.text || '',
+                channel: v.ownerText?.runs?.[0]?.text || '',
+                views: v.viewCountText?.simpleText || v.shortViewCountText?.simpleText || '',
+                duration: v.lengthText?.simpleText || 'LIVE',
+                published: v.publishedTimeText?.simpleText || '',
+                url: 'https://www.youtube.com/watch?v=' + v.videoId
+              });
+            } else if (item.reelItemRenderer) {
+              const r = item.reelItemRenderer;
+              videos.push({
+                rank: videos.length + 1,
+                title: r.headline?.simpleText || '',
+                channel: r.navigationEndpoint?.reelWatchEndpoint?.overlay?.reelPlayerOverlayRenderer?.reelPlayerHeaderSupportedRenderers?.reelPlayerHeaderRenderer?.channelTitleText?.runs?.[0]?.text || '',
+                views: r.viewCountText?.simpleText || '',
+                duration: 'SHORT',
+                published: r.publishedTimeText?.simpleText || '',
+                url: 'https://www.youtube.com/shorts/' + r.videoId
+              });
+            }
+          }
+        }
+        return videos;
+      })()
+    `);
+        if (!Array.isArray(data))
+            return [];
+        return data;
+    },
+});

package/vendor/opencli/dist/clis/youtube/transcript-group.d.ts

@@ -0,0 +1,44 @@
+/**
+ * Transcript grouping: sentence merging, speaker detection, and chapter support.
+ * Ported and simplified from Defuddle's YouTube extractor.
+ *
+ * Raw segments (2-3 second fragments) are grouped into readable paragraphs:
+ * - Sentence boundaries: merge until sentence-ending punctuation (.!?)
+ * - Speaker turns: detect ">>" markers from YouTube auto-captions
+ * - Chapters: optional chapter headings inserted at appropriate timestamps
+ */
+export interface RawSegment {
+    start: number;
+    end: number;
+    text: string;
+}
+export interface GroupedSegment {
+    start: number;
+    text: string;
+    speakerChange: boolean;
+    speaker?: number;
+}
+export interface Chapter {
+    title: string;
+    start: number;
+}
+/**
+ * Group raw transcript segments into readable blocks.
+ * If speaker markers (>>) are present, groups by speaker turn.
+ * Otherwise, groups by sentence boundaries.
+ */
+export declare function groupTranscriptSegments(segments: {
+    start: number;
+    text: string;
+}[]): GroupedSegment[];
+/**
+ * Format grouped segments + chapters into a final text output.
+ */
+export declare function formatGroupedTranscript(segments: GroupedSegment[], chapters?: Chapter[]): {
+    rows: Array<{
+        timestamp: string;
+        speaker: string;
+        text: string;
+    }>;
+    plainText: string;
+};

package/vendor/opencli/dist/clis/youtube/transcript-group.js

@@ -0,0 +1,226 @@
+/**
+ * Transcript grouping: sentence merging, speaker detection, and chapter support.
+ * Ported and simplified from Defuddle's YouTube extractor.
+ *
+ * Raw segments (2-3 second fragments) are grouped into readable paragraphs:
+ * - Sentence boundaries: merge until sentence-ending punctuation (.!?)
+ * - Speaker turns: detect ">>" markers from YouTube auto-captions
+ * - Chapters: optional chapter headings inserted at appropriate timestamps
+ */
+// Include CJK sentence-ending punctuation: 。！？ (fullwidth: ．！？)
+const SENTENCE_END = /[.!?\u3002\uFF01\uFF1F\uFF0E]["'\u2019\u201D)]*\s*$/;
+const QUESTION_END = /[?\uFF1F]["'\u2019\u201D)]*\s*$/;
+const TRANSCRIPT_GROUP_GAP_SECONDS = 20;
+const TURN_MERGE_MAX_WORDS = 80;
+const TURN_MERGE_MAX_SPAN_SECONDS = 45;
+const SHORT_UTTERANCE_MAX_WORDS = 3;
+const FIRST_GROUP_MERGE_MIN_WORDS = 8;
+function countWords(text) {
+    return text.split(/\s+/).filter(Boolean).length;
+}
+/**
+ * Group raw transcript segments into readable blocks.
+ * If speaker markers (>>) are present, groups by speaker turn.
+ * Otherwise, groups by sentence boundaries.
+ */
+export function groupTranscriptSegments(segments) {
+    if (segments.length === 0)
+        return [];
+    const hasSpeakerMarkers = segments.some(s => /^>>/.test(s.text));
+    return hasSpeakerMarkers ? groupBySpeaker(segments) : groupBySentence(segments);
+}
+/**
+ * Format grouped segments + chapters into a final text output.
+ */
+export function formatGroupedTranscript(segments, chapters = []) {
+    const sortedChapters = [...chapters].sort((a, b) => a.start - b.start);
+    let chapterIdx = 0;
+    const rows = [];
+    const textParts = [];
+    for (const segment of segments) {
+        // Insert chapter headings
+        while (chapterIdx < sortedChapters.length && sortedChapters[chapterIdx].start <= segment.start) {
+            const title = sortedChapters[chapterIdx].title;
+            rows.push({ timestamp: fmtTime(sortedChapters[chapterIdx].start), speaker: '', text: `[Chapter] ${title}` });
+            if (textParts.length > 0)
+                textParts.push('');
+            textParts.push(`### ${title}`);
+            textParts.push('');
+            chapterIdx++;
+        }
+        const timestamp = fmtTime(segment.start);
+        const speaker = segment.speaker !== undefined ? `Speaker ${segment.speaker + 1}` : '';
+        rows.push({ timestamp, speaker, text: segment.text });
+        if (segment.speakerChange && textParts.length > 0) {
+            textParts.push('');
+        }
+        textParts.push(`${timestamp} ${segment.text}`);
+    }
+    return { rows, plainText: textParts.join('\n') };
+}
+function fmtTime(sec) {
+    const h = Math.floor(sec / 3600);
+    const m = Math.floor((sec % 3600) / 60);
+    const s = Math.floor(sec % 60);
+    if (h > 0) {
+        return `${h}:${String(m).padStart(2, '0')}:${String(s).padStart(2, '0')}`;
+    }
+    return `${m}:${String(s).padStart(2, '0')}`;
+}
+// ── Sentence grouping ─────────────────────────────────────────────────────
+// Max time span (seconds) for a single group when no sentence boundaries are found.
+// Prevents unbounded merging for languages without punctuation (Chinese, etc.).
+const MAX_GROUP_SPAN_SECONDS = 30;
+function groupBySentence(segments) {
+    const groups = [];
+    let buffer = '';
+    let bufferStart = 0;
+    let lastStart = 0;
+    const flush = () => {
+        if (buffer.trim()) {
+            groups.push({ start: bufferStart, text: buffer.trim(), speakerChange: false });
+            buffer = '';
+        }
+    };
+    for (const seg of segments) {
+        // Large gap between segments — always flush
+        if (buffer && seg.start - lastStart > TRANSCRIPT_GROUP_GAP_SECONDS) {
+            flush();
+        }
+        // Time-based flush: prevent unbounded groups for unpunctuated languages
+        if (buffer && seg.start - bufferStart > MAX_GROUP_SPAN_SECONDS) {
+            flush();
+        }
+        if (!buffer)
+            bufferStart = seg.start;
+        buffer += (buffer ? ' ' : '') + seg.text;
+        lastStart = seg.start;
+        if (SENTENCE_END.test(seg.text))
+            flush();
+    }
+    flush();
+    return groups;
+}
+// ── Speaker grouping ──────────────────────────────────────────────────────
+function groupBySpeaker(segments) {
+    const turns = [];
+    let currentTurn = null;
+    let speakerIndex = -1;
+    let prevSegText = '';
+    for (const seg of segments) {
+        const isSpeakerChange = /^>>/.test(seg.text);
+        const cleanText = seg.text.replace(/^>>\s*/, '').replace(/^-\s+/, '');
+        const prevEndsWithComma = /,\s*$/.test(prevSegText);
+        const prevEndedSentence = (SENTENCE_END.test(prevSegText) || !prevSegText) && !prevEndsWithComma;
+        const isRealSpeakerChange = isSpeakerChange && prevEndedSentence;
+        if (isRealSpeakerChange) {
+            if (currentTurn)
+                turns.push(currentTurn);
+            speakerIndex = (speakerIndex + 1) % 2;
+            currentTurn = {
+                start: seg.start,
+                segments: [{ start: seg.start, text: cleanText }],
+                speakerChange: true,
+                speaker: speakerIndex,
+            };
+        }
+        else {
+            if (!currentTurn) {
+                currentTurn = { start: seg.start, segments: [], speakerChange: false };
+            }
+            currentTurn.segments.push({ start: seg.start, text: cleanText });
+        }
+        prevSegText = cleanText;
+    }
+    if (currentTurn)
+        turns.push(currentTurn);
+    splitAffirmativeTurns(turns);
+    const groups = [];
+    for (const turn of turns) {
+        const sentenceGroups = turn.speaker === undefined
+            ? groupBySentence(turn.segments)
+            : mergeSentenceGroupsWithinTurn(groupBySentence(turn.segments));
+        for (let i = 0; i < sentenceGroups.length; i++) {
+            groups.push({
+                ...sentenceGroups[i],
+                speakerChange: i === 0 && turn.speakerChange,
+                speaker: turn.speaker,
+            });
+        }
+    }
+    return groups;
+}
+function splitAffirmativeTurns(turns) {
+    const affirmativePattern = /^(mhm|yeah|yes|yep|right|okay|ok|absolutely|sure|exactly|uh-huh|mm-hmm)[.!,]?\s+/i;
+    for (let i = 0; i < turns.length; i++) {
+        const turn = turns[i];
+        if (turn.speaker === undefined || turn.segments.length === 0)
+            continue;
+        const firstSeg = turn.segments[0];
+        const match = affirmativePattern.exec(firstSeg.text);
+        if (!match)
+            continue;
+        if (/,\s*$/.test(match[0]))
+            continue;
+        const remainder = firstSeg.text.slice(match[0].length).trim();
+        const restSegments = turn.segments.slice(1);
+        const restWords = countWords(remainder) + restSegments.reduce((sum, s) => sum + countWords(s.text), 0);
+        if (restWords < 30)
+            continue;
+        const affirmativeText = match[0].trimEnd();
+        const newRestSegments = remainder
+            ? [{ start: firstSeg.start, text: remainder }, ...restSegments]
+            : restSegments;
+        turns.splice(i, 1, {
+            start: turn.start,
+            segments: [{ start: firstSeg.start, text: affirmativeText }],
+            speakerChange: turn.speakerChange,
+            speaker: turn.speaker,
+        }, {
+            start: newRestSegments[0].start,
+            segments: newRestSegments,
+            speakerChange: true,
+            speaker: turn.speaker === 0 ? 1 : 0,
+        });
+        i++;
+    }
+}
+function mergeSentenceGroupsWithinTurn(groups) {
+    if (groups.length <= 1)
+        return groups;
+    const merged = [];
+    let current = { ...groups[0] };
+    let currentIsFirstInTurn = true;
+    for (let i = 1; i < groups.length; i++) {
+        const next = groups[i];
+        if (shouldMergeSentenceGroups(current, next, currentIsFirstInTurn)) {
+            current.text = `${current.text} ${next.text}`;
+            continue;
+        }
+        merged.push(current);
+        current = { ...next };
+        currentIsFirstInTurn = false;
+    }
+    merged.push(current);
+    return merged;
+}
+function shouldMergeSentenceGroups(current, next, currentIsFirstInTurn) {
+    const currentWords = countWords(current.text);
+    const nextWords = countWords(next.text);
+    if (isShortStandaloneUtterance(current.text, currentWords)
+        || isShortStandaloneUtterance(next.text, nextWords))
+        return false;
+    if (currentIsFirstInTurn && currentWords < FIRST_GROUP_MERGE_MIN_WORDS)
+        return false;
+    if (QUESTION_END.test(current.text) || QUESTION_END.test(next.text))
+        return false;
+    if (currentWords + nextWords > TURN_MERGE_MAX_WORDS)
+        return false;
+    if (next.start - current.start > TURN_MERGE_MAX_SPAN_SECONDS)
+        return false;
+    return true;
+}
+function isShortStandaloneUtterance(text, words) {
+    const w = words ?? countWords(text);
+    return w > 0 && w <= SHORT_UTTERANCE_MAX_WORDS && SENTENCE_END.test(text);
+}

package/vendor/opencli/dist/clis/youtube/transcript-group.test.d.ts

@@ -0,0 +1 @@
+export {};

package/vendor/opencli/dist/clis/youtube/transcript-group.test.js

@@ -0,0 +1,99 @@
+import { describe, it, expect } from 'vitest';
+import { groupTranscriptSegments, formatGroupedTranscript } from './transcript-group.js';
+describe('groupTranscriptSegments', () => {
+    it('groups segments by sentence boundaries', () => {
+        const segments = [
+            { start: 0, text: 'Hello there.' },
+            { start: 2, text: 'How are you doing today?' },
+            { start: 5, text: 'I am' },
+            { start: 6, text: 'doing well.' },
+        ];
+        const result = groupTranscriptSegments(segments);
+        expect(result).toHaveLength(3);
+        expect(result[0].text).toBe('Hello there.');
+        expect(result[1].text).toBe('How are you doing today?');
+        expect(result[2].text).toBe('I am doing well.');
+    });
+    it('flushes on large time gaps', () => {
+        const segments = [
+            { start: 0, text: 'First part' },
+            { start: 2, text: 'still first' },
+            { start: 25, text: 'second part after gap' },
+        ];
+        const result = groupTranscriptSegments(segments);
+        expect(result).toHaveLength(2);
+        expect(result[0].text).toBe('First part still first');
+        expect(result[1].text).toBe('second part after gap');
+    });
+    it('respects 30s max group span for unpunctuated text', () => {
+        // Simulate CJK captions without punctuation
+        const segments = Array.from({ length: 20 }, (_, i) => ({
+            start: i * 2,
+            text: `segment${i}`,
+        }));
+        const result = groupTranscriptSegments(segments);
+        // 20 segments * 2s = 40s total, should be split into at least 2 groups
+        expect(result.length).toBeGreaterThanOrEqual(2);
+        // No single group should span more than ~30s
+        for (const g of result) {
+            const words = g.text.split(' ');
+            // With 2s per segment and 30s max, each group should have at most ~16 segments
+            expect(words.length).toBeLessThanOrEqual(16);
+        }
+    });
+    it('detects speaker changes via >> markers', () => {
+        const segments = [
+            { start: 0, text: '>> How are you?' },
+            { start: 3, text: '>> I am fine.' },
+        ];
+        const result = groupTranscriptSegments(segments);
+        expect(result.some(g => g.speakerChange)).toBe(true);
+        expect(result.some(g => g.speaker !== undefined)).toBe(true);
+    });
+    it('recognizes CJK sentence-ending punctuation', () => {
+        const segments = [
+            { start: 0, text: '你好世界。' },
+            { start: 2, text: '这是测试' },
+            { start: 4, text: '内容。' },
+        ];
+        const result = groupTranscriptSegments(segments);
+        expect(result).toHaveLength(2);
+        expect(result[0].text).toBe('你好世界。');
+        expect(result[1].text).toBe('这是测试 内容。');
+    });
+    it('returns empty array for empty input', () => {
+        expect(groupTranscriptSegments([])).toEqual([]);
+    });
+});
+describe('formatGroupedTranscript', () => {
+    it('formats timestamps correctly', () => {
+        const segments = [
+            { start: 65, text: 'One minute five.', speakerChange: false },
+            { start: 3661, text: 'One hour one minute.', speakerChange: false },
+        ];
+        const { rows } = formatGroupedTranscript(segments);
+        expect(rows[0].timestamp).toBe('1:05');
+        expect(rows[1].timestamp).toBe('1:01:01');
+    });
+    it('inserts chapter headings at correct positions', () => {
+        const segments = [
+            { start: 0, text: 'Intro text.', speakerChange: false },
+            { start: 60, text: 'Chapter content.', speakerChange: false },
+        ];
+        const chapters = [{ title: 'Introduction', start: 0 }, { title: 'Main', start: 50 }];
+        const { rows } = formatGroupedTranscript(segments, chapters);
+        expect(rows[0].text).toBe('[Chapter] Introduction');
+        expect(rows[1].text).toBe('Intro text.');
+        expect(rows[2].text).toBe('[Chapter] Main');
+        expect(rows[3].text).toBe('Chapter content.');
+    });
+    it('labels speakers', () => {
+        const segments = [
+            { start: 0, text: 'Hello.', speakerChange: true, speaker: 0 },
+            { start: 5, text: 'Hi there.', speakerChange: true, speaker: 1 },
+        ];
+        const { rows } = formatGroupedTranscript(segments);
+        expect(rows[0].speaker).toBe('Speaker 1');
+        expect(rows[1].speaker).toBe('Speaker 2');
+    });
+});

package/vendor/opencli/dist/clis/youtube/transcript.d.ts

@@ -0,0 +1 @@
+export {};