@mz1999/defuddle 0.14.1

package/dist/extractors/github.js

@@ -0,0 +1,251 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.GitHubExtractor = void 0;
+const _base_1 = require("./_base");
+const dom_1 = require("../utils/dom");
+const comments_1 = require("../utils/comments");
+class GitHubExtractor extends _base_1.BaseExtractor {
+    constructor(document, url) {
+        super(document, url);
+        this.isIssue = /\/issues\/\d+/.test(url);
+        this.isPR = /\/pull\/\d+/.test(url);
+    }
+    canExtract() {
+        const githubIndicators = [
+            'meta[name="expected-hostname"][content="github.com"]',
+            'meta[name="octolytics-url"]',
+            'meta[name="github-keyboard-shortcuts"]',
+            '.js-header-wrapper',
+            '#js-repo-pjax-container',
+        ];
+        if (!githubIndicators.some(selector => this.document.querySelector(selector) !== null)) {
+            return false;
+        }
+        if (this.isIssue) {
+            return [
+                '[data-testid="issue-metadata-sticky"]',
+                '[data-testid="issue-title"]',
+            ].some(selector => this.document.querySelector(selector) !== null);
+        }
+        if (this.isPR) {
+            return [
+                '.pull-discussion-timeline',
+                '.discussion-timeline',
+                '.gh-header-title',
+                '.js-issue-title',
+            ].some(selector => this.document.querySelector(selector) !== null);
+        }
+        return false;
+    }
+    extract() {
+        const repoInfo = this.extractRepoInfo();
+        const number = this.extractNumber();
+        const type = this.isPR ? 'pull' : 'issue';
+        const prBody = this.isPR ? this.getPRBody() : null;
+        const { content: postContent, author, published } = this.isPR
+            ? this.getPRContent(prBody)
+            : this.getIssueContent();
+        const comments = this.options.includeReplies !== false
+            ? (this.isPR ? this.extractPRComments(prBody) : this.extractComments())
+            : '';
+        const contentHtml = this.createContentHtml(postContent, comments);
+        return {
+            content: contentHtml,
+            contentHtml: contentHtml,
+            extractedContent: {
+                type,
+                number,
+                repository: repoInfo.repo,
+                owner: repoInfo.owner,
+            },
+            variables: {
+                title: this.document.title,
+                author,
+                published,
+                site: `GitHub - ${repoInfo.owner}/${repoInfo.repo}`,
+                description: this.createDescription(contentHtml),
+            }
+        };
+    }
+    createContentHtml(postContent, comments) {
+        return (0, comments_1.buildContentHtml)('github', postContent, comments);
+    }
+    getIssueContent() {
+        const issueContainer = this.document.querySelector('[data-testid="issue-viewer-issue-container"]');
+        if (!issueContainer)
+            return { content: '', author: '', published: '' };
+        const author = this.extractAuthor(issueContainer, [
+            'a[data-testid="issue-body-header-author"]',
+            '.IssueBodyHeaderAuthor-module__authorLoginLink--_S7aT',
+            '.ActivityHeader-module__AuthorLink--iofTU',
+            'a[href*="/users/"][data-hovercard-url*="/users/"]',
+            'a[aria-label*="profile"]'
+        ]);
+        const issueTimeElement = issueContainer.querySelector('relative-time');
+        const published = issueTimeElement?.getAttribute('datetime') || '';
+        const issueBodyElement = issueContainer.querySelector('[data-testid="issue-body-viewer"] .markdown-body');
+        if (!issueBodyElement)
+            return { content: '', author, published };
+        const content = this.cleanBodyContent(issueBodyElement);
+        return { content, author, published };
+    }
+    extractComments() {
+        const commentElements = Array.from(this.document.querySelectorAll('[data-wrapper-timeline-id]'));
+        const processedComments = new Set();
+        const commentData = [];
+        for (const commentElement of commentElements) {
+            const commentContainer = commentElement.querySelector('.react-issue-comment');
+            if (!commentContainer)
+                continue;
+            const commentId = commentElement.getAttribute('data-wrapper-timeline-id');
+            if (!commentId || processedComments.has(commentId))
+                continue;
+            processedComments.add(commentId);
+            const author = this.extractAuthor(commentContainer, [
+                '.ActivityHeader-module__AuthorLink--iofTU',
+                'a[data-testid="avatar-link"]',
+                'a[href^="/"][data-hovercard-url*="/users/"]'
+            ]);
+            const timeElement = commentContainer.querySelector('relative-time');
+            const timestamp = timeElement?.getAttribute('datetime') || '';
+            const date = timestamp ? new Date(timestamp).toISOString().split('T')[0] : '';
+            const bodyElement = commentContainer.querySelector('.markdown-body');
+            if (!bodyElement)
+                continue;
+            const bodyContent = this.cleanBodyContent(bodyElement);
+            if (!bodyContent)
+                continue;
+            commentData.push({
+                author,
+                date,
+                content: bodyContent,
+            });
+        }
+        return (0, comments_1.buildCommentTree)(commentData);
+    }
+    getPRBody() {
+        // PR body is in [id^="pullrequest-"] or the first .timeline-comment
+        return this.document.querySelector('[id^="pullrequest-"]')
+            || this.document.querySelector('.timeline-comment');
+    }
+    getPRContent(prBody) {
+        const bodyEl = prBody?.querySelector('.comment-body.markdown-body')
+            || this.document.querySelector('.comment-body.markdown-body');
+        const content = bodyEl ? this.cleanBodyContent(bodyEl) : '';
+        const authorEl = prBody?.querySelector('.author')
+            || this.document.querySelector('.gh-header-meta .author');
+        const author = authorEl?.textContent?.trim() || '';
+        const timeEl = prBody?.querySelector('relative-time');
+        const published = timeEl?.getAttribute('datetime') || '';
+        return { content, author, published };
+    }
+    extractPRComments(prBody) {
+        // Find all comment containers: regular comments (.timeline-comment)
+        // and code review comments (.review-comment)
+        const allComments = Array.from(this.document.querySelectorAll('.timeline-comment, .review-comment'));
+        const commentData = [];
+        for (const comment of allComments) {
+            // Skip the PR description
+            if (prBody && (comment === prBody || prBody.contains(comment)))
+                continue;
+            const authorEl = comment.querySelector('.author');
+            const author = authorEl?.textContent?.trim() || '';
+            const timeEl = comment.querySelector('relative-time');
+            const timestamp = timeEl?.getAttribute('datetime') || '';
+            const date = timestamp ? new Date(timestamp).toISOString().split('T')[0] : '';
+            const bodyEl = comment.querySelector('.comment-body.markdown-body');
+            if (!bodyEl)
+                continue;
+            const bodyContent = this.cleanBodyContent(bodyEl);
+            if (!bodyContent)
+                continue;
+            commentData.push({
+                author,
+                date,
+                content: bodyContent,
+            });
+        }
+        return (0, comments_1.buildCommentTree)(commentData);
+    }
+    extractAuthor(container, selectors) {
+        for (const selector of selectors) {
+            const authorLink = container.querySelector(selector);
+            if (authorLink) {
+                const href = authorLink.getAttribute('href');
+                if (href) {
+                    if (href.startsWith('/')) {
+                        return href.substring(1);
+                    }
+                    else if (href.includes('github.com/')) {
+                        const match = href.match(/github\.com\/([^\/\?#]+)/);
+                        if (match && match[1]) {
+                            return match[1];
+                        }
+                    }
+                }
+            }
+        }
+        return 'Unknown';
+    }
+    cleanBodyContent(bodyElement) {
+        const cleanBody = bodyElement.cloneNode(true);
+        cleanBody.querySelectorAll('button, [data-testid*="button"], [data-testid*="menu"]').forEach(el => el.remove());
+        cleanBody.querySelectorAll('.js-clipboard-copy, .zeroclipboard-container').forEach(el => el.remove());
+        // Convert GitHub's highlighted code blocks to standard <pre><code>
+        // GitHub uses <div class="highlight highlight-source-{lang}"><pre>spans...</pre></div>
+        // The <pre> has no <code> child, which breaks markdown conversion.
+        cleanBody.querySelectorAll('div.highlight[class*="highlight-source-"] pre, div.highlight pre').forEach(pre => {
+            const wrapper = pre.parentElement;
+            if (!wrapper)
+                return;
+            // Extract language from wrapper class (e.g. "highlight-source-ts")
+            const langMatch = wrapper.className.match(/highlight-source-(\w+)/);
+            const lang = langMatch?.[1] || '';
+            // Use data-snippet-clipboard-copy-content if available (clean text),
+            // otherwise fall back to textContent
+            const content = wrapper.getAttribute('data-snippet-clipboard-copy-content')
+                || pre.textContent || '';
+            const code = this.document.createElement('code');
+            if (lang) {
+                code.setAttribute('class', `language-${lang}`);
+                code.setAttribute('data-lang', lang);
+            }
+            code.textContent = content;
+            const newPre = this.document.createElement('pre');
+            newPre.appendChild(code);
+            wrapper.replaceWith(newPre);
+        });
+        return (0, dom_1.serializeHTML)(cleanBody).trim();
+    }
+    extractNumber() {
+        // Try URL first (most reliable)
+        const urlMatch = this.url.match(/\/(issues|pull)\/(\d+)/);
+        if (urlMatch)
+            return urlMatch[2];
+        // Fallback to HTML extraction
+        const titleElement = this.document.querySelector('h1');
+        const titleMatch = titleElement?.textContent?.match(/#(\d+)/);
+        return titleMatch ? titleMatch[1] : '';
+    }
+    extractRepoInfo() {
+        // Try URL first (most reliable)
+        const urlMatch = this.url.match(/github\.com\/([^\/]+)\/([^\/]+)/);
+        if (urlMatch) {
+            return { owner: urlMatch[1], repo: urlMatch[2] };
+        }
+        // Fallback to HTML extraction
+        const titleMatch = this.document.title.match(/([^\/\s]+)\/([^\/\s]+)/);
+        return titleMatch ? { owner: titleMatch[1], repo: titleMatch[2] } : { owner: '', repo: '' };
+    }
+    createDescription(content) {
+        if (!content)
+            return '';
+        const tempDiv = this.document.createElement('div');
+        tempDiv.appendChild((0, dom_1.parseHTML)(this.document, content));
+        return tempDiv.textContent?.trim()
+            .slice(0, 140)
+            .replace(/\s+/g, ' ') || '';
+    }
+}
+exports.GitHubExtractor = GitHubExtractor;
+//# sourceMappingURL=github.js.map

package/dist/extractors/github.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"github.js","sourceRoot":"","sources":["../../src/extractors/github.ts"],"names":[],"mappings":";;;AAAA,mCAAwC;AAExC,sCAAwD;AACxD,gDAAyF;AAEzF,MAAa,eAAgB,SAAQ,qBAAa;IAIjD,YAAY,QAAkB,EAAE,GAAW;QAC1C,KAAK,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QACrB,IAAI,CAAC,OAAO,GAAG,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACzC,IAAI,CAAC,IAAI,GAAG,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACrC,CAAC;IAED,UAAU;QACT,MAAM,gBAAgB,GAAG;YACxB,sDAAsD;YACtD,6BAA6B;YAC7B,wCAAwC;YACxC,oBAAoB;YACpB,yBAAyB;SACzB,CAAC;QAEF,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,QAAQ,CAAC,KAAK,IAAI,CAAC,EAAE,CAAC;YACxF,OAAO,KAAK,CAAC;QACd,CAAC;QAED,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YAClB,OAAO;gBACN,uCAAuC;gBACvC,6BAA6B;aAC7B,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,QAAQ,CAAC,KAAK,IAAI,CAAC,CAAC;QACpE,CAAC;QAED,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACf,OAAO;gBACN,2BAA2B;gBAC3B,sBAAsB;gBACtB,kBAAkB;gBAClB,iBAAiB;aACjB,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,QAAQ,CAAC,KAAK,IAAI,CAAC,CAAC;QACpE,CAAC;QAED,OAAO,KAAK,CAAC;IACd,CAAC;IAED,OAAO;QACN,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,EAAE,CAAC;QACxC,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QACpC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC;QAE1C,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QACnD,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC,IAAI;YAC5D,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC;YAC3B,CAAC,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;QAC1B,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,cAAc,KAAK,KAAK;YACrD,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;YACvE,CAAC,CAAC,EAAE,CAAC;QACN,MAAM,WAAW,GAAG,IAAI,CAAC,iBAAiB,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;QAElE,OAAO;YACN,OAAO,EAAE,WAAW;YACpB,WAAW,EAAE,WAAW;YACxB,gBAAgB,EAAE;gBACjB,IAAI;gBACJ,MAAM;gBACN,UAAU,EAAE,QAAQ,CAAC,IAAI;gBACzB,KAAK,EAAE,QAAQ,CAAC,KAAK;aACrB;YACD,SAAS,EAAE;gBACV,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,KAAK;gBAC1B,MAAM;gBACN,SAAS;gBACT,IAAI,EAAE,YAAY,QAAQ,CAAC,KAAK,IAAI,QAAQ,CAAC,IAAI,EAAE;gBACnD,WAAW,EAAE,IAAI,CAAC,iBAAiB,CAAC,WAAW,CAAC;aAChD;SACD,CAAC;IACH,CAAC;IAEO,iBAAiB,CAAC,WAAmB,EAAE,QAAgB;QAC9D,OAAO,IAAA,2BAAgB,EAAC,QAAQ,EAAE,WAAW,EAAE,QAAQ,CAAC,CAAC;IAC1D,CAAC;IAEO,eAAe;QACtB,MAAM,cAAc,GAAG,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,8CAA8C,CAAC,CAAC;QACnG,IAAI,CAAC,cAAc;YAAE,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC;QAEvE,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,cAAc,EAAE;YACjD,2CAA2C;YAC3C,uDAAuD;YACvD,2CAA2C;YAC3C,mDAAmD;YACnD,0BAA0B;SAC1B,CAAC,CAAC;QAEH,MAAM,gBAAgB,GAAG,cAAc,CAAC,aAAa,CAAC,eAAe,CAAC,CAAC;QACvE,MAAM,SAAS,GAAG,gBAAgB,EAAE,YAAY,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;QAEnE,MAAM,gBAAgB,GAAG,cAAc,CAAC,aAAa,CAAC,kDAAkD,CAAC,CAAC;QAC1G,IAAI,CAAC,gBAAgB;YAAE,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;QAEjE,MAAM,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,gBAAgB,CAAC,CAAC;QAExD,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;IACvC,CAAC;IAEO,eAAe;QACtB,MAAM,eAAe,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,4BAA4B,CAAC,CAAC,CAAC;QACjG,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAU,CAAC;QAC5C,MAAM,WAAW,GAAkB,EAAE,CAAC;QAEtC,KAAK,MAAM,cAAc,IAAI,eAAe,EAAE,CAAC;YAC9C,MAAM,gBAAgB,GAAG,cAAc,CAAC,aAAa,CAAC,sBAAsB,CAAC,CAAC;YAC9E,IAAI,CAAC,gBAAgB;gBAAE,SAAS;YAEhC,MAAM,SAAS,GAAG,cAAc,CAAC,YAAY,CAAC,0BAA0B,CAAC,CAAC;YAC1E,IAAI,CAAC,SAAS,IAAI,iBAAiB,CAAC,GAAG,CAAC,SAAS,CAAC;gBAAE,SAAS;YAC7D,iBAAiB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YAEjC,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,gBAAgB,EAAE;gBACnD,2CAA2C;gBAC3C,8BAA8B;gBAC9B,6CAA6C;aAC7C,CAAC,CAAC;YAEH,MAAM,WAAW,GAAG,gBAAgB,CAAC,aAAa,CAAC,eAAe,CAAC,CAAC;YACpE,MAAM,SAAS,GAAG,WAAW,EAAE,YAAY,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;YAC9D,MAAM,IAAI,GAAG,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAE9E,MAAM,WAAW,GAAG,gBAAgB,CAAC,aAAa,CAAC,gBAAgB,CAAC,CAAC;YACrE,IAAI,CAAC,WAAW;gBAAE,SAAS;YAE3B,MAAM,WAAW,GAAG,IAAI,CAAC,gBAAgB,CAAC,WAAW,CAAC,CAAC;YACvD,IAAI,CAAC,WAAW;gBAAE,SAAS;YAE3B,WAAW,CAAC,IAAI,CAAC;gBAChB,MAAM;gBACN,IAAI;gBACJ,OAAO,EAAE,WAAW;aACpB,CAAC,CAAC;QACJ,CAAC;QAED,OAAO,IAAA,2BAAgB,EAAC,WAAW,CAAC,CAAC;IACtC,CAAC;IAEO,SAAS;QAChB,oEAAoE;QACpE,OAAO,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,sBAAsB,CAAC;eACtD,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,mBAAmB,CAAC,CAAC;IACtD,CAAC;IAEO,YAAY,CAAC,MAAsB;QAE1C,MAAM,MAAM,GAAG,MAAM,EAAE,aAAa,CAAC,6BAA6B,CAAC;eAC/D,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,6BAA6B,CAAC,CAAC;QAC/D,MAAM,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAE5D,MAAM,QAAQ,GAAG,MAAM,EAAE,aAAa,CAAC,SAAS,CAAC;eAC7C,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,yBAAyB,CAAC,CAAC;QAC3D,MAAM,MAAM,GAAG,QAAQ,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QAEnD,MAAM,MAAM,GAAG,MAAM,EAAE,aAAa,CAAC,eAAe,CAAC,CAAC;QACtD,MAAM,SAAS,GAAG,MAAM,EAAE,YAAY,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;QAEzD,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;IACvC,CAAC;IAEO,iBAAiB,CAAC,MAAsB;QAC/C,oEAAoE;QACpE,6CAA6C;QAC7C,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,CAC7B,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,oCAAoC,CAAC,CACpE,CAAC;QACF,MAAM,WAAW,GAAkB,EAAE,CAAC;QAEtC,KAAK,MAAM,OAAO,IAAI,WAAW,EAAE,CAAC;YACnC,0BAA0B;YAC1B,IAAI,MAAM,IAAI,CAAC,OAAO,KAAK,MAAM,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;gBAAE,SAAS;YAEzE,MAAM,QAAQ,GAAG,OAAO,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;YAClD,MAAM,MAAM,GAAG,QAAQ,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;YAEnD,MAAM,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,eAAe,CAAC,CAAC;YACtD,MAAM,SAAS,GAAG,MAAM,EAAE,YAAY,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;YACzD,MAAM,IAAI,GAAG,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAE9E,MAAM,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,6BAA6B,CAAC,CAAC;YACpE,IAAI,CAAC,MAAM;gBAAE,SAAS;YAEtB,MAAM,WAAW,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;YAClD,IAAI,CAAC,WAAW;gBAAE,SAAS;YAE3B,WAAW,CAAC,IAAI,CAAC;gBAChB,MAAM;gBACN,IAAI;gBACJ,OAAO,EAAE,WAAW;aACpB,CAAC,CAAC;QACJ,CAAC;QAED,OAAO,IAAA,2BAAgB,EAAC,WAAW,CAAC,CAAC;IACtC,CAAC;IAEO,aAAa,CAAC,SAAkB,EAAE,SAAmB;QAC5D,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YAClC,MAAM,UAAU,GAAG,SAAS,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;YACrD,IAAI,UAAU,EAAE,CAAC;gBAChB,MAAM,IAAI,GAAG,UAAU,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;gBAC7C,IAAI,IAAI,EAAE,CAAC;oBACV,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;wBAC1B,OAAO,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;oBAC1B,CAAC;yBAAM,IAAI,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,EAAE,CAAC;wBACzC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC;wBACrD,IAAI,KAAK,IAAI,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;4BACvB,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC;wBACjB,CAAC;oBACF,CAAC;gBACF,CAAC;YACF,CAAC;QACF,CAAC;QACD,OAAO,SAAS,CAAC;IAClB,CAAC;IAEO,gBAAgB,CAAC,WAAoB;QAC5C,MAAM,SAAS,GAAG,WAAW,CAAC,SAAS,CAAC,IAAI,CAAY,CAAC;QACzD,SAAS,CAAC,gBAAgB,CAAC,wDAAwD,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC;QAChH,SAAS,CAAC,gBAAgB,CAAC,8CAA8C,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC;QAEtG,mEAAmE;QACnE,uFAAuF;QACvF,mEAAmE;QACnE,SAAS,CAAC,gBAAgB,CAAC,kEAAkE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE;YAC5G,MAAM,OAAO,GAAG,GAAG,CAAC,aAAa,CAAC;YAClC,IAAI,CAAC,OAAO;gBAAE,OAAO;YAErB,mEAAmE;YACnE,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC;YACpE,MAAM,IAAI,GAAG,SAAS,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAElC,qEAAqE;YACrE,qCAAqC;YACrC,MAAM,OAAO,GAAG,OAAO,CAAC,YAAY,CAAC,qCAAqC,CAAC;mBACvE,GAAG,CAAC,WAAW,IAAI,EAAE,CAAC;YAE1B,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;YACjD,IAAI,IAAI,EAAE,CAAC;gBACV,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,YAAY,IAAI,EAAE,CAAC,CAAC;gBAC/C,IAAI,CAAC,YAAY,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;YACtC,CAAC;YACD,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC;YAE3B,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;YAClD,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;YACzB,OAAO,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QAC7B,CAAC,CAAC,CAAC;QAEH,OAAO,IAAA,mBAAa,EAAC,SAAS,CAAC,CAAC,IAAI,EAAE,CAAC;IACxC,CAAC;IAEO,aAAa;QACpB,gCAAgC;QAChC,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC;QAC1D,IAAI,QAAQ;YAAE,OAAO,QAAQ,CAAC,CAAC,CAAC,CAAC;QAEjC,8BAA8B;QAC9B,MAAM,YAAY,GAAG,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;QACvD,MAAM,UAAU,GAAG,YAAY,EAAE,WAAW,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC9D,OAAO,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACxC,CAAC;IAEO,eAAe;QACtB,gCAAgC;QAChC,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,iCAAiC,CAAC,CAAC;QACnE,IAAI,QAAQ,EAAE,CAAC;YACd,OAAO,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;QAClD,CAAC;QAED,8BAA8B;QAC9B,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC;QACvE,OAAO,UAAU,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,UAAU,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;IAC7F,CAAC;IAGO,iBAAiB,CAAC,OAAe;QACxC,IAAI,CAAC,OAAO;YAAE,OAAO,EAAE,CAAC;QAExB,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;QACnD,OAAO,CAAC,WAAW,CAAC,IAAA,eAAS,EAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;QACvD,OAAO,OAAO,CAAC,WAAW,EAAE,IAAI,EAAE;aAChC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;aACb,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC;IAC9B,CAAC;CACD;AA/RD,0CA+RC"}

package/dist/extractors/grok.d.ts

@@ -0,0 +1,15 @@
+import { ConversationExtractor } from './_conversation';
+import { ConversationMessage, ConversationMetadata, Footnote } from '../types/extractors';
+export declare class GrokExtractor extends ConversationExtractor {
+    private messageContainerSelector;
+    private messageBubbles;
+    private footnotes;
+    private footnoteCounter;
+    constructor(document: Document, url: string);
+    canExtract(): boolean;
+    protected extractMessages(): ConversationMessage[];
+    protected getFootnotes(): Footnote[];
+    protected getMetadata(): ConversationMetadata;
+    private getTitle;
+    private processFootnotes;
+}

package/dist/extractors/grok.js

@@ -0,0 +1,142 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.GrokExtractor = void 0;
+const _conversation_1 = require("./_conversation");
+const dom_1 = require("../utils/dom");
+class GrokExtractor extends _conversation_1.ConversationExtractor {
+    constructor(document, url) {
+        super(document, url);
+        // Note: This selector relies heavily on CSS utility classes and may break if Grok's UI changes.
+        this.messageContainerSelector = '.relative.group.flex.flex-col.justify-center.w-full';
+        this.messageBubbles = document.querySelectorAll(this.messageContainerSelector);
+        this.footnotes = [];
+        this.footnoteCounter = 0;
+    }
+    canExtract() {
+        return !!this.messageBubbles && this.messageBubbles.length > 0;
+    }
+    extractMessages() {
+        const messages = [];
+        this.footnotes = [];
+        this.footnoteCounter = 0;
+        if (!this.messageBubbles || this.messageBubbles.length === 0)
+            return messages;
+        this.messageBubbles.forEach((container) => {
+            // Note: Relies on layout classes 'items-end' and 'items-start' which might change.
+            const isUserMessage = container.classList.contains('items-end');
+            const isGrokMessage = container.classList.contains('items-start');
+            if (!isUserMessage && !isGrokMessage)
+                return; // Skip elements that aren't clearly user or Grok messages
+            const messageBubble = container.querySelector('.message-bubble');
+            if (!messageBubble)
+                return; // Skip if the core message bubble isn't found
+            let content = '';
+            let role = '';
+            let author = '';
+            if (isUserMessage) {
+                // Assume user message bubble's textContent is the desired content.
+                // This is simpler and potentially less brittle than selecting specific spans.
+                content = messageBubble.textContent || '';
+                role = 'user';
+                author = 'You'; // Or potentially extract from an attribute if available later
+            }
+            else if (isGrokMessage) {
+                role = 'assistant';
+                author = 'Grok'; // Or potentially extract from an attribute if available later
+                // Clone the bubble to modify it without affecting the original page
+                const clonedBubble = messageBubble.cloneNode(true);
+                // Remove known non-content elements like the DeepSearch artifact
+                clonedBubble.querySelector('.relative.border.border-border-l1.bg-surface-base')?.remove();
+                // Add selectors here for any other known elements to remove (e.g., buttons, toolbars within the bubble)
+                content = (0, dom_1.serializeHTML)(clonedBubble);
+                // Process footnotes/links in the cleaned content
+                content = this.processFootnotes(content);
+            }
+            if (content.trim()) {
+                messages.push({
+                    author: author,
+                    content: content.trim(),
+                    metadata: {
+                        role: role
+                    }
+                });
+            }
+        });
+        return messages;
+    }
+    getFootnotes() {
+        return this.footnotes;
+    }
+    getMetadata() {
+        const title = this.getTitle();
+        const messageCount = this.messageBubbles?.length || 0;
+        return {
+            title,
+            site: 'Grok',
+            url: this.url,
+            messageCount: messageCount, // Use estimated count
+            description: `Grok conversation with ${messageCount} messages`
+        };
+    }
+    getTitle() {
+        // Try to get the page title first (more reliable)
+        const pageTitle = this.document.title?.trim();
+        if (pageTitle && pageTitle !== 'Grok' && !pageTitle.startsWith('Grok by ')) {
+            // Remove ' - Grok' suffix if present
+            return pageTitle.replace(/\s-\s*Grok$/, '').trim();
+        }
+        // Fallback: Find the first user message bubble and use its text content
+        // Note: Still relies on 'items-end' class.
+        const firstUserContainer = this.document.querySelector(`${this.messageContainerSelector}.items-end`);
+        if (firstUserContainer) {
+            const messageBubble = firstUserContainer.querySelector('.message-bubble');
+            if (messageBubble) {
+                const text = messageBubble.textContent?.trim() || '';
+                // Truncate to first 50 characters if longer
+                return text.length > 50 ? text.slice(0, 50) + '...' : text;
+            }
+        }
+        return 'Grok Conversation'; // Default fallback
+    }
+    processFootnotes(content) {
+        // Regex to find <a> tags, capture href and link text
+        const linkPattern = /<a\s+(?:[^>]*?\s+)?href="([^"]*)"[^>]*>(.*?)<\/a>/gi; // Use 'g' and 'i' flags
+        return content.replace(linkPattern, (match, url, linkText) => {
+            // Skip processing for internal anchor links, empty URLs, or non-http(s) protocols
+            if (!url || url.startsWith('#') || !url.match(/^https?:\/\//i)) {
+                return match;
+            }
+            // Check if this URL already exists in our footnotes
+            let footnote = this.footnotes.find(fn => fn.url === url);
+            let footnoteIndex;
+            if (!footnote) {
+                // Create a new footnote if URL doesn't exist
+                this.footnoteCounter++;
+                footnoteIndex = this.footnoteCounter;
+                let domainText = url; // Default to full URL if parsing fails
+                try {
+                    const domain = new URL(url).hostname.replace(/^www\./, '');
+                    domainText = `<a href="${url}" target="_blank" rel="noopener noreferrer">${domain}</a>`;
+                }
+                catch (e) {
+                    // Keep domainText as the original URL if parsing fails
+                    domainText = `<a href="${url}" target="_blank" rel="noopener noreferrer">${url}</a>`;
+                    console.warn(`GrokExtractor: Could not parse URL for footnote: ${url}`);
+                }
+                this.footnotes.push({
+                    url,
+                    text: domainText // Store the link HTML directly
+                });
+            }
+            else {
+                // Find the 1-based index of the existing footnote
+                footnoteIndex = this.footnotes.findIndex(fn => fn.url === url) + 1;
+            }
+            // Return the original link text wrapped with a footnote reference
+            // Ensure the link text itself is not clickable again if it was part of the original match
+            return `${linkText}<sup id="fnref:${footnoteIndex}" class="footnote-ref"><a href="#fn:${footnoteIndex}" class="footnote-link">${footnoteIndex}</a></sup>`;
+        });
+    }
+}
+exports.GrokExtractor = GrokExtractor;
+//# sourceMappingURL=grok.js.map

package/dist/extractors/grok.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"grok.js","sourceRoot":"","sources":["../../src/extractors/grok.ts"],"names":[],"mappings":";;;AAAA,mDAAwD;AAExD,sCAA6C;AAE7C,MAAa,aAAc,SAAQ,qCAAqB;IAOvD,YAAY,QAAkB,EAAE,GAAW;QAC1C,KAAK,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QAPtB,gGAAgG;QACxF,6BAAwB,GAAG,qDAAqD,CAAC;QAOxF,IAAI,CAAC,cAAc,GAAG,QAAQ,CAAC,gBAAgB,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;QAC/E,IAAI,CAAC,SAAS,GAAG,EAAE,CAAC;QACpB,IAAI,CAAC,eAAe,GAAG,CAAC,CAAC;IAC1B,CAAC;IAED,UAAU;QACT,OAAO,CAAC,CAAC,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC;IAChE,CAAC;IAES,eAAe;QACxB,MAAM,QAAQ,GAA0B,EAAE,CAAC;QAC3C,IAAI,CAAC,SAAS,GAAG,EAAE,CAAC;QACpB,IAAI,CAAC,eAAe,GAAG,CAAC,CAAC;QAEzB,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,cAAc,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,QAAQ,CAAC;QAE9E,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC,SAAS,EAAE,EAAE;YACzC,mFAAmF;YACnF,MAAM,aAAa,GAAG,SAAS,CAAC,SAAS,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;YAChE,MAAM,aAAa,GAAG,SAAS,CAAC,SAAS,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;YAElE,IAAI,CAAC,aAAa,IAAI,CAAC,aAAa;gBAAE,OAAO,CAAC,0DAA0D;YAExG,MAAM,aAAa,GAAG,SAAS,CAAC,aAAa,CAAC,iBAAiB,CAAC,CAAC;YACjE,IAAI,CAAC,aAAa;gBAAE,OAAO,CAAC,8CAA8C;YAE1E,IAAI,OAAO,GAAW,EAAE,CAAC;YACzB,IAAI,IAAI,GAAW,EAAE,CAAC;YACtB,IAAI,MAAM,GAAW,EAAE,CAAC;YAExB,IAAI,aAAa,EAAE,CAAC;gBACnB,mEAAmE;gBACnE,8EAA8E;gBAC9E,OAAO,GAAG,aAAa,CAAC,WAAW,IAAI,EAAE,CAAC;gBAC1C,IAAI,GAAG,MAAM,CAAC;gBACd,MAAM,GAAG,KAAK,CAAC,CAAC,8DAA8D;YAC/E,CAAC;iBAAM,IAAI,aAAa,EAAE,CAAC;gBAC1B,IAAI,GAAG,WAAW,CAAC;gBACnB,MAAM,GAAG,MAAM,CAAC,CAAC,8DAA8D;gBAE/E,oEAAoE;gBACpE,MAAM,YAAY,GAAG,aAAa,CAAC,SAAS,CAAC,IAAI,CAAY,CAAC;gBAE9D,iEAAiE;gBACjE,YAAY,CAAC,aAAa,CAAC,mDAAmD,CAAC,EAAE,MAAM,EAAE,CAAC;gBAC1F,wGAAwG;gBAExG,OAAO,GAAG,IAAA,mBAAa,EAAC,YAAY,CAAC,CAAC;gBAEtC,iDAAiD;gBACjD,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;YAC1C,CAAC;YAED,IAAI,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;gBACpB,QAAQ,CAAC,IAAI,CAAC;oBACb,MAAM,EAAE,MAAM;oBACd,OAAO,EAAE,OAAO,CAAC,IAAI,EAAE;oBACvB,QAAQ,EAAE;wBACT,IAAI,EAAE,IAAI;qBACV;iBACD,CAAC,CAAC;YACJ,CAAC;QACF,CAAC,CAAC,CAAC;QAEH,OAAO,QAAQ,CAAC;IACjB,CAAC;IAES,YAAY;QACrB,OAAO,IAAI,CAAC,SAAS,CAAC;IACvB,CAAC;IAES,WAAW;QACpB,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;QAC9B,MAAM,YAAY,GAAG,IAAI,CAAC,cAAc,EAAE,MAAM,IAAI,CAAC,CAAC;QAEtD,OAAO;YACN,KAAK;YACL,IAAI,EAAE,MAAM;YACZ,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,YAAY,EAAE,YAAY,EAAE,sBAAsB;YAClD,WAAW,EAAE,0BAA0B,YAAY,WAAW;SAC9D,CAAC;IACH,CAAC;IAEO,QAAQ;QACf,kDAAkD;QAClD,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC;QAC9C,IAAI,SAAS,IAAI,SAAS,KAAK,MAAM,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YAC5E,qCAAqC;YACrC,OAAO,SAAS,CAAC,OAAO,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QACpD,CAAC;QAED,wEAAwE;QACxE,2CAA2C;QAC3C,MAAM,kBAAkB,GAAG,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,GAAG,IAAI,CAAC,wBAAwB,YAAY,CAAC,CAAC;QACrG,IAAI,kBAAkB,EAAE,CAAC;YACxB,MAAM,aAAa,GAAG,kBAAkB,CAAC,aAAa,CAAC,iBAAiB,CAAC,CAAC;YAC1E,IAAI,aAAa,EAAE,CAAC;gBACnB,MAAM,IAAI,GAAG,aAAa,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;gBACrD,4CAA4C;gBAC5C,OAAO,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;YAC5D,CAAC;QACF,CAAC;QAED,OAAO,mBAAmB,CAAC,CAAC,mBAAmB;IAChD,CAAC;IAEO,gBAAgB,CAAC,OAAe;QACvC,qDAAqD;QACrD,MAAM,WAAW,GAAG,qDAAqD,CAAC,CAAC,wBAAwB;QAEnG,OAAO,OAAO,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE;YAC3D,kFAAkF;YACnF,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,eAAe,CAAC,EAAE,CAAC;gBAChE,OAAO,KAAK,CAAC;YACd,CAAC;YAED,oDAAoD;YACpD,IAAI,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC;YACzD,IAAI,aAAqB,CAAC;YAE1B,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACf,6CAA6C;gBAC7C,IAAI,CAAC,eAAe,EAAE,CAAC;gBACvB,aAAa,GAAG,IAAI,CAAC,eAAe,CAAC;gBAErC,IAAI,UAAU,GAAG,GAAG,CAAC,CAAC,uCAAuC;gBAC7D,IAAI,CAAC;oBACJ,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;oBAC3D,UAAU,GAAG,YAAY,GAAG,+CAA+C,MAAM,MAAM,CAAC;gBACzF,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACZ,uDAAuD;oBACvD,UAAU,GAAG,YAAY,GAAG,+CAA+C,GAAG,MAAM,CAAC;oBACrF,OAAO,CAAC,IAAI,CAAC,oDAAoD,GAAG,EAAE,CAAC,CAAC;gBACzE,CAAC;gBAED,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;oBACnB,GAAG;oBACH,IAAI,EAAE,UAAU,CAAC,+BAA+B;iBAChD,CAAC,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACP,kDAAkD;gBAClD,aAAa,GAAG,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,GAAG,KAAK,GAAG,CAAC,GAAG,CAAC,CAAC;YACpE,CAAC;YAED,kEAAkE;YAClE,0FAA0F;YAC1F,OAAO,GAAG,QAAQ,kBAAkB,aAAa,uCAAuC,aAAa,2BAA2B,aAAa,YAAY,CAAC;QAC3J,CAAC,CAAC,CAAC;IACJ,CAAC;CACD;AA/JD,sCA+JC"}

package/dist/extractors/hackernews.d.ts

@@ -0,0 +1,21 @@
+import { BaseExtractor } from './_base';
+import { ExtractorResult } from '../types/extractors';
+export declare class HackerNewsExtractor extends BaseExtractor {
+    private mainPost;
+    private isCommentPage;
+    private mainComment;
+    constructor(document: Document, url: string);
+    private detectCommentPage;
+    private findMainComment;
+    canExtract(): boolean;
+    extract(): ExtractorResult;
+    private createContentHtml;
+    private getPostContent;
+    private extractComments;
+    private processComments;
+    private getPostId;
+    private getPostTitle;
+    private getPostAuthor;
+    private createDescription;
+    private getPostDate;
+}

package/dist/extractors/hackernews.js

@@ -0,0 +1,155 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.HackerNewsExtractor = void 0;
+const _base_1 = require("./_base");
+const dom_1 = require("../utils/dom");
+const comments_1 = require("../utils/comments");
+class HackerNewsExtractor extends _base_1.BaseExtractor {
+    constructor(document, url) {
+        super(document, url);
+        this.mainPost = document.querySelector('.fatitem');
+        this.isCommentPage = this.detectCommentPage();
+        this.mainComment = this.isCommentPage ? this.findMainComment() : null;
+    }
+    detectCommentPage() {
+        // Comment pages have an "on: <story title>" link but no story title row
+        return !!this.mainPost?.querySelector('.onstory') && !this.mainPost?.querySelector('.titleline');
+    }
+    findMainComment() {
+        // Use the tr.athing row which contains both the comment metadata (.comhead)
+        // and the comment text (.commtext). The .comment div alone doesn't include
+        // the author (.hnuser) or timestamp (.age) which are in the sibling .comhead.
+        return this.mainPost?.querySelector('tr.athing') || null;
+    }
+    canExtract() {
+        return !!this.mainPost;
+    }
+    extract() {
+        const postContent = this.getPostContent();
+        const comments = this.options.includeReplies !== false ? this.extractComments() : '';
+        const contentHtml = this.createContentHtml(postContent, comments);
+        const postTitle = this.getPostTitle();
+        const postAuthor = this.getPostAuthor();
+        const description = this.createDescription();
+        const published = this.getPostDate();
+        return {
+            content: contentHtml,
+            contentHtml: contentHtml,
+            extractedContent: {
+                postId: this.getPostId(),
+                postAuthor,
+            },
+            variables: {
+                title: postTitle,
+                author: postAuthor,
+                site: 'Hacker News',
+                description,
+                published,
+            }
+        };
+    }
+    createContentHtml(postContent, comments) {
+        return (0, comments_1.buildContentHtml)('hackernews', postContent, comments);
+    }
+    getPostContent() {
+        if (!this.mainPost)
+            return '';
+        // If this is a comment page, use the comment as the main content
+        if (this.isCommentPage && this.mainComment) {
+            const author = this.mainComment.querySelector('.hnuser')?.textContent || '[deleted]';
+            const commtext = this.mainComment.querySelector('.commtext');
+            const commentText = commtext ? (0, dom_1.serializeHTML)(commtext) : '';
+            const timeElement = this.mainComment.querySelector('.age');
+            const timestamp = timeElement?.getAttribute('title') || '';
+            const date = timestamp.split('T')[0] || '';
+            const points = this.mainComment.querySelector('.score')?.textContent?.trim() || '';
+            return (0, comments_1.buildComment)({
+                author,
+                date,
+                content: commentText,
+                score: points || undefined,
+            });
+        }
+        // Otherwise handle regular post content
+        const titleRow = this.mainPost.querySelector('tr.athing');
+        const subRow = titleRow?.nextElementSibling;
+        const url = titleRow?.querySelector('.titleline a')?.getAttribute('href') || '';
+        let content = '';
+        if (url) {
+            content += `<p><a href="${url}" target="_blank">${url}</a></p>`;
+        }
+        const text = this.mainPost.querySelector('.toptext');
+        if (text) {
+            content += `<div class="post-text">${(0, dom_1.serializeHTML)(text)}</div>`;
+        }
+        return content;
+    }
+    extractComments() {
+        const comments = Array.from(this.document.querySelectorAll('tr.comtr'));
+        return this.processComments(comments);
+    }
+    processComments(comments) {
+        const commentData = [];
+        const processedIds = new Set();
+        for (const comment of comments) {
+            const id = comment.getAttribute('id');
+            if (!id || processedIds.has(id))
+                continue;
+            processedIds.add(id);
+            const indent = comment.querySelector('.ind img')?.getAttribute('width') || '0';
+            const depth = parseInt(indent) / 40;
+            const commentText = comment.querySelector('.commtext');
+            const author = comment.querySelector('.hnuser')?.textContent || '[deleted]';
+            const timeElement = comment.querySelector('.age');
+            const points = comment.querySelector('.score')?.textContent?.trim() || '';
+            if (!commentText)
+                continue;
+            const commentUrl = `https://news.ycombinator.com/item?id=${id}`;
+            const timestamp = timeElement?.getAttribute('title') || '';
+            const date = timestamp.split('T')[0] || '';
+            commentData.push({
+                author,
+                date,
+                content: (0, dom_1.serializeHTML)(commentText),
+                depth,
+                score: points || undefined,
+                url: commentUrl,
+            });
+        }
+        return (0, comments_1.buildCommentTree)(commentData);
+    }
+    getPostId() {
+        const match = this.url.match(/id=(\d+)/);
+        return match?.[1] || '';
+    }
+    getPostTitle() {
+        if (this.isCommentPage && this.mainComment) {
+            const author = this.mainComment.querySelector('.hnuser')?.textContent || '[deleted]';
+            const commentText = this.mainComment.querySelector('.commtext')?.textContent || '';
+            // Use first 50 characters of comment as title
+            const preview = commentText.trim().slice(0, 50) + (commentText.length > 50 ? '...' : '');
+            return `Comment by ${author}: ${preview}`;
+        }
+        return this.mainPost?.querySelector('.titleline')?.textContent?.trim() || '';
+    }
+    getPostAuthor() {
+        return this.mainPost?.querySelector('.hnuser')?.textContent?.trim() || '';
+    }
+    createDescription() {
+        const title = this.getPostTitle();
+        const author = this.getPostAuthor();
+        if (this.isCommentPage) {
+            return `Comment by ${author} on Hacker News`;
+        }
+        return `${title} - by ${author} on Hacker News`;
+    }
+    getPostDate() {
+        if (!this.mainPost)
+            return '';
+        const timeElement = this.mainPost.querySelector('.age');
+        const timestamp = timeElement?.getAttribute('title') || '';
+        return timestamp.split('T')[0] || '';
+    }
+}
+exports.HackerNewsExtractor = HackerNewsExtractor;
+//# sourceMappingURL=hackernews.js.map

package/dist/extractors/hackernews.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"hackernews.js","sourceRoot":"","sources":["../../src/extractors/hackernews.ts"],"names":[],"mappings":";;;AAAA,mCAAwC;AAExC,sCAA6C;AAC7C,gDAAkG;AAElG,MAAa,mBAAoB,SAAQ,qBAAa;IAKrD,YAAY,QAAkB,EAAE,GAAW;QAC1C,KAAK,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QACrB,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;QACnD,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC9C,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;IACvE,CAAC;IAEO,iBAAiB;QACxB,wEAAwE;QACxE,OAAO,CAAC,CAAC,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,YAAY,CAAC,CAAC;IAClG,CAAC;IAEO,eAAe;QACtB,4EAA4E;QAC5E,2EAA2E;QAC3E,8EAA8E;QAC9E,OAAO,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,WAAW,CAAC,IAAI,IAAI,CAAC;IAC1D,CAAC;IAED,UAAU;QACT,OAAO,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC;IACxB,CAAC;IAED,OAAO;QACN,MAAM,WAAW,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;QAC1C,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,cAAc,KAAK,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAErF,MAAM,WAAW,GAAG,IAAI,CAAC,iBAAiB,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;QAClE,MAAM,SAAS,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;QACtC,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QACxC,MAAM,WAAW,GAAG,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC7C,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QAErC,OAAO;YACN,OAAO,EAAE,WAAW;YACpB,WAAW,EAAE,WAAW;YACxB,gBAAgB,EAAE;gBACjB,MAAM,EAAE,IAAI,CAAC,SAAS,EAAE;gBACxB,UAAU;aACV;YACD,SAAS,EAAE;gBACV,KAAK,EAAE,SAAS;gBAChB,MAAM,EAAE,UAAU;gBAClB,IAAI,EAAE,aAAa;gBACnB,WAAW;gBACX,SAAS;aACT;SACD,CAAC;IACH,CAAC;IAEO,iBAAiB,CAAC,WAAmB,EAAE,QAAgB;QAC9D,OAAO,IAAA,2BAAgB,EAAC,YAAY,EAAE,WAAW,EAAE,QAAQ,CAAC,CAAC;IAC9D,CAAC;IAEO,cAAc;QACrB,IAAI,CAAC,IAAI,CAAC,QAAQ;YAAE,OAAO,EAAE,CAAC;QAE9B,iEAAiE;QACjE,IAAI,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YAC5C,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,aAAa,CAAC,SAAS,CAAC,EAAE,WAAW,IAAI,WAAW,CAAC;YACrF,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,aAAa,CAAC,WAAW,CAAC,CAAC;YAC7D,MAAM,WAAW,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAA,mBAAa,EAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAC5D,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;YAC3D,MAAM,SAAS,GAAG,WAAW,EAAE,YAAY,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;YAC3D,MAAM,IAAI,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAC3C,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,aAAa,CAAC,QAAQ,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;YAEnF,OAAO,IAAA,uBAAY,EAAC;gBACnB,MAAM;gBACN,IAAI;gBACJ,OAAO,EAAE,WAAW;gBACpB,KAAK,EAAE,MAAM,IAAI,SAAS;aAC1B,CAAC,CAAC;QACJ,CAAC;QAED,wCAAwC;QACxC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,WAAW,CAAC,CAAC;QAC1D,MAAM,MAAM,GAAG,QAAQ,EAAE,kBAAkB,CAAC;QAC5C,MAAM,GAAG,GAAG,QAAQ,EAAE,aAAa,CAAC,cAAc,CAAC,EAAE,YAAY,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QAEhF,IAAI,OAAO,GAAG,EAAE,CAAC;QACjB,IAAI,GAAG,EAAE,CAAC;YACT,OAAO,IAAI,eAAe,GAAG,qBAAqB,GAAG,UAAU,CAAC;QACjE,CAAC;QAED,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;QACrD,IAAI,IAAI,EAAE,CAAC;YACV,OAAO,IAAI,0BAA0B,IAAA,mBAAa,EAAC,IAAI,CAAC,QAAQ,CAAC;QAClE,CAAC;QAED,OAAO,OAAO,CAAC;IAChB,CAAC;IAEO,eAAe;QACtB,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,CAAC;QACxE,OAAO,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC;IACvC,CAAC;IAEO,eAAe,CAAC,QAAmB;QAC1C,MAAM,WAAW,GAAkB,EAAE,CAAC;QACtC,MAAM,YAAY,GAAG,IAAI,GAAG,EAAU,CAAC;QAEvC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAChC,MAAM,EAAE,GAAG,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;YACtC,IAAI,CAAC,EAAE,IAAI,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC;gBAAE,SAAS;YAC1C,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAErB,MAAM,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,UAAU,CAAC,EAAE,YAAY,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC;YAC/E,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;YACpC,MAAM,WAAW,GAAG,OAAO,CAAC,aAAa,CAAC,WAAW,CAAC,CAAC;YACvD,MAAM,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,SAAS,CAAC,EAAE,WAAW,IAAI,WAAW,CAAC;YAC5E,MAAM,WAAW,GAAG,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;YAClD,MAAM,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,QAAQ,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;YAE1E,IAAI,CAAC,WAAW;gBAAE,SAAS;YAE3B,MAAM,UAAU,GAAG,wCAAwC,EAAE,EAAE,CAAC;YAChE,MAAM,SAAS,GAAG,WAAW,EAAE,YAAY,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;YAC3D,MAAM,IAAI,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAE3C,WAAW,CAAC,IAAI,CAAC;gBAChB,MAAM;gBACN,IAAI;gBACJ,OAAO,EAAE,IAAA,mBAAa,EAAC,WAAW,CAAC;gBACnC,KAAK;gBACL,KAAK,EAAE,MAAM,IAAI,SAAS;gBAC1B,GAAG,EAAE,UAAU;aACf,CAAC,CAAC;QACJ,CAAC;QAED,OAAO,IAAA,2BAAgB,EAAC,WAAW,CAAC,CAAC;IACtC,CAAC;IAEO,SAAS;QAChB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QACzC,OAAO,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACzB,CAAC;IAEO,YAAY;QACnB,IAAI,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YAC5C,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,aAAa,CAAC,SAAS,CAAC,EAAE,WAAW,IAAI,WAAW,CAAC;YACrF,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,aAAa,CAAC,WAAW,CAAC,EAAE,WAAW,IAAI,EAAE,CAAC;YACnF,8CAA8C;YAC9C,MAAM,OAAO,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACzF,OAAO,cAAc,MAAM,KAAK,OAAO,EAAE,CAAC;QAC3C,CAAC;QACD,OAAO,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,YAAY,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IAC9E,CAAC;IAEO,aAAa;QACpB,OAAO,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,SAAS,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IAC3E,CAAC;IAEO,iBAAiB;QACxB,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;QAClC,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QACpC,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACxB,OAAO,cAAc,MAAM,iBAAiB,CAAC;QAC9C,CAAC;QACD,OAAO,GAAG,KAAK,SAAS,MAAM,iBAAiB,CAAC;IACjD,CAAC;IAEO,WAAW;QAClB,IAAI,CAAC,IAAI,CAAC,QAAQ;YAAE,OAAO,EAAE,CAAC;QAE9B,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;QACxD,MAAM,SAAS,GAAG,WAAW,EAAE,YAAY,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAC3D,OAAO,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACtC,CAAC;CACD;AA9KD,kDA8KC"}

package/dist/extractors/reddit.d.ts

@@ -0,0 +1,22 @@
+import { BaseExtractor } from './_base';
+import { ExtractorResult } from '../types/extractors';
+export declare class RedditExtractor extends BaseExtractor {
+    private shredditPost;
+    private isOldReddit;
+    constructor(document: Document, url: string);
+    canExtract(): boolean;
+    canExtractAsync(): boolean;
+    private isCommentsPage;
+    extractAsync(): Promise<ExtractorResult>;
+    extract(): ExtractorResult;
+    private extractOldReddit;
+    private getPostContent;
+    private createContentHtml;
+    private extractComments;
+    private getPostId;
+    private getSubreddit;
+    private getPostAuthor;
+    private createDescription;
+    private collectOldRedditComments;
+    private processComments;
+}