@mz1999/defuddle 0.14.1

package/dist/extractors/reddit.js

@@ -0,0 +1,197 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RedditExtractor = void 0;
+const _base_1 = require("./_base");
+const dom_1 = require("../utils/dom");
+const comments_1 = require("../utils/comments");
+class RedditExtractor extends _base_1.BaseExtractor {
+    constructor(document, url) {
+        super(document, url);
+        this.shredditPost = document.querySelector('shreddit-post');
+        this.isOldReddit = !!document.querySelector('.thing.link');
+    }
+    canExtract() {
+        return !!this.shredditPost || this.isOldReddit;
+    }
+    canExtractAsync() {
+        // For new reddit comment pages, extract() returns empty content
+        // when shreddit-comment elements are missing (server-side fetch),
+        // causing parseAsync() to fall through to this async path.
+        return this.isCommentsPage() && !this.isOldReddit;
+    }
+    isCommentsPage() {
+        return /\/r\/.+\/comments\//.test(this.url);
+    }
+    async extractAsync() {
+        // Convert URL to old.reddit.com
+        const oldUrl = new URL(this.url);
+        oldUrl.hostname = 'old.reddit.com';
+        const response = await fetch(oldUrl.toString(), {
+            headers: {
+                'User-Agent': 'Mozilla/5.0 (compatible; Defuddle/1.0)',
+            },
+        });
+        if (!response.ok) {
+            throw new Error(`Failed to fetch old.reddit.com: ${response.status}`);
+        }
+        const html = await response.text();
+        const Parser = this.document.defaultView?.DOMParser ?? (typeof DOMParser !== 'undefined' ? DOMParser : null);
+        if (!Parser) {
+            throw new Error('DOMParser is not available in this environment');
+        }
+        const doc = new Parser().parseFromString(html, 'text/html');
+        return this.extractOldReddit(doc);
+    }
+    extract() {
+        if (this.isOldReddit) {
+            return this.extractOldReddit(this.document);
+        }
+        // New reddit server-side HTML includes shreddit-post but not
+        // shreddit-comment elements (those require JS). Return empty
+        // so parseAsync() falls through to extractAsync() which fetches
+        // old.reddit.com with full content.
+        const hasComments = this.document.querySelectorAll('shreddit-comment').length > 0;
+        if (this.isCommentsPage() && !hasComments) {
+            return { content: '', contentHtml: '' };
+        }
+        const postContent = this.getPostContent();
+        const comments = this.options.includeReplies !== false ? this.extractComments() : '';
+        const contentHtml = this.createContentHtml(postContent, comments);
+        const postTitle = this.document.querySelector('h1')?.textContent?.trim() || '';
+        const subreddit = this.getSubreddit();
+        const postAuthor = this.getPostAuthor();
+        const description = this.createDescription(postContent);
+        return {
+            content: contentHtml,
+            contentHtml: contentHtml,
+            extractedContent: {
+                postId: this.getPostId(),
+                subreddit,
+                postAuthor,
+            },
+            variables: {
+                title: postTitle,
+                author: postAuthor,
+                site: `r/${subreddit}`,
+                description,
+            }
+        };
+    }
+    extractOldReddit(root) {
+        const thingLink = root.querySelector('.thing.link');
+        const postTitle = thingLink?.querySelector('a.title')?.textContent?.trim() || '';
+        const postAuthor = thingLink?.getAttribute('data-author') || '';
+        const subreddit = thingLink?.getAttribute('data-subreddit') || '';
+        const postBodyEl = thingLink?.querySelector('.usertext-body .md');
+        const postBody = postBodyEl ? (0, dom_1.serializeHTML)(postBodyEl) : '';
+        let comments = '';
+        if (this.options.includeReplies !== false) {
+            const commentArea = root.querySelector('.commentarea .sitetable');
+            const commentData = commentArea ? this.collectOldRedditComments(commentArea) : [];
+            comments = commentData.length > 0 ? (0, comments_1.buildCommentTree)(commentData) : '';
+        }
+        const contentHtml = this.createContentHtml(postBody, comments);
+        const description = this.createDescription(postBody);
+        return {
+            content: contentHtml,
+            contentHtml: contentHtml,
+            extractedContent: {
+                postId: this.getPostId(),
+                subreddit,
+                postAuthor,
+            },
+            variables: {
+                title: postTitle,
+                author: postAuthor,
+                site: `r/${subreddit}`,
+                description,
+            }
+        };
+    }
+    getPostContent() {
+        const textBodyEl = this.shredditPost?.querySelector('[slot="text-body"]');
+        const textBody = textBodyEl ? (0, dom_1.serializeHTML)(textBodyEl) : '';
+        const mediaBody = this.shredditPost?.querySelector('#post-image')?.outerHTML || '';
+        return textBody + mediaBody;
+    }
+    createContentHtml(postContent, comments) {
+        return (0, comments_1.buildContentHtml)('reddit', postContent, comments);
+    }
+    extractComments() {
+        const comments = Array.from(this.document.querySelectorAll('shreddit-comment'));
+        return this.processComments(comments);
+    }
+    getPostId() {
+        const match = this.url.match(/comments\/([a-zA-Z0-9]+)/);
+        return match?.[1] || '';
+    }
+    getSubreddit() {
+        const match = this.url.match(/\/r\/([^/]+)/);
+        return match?.[1] || '';
+    }
+    getPostAuthor() {
+        return this.shredditPost?.getAttribute('author') || '';
+    }
+    createDescription(postContent) {
+        if (!postContent)
+            return '';
+        const tempDiv = this.document.createElement('div');
+        tempDiv.appendChild((0, dom_1.parseHTML)(this.document, postContent));
+        return tempDiv.textContent?.trim()
+            .slice(0, 140)
+            .replace(/\s+/g, ' ') || '';
+    }
+    collectOldRedditComments(container, depth = 0) {
+        const result = [];
+        const comments = Array.from(container.querySelectorAll(':scope > .thing.comment'));
+        for (const comment of comments) {
+            const author = comment.getAttribute('data-author') || '';
+            const permalink = comment.getAttribute('data-permalink') || '';
+            const score = comment.querySelector('.entry .tagline .score.unvoted')?.textContent?.trim() || '';
+            const timeEl = comment.querySelector('.entry .tagline time[datetime]');
+            const datetime = timeEl?.getAttribute('datetime') || '';
+            const date = datetime ? new Date(datetime).toISOString().split('T')[0] : '';
+            const bodyEl = comment.querySelector('.entry .usertext-body .md');
+            const body = bodyEl ? (0, dom_1.serializeHTML)(bodyEl) : '';
+            result.push({
+                author,
+                date,
+                content: body,
+                depth,
+                score: score || undefined,
+                url: permalink ? `https://reddit.com${permalink}` : undefined,
+            });
+            const childContainer = comment.querySelector('.child > .sitetable');
+            if (childContainer) {
+                result.push(...this.collectOldRedditComments(childContainer, depth + 1));
+            }
+        }
+        return result;
+    }
+    processComments(comments) {
+        const commentData = [];
+        for (const comment of comments) {
+            const depth = parseInt(comment.getAttribute('depth') || '0');
+            const author = comment.getAttribute('author') || '';
+            const score = comment.getAttribute('score') || '0';
+            const permalink = comment.getAttribute('permalink') || '';
+            const commentEl = comment.querySelector('[slot="comment"]');
+            const content = commentEl ? (0, dom_1.serializeHTML)(commentEl) : '';
+            const timestamp = comment.getAttribute('created')
+                || comment.querySelector('time')?.getAttribute('datetime')
+                || '';
+            const date = timestamp ? new Date(timestamp).toISOString().split('T')[0] : '';
+            commentData.push({
+                author,
+                date,
+                content,
+                depth,
+                score: `${score} points`,
+                url: permalink ? `https://reddit.com${permalink}` : undefined,
+            });
+        }
+        return (0, comments_1.buildCommentTree)(commentData);
+    }
+}
+exports.RedditExtractor = RedditExtractor;
+//# sourceMappingURL=reddit.js.map

package/dist/extractors/reddit.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"reddit.js","sourceRoot":"","sources":["../../src/extractors/reddit.ts"],"names":[],"mappings":";;;AAAA,mCAAwC;AAExC,sCAAwD;AACxD,gDAAyF;AAEzF,MAAa,eAAgB,SAAQ,qBAAa;IAIjD,YAAY,QAAkB,EAAE,GAAW;QAC1C,KAAK,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QACrB,IAAI,CAAC,YAAY,GAAG,QAAQ,CAAC,aAAa,CAAC,eAAe,CAAC,CAAC;QAC5D,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,aAAa,CAAC,CAAC;IAC5D,CAAC;IAED,UAAU;QACT,OAAO,CAAC,CAAC,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,WAAW,CAAC;IAChD,CAAC;IAED,eAAe;QACd,gEAAgE;QAChE,kEAAkE;QAClE,2DAA2D;QAC3D,OAAO,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC;IACnD,CAAC;IAEO,cAAc;QACrB,OAAO,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC7C,CAAC;IAED,KAAK,CAAC,YAAY;QACjB,gCAAgC;QAChC,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACjC,MAAM,CAAC,QAAQ,GAAG,gBAAgB,CAAC;QAEnC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE;YAC/C,OAAO,EAAE;gBACR,YAAY,EAAE,wCAAwC;aACtD;SACD,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YAClB,MAAM,IAAI,KAAK,CAAC,mCAAmC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;QACvE,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACnC,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,SAAS,IAAI,CAAC,OAAO,SAAS,KAAK,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC7G,IAAI,CAAC,MAAM,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;QACnE,CAAC;QACD,MAAM,GAAG,GAAG,IAAI,MAAM,EAAE,CAAC,eAAe,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;QAE5D,OAAO,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;IACnC,CAAC;IAED,OAAO;QACN,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACtB,OAAO,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC7C,CAAC;QAED,6DAA6D;QAC7D,6DAA6D;QAC7D,gEAAgE;QAChE,oCAAoC;QACpC,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,kBAAkB,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;QAClF,IAAI,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC;YAC3C,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,WAAW,EAAE,EAAE,EAAE,CAAC;QACzC,CAAC;QAED,MAAM,WAAW,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;QAC1C,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,cAAc,KAAK,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAErF,MAAM,WAAW,GAAG,IAAI,CAAC,iBAAiB,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;QAClE,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QAC/E,MAAM,SAAS,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;QACtC,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QACxC,MAAM,WAAW,GAAG,IAAI,CAAC,iBAAiB,CAAC,WAAW,CAAC,CAAC;QAExD,OAAO;YACN,OAAO,EAAE,WAAW;YACpB,WAAW,EAAE,WAAW;YACxB,gBAAgB,EAAE;gBACjB,MAAM,EAAE,IAAI,CAAC,SAAS,EAAE;gBACxB,SAAS;gBACT,UAAU;aACV;YACD,SAAS,EAAE;gBACV,KAAK,EAAE,SAAS;gBAChB,MAAM,EAAE,UAAU;gBAClB,IAAI,EAAE,KAAK,SAAS,EAAE;gBACtB,WAAW;aACX;SACD,CAAC;IACH,CAAC;IAEO,gBAAgB,CAAC,IAAwB;QAChD,MAAM,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC,aAAa,CAAC,CAAC;QACpD,MAAM,SAAS,GAAG,SAAS,EAAE,aAAa,CAAC,SAAS,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QACjF,MAAM,UAAU,GAAG,SAAS,EAAE,YAAY,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC;QAChE,MAAM,SAAS,GAAG,SAAS,EAAE,YAAY,CAAC,gBAAgB,CAAC,IAAI,EAAE,CAAC;QAClE,MAAM,UAAU,GAAG,SAAS,EAAE,aAAa,CAAC,oBAAoB,CAAC,CAAC;QAClE,MAAM,QAAQ,GAAG,UAAU,CAAC,CAAC,CAAC,IAAA,mBAAa,EAAC,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAE7D,IAAI,QAAQ,GAAG,EAAE,CAAC;QAClB,IAAI,IAAI,CAAC,OAAO,CAAC,cAAc,KAAK,KAAK,EAAE,CAAC;YAC3C,MAAM,WAAW,GAAG,IAAI,CAAC,aAAa,CAAC,yBAAyB,CAAC,CAAC;YAClE,MAAM,WAAW,GAAG,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,wBAAwB,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAClF,QAAQ,GAAG,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAA,2BAAgB,EAAC,WAAW,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACxE,CAAC;QAED,MAAM,WAAW,GAAG,IAAI,CAAC,iBAAiB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QAC/D,MAAM,WAAW,GAAG,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QAErD,OAAO;YACN,OAAO,EAAE,WAAW;YACpB,WAAW,EAAE,WAAW;YACxB,gBAAgB,EAAE;gBACjB,MAAM,EAAE,IAAI,CAAC,SAAS,EAAE;gBACxB,SAAS;gBACT,UAAU;aACV;YACD,SAAS,EAAE;gBACV,KAAK,EAAE,SAAS;gBAChB,MAAM,EAAE,UAAU;gBAClB,IAAI,EAAE,KAAK,SAAS,EAAE;gBACtB,WAAW;aACX;SACD,CAAC;IACH,CAAC;IAEO,cAAc;QACrB,MAAM,UAAU,GAAG,IAAI,CAAC,YAAY,EAAE,aAAa,CAAC,oBAAoB,CAAC,CAAC;QAC1E,MAAM,QAAQ,GAAG,UAAU,CAAC,CAAC,CAAC,IAAA,mBAAa,EAAC,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7D,MAAM,SAAS,GAAG,IAAI,CAAC,YAAY,EAAE,aAAa,CAAC,aAAa,CAAC,EAAE,SAAS,IAAI,EAAE,CAAC;QAEnF,OAAO,QAAQ,GAAG,SAAS,CAAC;IAC7B,CAAC;IAEO,iBAAiB,CAAC,WAAmB,EAAE,QAAgB;QAC9D,OAAO,IAAA,2BAAgB,EAAC,QAAQ,EAAE,WAAW,EAAE,QAAQ,CAAC,CAAC;IAC1D,CAAC;IAEO,eAAe;QACtB,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,kBAAkB,CAAC,CAAC,CAAC;QAChF,OAAO,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC;IACvC,CAAC;IAEO,SAAS;QAChB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC;QACzD,OAAO,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACzB,CAAC;IAEO,YAAY;QACnB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;QAC7C,OAAO,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACzB,CAAC;IAEO,aAAa;QACpB,OAAO,IAAI,CAAC,YAAY,EAAE,YAAY,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;IACxD,CAAC;IAEO,iBAAiB,CAAC,WAAmB;QAC5C,IAAI,CAAC,WAAW;YAAE,OAAO,EAAE,CAAC;QAE5B,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;QACnD,OAAO,CAAC,WAAW,CAAC,IAAA,eAAS,EAAC,IAAI,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC,CAAC;QAC3D,OAAO,OAAO,CAAC,WAAW,EAAE,IAAI,EAAE;aAChC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;aACb,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC;IAC9B,CAAC;IAEO,wBAAwB,CAAC,SAAkB,EAAE,QAAgB,CAAC;QACrE,MAAM,MAAM,GAAkB,EAAE,CAAC;QACjC,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,gBAAgB,CAAC,yBAAyB,CAAC,CAAC,CAAC;QAEnF,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAChC,MAAM,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC;YACzD,MAAM,SAAS,GAAG,OAAO,CAAC,YAAY,CAAC,gBAAgB,CAAC,IAAI,EAAE,CAAC;YAC/D,MAAM,KAAK,GAAG,OAAO,CAAC,aAAa,CAAC,gCAAgC,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;YACjG,MAAM,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,gCAAgC,CAAC,CAAC;YACvE,MAAM,QAAQ,GAAG,MAAM,EAAE,YAAY,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;YACxD,MAAM,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAC5E,MAAM,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,2BAA2B,CAAC,CAAC;YAClE,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,IAAA,mBAAa,EAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAEjD,MAAM,CAAC,IAAI,CAAC;gBACX,MAAM;gBACN,IAAI;gBACJ,OAAO,EAAE,IAAI;gBACb,KAAK;gBACL,KAAK,EAAE,KAAK,IAAI,SAAS;gBACzB,GAAG,EAAE,SAAS,CAAC,CAAC,CAAC,qBAAqB,SAAS,EAAE,CAAC,CAAC,CAAC,SAAS;aAC7D,CAAC,CAAC;YAEH,MAAM,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC,qBAAqB,CAAC,CAAC;YACpE,IAAI,cAAc,EAAE,CAAC;gBACpB,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,wBAAwB,CAAC,cAAc,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC;YAC1E,CAAC;QACF,CAAC;QAED,OAAO,MAAM,CAAC;IACf,CAAC;IAEO,eAAe,CAAC,QAAmB;QAC1C,MAAM,WAAW,GAAkB,EAAE,CAAC;QAEtC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAChC,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC,CAAC;YAC7D,MAAM,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;YACpD,MAAM,KAAK,GAAG,OAAO,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC;YACnD,MAAM,SAAS,GAAG,OAAO,CAAC,YAAY,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC;YAC1D,MAAM,SAAS,GAAG,OAAO,CAAC,aAAa,CAAC,kBAAkB,CAAC,CAAC;YAC5D,MAAM,OAAO,GAAG,SAAS,CAAC,CAAC,CAAC,IAAA,mBAAa,EAAC,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAE1D,MAAM,SAAS,GAAG,OAAO,CAAC,YAAY,CAAC,SAAS,CAAC;mBAC7C,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE,YAAY,CAAC,UAAU,CAAC;mBACvD,EAAE,CAAC;YACP,MAAM,IAAI,GAAG,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAE9E,WAAW,CAAC,IAAI,CAAC;gBAChB,MAAM;gBACN,IAAI;gBACJ,OAAO;gBACP,KAAK;gBACL,KAAK,EAAE,GAAG,KAAK,SAAS;gBACxB,GAAG,EAAE,SAAS,CAAC,CAAC,CAAC,qBAAqB,SAAS,EAAE,CAAC,CAAC,CAAC,SAAS;aAC7D,CAAC,CAAC;QACJ,CAAC;QAED,OAAO,IAAA,2BAAgB,EAAC,WAAW,CAAC,CAAC;IACtC,CAAC;CACD;AAlOD,0CAkOC"}

package/dist/extractors/twitter.d.ts

@@ -0,0 +1,16 @@
+import { BaseExtractor } from './_base';
+import { ExtractorResult } from '../types/extractors';
+export declare class TwitterExtractor extends BaseExtractor {
+    private mainTweet;
+    private threadTweets;
+    constructor(document: Document, url: string);
+    canExtract(): boolean;
+    extract(): ExtractorResult;
+    private formatTweetText;
+    private extractTweet;
+    private extractUserInfo;
+    private extractImages;
+    private getTweetId;
+    private getTweetAuthor;
+    private createDescription;
+}

package/dist/extractors/twitter.js

@@ -0,0 +1,204 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TwitterExtractor = void 0;
+const _base_1 = require("./_base");
+const dom_1 = require("../utils/dom");
+class TwitterExtractor extends _base_1.BaseExtractor {
+    constructor(document, url) {
+        super(document, url);
+        this.mainTweet = null;
+        this.threadTweets = [];
+        // Get all tweets from the timeline
+        const timeline = document.querySelector('[aria-label="Timeline: Conversation"]');
+        if (!timeline) {
+            // Try to find a single tweet if not in timeline view
+            const singleTweet = document.querySelector('article[data-testid="tweet"]');
+            if (singleTweet) {
+                this.mainTweet = singleTweet;
+            }
+            return;
+        }
+        // Get all tweets before any section with "Discover more" or similar headings
+        let allTweets = Array.from(timeline.querySelectorAll('article[data-testid="tweet"]'));
+        const firstSection = timeline.querySelector('section, h2')?.parentElement;
+        if (firstSection) {
+            // Filter out tweets that appear after the first section
+            const cutoffIndex = allTweets.findIndex(tweet => firstSection.compareDocumentPosition(tweet) & Node.DOCUMENT_POSITION_FOLLOWING);
+            if (cutoffIndex !== -1) {
+                allTweets = allTweets.slice(0, cutoffIndex);
+            }
+        }
+        // Set main tweet and thread tweets
+        this.mainTweet = allTweets[0] || null;
+        this.threadTweets = allTweets.slice(1);
+    }
+    canExtract() {
+        return !!this.mainTweet;
+    }
+    extract() {
+        const mainContent = this.extractTweet(this.mainTweet);
+        const threadContent = this.options.includeReplies !== false
+            ? this.threadTweets.map(tweet => this.extractTweet(tweet)).join('\n<hr>\n')
+            : '';
+        const contentHtml = `
+			<div class="tweet-thread">
+				<div class="main-tweet">
+					${mainContent}
+				</div>
+				${threadContent ? `
+					<hr>
+					<div class="thread-tweets">
+						${threadContent}
+					</div>
+				` : ''}
+			</div>
+		`.trim();
+        const tweetId = this.getTweetId();
+        const tweetAuthor = this.getTweetAuthor();
+        const description = this.createDescription(this.mainTweet);
+        return {
+            content: contentHtml,
+            contentHtml: contentHtml,
+            extractedContent: {
+                tweetId,
+                tweetAuthor,
+            },
+            variables: {
+                title: `Thread by ${tweetAuthor}`,
+                author: tweetAuthor,
+                site: 'X (Twitter)',
+                description,
+            }
+        };
+    }
+    formatTweetText(text) {
+        if (!text)
+            return '';
+        // Create a temporary div to parse and clean the HTML
+        const tempDiv = this.document.createElement('div');
+        tempDiv.appendChild((0, dom_1.parseHTML)(this.document, text));
+        // Convert links to plain text with @ handles
+        tempDiv.querySelectorAll('a').forEach(link => {
+            const handle = link.textContent?.trim() || '';
+            link.replaceWith(handle);
+        });
+        // Remove unnecessary spans and divs but keep their content
+        tempDiv.querySelectorAll('span, div').forEach(element => {
+            element.replaceWith(...Array.from(element.childNodes));
+        });
+        // Get cleaned text and split into paragraphs
+        const cleanText = (0, dom_1.serializeHTML)(tempDiv);
+        const paragraphs = cleanText.split('\n')
+            .map(line => line.trim())
+            .filter(line => line);
+        // Wrap each paragraph in <p> tags
+        return paragraphs.map(p => `<p>${p}</p>`).join('\n');
+    }
+    extractTweet(tweet) {
+        if (!tweet)
+            return '';
+        // Clone the tweet element to modify it
+        const tweetClone = tweet.cloneNode(true);
+        // Convert emoji images to text
+        tweetClone.querySelectorAll('img[src*="/emoji/"]').forEach(img => {
+            if (img.tagName.toLowerCase() === 'img' && img.getAttribute('alt')) {
+                const altText = img.getAttribute('alt');
+                if (altText) {
+                    img.replaceWith(altText);
+                }
+            }
+        });
+        const tweetTextEl = tweetClone.querySelector('[data-testid="tweetText"]');
+        const tweetText = tweetTextEl ? (0, dom_1.serializeHTML)(tweetTextEl) : '';
+        const formattedText = this.formatTweetText(tweetText);
+        const images = this.extractImages(tweet);
+        // Get author info and date
+        const userInfo = this.extractUserInfo(tweet);
+        // Extract quoted tweet if present
+        const quotedTweet = tweet.querySelector('[aria-labelledby*="id__"]')?.querySelector('[data-testid="User-Name"]')?.closest('[aria-labelledby*="id__"]');
+        const quotedContent = quotedTweet ? this.extractTweet(quotedTweet) : '';
+        return `
+			<div class="tweet">
+				<div class="tweet-header">
+					<span class="tweet-author"><strong>${userInfo.fullName}</strong> <span class="tweet-handle">${userInfo.handle}</span></span>
+					${userInfo.date ? `<a href="${userInfo.permalink}" class="tweet-date">${userInfo.date}</a>` : ''}
+				</div>
+				${formattedText ? `<div class="tweet-text">${formattedText}</div>` : ''}
+				${images.length ? `
+					<div class="tweet-media">
+						${images.join('\n')}
+					</div>
+				` : ''}
+				${quotedContent ? `
+					<blockquote class="quoted-tweet">
+						${quotedContent}
+					</blockquote>
+				` : ''}
+			</div>
+		`.trim();
+    }
+    extractUserInfo(tweet) {
+        const nameElement = tweet.querySelector('[data-testid="User-Name"]');
+        if (!nameElement)
+            return { fullName: '', handle: '', date: '', permalink: '' };
+        // Try to get name and handle from links first (main tweet structure)
+        const links = nameElement.querySelectorAll('a');
+        let fullName = links?.[0]?.textContent?.trim() || '';
+        let handle = links?.[1]?.textContent?.trim() || '';
+        // If links don't have the info, try to get from spans (quoted tweet structure)
+        if (!fullName || !handle) {
+            fullName = nameElement.querySelector('span[style*="color: rgb(15, 20, 25)"] span')?.textContent?.trim() || '';
+            handle = nameElement.querySelector('span[style*="color: rgb(83, 100, 113)"]')?.textContent?.trim() || '';
+        }
+        const timestamp = tweet.querySelector('time');
+        const datetime = timestamp?.getAttribute('datetime') || '';
+        const date = datetime ? new Date(datetime).toISOString().split('T')[0] : '';
+        const permalink = timestamp?.closest('a')?.href || '';
+        return { fullName, handle, date, permalink };
+    }
+    extractImages(tweet) {
+        // Look for images in different containers
+        const imageContainers = [
+            '[data-testid="tweetPhoto"]',
+            '[data-testid="tweet-image"]',
+            'img[src*="media"]'
+        ];
+        const images = [];
+        // Skip images that are inside quoted tweets
+        const quotedTweet = tweet.querySelector('[aria-labelledby*="id__"]')?.querySelector('[data-testid="User-Name"]')?.closest('[aria-labelledby*="id__"]');
+        for (const selector of imageContainers) {
+            const elements = tweet.querySelectorAll(selector);
+            elements.forEach(img => {
+                // Skip if the image is inside a quoted tweet
+                if (quotedTweet?.contains(img)) {
+                    return;
+                }
+                // Check if element is an image by checking tag name and required properties
+                if (img.tagName.toLowerCase() === 'img' && img.getAttribute('alt')) {
+                    const highQualitySrc = img.getAttribute('src')?.replace(/&name=\w+$/, '&name=large') || '';
+                    const cleanAlt = img.getAttribute('alt')?.replace(/\s+/g, ' ').trim() || '';
+                    images.push(`<img src="${highQualitySrc}" alt="${cleanAlt}" />`);
+                }
+            });
+        }
+        return images;
+    }
+    getTweetId() {
+        const match = this.url.match(/status\/(\d+)/);
+        return match?.[1] || '';
+    }
+    getTweetAuthor() {
+        const nameElement = this.mainTweet?.querySelector('[data-testid="User-Name"]');
+        const links = nameElement?.querySelectorAll('a');
+        const handle = links?.[1]?.textContent?.trim() || '';
+        return handle.startsWith('@') ? handle : `@${handle}`;
+    }
+    createDescription(tweet) {
+        if (!tweet)
+            return '';
+        const tweetText = tweet.querySelector('[data-testid="tweetText"]')?.textContent || '';
+        return tweetText.trim().slice(0, 140).replace(/\s+/g, ' ');
+    }
+}
+exports.TwitterExtractor = TwitterExtractor;
+//# sourceMappingURL=twitter.js.map

package/dist/extractors/twitter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"twitter.js","sourceRoot":"","sources":["../../src/extractors/twitter.ts"],"names":[],"mappings":";;;AAAA,mCAAwC;AAExC,sCAAwD;AAExD,MAAa,gBAAiB,SAAQ,qBAAa;IAIlD,YAAY,QAAkB,EAAE,GAAW;QAC1C,KAAK,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QAJd,cAAS,GAAmB,IAAI,CAAC;QACjC,iBAAY,GAAc,EAAE,CAAC;QAKpC,mCAAmC;QACnC,MAAM,QAAQ,GAAG,QAAQ,CAAC,aAAa,CAAC,uCAAuC,CAAC,CAAC;QACjF,IAAI,CAAC,QAAQ,EAAE,CAAC;YACf,qDAAqD;YACrD,MAAM,WAAW,GAAG,QAAQ,CAAC,aAAa,CAAC,8BAA8B,CAAC,CAAC;YAC3E,IAAI,WAAW,EAAE,CAAC;gBACjB,IAAI,CAAC,SAAS,GAAG,WAAW,CAAC;YAC9B,CAAC;YACD,OAAO;QACR,CAAC;QAED,6EAA6E;QAC7E,IAAI,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,8BAA8B,CAAC,CAAC,CAAC;QACtF,MAAM,YAAY,GAAG,QAAQ,CAAC,aAAa,CAAC,aAAa,CAAC,EAAE,aAAa,CAAC;QAE1E,IAAI,YAAY,EAAE,CAAC;YAClB,wDAAwD;YACxD,MAAM,WAAW,GAAG,SAAS,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAC/C,YAAY,CAAC,uBAAuB,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,2BAA2B,CAC9E,CAAC;YACF,IAAI,WAAW,KAAK,CAAC,CAAC,EAAE,CAAC;gBACxB,SAAS,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC;YAC7C,CAAC;QACF,CAAC;QAED,mCAAmC;QACnC,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;QACtC,IAAI,CAAC,YAAY,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACxC,CAAC;IAED,UAAU;QACT,OAAO,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC;IACzB,CAAC;IAED,OAAO;QACN,MAAM,WAAW,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACtD,MAAM,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,cAAc,KAAK,KAAK;YAC1D,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC;YAC3E,CAAC,CAAC,EAAE,CAAC;QAEN,MAAM,WAAW,GAAG;;;OAGf,WAAW;;MAEZ,aAAa,CAAC,CAAC,CAAC;;;QAGd,aAAa;;KAEhB,CAAC,CAAC,CAAC,EAAE;;GAEP,CAAC,IAAI,EAAE,CAAC;QAET,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;QAClC,MAAM,WAAW,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;QAC1C,MAAM,WAAW,GAAG,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAE3D,OAAO;YACN,OAAO,EAAE,WAAW;YACpB,WAAW,EAAE,WAAW;YACxB,gBAAgB,EAAE;gBACjB,OAAO;gBACP,WAAW;aACX;YACD,SAAS,EAAE;gBACV,KAAK,EAAE,aAAa,WAAW,EAAE;gBACjC,MAAM,EAAE,WAAW;gBACnB,IAAI,EAAE,aAAa;gBACnB,WAAW;aACX;SACD,CAAC;IACH,CAAC;IAEO,eAAe,CAAC,IAAY;QACnC,IAAI,CAAC,IAAI;YAAE,OAAO,EAAE,CAAC;QAErB,qDAAqD;QACrD,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;QACnD,OAAO,CAAC,WAAW,CAAC,IAAA,eAAS,EAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC;QAEpD,6CAA6C;QAC7C,OAAO,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE;YAC5C,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;YAC9C,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QAC1B,CAAC,CAAC,CAAC;QAEH,2DAA2D;QAC3D,OAAO,CAAC,gBAAgB,CAAC,WAAW,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE;YACvD,OAAO,CAAC,WAAW,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;QACxD,CAAC,CAAC,CAAC;QAEH,6CAA6C;QAC7C,MAAM,SAAS,GAAG,IAAA,mBAAa,EAAC,OAAO,CAAC,CAAC;QACzC,MAAM,UAAU,GAAG,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC;aACtC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;aACxB,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;QAEvB,kCAAkC;QAClC,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACtD,CAAC;IAEO,YAAY,CAAC,KAAqB;QACzC,IAAI,CAAC,KAAK;YAAE,OAAO,EAAE,CAAC;QAEtB,uCAAuC;QACvC,MAAM,UAAU,GAAG,KAAK,CAAC,SAAS,CAAC,IAAI,CAAY,CAAC;QAEpD,+BAA+B;QAC/B,UAAU,CAAC,gBAAgB,CAAC,qBAAqB,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE;YAChE,IAAI,GAAG,CAAC,OAAO,CAAC,WAAW,EAAE,KAAK,KAAK,IAAI,GAAG,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;gBACpE,MAAM,OAAO,GAAG,GAAG,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;gBACxC,IAAI,OAAO,EAAE,CAAC;oBACb,GAAG,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;gBAC1B,CAAC;YACF,CAAC;QACF,CAAC,CAAC,CAAC;QAEH,MAAM,WAAW,GAAG,UAAU,CAAC,aAAa,CAAC,2BAA2B,CAAC,CAAC;QAC1E,MAAM,SAAS,GAAG,WAAW,CAAC,CAAC,CAAC,IAAA,mBAAa,EAAC,WAAW,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAChE,MAAM,aAAa,GAAG,IAAI,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC;QACtD,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;QAEzC,2BAA2B;QAC3B,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;QAE7C,kCAAkC;QAClC,MAAM,WAAW,GAAG,KAAK,CAAC,aAAa,CAAC,2BAA2B,CAAC,EAAE,aAAa,CAAC,2BAA2B,CAAC,EAAE,OAAO,CAAC,2BAA2B,CAAC,CAAC;QACvJ,MAAM,aAAa,GAAG,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAExE,OAAO;;;0CAGiC,QAAQ,CAAC,QAAQ,wCAAwC,QAAQ,CAAC,MAAM;OAC3G,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,YAAY,QAAQ,CAAC,SAAS,wBAAwB,QAAQ,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,EAAE;;MAE/F,aAAa,CAAC,CAAC,CAAC,2BAA2B,aAAa,QAAQ,CAAC,CAAC,CAAC,EAAE;MACrE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;;QAEd,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC;;KAEpB,CAAC,CAAC,CAAC,EAAE;MACJ,aAAa,CAAC,CAAC,CAAC;;QAEd,aAAa;;KAEhB,CAAC,CAAC,CAAC,EAAE;;GAEP,CAAC,IAAI,EAAE,CAAC;IACV,CAAC;IAEO,eAAe,CAAC,KAAc;QACrC,MAAM,WAAW,GAAG,KAAK,CAAC,aAAa,CAAC,2BAA2B,CAAC,CAAC;QACrE,IAAI,CAAC,WAAW;YAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC;QAE/E,qEAAqE;QACrE,MAAM,KAAK,GAAG,WAAW,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;QAChD,IAAI,QAAQ,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QACrD,IAAI,MAAM,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QAEnD,+EAA+E;QAC/E,IAAI,CAAC,QAAQ,IAAI,CAAC,MAAM,EAAE,CAAC;YAC1B,QAAQ,GAAG,WAAW,CAAC,aAAa,CAAC,4CAA4C,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;YAC9G,MAAM,GAAG,WAAW,CAAC,aAAa,CAAC,yCAAyC,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QAC1G,CAAC;QAED,MAAM,SAAS,GAAG,KAAK,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;QAC9C,MAAM,QAAQ,GAAG,SAAS,EAAE,YAAY,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;QAC3D,MAAM,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5E,MAAM,SAAS,GAAG,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,EAAE,IAAI,IAAI,EAAE,CAAC;QAEtD,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;IAC9C,CAAC;IAEO,aAAa,CAAC,KAAc;QACnC,0CAA0C;QAC1C,MAAM,eAAe,GAAG;YACvB,4BAA4B;YAC5B,6BAA6B;YAC7B,mBAAmB;SACnB,CAAC;QAEF,MAAM,MAAM,GAAa,EAAE,CAAC;QAE5B,4CAA4C;QAC5C,MAAM,WAAW,GAAG,KAAK,CAAC,aAAa,CAAC,2BAA2B,CAAC,EAAE,aAAa,CAAC,2BAA2B,CAAC,EAAE,OAAO,CAAC,2BAA2B,CAAC,CAAC;QAEvJ,KAAK,MAAM,QAAQ,IAAI,eAAe,EAAE,CAAC;YACxC,MAAM,QAAQ,GAAG,KAAK,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;YAElD,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE;gBACtB,6CAA6C;gBAC7C,IAAI,WAAW,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;oBAChC,OAAO;gBACR,CAAC;gBAED,4EAA4E;gBAC5E,IAAI,GAAG,CAAC,OAAO,CAAC,WAAW,EAAE,KAAK,KAAK,IAAI,GAAG,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;oBACpE,MAAM,cAAc,GAAG,GAAG,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC,YAAY,EAAE,aAAa,CAAC,IAAI,EAAE,CAAC;oBAC3F,MAAM,QAAQ,GAAG,GAAG,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC;oBAC5E,MAAM,CAAC,IAAI,CAAC,aAAa,cAAc,UAAU,QAAQ,MAAM,CAAC,CAAC;gBAClE,CAAC;YACF,CAAC,CAAC,CAAC;QACJ,CAAC;QAED,OAAO,MAAM,CAAC;IACf,CAAC;IAEO,UAAU;QACjB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;QAC9C,OAAO,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACzB,CAAC;IAEO,cAAc;QACrB,MAAM,WAAW,GAAG,IAAI,CAAC,SAAS,EAAE,aAAa,CAAC,2BAA2B,CAAC,CAAC;QAC/E,MAAM,KAAK,GAAG,WAAW,EAAE,gBAAgB,CAAC,GAAG,CAAC,CAAC;QACjD,MAAM,MAAM,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QACrD,OAAO,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,MAAM,EAAE,CAAC;IACvD,CAAC;IAEO,iBAAiB,CAAC,KAAqB;QAC9C,IAAI,CAAC,KAAK;YAAE,OAAO,EAAE,CAAC;QAEtB,MAAM,SAAS,GAAG,KAAK,CAAC,aAAa,CAAC,2BAA2B,CAAC,EAAE,WAAW,IAAI,EAAE,CAAC;QACtF,OAAO,SAAS,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC5D,CAAC;CACD;AAzOD,4CAyOC"}

package/dist/extractors/x-article.d.ts

@@ -0,0 +1,24 @@
+import { BaseExtractor } from './_base';
+import { ExtractorResult } from '../types/extractors';
+export declare class XArticleExtractor extends BaseExtractor {
+    private articleContainer;
+    constructor(document: Document, url: string, schemaOrgData?: any);
+    canExtract(): boolean;
+    extract(): ExtractorResult;
+    private extractTitle;
+    private extractAuthor;
+    private getAuthorFromUrl;
+    private getAuthorFromOgTitle;
+    private getArticleId;
+    private extractContent;
+    private cleanContent;
+    private convertEmbeddedTweets;
+    private convertCodeBlocks;
+    private convertHeaders;
+    private unwrapLinkedImages;
+    private upgradeImageQuality;
+    private convertDraftParagraphs;
+    private convertBoldSpans;
+    private removeDraftAttributes;
+    private createDescription;
+}