@mz1999/defuddle 0.14.1

package/dist/extractors/x-article.js

@@ -0,0 +1,267 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.XArticleExtractor = void 0;
+const _base_1 = require("./_base");
+const dom_1 = require("../utils/dom");
+const SELECTORS = {
+    ARTICLE_CONTAINER: '[data-testid="twitterArticleRichTextView"]',
+    TITLE: '[data-testid="twitter-article-title"]',
+    AUTHOR: '[itemprop="author"]',
+    AUTHOR_NAME: 'meta[itemprop="name"]',
+    AUTHOR_HANDLE: 'meta[itemprop="additionalName"]',
+    IMAGES: '[data-testid="tweetPhoto"] img',
+    DRAFT_PARAGRAPHS: '.longform-unstyled, .public-DraftStyleDefault-block',
+    BOLD_SPANS: 'span[style*="font-weight: bold"]',
+    DRAFT_ATTRIBUTES: '[data-offset-key]',
+    EMBEDDED_TWEET: '[data-testid="simpleTweet"]',
+    TWEET_TEXT: '[data-testid="tweetText"]',
+    USER_NAME: '[data-testid="User-Name"]',
+    CODE_BLOCK: '[data-testid="markdown-code-block"]',
+    HEADER_BLOCK: '[data-testid="longform-header"]',
+};
+class XArticleExtractor extends _base_1.BaseExtractor {
+    constructor(document, url, schemaOrgData) {
+        super(document, url, schemaOrgData);
+        this.articleContainer = document.querySelector(SELECTORS.ARTICLE_CONTAINER);
+    }
+    canExtract() {
+        return !!this.articleContainer;
+    }
+    extract() {
+        const title = this.extractTitle();
+        const author = this.extractAuthor();
+        const contentHtml = this.extractContent();
+        const description = this.createDescription();
+        return {
+            content: contentHtml,
+            contentHtml,
+            extractedContent: {
+                articleId: this.getArticleId(),
+            },
+            variables: {
+                title,
+                author,
+                site: 'X (Twitter)',
+                description,
+            }
+        };
+    }
+    extractTitle() {
+        const titleEl = this.document.querySelector(SELECTORS.TITLE);
+        return titleEl?.textContent?.trim() || 'Untitled X Article';
+    }
+    extractAuthor() {
+        const authorContainer = this.document.querySelector(SELECTORS.AUTHOR);
+        if (!authorContainer)
+            return this.getAuthorFromUrl();
+        const name = authorContainer.querySelector(SELECTORS.AUTHOR_NAME)?.getAttribute('content');
+        const handle = authorContainer.querySelector(SELECTORS.AUTHOR_HANDLE)?.getAttribute('content');
+        if (name && handle)
+            return `${name} (@${handle})`;
+        return name || handle || this.getAuthorFromUrl();
+    }
+    getAuthorFromUrl() {
+        // match username before /article/, excluding system paths like /i/
+        const match = this.url.match(/\/([a-zA-Z][a-zA-Z0-9_]{0,14})\/article\/\d+/);
+        return match ? `@${match[1]}` : this.getAuthorFromOgTitle();
+    }
+    getAuthorFromOgTitle() {
+        const ogTitle = this.document.querySelector('meta[property="og:title"]')?.getAttribute('content') || '';
+        // Match patterns like "(4) Heinrich on X: ..." or "Heinrich on X: ..."
+        const match = ogTitle.match(/^(?:\(\d+\)\s+)?(.+?)\s+on\s+X\s*:/);
+        return match ? match[1].trim() : 'Unknown';
+    }
+    getArticleId() {
+        const match = this.url.match(/article\/(\d+)/);
+        return match ? match[1] : '';
+    }
+    extractContent() {
+        if (!this.articleContainer)
+            return '';
+        const clone = this.articleContainer.cloneNode(true);
+        this.cleanContent(clone);
+        return `<article class="x-article">${(0, dom_1.serializeHTML)(clone)}</article>`;
+    }
+    cleanContent(container) {
+        const ownerDoc = container.ownerDocument || this.document;
+        // convert complex elements first (before other transformations)
+        this.convertEmbeddedTweets(container, ownerDoc);
+        this.convertCodeBlocks(container, ownerDoc);
+        this.convertHeaders(container, ownerDoc);
+        this.unwrapLinkedImages(container, ownerDoc);
+        this.upgradeImageQuality(container);
+        // convert bold spans BEFORE paragraphs so formatting is preserved
+        this.convertBoldSpans(container, ownerDoc);
+        this.convertDraftParagraphs(container, ownerDoc);
+        this.removeDraftAttributes(container);
+    }
+    convertEmbeddedTweets(container, ownerDoc) {
+        container.querySelectorAll(SELECTORS.EMBEDDED_TWEET).forEach(tweet => {
+            const blockquote = ownerDoc.createElement('blockquote');
+            blockquote.className = 'embedded-tweet';
+            // extract author info
+            const userNameEl = tweet.querySelector(SELECTORS.USER_NAME);
+            const authorLinks = userNameEl?.querySelectorAll('a');
+            const fullName = authorLinks?.[0]?.textContent?.trim() || '';
+            const handle = authorLinks?.[1]?.textContent?.trim() || '';
+            // extract tweet text
+            const tweetTextEl = tweet.querySelector(SELECTORS.TWEET_TEXT);
+            const tweetText = tweetTextEl?.textContent?.trim() || '';
+            // build clean blockquote content
+            if (fullName || handle) {
+                const cite = ownerDoc.createElement('cite');
+                cite.textContent = handle ? `${fullName} ${handle}` : fullName;
+                blockquote.appendChild(cite);
+            }
+            if (tweetText) {
+                const p = ownerDoc.createElement('p');
+                p.textContent = tweetText;
+                blockquote.appendChild(p);
+            }
+            tweet.replaceWith(blockquote);
+        });
+    }
+    convertCodeBlocks(container, ownerDoc) {
+        container.querySelectorAll(SELECTORS.CODE_BLOCK).forEach(block => {
+            const pre = block.querySelector('pre');
+            const code = block.querySelector('code');
+            if (!pre || !code)
+                return;
+            // extract language from class (e.g., "language-bash") or from span
+            let language = '';
+            const langClass = code.className.match(/language-(\w+)/);
+            if (langClass) {
+                language = langClass[1];
+            }
+            else {
+                // fallback: look for language label in the block header
+                const langSpan = block.querySelector('span');
+                language = langSpan?.textContent?.trim() || '';
+            }
+            // create clean pre/code structure
+            const newPre = ownerDoc.createElement('pre');
+            const newCode = ownerDoc.createElement('code');
+            if (language) {
+                newCode.setAttribute('data-lang', language);
+                newCode.className = `language-${language}`;
+            }
+            newCode.textContent = code.textContent || '';
+            newPre.appendChild(newCode);
+            // replace the entire block container
+            block.replaceWith(newPre);
+        });
+    }
+    convertHeaders(container, ownerDoc) {
+        // X articles use h2/h3 elements but content may be nested in spans/divs
+        container.querySelectorAll('h1, h2, h3, h4, h5, h6').forEach(header => {
+            const level = header.tagName.toLowerCase();
+            const text = header.textContent?.trim() || '';
+            if (!text)
+                return;
+            const newHeader = ownerDoc.createElement(level);
+            newHeader.textContent = text;
+            header.replaceWith(newHeader);
+        });
+    }
+    unwrapLinkedImages(container, ownerDoc) {
+        // find all tweetPhoto images and extract them from any ancestor anchors
+        container.querySelectorAll(SELECTORS.IMAGES).forEach(img => {
+            // find closest anchor ancestor
+            const anchor = img.closest('a');
+            if (!anchor || !container.contains(anchor))
+                return;
+            // create clean img tag with upgraded quality (like TwitterExtractor does)
+            let src = img.getAttribute('src') || '';
+            const alt = img.getAttribute('alt')?.replace(/\s+/g, ' ').trim() || 'Image';
+            // upgrade image quality
+            if (src.includes('&name=')) {
+                src = src.replace(/&name=\w+/, '&name=large');
+            }
+            else if (src.includes('?')) {
+                src = `${src}&name=large`;
+            }
+            else {
+                src = `${src}?name=large`;
+            }
+            const cleanImg = ownerDoc.createElement('img');
+            cleanImg.setAttribute('src', src);
+            cleanImg.setAttribute('alt', alt);
+            // replace anchor with clean image
+            anchor.replaceWith(cleanImg);
+        });
+    }
+    upgradeImageQuality(container) {
+        container.querySelectorAll(SELECTORS.IMAGES).forEach(img => {
+            const src = img.getAttribute('src');
+            if (!src)
+                return;
+            if (src.includes('&name=')) {
+                img.setAttribute('src', src.replace(/&name=\w+/, '&name=large'));
+            }
+            else if (src.includes('?')) {
+                img.setAttribute('src', `${src}&name=large`);
+            }
+            else {
+                img.setAttribute('src', `${src}?name=large`);
+            }
+        });
+    }
+    convertDraftParagraphs(container, ownerDoc) {
+        // node type constants (avoid using Node global which isn't available in all environments)
+        const TEXT_NODE = 3;
+        const ELEMENT_NODE = 1;
+        container.querySelectorAll(SELECTORS.DRAFT_PARAGRAPHS).forEach(div => {
+            const p = ownerDoc.createElement('p');
+            // preserve formatting (strong, links, code) by processing children
+            const processNode = (node) => {
+                if (node.nodeType === TEXT_NODE) {
+                    p.appendChild(ownerDoc.createTextNode(node.textContent || ''));
+                }
+                else if (node.nodeType === ELEMENT_NODE) {
+                    const el = node;
+                    const tag = el.tagName.toLowerCase();
+                    if (tag === 'strong') {
+                        const strong = ownerDoc.createElement('strong');
+                        strong.textContent = el.textContent || '';
+                        p.appendChild(strong);
+                    }
+                    else if (tag === 'a') {
+                        const link = ownerDoc.createElement('a');
+                        link.setAttribute('href', el.getAttribute('href') || '');
+                        link.textContent = el.textContent || '';
+                        p.appendChild(link);
+                    }
+                    else if (tag === 'code') {
+                        const code = ownerDoc.createElement('code');
+                        code.textContent = el.textContent || '';
+                        p.appendChild(code);
+                    }
+                    else {
+                        // recurse into other elements (spans, divs, etc.)
+                        el.childNodes.forEach(child => processNode(child));
+                    }
+                }
+            };
+            div.childNodes.forEach(child => processNode(child));
+            div.replaceWith(p);
+        });
+    }
+    convertBoldSpans(container, ownerDoc) {
+        container.querySelectorAll(SELECTORS.BOLD_SPANS).forEach(span => {
+            const strong = ownerDoc.createElement('strong');
+            strong.textContent = span.textContent || '';
+            span.replaceWith(strong);
+        });
+    }
+    removeDraftAttributes(container) {
+        container.querySelectorAll(SELECTORS.DRAFT_ATTRIBUTES).forEach(el => {
+            el.removeAttribute('data-offset-key');
+        });
+    }
+    createDescription() {
+        const text = this.articleContainer?.textContent?.trim() || '';
+        return text.slice(0, 140) + (text.length > 140 ? '...' : '');
+    }
+}
+exports.XArticleExtractor = XArticleExtractor;
+//# sourceMappingURL=x-article.js.map

package/dist/extractors/x-article.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"x-article.js","sourceRoot":"","sources":["../../src/extractors/x-article.ts"],"names":[],"mappings":";;;AAAA,mCAAwC;AAExC,sCAA6C;AAE7C,MAAM,SAAS,GAAG;IACjB,iBAAiB,EAAE,4CAA4C;IAC/D,KAAK,EAAE,uCAAuC;IAC9C,MAAM,EAAE,qBAAqB;IAC7B,WAAW,EAAE,uBAAuB;IACpC,aAAa,EAAE,iCAAiC;IAChD,MAAM,EAAE,gCAAgC;IACxC,gBAAgB,EAAE,qDAAqD;IACvE,UAAU,EAAE,kCAAkC;IAC9C,gBAAgB,EAAE,mBAAmB;IACrC,cAAc,EAAE,6BAA6B;IAC7C,UAAU,EAAE,2BAA2B;IACvC,SAAS,EAAE,2BAA2B;IACtC,UAAU,EAAE,qCAAqC;IACjD,YAAY,EAAE,iCAAiC;CACtC,CAAC;AAEX,MAAa,iBAAkB,SAAQ,qBAAa;IAGnD,YAAY,QAAkB,EAAE,GAAW,EAAE,aAAmB;QAC/D,KAAK,CAAC,QAAQ,EAAE,GAAG,EAAE,aAAa,CAAC,CAAC;QACpC,IAAI,CAAC,gBAAgB,GAAG,QAAQ,CAAC,aAAa,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;IAC7E,CAAC;IAED,UAAU;QACT,OAAO,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC;IAChC,CAAC;IAED,OAAO;QACN,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;QAClC,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QACpC,MAAM,WAAW,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;QAC1C,MAAM,WAAW,GAAG,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAE7C,OAAO;YACN,OAAO,EAAE,WAAW;YACpB,WAAW;YACX,gBAAgB,EAAE;gBACjB,SAAS,EAAE,IAAI,CAAC,YAAY,EAAE;aAC9B;YACD,SAAS,EAAE;gBACV,KAAK;gBACL,MAAM;gBACN,IAAI,EAAE,aAAa;gBACnB,WAAW;aACX;SACD,CAAC;IACH,CAAC;IAEO,YAAY;QACnB,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QAC7D,OAAO,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,oBAAoB,CAAC;IAC7D,CAAC;IAEO,aAAa;QACpB,MAAM,eAAe,GAAG,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QACtE,IAAI,CAAC,eAAe;YAAE,OAAO,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAErD,MAAM,IAAI,GAAG,eAAe,CAAC,aAAa,CAAC,SAAS,CAAC,WAAW,CAAC,EAAE,YAAY,CAAC,SAAS,CAAC,CAAC;QAC3F,MAAM,MAAM,GAAG,eAAe,CAAC,aAAa,CAAC,SAAS,CAAC,aAAa,CAAC,EAAE,YAAY,CAAC,SAAS,CAAC,CAAC;QAE/F,IAAI,IAAI,IAAI,MAAM;YAAE,OAAO,GAAG,IAAI,MAAM,MAAM,GAAG,CAAC;QAClD,OAAO,IAAI,IAAI,MAAM,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;IAClD,CAAC;IAEO,gBAAgB;QACvB,mEAAmE;QACnE,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,8CAA8C,CAAC,CAAC;QAC7E,OAAO,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,oBAAoB,EAAE,CAAC;IAC7D,CAAC;IAEO,oBAAoB;QAC3B,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,2BAA2B,CAAC,EAAE,YAAY,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;QACxG,uEAAuE;QACvE,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,oCAAoC,CAAC,CAAC;QAClE,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;IAC5C,CAAC;IAEO,YAAY;QACnB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;QAC/C,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAC9B,CAAC;IAEO,cAAc;QACrB,IAAI,CAAC,IAAI,CAAC,gBAAgB;YAAE,OAAO,EAAE,CAAC;QAEtC,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,IAAI,CAAgB,CAAC;QACnE,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;QAEzB,OAAO,8BAA8B,IAAA,mBAAa,EAAC,KAAK,CAAC,YAAY,CAAC;IACvE,CAAC;IAEO,YAAY,CAAC,SAAsB;QAC1C,MAAM,QAAQ,GAAG,SAAS,CAAC,aAAa,IAAI,IAAI,CAAC,QAAQ,CAAC;QAE1D,gEAAgE;QAChE,IAAI,CAAC,qBAAqB,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QAChD,IAAI,CAAC,iBAAiB,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QAC5C,IAAI,CAAC,cAAc,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QACzC,IAAI,CAAC,kBAAkB,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QAC7C,IAAI,CAAC,mBAAmB,CAAC,SAAS,CAAC,CAAC;QACpC,kEAAkE;QAClE,IAAI,CAAC,gBAAgB,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QAC3C,IAAI,CAAC,sBAAsB,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QACjD,IAAI,CAAC,qBAAqB,CAAC,SAAS,CAAC,CAAC;IACvC,CAAC;IAEO,qBAAqB,CAAC,SAAsB,EAAE,QAAkB;QACvE,SAAS,CAAC,gBAAgB,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE;YACpE,MAAM,UAAU,GAAG,QAAQ,CAAC,aAAa,CAAC,YAAY,CAAC,CAAC;YACxD,UAAU,CAAC,SAAS,GAAG,gBAAgB,CAAC;YAExC,sBAAsB;YACtB,MAAM,UAAU,GAAG,KAAK,CAAC,aAAa,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;YAC5D,MAAM,WAAW,GAAG,UAAU,EAAE,gBAAgB,CAAC,GAAG,CAAC,CAAC;YACtD,MAAM,QAAQ,GAAG,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;YAC7D,MAAM,MAAM,GAAG,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;YAE3D,qBAAqB;YACrB,MAAM,WAAW,GAAG,KAAK,CAAC,aAAa,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;YAC9D,MAAM,SAAS,GAAG,WAAW,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;YAEzD,iCAAiC;YACjC,IAAI,QAAQ,IAAI,MAAM,EAAE,CAAC;gBACxB,MAAM,IAAI,GAAG,QAAQ,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;gBAC5C,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,CAAC,CAAC,GAAG,QAAQ,IAAI,MAAM,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC;gBAC/D,UAAU,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;YAC9B,CAAC;YAED,IAAI,SAAS,EAAE,CAAC;gBACf,MAAM,CAAC,GAAG,QAAQ,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;gBACtC,CAAC,CAAC,WAAW,GAAG,SAAS,CAAC;gBAC1B,UAAU,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;YAC3B,CAAC;YAED,KAAK,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;QAC/B,CAAC,CAAC,CAAC;IACJ,CAAC;IAEO,iBAAiB,CAAC,SAAsB,EAAE,QAAkB;QACnE,SAAS,CAAC,gBAAgB,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE;YAChE,MAAM,GAAG,GAAG,KAAK,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;YACvC,MAAM,IAAI,GAAG,KAAK,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;YACzC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI;gBAAE,OAAO;YAE1B,mEAAmE;YACnE,IAAI,QAAQ,GAAG,EAAE,CAAC;YAClB,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;YACzD,IAAI,SAAS,EAAE,CAAC;gBACf,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;YACzB,CAAC;iBAAM,CAAC;gBACP,wDAAwD;gBACxD,MAAM,QAAQ,GAAG,KAAK,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;gBAC7C,QAAQ,GAAG,QAAQ,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;YAChD,CAAC;YAED,kCAAkC;YAClC,MAAM,MAAM,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;YAC7C,MAAM,OAAO,GAAG,QAAQ,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;YAC/C,IAAI,QAAQ,EAAE,CAAC;gBACd,OAAO,CAAC,YAAY,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;gBAC5C,OAAO,CAAC,SAAS,GAAG,YAAY,QAAQ,EAAE,CAAC;YAC5C,CAAC;YACD,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC;YAC7C,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;YAE5B,qCAAqC;YACrC,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QAC3B,CAAC,CAAC,CAAC;IACJ,CAAC;IAEO,cAAc,CAAC,SAAsB,EAAE,QAAkB;QAChE,wEAAwE;QACxE,SAAS,CAAC,gBAAgB,CAAC,wBAAwB,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE;YACrE,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;YAC3C,MAAM,IAAI,GAAG,MAAM,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;YAC9C,IAAI,CAAC,IAAI;gBAAE,OAAO;YAElB,MAAM,SAAS,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;YAChD,SAAS,CAAC,WAAW,GAAG,IAAI,CAAC;YAC7B,MAAM,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;QAC/B,CAAC,CAAC,CAAC;IACJ,CAAC;IAEO,kBAAkB,CAAC,SAAsB,EAAE,QAAkB;QACpE,wEAAwE;QACxE,SAAS,CAAC,gBAAgB,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE;YAC1D,+BAA+B;YAC/B,MAAM,MAAM,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YAChC,IAAI,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAAE,OAAO;YAEnD,0EAA0E;YAC1E,IAAI,GAAG,GAAG,GAAG,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;YACxC,MAAM,GAAG,GAAG,GAAG,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,IAAI,OAAO,CAAC;YAE5E,wBAAwB;YACxB,IAAI,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC5B,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,WAAW,EAAE,aAAa,CAAC,CAAC;YAC/C,CAAC;iBAAM,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC9B,GAAG,GAAG,GAAG,GAAG,aAAa,CAAC;YAC3B,CAAC;iBAAM,CAAC;gBACP,GAAG,GAAG,GAAG,GAAG,aAAa,CAAC;YAC3B,CAAC;YAED,MAAM,QAAQ,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;YAC/C,QAAQ,CAAC,YAAY,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;YAClC,QAAQ,CAAC,YAAY,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;YAElC,kCAAkC;YAClC,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;QAC9B,CAAC,CAAC,CAAC;IACJ,CAAC;IAEO,mBAAmB,CAAC,SAAsB;QACjD,SAAS,CAAC,gBAAgB,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE;YAC1D,MAAM,GAAG,GAAG,GAAG,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;YACpC,IAAI,CAAC,GAAG;gBAAE,OAAO;YAEjB,IAAI,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC5B,GAAG,CAAC,YAAY,CAAC,KAAK,EAAE,GAAG,CAAC,OAAO,CAAC,WAAW,EAAE,aAAa,CAAC,CAAC,CAAC;YAClE,CAAC;iBAAM,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC9B,GAAG,CAAC,YAAY,CAAC,KAAK,EAAE,GAAG,GAAG,aAAa,CAAC,CAAC;YAC9C,CAAC;iBAAM,CAAC;gBACP,GAAG,CAAC,YAAY,CAAC,KAAK,EAAE,GAAG,GAAG,aAAa,CAAC,CAAC;YAC9C,CAAC;QACF,CAAC,CAAC,CAAC;IACJ,CAAC;IAEO,sBAAsB,CAAC,SAAsB,EAAE,QAAkB;QACxE,0FAA0F;QAC1F,MAAM,SAAS,GAAG,CAAC,CAAC;QACpB,MAAM,YAAY,GAAG,CAAC,CAAC;QAEvB,SAAS,CAAC,gBAAgB,CAAC,SAAS,CAAC,gBAAgB,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE;YACpE,MAAM,CAAC,GAAG,QAAQ,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;YAEtC,mEAAmE;YACnE,MAAM,WAAW,GAAG,CAAC,IAAU,EAAQ,EAAE;gBACxC,IAAI,IAAI,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;oBACjC,CAAC,CAAC,WAAW,CAAC,QAAQ,CAAC,cAAc,CAAC,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,CAAC;gBAChE,CAAC;qBAAM,IAAI,IAAI,CAAC,QAAQ,KAAK,YAAY,EAAE,CAAC;oBAC3C,MAAM,EAAE,GAAG,IAAe,CAAC;oBAC3B,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;oBAErC,IAAI,GAAG,KAAK,QAAQ,EAAE,CAAC;wBACtB,MAAM,MAAM,GAAG,QAAQ,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;wBAChD,MAAM,CAAC,WAAW,GAAG,EAAE,CAAC,WAAW,IAAI,EAAE,CAAC;wBAC1C,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;oBACvB,CAAC;yBAAM,IAAI,GAAG,KAAK,GAAG,EAAE,CAAC;wBACxB,MAAM,IAAI,GAAG,QAAQ,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;wBACzC,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,EAAE,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;wBACzD,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC,WAAW,IAAI,EAAE,CAAC;wBACxC,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;oBACrB,CAAC;yBAAM,IAAI,GAAG,KAAK,MAAM,EAAE,CAAC;wBAC3B,MAAM,IAAI,GAAG,QAAQ,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;wBAC5C,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC,WAAW,IAAI,EAAE,CAAC;wBACxC,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;oBACrB,CAAC;yBAAM,CAAC;wBACP,kDAAkD;wBAClD,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC;oBACpD,CAAC;gBACF,CAAC;YACF,CAAC,CAAC;YAEF,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC;YACpD,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;QACpB,CAAC,CAAC,CAAC;IACJ,CAAC;IAEO,gBAAgB,CAAC,SAAsB,EAAE,QAAkB;QAClE,SAAS,CAAC,gBAAgB,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE;YAC/D,MAAM,MAAM,GAAG,QAAQ,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;YAChD,MAAM,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC;YAC5C,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QAC1B,CAAC,CAAC,CAAC;IACJ,CAAC;IAEO,qBAAqB,CAAC,SAAsB;QACnD,SAAS,CAAC,gBAAgB,CAAC,SAAS,CAAC,gBAAgB,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE;YACnE,EAAE,CAAC,eAAe,CAAC,iBAAiB,CAAC,CAAC;QACvC,CAAC,CAAC,CAAC;IACJ,CAAC;IAEO,iBAAiB;QACxB,MAAM,IAAI,GAAG,IAAI,CAAC,gBAAgB,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QAC9D,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAC9D,CAAC;CACD;AA/QD,8CA+QC"}

package/dist/extractors/x-oembed.d.ts

@@ -0,0 +1,20 @@
+import { BaseExtractor } from './_base';
+import { ExtractorResult } from '../types/extractors';
+export declare class XOembedExtractor extends BaseExtractor {
+    canExtract(): boolean;
+    extract(): ExtractorResult;
+    canExtractAsync(): boolean;
+    extractAsync(): Promise<ExtractorResult>;
+    private extractOembed;
+    private tryExtractFxTwitter;
+    private fetchFxTwitter;
+    private buildArticleResult;
+    private buildTweetResult;
+    private renderTweet;
+    private applyMarkers;
+    private applyFacets;
+    private renderArticle;
+    private renderBlock;
+    private renderAtomicBlock;
+    private renderInlineContent;
+}

package/dist/extractors/x-oembed.js

@@ -0,0 +1,350 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.XOembedExtractor = void 0;
+const _base_1 = require("./_base");
+const dom_1 = require("../utils/dom");
+class XOembedExtractor extends _base_1.BaseExtractor {
+    canExtract() {
+        return false;
+    }
+    extract() {
+        return {
+            content: '',
+            contentHtml: '',
+        };
+    }
+    canExtractAsync() {
+        return /\/(status|article)\/\d+/.test(this.url);
+    }
+    async extractAsync() {
+        // Try FxTwitter first — it has full tweet text and media
+        const fxResult = await this.tryExtractFxTwitter();
+        if (fxResult) {
+            return fxResult;
+        }
+        // Fall back to oEmbed (truncates long tweets but always available)
+        return this.extractOembed();
+    }
+    async extractOembed() {
+        const oembedUrl = `https://publish.twitter.com/oembed?url=${encodeURIComponent(this.url)}&omit_script=true`;
+        const response = await fetch(oembedUrl);
+        if (!response.ok) {
+            throw new Error(`oEmbed request failed: ${response.status}`);
+        }
+        const data = await response.json();
+        // Parse the oEmbed HTML to extract tweet text
+        const div = this.document.createElement('div');
+        div.appendChild((0, dom_1.parseHTML)(this.document, data.html));
+        // The oEmbed HTML contains a <blockquote> with <p> tags for text
+        // and an <a> tag for the date
+        const blockquote = div.querySelector('blockquote');
+        const paragraphs = blockquote?.querySelectorAll('p') || [];
+        const tweetText = Array.from(paragraphs)
+            .map(p => `<p>${(0, dom_1.serializeHTML)(p)}</p>`)
+            .join('\n');
+        const handle = data.author_url
+            ? `@${data.author_url.split('/').pop()}`
+            : '';
+        const dateLink = blockquote?.querySelector('a:last-child');
+        const dateText = dateLink?.textContent?.trim() || '';
+        const permalink = dateLink?.getAttribute('href') || this.url;
+        const escapedAuthorName = (0, dom_1.escapeHtml)(data.author_name);
+        const escapedHandle = (0, dom_1.escapeHtml)(handle);
+        const escapedDateText = (0, dom_1.escapeHtml)(dateText);
+        const escapedPermalink = (0, dom_1.escapeHtml)(permalink);
+        const contentHtml = `
+			<div class="tweet-thread">
+				<div class="main-tweet">
+					<div class="tweet">
+						<div class="tweet-header">
+							<span class="tweet-author"><strong>${escapedAuthorName}</strong> <span class="tweet-handle">${escapedHandle}</span></span>
+							${dateText ? `<a href="${escapedPermalink}" class="tweet-date">${escapedDateText}</a>` : ''}
+						</div>
+						${tweetText ? `<div class="tweet-text">${tweetText}</div>` : ''}
+					</div>
+				</div>
+			</div>
+		`.trim();
+        return {
+            content: contentHtml,
+            contentHtml: contentHtml,
+            variables: {
+                title: `Post by ${handle || data.author_name}`,
+                author: handle || data.author_name,
+                site: 'X (Twitter)',
+            }
+        };
+    }
+    async tryExtractFxTwitter() {
+        const match = this.url.match(/\/([a-zA-Z][a-zA-Z0-9_]{0,14})\/(status|article)\/(\d+)/);
+        if (!match)
+            return null;
+        try {
+            const data = await this.fetchFxTwitter(match[1], match[3]);
+            // If it's an article, use the rich article renderer
+            if (data.tweet?.article) {
+                return this.buildArticleResult(data);
+            }
+            // Otherwise use the full tweet text from FxTwitter
+            if (data.tweet?.text) {
+                return this.buildTweetResult(data);
+            }
+            return null;
+        }
+        catch {
+            return null;
+        }
+    }
+    async fetchFxTwitter(username, id) {
+        const apiUrl = `https://api.fxtwitter.com/${username}/status/${id}`;
+        const response = await fetch(apiUrl, {
+            headers: {
+                'User-Agent': 'Mozilla/5.0 (compatible; Defuddle/1.0; +https://defuddle.md)',
+            },
+        });
+        if (!response.ok) {
+            throw new Error(`FxTwitter API request failed: ${response.status}`);
+        }
+        return response.json();
+    }
+    buildArticleResult(data) {
+        const article = data.tweet.article;
+        const { blocks, entityMap } = article.content;
+        const contentHtml = this.renderArticle(blocks, entityMap, article.cover_media);
+        const handle = `@${data.tweet.author.screen_name}`;
+        return {
+            content: contentHtml,
+            contentHtml,
+            variables: {
+                title: article.title,
+                author: handle,
+                site: 'X (Twitter)',
+                description: article.preview_text,
+            }
+        };
+    }
+    buildTweetResult(data) {
+        const tweet = data.tweet;
+        const handle = `@${tweet.author.screen_name}`;
+        const contentHtml = this.renderTweet(tweet);
+        return {
+            content: contentHtml,
+            contentHtml,
+            variables: {
+                title: `Post by ${handle}`,
+                author: handle,
+                site: 'X (Twitter)',
+            }
+        };
+    }
+    renderTweet(tweet) {
+        const text = tweet.raw_text?.text || tweet.text;
+        // Filter out media facets — FxTwitter already strips pic.twitter.com
+        // links from the text, so media facet indices are stale
+        const facets = (tweet.raw_text?.facets || []).filter(f => f.type !== 'media');
+        // Split text into paragraphs on double newlines
+        const paragraphs = text.split(/\n\n+/);
+        let offset = 0;
+        const htmlParts = [];
+        for (const para of paragraphs) {
+            const paraStart = text.indexOf(para, offset);
+            const paraEnd = paraStart + para.length;
+            offset = paraEnd;
+            // Check if this paragraph is a blockquote (starts with >)
+            const isBlockquote = para.trimStart().startsWith('>');
+            let paraText = isBlockquote ? para.trimStart().slice(1).trimStart() : para;
+            const paraTextStart = isBlockquote
+                ? paraStart + (para.length - para.trimStart().length) + 1 + (para.trimStart().slice(1).length - para.trimStart().slice(1).trimStart().length)
+                : paraStart;
+            // Apply facets within this paragraph
+            const rendered = this.applyFacets(paraText, paraTextStart, paraEnd, facets);
+            // Handle line breaks within paragraph
+            const withBreaks = rendered.replace(/\n/g, '<br>');
+            if (isBlockquote) {
+                htmlParts.push(`<blockquote><p>${withBreaks}</p></blockquote>`);
+            }
+            else if (withBreaks.trim()) {
+                htmlParts.push(`<p>${withBreaks}</p>`);
+            }
+        }
+        // Append media images
+        if (tweet.media?.photos) {
+            for (const photo of tweet.media.photos) {
+                htmlParts.push(`<img src="${(0, dom_1.escapeHtml)(photo.url)}" alt="">`);
+            }
+        }
+        const handle = (0, dom_1.escapeHtml)(`@${tweet.author.screen_name}`);
+        const authorName = (0, dom_1.escapeHtml)(tweet.author.name);
+        return `<div class="tweet-thread"><div class="main-tweet"><div class="tweet">` +
+            `<div class="tweet-header"><span class="tweet-author"><strong>${authorName}</strong> <span class="tweet-handle">${handle}</span></span></div>` +
+            `<div class="tweet-text">${htmlParts.join('\n')}</div>` +
+            `</div></div></div>`;
+    }
+    applyMarkers(text, markers) {
+        if (markers.length === 0) {
+            return (0, dom_1.escapeHtml)(text);
+        }
+        markers.sort((a, b) => {
+            if (a.offset !== b.offset)
+                return a.offset - b.offset;
+            if (a.type === 'close' && b.type === 'open')
+                return -1;
+            if (a.type === 'open' && b.type === 'close')
+                return 1;
+            return 0;
+        });
+        let result = '';
+        let pos = 0;
+        for (const marker of markers) {
+            if (marker.offset > pos) {
+                result += (0, dom_1.escapeHtml)(text.slice(pos, marker.offset));
+            }
+            result += marker.tag;
+            pos = marker.offset;
+        }
+        if (pos < text.length) {
+            result += (0, dom_1.escapeHtml)(text.slice(pos));
+        }
+        return result;
+    }
+    applyFacets(text, textStart, textEnd, facets) {
+        const markers = [];
+        for (const facet of facets) {
+            const [fStart, fEnd] = facet.indices;
+            if (fEnd <= textStart || fStart >= textEnd)
+                continue;
+            const relStart = Math.max(0, fStart - textStart);
+            const relEnd = Math.min(text.length, fEnd - textStart);
+            if (facet.type === 'italic') {
+                markers.push({ offset: relStart, type: 'open', tag: '<em>' });
+                markers.push({ offset: relEnd, type: 'close', tag: '</em>' });
+            }
+            else if (facet.type === 'mention' && facet.text) {
+                const url = `https://x.com/${(0, dom_1.escapeHtml)(facet.text)}`;
+                markers.push({ offset: relStart, type: 'open', tag: `<a href="${url}">` });
+                markers.push({ offset: relEnd, type: 'close', tag: '</a>' });
+            }
+            else if (facet.type === 'url' && facet.original) {
+                const url = (0, dom_1.escapeHtml)(facet.original);
+                markers.push({ offset: relStart, type: 'open', tag: `<a href="${url}">` });
+                markers.push({ offset: relEnd, type: 'close', tag: '</a>' });
+            }
+        }
+        return this.applyMarkers(text, markers);
+    }
+    renderArticle(blocks, entityMap, coverMedia) {
+        const parts = [];
+        // Add cover image if available
+        if (coverMedia?.media_info?.original_img_url) {
+            parts.push(`<img src="${(0, dom_1.escapeHtml)(coverMedia.media_info.original_img_url)}" alt="Cover image">`);
+        }
+        let i = 0;
+        while (i < blocks.length) {
+            const block = blocks[i];
+            if (block.type === 'unordered-list-item') {
+                // Group consecutive list items into a <ul>
+                const items = [];
+                while (i < blocks.length && blocks[i].type === 'unordered-list-item') {
+                    items.push(`<li>${this.renderInlineContent(blocks[i], entityMap)}</li>`);
+                    i++;
+                }
+                parts.push(`<ul>${items.join('')}</ul>`);
+                continue;
+            }
+            const html = this.renderBlock(block, entityMap);
+            if (html) {
+                parts.push(html);
+            }
+            i++;
+        }
+        return `<article class="x-article">${parts.join('')}</article>`;
+    }
+    renderBlock(block, entityMap) {
+        switch (block.type) {
+            case 'unstyled': {
+                if (!block.text.trim())
+                    return '';
+                return `<p>${this.renderInlineContent(block, entityMap)}</p>`;
+            }
+            case 'header-two':
+                return `<h2>${this.renderInlineContent(block, entityMap)}</h2>`;
+            case 'header-three':
+                return `<h3>${this.renderInlineContent(block, entityMap)}</h3>`;
+            case 'atomic':
+                return this.renderAtomicBlock(block, entityMap);
+            default: {
+                if (!block.text.trim())
+                    return '';
+                return `<p>${this.renderInlineContent(block, entityMap)}</p>`;
+            }
+        }
+    }
+    renderAtomicBlock(block, entityMap) {
+        if (block.entityRanges.length === 0)
+            return '';
+        const entityEntry = entityMap.find(e => e.key === String(block.entityRanges[0].key));
+        if (!entityEntry)
+            return '';
+        const entity = entityEntry.value;
+        switch (entity.type) {
+            case 'MEDIA': {
+                const caption = entity.data.caption;
+                if (caption) {
+                    return `<figure><figcaption>${(0, dom_1.escapeHtml)(caption)}</figcaption></figure>`;
+                }
+                return '';
+            }
+            case 'MARKDOWN': {
+                const markdown = entity.data.markdown || '';
+                // Strip the wrapping ```...``` fences
+                const codeMatch = markdown.match(/^```(\w*)\n([\s\S]*?)\n?```$/);
+                if (codeMatch) {
+                    const lang = codeMatch[1];
+                    const code = codeMatch[2];
+                    const langAttr = lang ? ` class="language-${(0, dom_1.escapeHtml)(lang)}" data-lang="${(0, dom_1.escapeHtml)(lang)}"` : '';
+                    return `<pre><code${langAttr}>${(0, dom_1.escapeHtml)(code)}</code></pre>`;
+                }
+                return `<pre><code>${(0, dom_1.escapeHtml)(markdown)}</code></pre>`;
+            }
+            default:
+                return '';
+        }
+    }
+    renderInlineContent(block, entityMap) {
+        const text = block.text;
+        if (!text)
+            return '';
+        const markers = [];
+        for (const range of block.inlineStyleRanges) {
+            if (range.style === 'Bold') {
+                markers.push({ offset: range.offset, type: 'open', tag: '<strong>' });
+                markers.push({ offset: range.offset + range.length, type: 'close', tag: '</strong>' });
+            }
+        }
+        for (const range of block.entityRanges) {
+            const entityEntry = entityMap.find(e => e.key === String(range.key));
+            if (entityEntry?.value.type === 'LINK' && entityEntry.value.data.url) {
+                const url = (0, dom_1.escapeHtml)(entityEntry.value.data.url);
+                markers.push({ offset: range.offset, type: 'open', tag: `<a href="${url}">` });
+                markers.push({ offset: range.offset + range.length, type: 'close', tag: '</a>' });
+            }
+        }
+        if (block.data?.mentions) {
+            for (const mention of block.data.mentions) {
+                const url = `https://x.com/${(0, dom_1.escapeHtml)(mention.text)}`;
+                markers.push({ offset: mention.fromIndex, type: 'open', tag: `<a href="${url}">` });
+                markers.push({ offset: mention.toIndex, type: 'close', tag: '</a>' });
+            }
+        }
+        if (block.data?.urls) {
+            for (const urlData of block.data.urls) {
+                const url = (0, dom_1.escapeHtml)(urlData.text);
+                markers.push({ offset: urlData.fromIndex, type: 'open', tag: `<a href="${url}">` });
+                markers.push({ offset: urlData.toIndex, type: 'close', tag: '</a>' });
+            }
+        }
+        return this.applyMarkers(text, markers);
+    }
+}
+exports.XOembedExtractor = XOembedExtractor;
+//# sourceMappingURL=x-oembed.js.map