@claudeink/mcp-server 0.0.1 → 0.0.3

package/workflow/tools/crawler/node_modules/defuddle/dist/extractors/grok.js

@@ -0,0 +1,141 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.GrokExtractor = void 0;
+const _conversation_1 = require("./_conversation");
+class GrokExtractor extends _conversation_1.ConversationExtractor {
+    constructor(document, url) {
+        super(document, url);
+        // Note: This selector relies heavily on CSS utility classes and may break if Grok's UI changes.
+        this.messageContainerSelector = '.relative.group.flex.flex-col.justify-center.w-full';
+        this.messageBubbles = document.querySelectorAll(this.messageContainerSelector);
+        this.footnotes = [];
+        this.footnoteCounter = 0;
+    }
+    canExtract() {
+        return !!this.messageBubbles && this.messageBubbles.length > 0;
+    }
+    extractMessages() {
+        const messages = [];
+        this.footnotes = [];
+        this.footnoteCounter = 0;
+        if (!this.messageBubbles || this.messageBubbles.length === 0)
+            return messages;
+        this.messageBubbles.forEach((container) => {
+            // Note: Relies on layout classes 'items-end' and 'items-start' which might change.
+            const isUserMessage = container.classList.contains('items-end');
+            const isGrokMessage = container.classList.contains('items-start');
+            if (!isUserMessage && !isGrokMessage)
+                return; // Skip elements that aren't clearly user or Grok messages
+            const messageBubble = container.querySelector('.message-bubble');
+            if (!messageBubble)
+                return; // Skip if the core message bubble isn't found
+            let content = '';
+            let role = '';
+            let author = '';
+            if (isUserMessage) {
+                // Assume user message bubble's textContent is the desired content.
+                // This is simpler and potentially less brittle than selecting specific spans.
+                content = messageBubble.textContent || '';
+                role = 'user';
+                author = 'You'; // Or potentially extract from an attribute if available later
+            }
+            else if (isGrokMessage) {
+                role = 'assistant';
+                author = 'Grok'; // Or potentially extract from an attribute if available later
+                // Clone the bubble to modify it without affecting the original page
+                const clonedBubble = messageBubble.cloneNode(true);
+                // Remove known non-content elements like the DeepSearch artifact
+                clonedBubble.querySelector('.relative.border.border-border-l1.bg-surface-base')?.remove();
+                // Add selectors here for any other known elements to remove (e.g., buttons, toolbars within the bubble)
+                content = clonedBubble.innerHTML;
+                // Process footnotes/links in the cleaned content
+                content = this.processFootnotes(content);
+            }
+            if (content.trim()) {
+                messages.push({
+                    author: author,
+                    content: content.trim(),
+                    metadata: {
+                        role: role
+                    }
+                });
+            }
+        });
+        return messages;
+    }
+    getFootnotes() {
+        return this.footnotes;
+    }
+    getMetadata() {
+        const title = this.getTitle();
+        const messageCount = this.messageBubbles?.length || 0;
+        return {
+            title,
+            site: 'Grok',
+            url: this.url,
+            messageCount: messageCount, // Use estimated count
+            description: `Grok conversation with ${messageCount} messages`
+        };
+    }
+    getTitle() {
+        // Try to get the page title first (more reliable)
+        const pageTitle = this.document.title?.trim();
+        if (pageTitle && pageTitle !== 'Grok' && !pageTitle.startsWith('Grok by ')) {
+            // Remove ' - Grok' suffix if present
+            return pageTitle.replace(/\s-\s*Grok$/, '').trim();
+        }
+        // Fallback: Find the first user message bubble and use its text content
+        // Note: Still relies on 'items-end' class.
+        const firstUserContainer = this.document.querySelector(`${this.messageContainerSelector}.items-end`);
+        if (firstUserContainer) {
+            const messageBubble = firstUserContainer.querySelector('.message-bubble');
+            if (messageBubble) {
+                const text = messageBubble.textContent?.trim() || '';
+                // Truncate to first 50 characters if longer
+                return text.length > 50 ? text.slice(0, 50) + '...' : text;
+            }
+        }
+        return 'Grok Conversation'; // Default fallback
+    }
+    processFootnotes(content) {
+        // Regex to find <a> tags, capture href and link text
+        const linkPattern = /<a\s+(?:[^>]*?\s+)?href="([^"]*)"[^>]*>(.*?)<\/a>/gi; // Use 'g' and 'i' flags
+        return content.replace(linkPattern, (match, url, linkText) => {
+            // Skip processing for internal anchor links, empty URLs, or non-http(s) protocols
+            if (!url || url.startsWith('#') || !url.match(/^https?:\/\//i)) {
+                return match;
+            }
+            // Check if this URL already exists in our footnotes
+            let footnote = this.footnotes.find(fn => fn.url === url);
+            let footnoteIndex;
+            if (!footnote) {
+                // Create a new footnote if URL doesn't exist
+                this.footnoteCounter++;
+                footnoteIndex = this.footnoteCounter;
+                let domainText = url; // Default to full URL if parsing fails
+                try {
+                    const domain = new URL(url).hostname.replace(/^www\./, '');
+                    domainText = `<a href="${url}" target="_blank" rel="noopener noreferrer">${domain}</a>`;
+                }
+                catch (e) {
+                    // Keep domainText as the original URL if parsing fails
+                    domainText = `<a href="${url}" target="_blank" rel="noopener noreferrer">${url}</a>`;
+                    console.warn(`GrokExtractor: Could not parse URL for footnote: ${url}`);
+                }
+                this.footnotes.push({
+                    url,
+                    text: domainText // Store the link HTML directly
+                });
+            }
+            else {
+                // Find the 1-based index of the existing footnote
+                footnoteIndex = this.footnotes.findIndex(fn => fn.url === url) + 1;
+            }
+            // Return the original link text wrapped with a footnote reference
+            // Ensure the link text itself is not clickable again if it was part of the original match
+            return `${linkText}<sup id="fnref:${footnoteIndex}" class="footnote-ref"><a href="#fn:${footnoteIndex}" class="footnote-link">${footnoteIndex}</a></sup>`;
+        });
+    }
+}
+exports.GrokExtractor = GrokExtractor;
+//# sourceMappingURL=grok.js.map

package/workflow/tools/crawler/node_modules/defuddle/dist/extractors/grok.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"grok.js","sourceRoot":"","sources":["../../src/extractors/grok.ts"],"names":[],"mappings":";;;AAAA,mDAAwD;AAGxD,MAAa,aAAc,SAAQ,qCAAqB;IAOvD,YAAY,QAAkB,EAAE,GAAW;QAC1C,KAAK,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QAPtB,gGAAgG;QACxF,6BAAwB,GAAG,qDAAqD,CAAC;QAOxF,IAAI,CAAC,cAAc,GAAG,QAAQ,CAAC,gBAAgB,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;QAC/E,IAAI,CAAC,SAAS,GAAG,EAAE,CAAC;QACpB,IAAI,CAAC,eAAe,GAAG,CAAC,CAAC;IAC1B,CAAC;IAED,UAAU;QACT,OAAO,CAAC,CAAC,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC;IAChE,CAAC;IAES,eAAe;QACxB,MAAM,QAAQ,GAA0B,EAAE,CAAC;QAC3C,IAAI,CAAC,SAAS,GAAG,EAAE,CAAC;QACpB,IAAI,CAAC,eAAe,GAAG,CAAC,CAAC;QAEzB,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,cAAc,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,QAAQ,CAAC;QAE9E,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC,SAAS,EAAE,EAAE;YACzC,mFAAmF;YACnF,MAAM,aAAa,GAAG,SAAS,CAAC,SAAS,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;YAChE,MAAM,aAAa,GAAG,SAAS,CAAC,SAAS,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;YAElE,IAAI,CAAC,aAAa,IAAI,CAAC,aAAa;gBAAE,OAAO,CAAC,0DAA0D;YAExG,MAAM,aAAa,GAAG,SAAS,CAAC,aAAa,CAAC,iBAAiB,CAAC,CAAC;YACjE,IAAI,CAAC,aAAa;gBAAE,OAAO,CAAC,8CAA8C;YAE1E,IAAI,OAAO,GAAW,EAAE,CAAC;YACzB,IAAI,IAAI,GAAW,EAAE,CAAC;YACtB,IAAI,MAAM,GAAW,EAAE,CAAC;YAExB,IAAI,aAAa,EAAE,CAAC;gBACnB,mEAAmE;gBACnE,8EAA8E;gBAC9E,OAAO,GAAG,aAAa,CAAC,WAAW,IAAI,EAAE,CAAC;gBAC1C,IAAI,GAAG,MAAM,CAAC;gBACd,MAAM,GAAG,KAAK,CAAC,CAAC,8DAA8D;YAC/E,CAAC;iBAAM,IAAI,aAAa,EAAE,CAAC;gBAC1B,IAAI,GAAG,WAAW,CAAC;gBACnB,MAAM,GAAG,MAAM,CAAC,CAAC,8DAA8D;gBAE/E,oEAAoE;gBACpE,MAAM,YAAY,GAAG,aAAa,CAAC,SAAS,CAAC,IAAI,CAAY,CAAC;gBAE9D,iEAAiE;gBACjE,YAAY,CAAC,aAAa,CAAC,mDAAmD,CAAC,EAAE,MAAM,EAAE,CAAC;gBAC1F,wGAAwG;gBAExG,OAAO,GAAG,YAAY,CAAC,SAAS,CAAC;gBAEjC,iDAAiD;gBACjD,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;YAC1C,CAAC;YAED,IAAI,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;gBACpB,QAAQ,CAAC,IAAI,CAAC;oBACb,MAAM,EAAE,MAAM;oBACd,OAAO,EAAE,OAAO,CAAC,IAAI,EAAE;oBACvB,QAAQ,EAAE;wBACT,IAAI,EAAE,IAAI;qBACV;iBACD,CAAC,CAAC;YACJ,CAAC;QACF,CAAC,CAAC,CAAC;QAEH,OAAO,QAAQ,CAAC;IACjB,CAAC;IAES,YAAY;QACrB,OAAO,IAAI,CAAC,SAAS,CAAC;IACvB,CAAC;IAES,WAAW;QACpB,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;QAC9B,MAAM,YAAY,GAAG,IAAI,CAAC,cAAc,EAAE,MAAM,IAAI,CAAC,CAAC;QAEtD,OAAO;YACN,KAAK;YACL,IAAI,EAAE,MAAM;YACZ,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,YAAY,EAAE,YAAY,EAAE,sBAAsB;YAClD,WAAW,EAAE,0BAA0B,YAAY,WAAW;SAC9D,CAAC;IACH,CAAC;IAEO,QAAQ;QACf,kDAAkD;QAClD,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC;QAC9C,IAAI,SAAS,IAAI,SAAS,KAAK,MAAM,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YAC5E,qCAAqC;YACrC,OAAO,SAAS,CAAC,OAAO,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QACpD,CAAC;QAED,wEAAwE;QACxE,2CAA2C;QAC3C,MAAM,kBAAkB,GAAG,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,GAAG,IAAI,CAAC,wBAAwB,YAAY,CAAC,CAAC;QACrG,IAAI,kBAAkB,EAAE,CAAC;YACxB,MAAM,aAAa,GAAG,kBAAkB,CAAC,aAAa,CAAC,iBAAiB,CAAC,CAAC;YAC1E,IAAI,aAAa,EAAE,CAAC;gBACnB,MAAM,IAAI,GAAG,aAAa,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;gBACrD,4CAA4C;gBAC5C,OAAO,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;YAC5D,CAAC;QACF,CAAC;QAED,OAAO,mBAAmB,CAAC,CAAC,mBAAmB;IAChD,CAAC;IAEO,gBAAgB,CAAC,OAAe;QACvC,qDAAqD;QACrD,MAAM,WAAW,GAAG,qDAAqD,CAAC,CAAC,wBAAwB;QAEnG,OAAO,OAAO,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE;YAC3D,kFAAkF;YACnF,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,eAAe,CAAC,EAAE,CAAC;gBAChE,OAAO,KAAK,CAAC;YACd,CAAC;YAED,oDAAoD;YACpD,IAAI,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC;YACzD,IAAI,aAAqB,CAAC;YAE1B,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACf,6CAA6C;gBAC7C,IAAI,CAAC,eAAe,EAAE,CAAC;gBACvB,aAAa,GAAG,IAAI,CAAC,eAAe,CAAC;gBAErC,IAAI,UAAU,GAAG,GAAG,CAAC,CAAC,uCAAuC;gBAC7D,IAAI,CAAC;oBACJ,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;oBAC3D,UAAU,GAAG,YAAY,GAAG,+CAA+C,MAAM,MAAM,CAAC;gBACzF,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACZ,uDAAuD;oBACvD,UAAU,GAAG,YAAY,GAAG,+CAA+C,GAAG,MAAM,CAAC;oBACrF,OAAO,CAAC,IAAI,CAAC,oDAAoD,GAAG,EAAE,CAAC,CAAC;gBACzE,CAAC;gBAED,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;oBACnB,GAAG;oBACH,IAAI,EAAE,UAAU,CAAC,+BAA+B;iBAChD,CAAC,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACP,kDAAkD;gBAClD,aAAa,GAAG,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,GAAG,KAAK,GAAG,CAAC,GAAG,CAAC,CAAC;YACpE,CAAC;YAED,kEAAkE;YAClE,0FAA0F;YAC1F,OAAO,GAAG,QAAQ,kBAAkB,aAAa,uCAAuC,aAAa,2BAA2B,aAAa,YAAY,CAAC;QAC3J,CAAC,CAAC,CAAC;IACJ,CAAC;CACD;AA/JD,sCA+JC"}

package/workflow/tools/crawler/node_modules/defuddle/dist/extractors/hackernews.d.ts

@@ -0,0 +1,21 @@
+import { BaseExtractor } from './_base';
+import { ExtractorResult } from '../types/extractors';
+export declare class HackerNewsExtractor extends BaseExtractor {
+    private mainPost;
+    private isCommentPage;
+    private mainComment;
+    constructor(document: Document, url: string);
+    private detectCommentPage;
+    private findMainComment;
+    canExtract(): boolean;
+    extract(): ExtractorResult;
+    private createContentHtml;
+    private getPostContent;
+    private extractComments;
+    private processComments;
+    private getPostId;
+    private getPostTitle;
+    private getPostAuthor;
+    private createDescription;
+    private getPostDate;
+}

package/workflow/tools/crawler/node_modules/defuddle/dist/extractors/hackernews.js

@@ -0,0 +1,206 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.HackerNewsExtractor = void 0;
+const _base_1 = require("./_base");
+class HackerNewsExtractor extends _base_1.BaseExtractor {
+    constructor(document, url) {
+        super(document, url);
+        this.mainPost = document.querySelector('.fatitem');
+        this.isCommentPage = this.detectCommentPage();
+        this.mainComment = this.isCommentPage ? this.findMainComment() : null;
+    }
+    detectCommentPage() {
+        // Check if we're on a comment page by looking for a parent link in the navigation
+        return !!this.mainPost?.querySelector('.navs a[href*="parent"]');
+    }
+    findMainComment() {
+        // The main comment is the first comment in the fatitem
+        const comment = this.mainPost?.querySelector('.comment');
+        return comment || null;
+    }
+    canExtract() {
+        return !!this.mainPost;
+    }
+    extract() {
+        const postContent = this.getPostContent();
+        const comments = this.extractComments();
+        const contentHtml = this.createContentHtml(postContent, comments);
+        const postTitle = this.getPostTitle();
+        const postAuthor = this.getPostAuthor();
+        const description = this.createDescription();
+        const published = this.getPostDate();
+        return {
+            content: contentHtml,
+            contentHtml: contentHtml,
+            extractedContent: {
+                postId: this.getPostId(),
+                postAuthor,
+            },
+            variables: {
+                title: postTitle,
+                author: postAuthor,
+                site: 'Hacker News',
+                description,
+                published,
+            }
+        };
+    }
+    createContentHtml(postContent, comments) {
+        return `
+			<div class="hackernews-post">
+				<div class="post-content">
+					${postContent}
+				</div>
+				${comments ? `
+					<hr>
+					<h2>Comments</h2>
+					<div class="hackernews-comments">
+						${comments}
+					</div>
+				` : ''}
+			</div>
+		`.trim();
+    }
+    getPostContent() {
+        if (!this.mainPost)
+            return '';
+        // If this is a comment page, use the comment as the main content
+        if (this.isCommentPage && this.mainComment) {
+            const author = this.mainComment.querySelector('.hnuser')?.textContent || '[deleted]';
+            const commentText = this.mainComment.querySelector('.commtext')?.innerHTML || '';
+            const timeElement = this.mainComment.querySelector('.age');
+            const timestamp = timeElement?.getAttribute('title') || '';
+            const date = timestamp.split('T')[0] || '';
+            const points = this.mainComment.querySelector('.score')?.textContent?.trim() || '';
+            const parentUrl = this.mainPost.querySelector('.navs a[href*="parent"]')?.getAttribute('href') || '';
+            return `
+				<div class="comment main-comment">
+					<div class="comment-metadata">
+						<span class="comment-author"><strong>${author}</strong></span> •
+						<span class="comment-date">${date}</span>
+						${points ? ` • <span class="comment-points">${points}</span>` : ''}
+						${parentUrl ? ` • <a href="https://news.ycombinator.com/${parentUrl}" class="parent-link">parent</a>` : ''}
+					</div>
+					<div class="comment-content">${commentText}</div>
+				</div>
+			`.trim();
+        }
+        // Otherwise handle regular post content
+        const titleRow = this.mainPost.querySelector('tr.athing');
+        const subRow = titleRow?.nextElementSibling;
+        const url = titleRow?.querySelector('.titleline a')?.getAttribute('href') || '';
+        let content = '';
+        if (url) {
+            content += `<p><a href="${url}" target="_blank">${url}</a></p>`;
+        }
+        const text = this.mainPost.querySelector('.toptext');
+        if (text) {
+            content += `<div class="post-text">${text.innerHTML}</div>`;
+        }
+        return content;
+    }
+    extractComments() {
+        const comments = Array.from(this.document.querySelectorAll('tr.comtr'));
+        return this.processComments(comments);
+    }
+    processComments(comments) {
+        let html = '';
+        const processedIds = new Set();
+        let currentDepth = -1;
+        let blockquoteStack = [];
+        for (const comment of comments) {
+            const id = comment.getAttribute('id');
+            if (!id || processedIds.has(id))
+                continue;
+            processedIds.add(id);
+            const indent = comment.querySelector('.ind img')?.getAttribute('width') || '0';
+            const depth = parseInt(indent) / 40;
+            const commentText = comment.querySelector('.commtext');
+            const author = comment.querySelector('.hnuser')?.textContent || '[deleted]';
+            const timeElement = comment.querySelector('.age');
+            const points = comment.querySelector('.score')?.textContent?.trim() || '';
+            if (!commentText)
+                continue;
+            // Get the comment URL
+            const commentUrl = `https://news.ycombinator.com/item?id=${id}`;
+            // Get the timestamp from the title attribute and extract the date portion
+            const timestamp = timeElement?.getAttribute('title') || '';
+            const date = timestamp.split('T')[0] || '';
+            // For top-level comments, close all previous blockquotes and start fresh
+            if (depth === 0) {
+                while (blockquoteStack.length > 0) {
+                    html += '</blockquote>';
+                    blockquoteStack.pop();
+                }
+                html += '<blockquote>';
+                blockquoteStack = [0];
+                currentDepth = 0;
+            }
+            // For nested comments
+            else {
+                // If we're moving back up the tree
+                if (depth < currentDepth) {
+                    while (blockquoteStack.length > 0 && blockquoteStack[blockquoteStack.length - 1] >= depth) {
+                        html += '</blockquote>';
+                        blockquoteStack.pop();
+                    }
+                }
+                // If we're going deeper
+                else if (depth > currentDepth) {
+                    html += '<blockquote>';
+                    blockquoteStack.push(depth);
+                }
+                // If we're at the same depth, no need to close or open blockquotes
+            }
+            html += `<div class="comment">
+	<div class="comment-metadata">
+		<span class="comment-author"><strong>${author}</strong></span> •
+		<a href="${commentUrl}" class="comment-link">${date}</a>
+		${points ? ` • <span class="comment-points">${points}</span>` : ''}
+	</div>
+	<div class="comment-content">${commentText.innerHTML}</div>
+</div>`;
+            currentDepth = depth;
+        }
+        // Close any remaining blockquotes
+        while (blockquoteStack.length > 0) {
+            html += '</blockquote>';
+            blockquoteStack.pop();
+        }
+        return html;
+    }
+    getPostId() {
+        const match = this.url.match(/id=(\d+)/);
+        return match?.[1] || '';
+    }
+    getPostTitle() {
+        if (this.isCommentPage && this.mainComment) {
+            const author = this.mainComment.querySelector('.hnuser')?.textContent || '[deleted]';
+            const commentText = this.mainComment.querySelector('.commtext')?.textContent || '';
+            // Use first 50 characters of comment as title
+            const preview = commentText.trim().slice(0, 50) + (commentText.length > 50 ? '...' : '');
+            return `Comment by ${author}: ${preview}`;
+        }
+        return this.mainPost?.querySelector('.titleline')?.textContent?.trim() || '';
+    }
+    getPostAuthor() {
+        return this.mainPost?.querySelector('.hnuser')?.textContent?.trim() || '';
+    }
+    createDescription() {
+        const title = this.getPostTitle();
+        const author = this.getPostAuthor();
+        if (this.isCommentPage) {
+            return `Comment by ${author} on Hacker News`;
+        }
+        return `${title} - by ${author} on Hacker News`;
+    }
+    getPostDate() {
+        if (!this.mainPost)
+            return '';
+        const timeElement = this.mainPost.querySelector('.age');
+        const timestamp = timeElement?.getAttribute('title') || '';
+        return timestamp.split('T')[0] || '';
+    }
+}
+exports.HackerNewsExtractor = HackerNewsExtractor;
+//# sourceMappingURL=hackernews.js.map

package/workflow/tools/crawler/node_modules/defuddle/dist/extractors/hackernews.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"hackernews.js","sourceRoot":"","sources":["../../src/extractors/hackernews.ts"],"names":[],"mappings":";;;AAAA,mCAAwC;AAGxC,MAAa,mBAAoB,SAAQ,qBAAa;IAKrD,YAAY,QAAkB,EAAE,GAAW;QAC1C,KAAK,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QACrB,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;QACnD,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC9C,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;IACvE,CAAC;IAEO,iBAAiB;QACxB,kFAAkF;QAClF,OAAO,CAAC,CAAC,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,yBAAyB,CAAC,CAAC;IAClE,CAAC;IAEO,eAAe;QACtB,uDAAuD;QACvD,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;QACzD,OAAO,OAAO,IAAI,IAAI,CAAC;IACxB,CAAC;IAED,UAAU;QACT,OAAO,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC;IACxB,CAAC;IAED,OAAO;QACN,MAAM,WAAW,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;QAC1C,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,EAAE,CAAC;QAExC,MAAM,WAAW,GAAG,IAAI,CAAC,iBAAiB,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;QAClE,MAAM,SAAS,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;QACtC,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QACxC,MAAM,WAAW,GAAG,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC7C,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QAErC,OAAO;YACN,OAAO,EAAE,WAAW;YACpB,WAAW,EAAE,WAAW;YACxB,gBAAgB,EAAE;gBACjB,MAAM,EAAE,IAAI,CAAC,SAAS,EAAE;gBACxB,UAAU;aACV;YACD,SAAS,EAAE;gBACV,KAAK,EAAE,SAAS;gBAChB,MAAM,EAAE,UAAU;gBAClB,IAAI,EAAE,aAAa;gBACnB,WAAW;gBACX,SAAS;aACT;SACD,CAAC;IACH,CAAC;IAEO,iBAAiB,CAAC,WAAmB,EAAE,QAAgB;QAC9D,OAAO;;;OAGF,WAAW;;MAEZ,QAAQ,CAAC,CAAC,CAAC;;;;QAIT,QAAQ;;KAEX,CAAC,CAAC,CAAC,EAAE;;GAEP,CAAC,IAAI,EAAE,CAAC;IACV,CAAC;IAEO,cAAc;QACrB,IAAI,CAAC,IAAI,CAAC,QAAQ;YAAE,OAAO,EAAE,CAAC;QAE9B,iEAAiE;QACjE,IAAI,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YAC5C,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,aAAa,CAAC,SAAS,CAAC,EAAE,WAAW,IAAI,WAAW,CAAC;YACrF,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,aAAa,CAAC,WAAW,CAAC,EAAE,SAAS,IAAI,EAAE,CAAC;YACjF,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;YAC3D,MAAM,SAAS,GAAG,WAAW,EAAE,YAAY,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;YAC3D,MAAM,IAAI,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAC3C,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,aAAa,CAAC,QAAQ,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;YACnF,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,yBAAyB,CAAC,EAAE,YAAY,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YAErG,OAAO;;;6CAGmC,MAAM;mCAChB,IAAI;QAC/B,MAAM,CAAC,CAAC,CAAC,mCAAmC,MAAM,SAAS,CAAC,CAAC,CAAC,EAAE;QAChE,SAAS,CAAC,CAAC,CAAC,4CAA4C,SAAS,kCAAkC,CAAC,CAAC,CAAC,EAAE;;oCAE5E,WAAW;;IAE3C,CAAC,IAAI,EAAE,CAAC;QACV,CAAC;QAED,wCAAwC;QACxC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,WAAW,CAAC,CAAC;QAC1D,MAAM,MAAM,GAAG,QAAQ,EAAE,kBAAkB,CAAC;QAC5C,MAAM,GAAG,GAAG,QAAQ,EAAE,aAAa,CAAC,cAAc,CAAC,EAAE,YAAY,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QAEhF,IAAI,OAAO,GAAG,EAAE,CAAC;QACjB,IAAI,GAAG,EAAE,CAAC;YACT,OAAO,IAAI,eAAe,GAAG,qBAAqB,GAAG,UAAU,CAAC;QACjE,CAAC;QAED,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;QACrD,IAAI,IAAI,EAAE,CAAC;YACV,OAAO,IAAI,0BAA0B,IAAI,CAAC,SAAS,QAAQ,CAAC;QAC7D,CAAC;QAED,OAAO,OAAO,CAAC;IAChB,CAAC;IAEO,eAAe;QACtB,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,CAAC;QACxE,OAAO,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC;IACvC,CAAC;IAEO,eAAe,CAAC,QAAmB;QAC1C,IAAI,IAAI,GAAG,EAAE,CAAC;QACd,MAAM,YAAY,GAAG,IAAI,GAAG,EAAU,CAAC;QACvC,IAAI,YAAY,GAAG,CAAC,CAAC,CAAC;QACtB,IAAI,eAAe,GAAa,EAAE,CAAC;QAEnC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAChC,MAAM,EAAE,GAAG,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;YACtC,IAAI,CAAC,EAAE,IAAI,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC;gBAAE,SAAS;YAC1C,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAErB,MAAM,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,UAAU,CAAC,EAAE,YAAY,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC;YAC/E,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;YACpC,MAAM,WAAW,GAAG,OAAO,CAAC,aAAa,CAAC,WAAW,CAAC,CAAC;YACvD,MAAM,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,SAAS,CAAC,EAAE,WAAW,IAAI,WAAW,CAAC;YAC5E,MAAM,WAAW,GAAG,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;YAClD,MAAM,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,QAAQ,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;YAE1E,IAAI,CAAC,WAAW;gBAAE,SAAS;YAE3B,sBAAsB;YACtB,MAAM,UAAU,GAAG,wCAAwC,EAAE,EAAE,CAAC;YAEhE,0EAA0E;YAC1E,MAAM,SAAS,GAAG,WAAW,EAAE,YAAY,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;YAC3D,MAAM,IAAI,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAE3C,yEAAyE;YACzE,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;gBACjB,OAAO,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACnC,IAAI,IAAI,eAAe,CAAC;oBACxB,eAAe,CAAC,GAAG,EAAE,CAAC;gBACvB,CAAC;gBACD,IAAI,IAAI,cAAc,CAAC;gBACvB,eAAe,GAAG,CAAC,CAAC,CAAC,CAAC;gBACtB,YAAY,GAAG,CAAC,CAAC;YAClB,CAAC;YACD,sBAAsB;iBACjB,CAAC;gBACL,mCAAmC;gBACnC,IAAI,KAAK,GAAG,YAAY,EAAE,CAAC;oBAC1B,OAAO,eAAe,CAAC,MAAM,GAAG,CAAC,IAAI,eAAe,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,KAAK,EAAE,CAAC;wBAC3F,IAAI,IAAI,eAAe,CAAC;wBACxB,eAAe,CAAC,GAAG,EAAE,CAAC;oBACvB,CAAC;gBACF,CAAC;gBACD,wBAAwB;qBACnB,IAAI,KAAK,GAAG,YAAY,EAAE,CAAC;oBAC/B,IAAI,IAAI,cAAc,CAAC;oBACvB,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAC7B,CAAC;gBACD,mEAAmE;YACpE,CAAC;YAED,IAAI,IAAI;;yCAE8B,MAAM;aAClC,UAAU,0BAA0B,IAAI;IACjD,MAAM,CAAC,CAAC,CAAC,mCAAmC,MAAM,SAAS,CAAC,CAAC,CAAC,EAAE;;gCAEpC,WAAW,CAAC,SAAS;OAC9C,CAAC;YAEL,YAAY,GAAG,KAAK,CAAC;QACtB,CAAC;QAED,kCAAkC;QAClC,OAAO,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnC,IAAI,IAAI,eAAe,CAAC;YACxB,eAAe,CAAC,GAAG,EAAE,CAAC;QACvB,CAAC;QAED,OAAO,IAAI,CAAC;IACb,CAAC;IAEO,SAAS;QAChB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QACzC,OAAO,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACzB,CAAC;IAEO,YAAY;QACnB,IAAI,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YAC5C,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,aAAa,CAAC,SAAS,CAAC,EAAE,WAAW,IAAI,WAAW,CAAC;YACrF,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,aAAa,CAAC,WAAW,CAAC,EAAE,WAAW,IAAI,EAAE,CAAC;YACnF,8CAA8C;YAC9C,MAAM,OAAO,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACzF,OAAO,cAAc,MAAM,KAAK,OAAO,EAAE,CAAC;QAC3C,CAAC;QACD,OAAO,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,YAAY,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IAC9E,CAAC;IAEO,aAAa;QACpB,OAAO,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,SAAS,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IAC3E,CAAC;IAEO,iBAAiB;QACxB,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;QAClC,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QACpC,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACxB,OAAO,cAAc,MAAM,iBAAiB,CAAC;QAC9C,CAAC;QACD,OAAO,GAAG,KAAK,SAAS,MAAM,iBAAiB,CAAC;IACjD,CAAC;IAEO,WAAW;QAClB,IAAI,CAAC,IAAI,CAAC,QAAQ;YAAE,OAAO,EAAE,CAAC;QAE9B,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;QACxD,MAAM,SAAS,GAAG,WAAW,EAAE,YAAY,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAC3D,OAAO,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACtC,CAAC;CACD;AAvOD,kDAuOC"}

package/workflow/tools/crawler/node_modules/defuddle/dist/extractors/reddit.d.ts

@@ -0,0 +1,16 @@
+import { BaseExtractor } from './_base';
+import { ExtractorResult } from '../types/extractors';
+export declare class RedditExtractor extends BaseExtractor {
+    private shredditPost;
+    constructor(document: Document, url: string);
+    canExtract(): boolean;
+    extract(): ExtractorResult;
+    private getPostContent;
+    private createContentHtml;
+    private extractComments;
+    private getPostId;
+    private getSubreddit;
+    private getPostAuthor;
+    private createDescription;
+    private processComments;
+}

package/workflow/tools/crawler/node_modules/defuddle/dist/extractors/reddit.js

@@ -0,0 +1,143 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RedditExtractor = void 0;
+const _base_1 = require("./_base");
+class RedditExtractor extends _base_1.BaseExtractor {
+    constructor(document, url) {
+        super(document, url);
+        this.shredditPost = document.querySelector('shreddit-post');
+    }
+    canExtract() {
+        return !!this.shredditPost;
+    }
+    extract() {
+        const postContent = this.getPostContent();
+        const comments = this.extractComments();
+        const contentHtml = this.createContentHtml(postContent, comments);
+        const postTitle = this.document.querySelector('h1')?.textContent?.trim() || '';
+        const subreddit = this.getSubreddit();
+        const postAuthor = this.getPostAuthor();
+        const description = this.createDescription(postContent);
+        return {
+            content: contentHtml,
+            contentHtml: contentHtml,
+            extractedContent: {
+                postId: this.getPostId(),
+                subreddit,
+                postAuthor,
+            },
+            variables: {
+                title: postTitle,
+                author: postAuthor,
+                site: `r/${subreddit}`,
+                description,
+            }
+        };
+    }
+    getPostContent() {
+        const textBody = this.shredditPost?.querySelector('[slot="text-body"]')?.innerHTML || '';
+        const mediaBody = this.shredditPost?.querySelector('#post-image')?.outerHTML || '';
+        return textBody + mediaBody;
+    }
+    createContentHtml(postContent, comments) {
+        return `
+			<div class="reddit-post">
+				<div class="post-content">
+					${postContent}
+				</div>
+			</div>
+			${comments ? `
+				<hr>
+				<h2>Comments</h2>
+				<div class="reddit-comments">
+					${comments}
+				</div>
+			` : ''}
+		`.trim();
+    }
+    extractComments() {
+        const comments = Array.from(this.document.querySelectorAll('shreddit-comment'));
+        return this.processComments(comments);
+    }
+    getPostId() {
+        const match = this.url.match(/comments\/([a-zA-Z0-9]+)/);
+        return match?.[1] || '';
+    }
+    getSubreddit() {
+        const match = this.url.match(/\/r\/([^/]+)/);
+        return match?.[1] || '';
+    }
+    getPostAuthor() {
+        return this.shredditPost?.getAttribute('author') || '';
+    }
+    createDescription(postContent) {
+        if (!postContent)
+            return '';
+        const tempDiv = document.createElement('div');
+        tempDiv.innerHTML = postContent;
+        return tempDiv.textContent?.trim()
+            .slice(0, 140)
+            .replace(/\s+/g, ' ') || '';
+    }
+    processComments(comments) {
+        let html = '';
+        let currentDepth = -1;
+        let blockquoteStack = []; // Keep track of open blockquotes at each depth
+        for (const comment of comments) {
+            const depth = parseInt(comment.getAttribute('depth') || '0');
+            const author = comment.getAttribute('author') || '';
+            const score = comment.getAttribute('score') || '0';
+            const permalink = comment.getAttribute('permalink') || '';
+            const content = comment.querySelector('[slot="comment"]')?.innerHTML || '';
+            // Get timestamp from faceplate-timeago element
+            const timeElement = comment.querySelector('faceplate-timeago');
+            const timestamp = timeElement?.getAttribute('ts') || '';
+            const date = timestamp ? new Date(timestamp).toISOString().split('T')[0] : '';
+            // For top-level comments, close all previous blockquotes and start fresh
+            if (depth === 0) {
+                // Close all open blockquotes
+                while (blockquoteStack.length > 0) {
+                    html += '</blockquote>';
+                    blockquoteStack.pop();
+                }
+                html += '<blockquote>';
+                blockquoteStack = [0];
+                currentDepth = 0;
+            }
+            // For nested comments
+            else {
+                // If we're moving back up the tree
+                if (depth < currentDepth) {
+                    // Close blockquotes until we reach the current depth
+                    while (blockquoteStack.length > 0 && blockquoteStack[blockquoteStack.length - 1] >= depth) {
+                        html += '</blockquote>';
+                        blockquoteStack.pop();
+                    }
+                }
+                // If we're going deeper
+                else if (depth > currentDepth) {
+                    html += '<blockquote>';
+                    blockquoteStack.push(depth);
+                }
+                // If we're at the same depth, no need to close or open blockquotes
+            }
+            html += `<div class="comment">
+	<div class="comment-metadata">
+		<span class="comment-author"><strong>${author}</strong></span> •
+		<a href="https://reddit.com${permalink}" class="comment-link">${score} points</a> •
+		<span class="comment-date">${date}</span>
+	</div>
+	<div class="comment-content">${content}</div>
+</div>`;
+            currentDepth = depth;
+        }
+        // Close any remaining blockquotes
+        while (blockquoteStack.length > 0) {
+            html += '</blockquote>';
+            blockquoteStack.pop();
+        }
+        return html;
+    }
+}
+exports.RedditExtractor = RedditExtractor;
+//# sourceMappingURL=reddit.js.map

package/workflow/tools/crawler/node_modules/defuddle/dist/extractors/reddit.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"reddit.js","sourceRoot":"","sources":["../../src/extractors/reddit.ts"],"names":[],"mappings":";;;AAAA,mCAAwC;AAGxC,MAAa,eAAgB,SAAQ,qBAAa;IAGjD,YAAY,QAAkB,EAAE,GAAW;QAC1C,KAAK,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QACrB,IAAI,CAAC,YAAY,GAAG,QAAQ,CAAC,aAAa,CAAC,eAAe,CAAC,CAAC;IAC7D,CAAC;IAED,UAAU;QACT,OAAO,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC;IAC5B,CAAC;IAED,OAAO;QACN,MAAM,WAAW,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;QAC1C,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,EAAE,CAAC;QAExC,MAAM,WAAW,GAAG,IAAI,CAAC,iBAAiB,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;QAClE,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QAC/E,MAAM,SAAS,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;QACtC,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QACxC,MAAM,WAAW,GAAG,IAAI,CAAC,iBAAiB,CAAC,WAAW,CAAC,CAAC;QAExD,OAAO;YACN,OAAO,EAAE,WAAW;YACpB,WAAW,EAAE,WAAW;YACxB,gBAAgB,EAAE;gBACjB,MAAM,EAAE,IAAI,CAAC,SAAS,EAAE;gBACxB,SAAS;gBACR,UAAU;aACX;YACD,SAAS,EAAE;gBACV,KAAK,EAAE,SAAS;gBAChB,MAAM,EAAE,UAAU;gBAClB,IAAI,EAAE,KAAK,SAAS,EAAE;gBACtB,WAAW;aACX;SACD,CAAC;IACH,CAAC;IAEO,cAAc;QACrB,MAAM,QAAQ,GAAG,IAAI,CAAC,YAAY,EAAE,aAAa,CAAC,oBAAoB,CAAC,EAAE,SAAS,IAAI,EAAE,CAAC;QACzF,MAAM,SAAS,GAAG,IAAI,CAAC,YAAY,EAAE,aAAa,CAAC,aAAa,CAAC,EAAE,SAAS,IAAI,EAAE,CAAC;QAEnF,OAAO,QAAQ,GAAG,SAAS,CAAC;IAC7B,CAAC;IAEO,iBAAiB,CAAC,WAAmB,EAAE,QAAgB;QAC9D,OAAO;;;OAGF,WAAW;;;KAGb,QAAQ,CAAC,CAAC,CAAC;;;;OAIT,QAAQ;;IAEX,CAAC,CAAC,CAAC,EAAE;GACN,CAAC,IAAI,EAAE,CAAC;IACV,CAAC;IAEO,eAAe;QACtB,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,kBAAkB,CAAC,CAAC,CAAC;QAChF,OAAO,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC;IACvC,CAAC;IAEO,SAAS;QAChB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC;QACzD,OAAO,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACzB,CAAC;IAEO,YAAY;QACnB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;QAC7C,OAAO,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACzB,CAAC;IAEO,aAAa;QACpB,OAAO,IAAI,CAAC,YAAY,EAAE,YAAY,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;IACxD,CAAC;IAEO,iBAAiB,CAAC,WAAmB;QAC5C,IAAI,CAAC,WAAW;YAAE,OAAO,EAAE,CAAC;QAE5B,MAAM,OAAO,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;QAC9C,OAAO,CAAC,SAAS,GAAG,WAAW,CAAC;QAChC,OAAO,OAAO,CAAC,WAAW,EAAE,IAAI,EAAE;aAChC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;aACb,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC;IAC9B,CAAC;IAEO,eAAe,CAAC,QAAmB;QAC1C,IAAI,IAAI,GAAG,EAAE,CAAC;QACd,IAAI,YAAY,GAAG,CAAC,CAAC,CAAC;QACtB,IAAI,eAAe,GAAa,EAAE,CAAC,CAAC,+CAA+C;QAEnF,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAChC,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC,CAAC;YAC7D,MAAM,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;YACpD,MAAM,KAAK,GAAG,OAAO,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC;YACnD,MAAM,SAAS,GAAG,OAAO,CAAC,YAAY,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC;YAC1D,MAAM,OAAO,GAAG,OAAO,CAAC,aAAa,CAAC,kBAAkB,CAAC,EAAE,SAAS,IAAI,EAAE,CAAC;YAE3E,+CAA+C;YAC/C,MAAM,WAAW,GAAG,OAAO,CAAC,aAAa,CAAC,mBAAmB,CAAC,CAAC;YAC/D,MAAM,SAAS,GAAG,WAAW,EAAE,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;YACxD,MAAM,IAAI,GAAG,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAE9E,yEAAyE;YACzE,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;gBACjB,6BAA6B;gBAC7B,OAAO,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACnC,IAAI,IAAI,eAAe,CAAC;oBACxB,eAAe,CAAC,GAAG,EAAE,CAAC;gBACvB,CAAC;gBACD,IAAI,IAAI,cAAc,CAAC;gBACvB,eAAe,GAAG,CAAC,CAAC,CAAC,CAAC;gBACtB,YAAY,GAAG,CAAC,CAAC;YAClB,CAAC;YACD,sBAAsB;iBACjB,CAAC;gBACL,mCAAmC;gBACnC,IAAI,KAAK,GAAG,YAAY,EAAE,CAAC;oBAC1B,qDAAqD;oBACrD,OAAO,eAAe,CAAC,MAAM,GAAG,CAAC,IAAI,eAAe,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,KAAK,EAAE,CAAC;wBAC3F,IAAI,IAAI,eAAe,CAAC;wBACxB,eAAe,CAAC,GAAG,EAAE,CAAC;oBACvB,CAAC;gBACF,CAAC;gBACD,wBAAwB;qBACnB,IAAI,KAAK,GAAG,YAAY,EAAE,CAAC;oBAC/B,IAAI,IAAI,cAAc,CAAC;oBACvB,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAC7B,CAAC;gBACD,mEAAmE;YACpE,CAAC;YAED,IAAI,IAAI;;yCAE8B,MAAM;+BAChB,SAAS,0BAA0B,KAAK;+BACxC,IAAI;;gCAEH,OAAO;OAChC,CAAC;YAEL,YAAY,GAAG,KAAK,CAAC;QACtB,CAAC;QAED,kCAAkC;QAClC,OAAO,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnC,IAAI,IAAI,eAAe,CAAC;YACxB,eAAe,CAAC,GAAG,EAAE,CAAC;QACvB,CAAC;QAED,OAAO,IAAI,CAAC;IACb,CAAC;CACD;AA9JD,0CA8JC"}

package/workflow/tools/crawler/node_modules/defuddle/dist/extractors/twitter.d.ts

@@ -0,0 +1,16 @@
+import { BaseExtractor } from './_base';
+import { ExtractorResult } from '../types/extractors';
+export declare class TwitterExtractor extends BaseExtractor {
+    private mainTweet;
+    private threadTweets;
+    constructor(document: Document, url: string);
+    canExtract(): boolean;
+    extract(): ExtractorResult;
+    private formatTweetText;
+    private extractTweet;
+    private extractUserInfo;
+    private extractImages;
+    private getTweetId;
+    private getTweetAuthor;
+    private createDescription;
+}