@apitap/core 1.0.0

package/dist/read/decoders/grokipedia.js

@@ -0,0 +1,210 @@
+import { safeFetch } from '../../discovery/fetch.js';
+function estimateTokens(text) {
+    return Math.ceil(text.length / 4);
+}
+/**
+ * Grokipedia decoder — xAI's open knowledge base (6M+ articles)
+ *
+ * API endpoints (all public, no auth):
+ *   /api/page?slug=X&includeContent=true  — Full article with citations
+ *   /api/full-text-search?query=X&limit=N — Search with relevance scoring
+ *   /api/stats                            — Site-wide stats
+ *   /api/typeahead?query=X                — Autocomplete
+ *   /api/list-pages?limit=N               — Browse articles
+ *   /api/top-contributors?limit=N         — Top editors
+ *   /api/list-edit-requests?limit=N       — Recent edits
+ */
+const GROKIPEDIA_API = 'https://grokipedia.com/api';
+export const grokipediaDecoder = {
+    name: 'grokipedia',
+    patterns: [
+        /grokipedia\.com\/wiki\/([^#?]+)/,
+        /grokipedia\.com\/article\/([^#?]+)/,
+        /grokipedia\.com\/search\?/,
+        /grokipedia\.com\/?$/,
+        /grokipedia\.com\/?(?:\?|#|$)/,
+    ],
+    async decode(url, options = {}) {
+        try {
+            const apiBase = options._apiBaseUrl || GROKIPEDIA_API;
+            // Search URL: /search?q=query
+            const searchMatch = url.match(/grokipedia\.com\/search\?.*q=([^&#]+)/);
+            if (searchMatch) {
+                return decodeSearch(apiBase, decodeURIComponent(searchMatch[1]), url, options);
+            }
+            // Article URL: /wiki/Slug or /article/Slug
+            const articleMatch = url.match(/grokipedia\.com\/(?:wiki|article)\/([^#?]+)/);
+            if (articleMatch) {
+                return decodeArticle(apiBase, articleMatch[1], url, options);
+            }
+            // Homepage: return stats + trending/recent
+            if (/grokipedia\.com\/?(?:\?|#|$)/.test(url)) {
+                return decodeHomepage(apiBase, url, options);
+            }
+            return null;
+        }
+        catch {
+            return null;
+        }
+    },
+};
+async function decodeArticle(apiBase, slug, url, options) {
+    const apiUrl = `${apiBase}/page?slug=${encodeURIComponent(slug)}&includeContent=true`;
+    // Grokipedia articles can be very large (743KB+ for Elon Musk) — raise body limit to 2MB
+    const result = await safeFetch(apiUrl, { skipSsrf: options.skipSsrf, maxBodySize: 2 * 1024 * 1024 });
+    if (!result || result.status !== 200)
+        return null;
+    let data;
+    try {
+        data = JSON.parse(result.body);
+    }
+    catch {
+        return null;
+    }
+    const page = data?.page;
+    if (!page)
+        return null;
+    const title = page.title || decodeURIComponent(slug).replace(/_/g, ' ');
+    const content = page.content || page.description || '';
+    const citations = page.citations || [];
+    const images = page.images || [];
+    const metadata = page.metadata || {};
+    const stats = page.stats || {};
+    // Truncate content if maxBytes specified
+    const maxChars = options.maxBytes ? options.maxBytes : 20000;
+    const truncatedContent = content.length > maxChars
+        ? content.slice(0, maxChars) + `\n\n[Truncated — full article is ${content.length} chars. ${citations.length} citations available.]`
+        : content;
+    // Build citations section (top 10)
+    const topCitations = citations.slice(0, 10);
+    const citationBlock = topCitations.length > 0
+        ? '\n\n## Sources\n' + topCitations.map((c, i) => `${i + 1}. [${c.title || 'Source'}](${c.url})`).join('\n')
+        : '';
+    // Build stats line
+    const statsLine = stats.totalViews
+        ? `\n\nViews: ${Number(stats.totalViews).toLocaleString()} | Quality: ${stats.qualityScore || 'N/A'} | Language: ${metadata.language || 'en'}`
+        : '';
+    const resultImages = images.slice(0, 5).map((img) => ({
+        alt: img.caption || title,
+        src: img.url || '',
+    }));
+    const resultLinks = [
+        { text: 'Full article', href: `https://grokipedia.com/wiki/${slug}` },
+    ];
+    // Add citation links
+    topCitations.forEach((c) => {
+        if (c.url) {
+            resultLinks.push({ text: c.title || 'Source', href: c.url });
+        }
+    });
+    return {
+        url,
+        title,
+        author: metadata.lastEditor || null,
+        description: page.description || null,
+        content: truncatedContent + citationBlock + statsLine,
+        links: resultLinks,
+        images: resultImages,
+        metadata: {
+            type: 'article',
+            publishedAt: metadata.lastModified ? new Date(metadata.lastModified * 1000).toISOString() : null,
+            source: 'grokipedia-api',
+            canonical: `https://grokipedia.com/wiki/${slug}`,
+            siteName: 'Grokipedia',
+        },
+        cost: { tokens: estimateTokens(truncatedContent + citationBlock + statsLine) },
+    };
+}
+async function decodeSearch(apiBase, query, url, options) {
+    const apiUrl = `${apiBase}/full-text-search?query=${encodeURIComponent(query)}&limit=10`;
+    const result = await safeFetch(apiUrl, { skipSsrf: options.skipSsrf });
+    if (!result || result.status !== 200)
+        return null;
+    let data;
+    try {
+        data = JSON.parse(result.body);
+    }
+    catch {
+        return null;
+    }
+    const results = data?.results || [];
+    if (results.length === 0)
+        return null;
+    const content = results.map((r, i) => {
+        const views = r.viewCount ? ` (${Number(r.viewCount).toLocaleString()} views)` : '';
+        const snippet = (r.snippet || '').replace(/<\/?em>/g, '**').replace(/\n/g, ' ').trim();
+        return `${i + 1}. **[${r.title}](https://grokipedia.com/wiki/${r.slug})**${views}\n   ${snippet}`;
+    }).join('\n\n');
+    const links = results.map((r) => ({
+        text: r.title || r.slug,
+        href: `https://grokipedia.com/wiki/${r.slug}`,
+    }));
+    return {
+        url,
+        title: `Grokipedia search: "${query}"`,
+        author: null,
+        description: `${results.length} results for "${query}"`,
+        content,
+        links,
+        images: [],
+        metadata: {
+            type: 'search-results',
+            publishedAt: null,
+            source: 'grokipedia-api',
+            canonical: null,
+            siteName: 'Grokipedia',
+        },
+        cost: { tokens: estimateTokens(content) },
+    };
+}
+async function decodeHomepage(apiBase, url, options) {
+    // Fetch stats
+    const statsResult = await safeFetch(`${apiBase}/stats`, { skipSsrf: options.skipSsrf });
+    let statsData = {};
+    if (statsResult?.status === 200) {
+        try {
+            statsData = JSON.parse(statsResult.body);
+        }
+        catch { }
+    }
+    // Fetch recent edits
+    const editsResult = await safeFetch(`${apiBase}/list-edit-requests?limit=5`, { skipSsrf: options.skipSsrf });
+    let editsData = {};
+    if (editsResult?.status === 200) {
+        try {
+            editsData = JSON.parse(editsResult.body);
+        }
+        catch { }
+    }
+    const totalPages = Number(statsData.totalPages || 0).toLocaleString();
+    const indexGB = (Number(statsData.indexSizeBytes || 0) / (1024 ** 3)).toFixed(1);
+    let content = `# Grokipedia\n\nAn open source, comprehensive collection of all knowledge.\n\n`;
+    content += `**${totalPages} articles** | **${indexGB} GB index**\n\n`;
+    const edits = editsData.editRequests || [];
+    if (edits.length > 0) {
+        content += `## Recent Activity\n`;
+        for (const edit of edits) {
+            const article = edit.slug?.replace(/_/g, ' ') || 'Unknown';
+            const editor = edit.userId || 'Anonymous';
+            content += `- **${article}** — edited by ${editor} (${edit.type?.replace('EDIT_REQUEST_TYPE_', '').toLowerCase().replace(/_/g, ' ')})\n`;
+        }
+    }
+    return {
+        url,
+        title: 'Grokipedia',
+        author: null,
+        description: `Open knowledge base with ${totalPages} articles`,
+        content,
+        links: [],
+        images: [],
+        metadata: {
+            type: 'website',
+            publishedAt: null,
+            source: 'grokipedia-api',
+            canonical: 'https://grokipedia.com',
+            siteName: 'Grokipedia',
+        },
+        cost: { tokens: estimateTokens(content) },
+    };
+}
+//# sourceMappingURL=grokipedia.js.map

package/dist/read/decoders/grokipedia.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"grokipedia.js","sourceRoot":"","sources":["../../../src/read/decoders/grokipedia.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AAErD,SAAS,cAAc,CAAC,IAAY;IAClC,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACpC,CAAC;AAED;;;;;;;;;;;GAWG;AAEH,MAAM,cAAc,GAAG,4BAA4B,CAAC;AAEpD,MAAM,CAAC,MAAM,iBAAiB,GAAY;IACxC,IAAI,EAAE,YAAY;IAClB,QAAQ,EAAE;QACR,iCAAiC;QACjC,oCAAoC;QACpC,2BAA2B;QAC3B,qBAAqB;QACrB,8BAA8B;KAC/B;IAED,KAAK,CAAC,MAAM,CAAC,GAAW,EAAE,UAAyE,EAAE;QACnG,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,OAAO,CAAC,WAAW,IAAI,cAAc,CAAC;YAEtD,8BAA8B;YAC9B,MAAM,WAAW,GAAG,GAAG,CAAC,KAAK,CAAC,uCAAuC,CAAC,CAAC;YACvE,IAAI,WAAW,EAAE,CAAC;gBAChB,OAAO,YAAY,CAAC,OAAO,EAAE,kBAAkB,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;YACjF,CAAC;YAED,2CAA2C;YAC3C,MAAM,YAAY,GAAG,GAAG,CAAC,KAAK,CAAC,6CAA6C,CAAC,CAAC;YAC9E,IAAI,YAAY,EAAE,CAAC;gBACjB,OAAO,aAAa,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;YAC/D,CAAC;YAED,2CAA2C;YAC3C,IAAI,8BAA8B,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC7C,OAAO,cAAc,CAAC,OAAO,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;YAC/C,CAAC;YAED,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;CACF,CAAC;AAEF,KAAK,UAAU,aAAa,CAC1B,OAAe,EACf,IAAY,EACZ,GAAW,EACX,OAAsE;IAEtE,MAAM,MAAM,GAAG,GAAG,OAAO,cAAc,kBAAkB,CAAC,IAAI,CAAC,sBAAsB,CAAC;IACtF,yFAAyF;IACzF,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,MAAM,EAAE,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,WAAW,EAAE,CAAC,GAAG,IAAI,GAAG,IAAI,EAAE,CAAC,CAAC;IACrG,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,KAAK,GAAG;QAAE,OAAO,IAAI,CAAC;IAElD,IAAI,IAAS,CAAC;IACd,IAAI,CAAC;QACH,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACjC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,EAAE,IAAI,CAAC;IACxB,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAC;IAEvB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,kBAAkB,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IACxE,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC;IACvD,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC;IACvC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC;IACjC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC;IACrC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;IAE/B,yCAAyC;IACzC,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC;IAC7D,MAAM,gBAAgB,GAAG,OAAO,CAAC,MAAM,GAAG,QAAQ;QAChD,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,GAAG,oCAAoC,OAAO,CAAC,MAAM,WAAW,SAAS,CAAC,MAAM,wBAAwB;QACpI,CAAC,CAAC,OAAO,CAAC;IAEZ,mCAAmC;IACnC,MAAM,YAAY,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC5C,MAAM,aAAa,GAAG,YAAY,CAAC,MAAM,GAAG,CAAC;QAC3C,CAAC,CAAC,kBAAkB,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,CAAS,EAAE,EAAE,CAC1D,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,KAAK,IAAI,QAAQ,KAAK,CAAC,CAAC,GAAG,GAAG,CAC/C,CAAC,IAAI,CAAC,IAAI,CAAC;QACd,CAAC,CAAC,EAAE,CAAC;IAEP,mBAAmB;IACnB,MAAM,SAAS,GAAG,KAAK,CAAC,UAAU;QAChC,CAAC,CAAC,cAAc,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,cAAc,EAAE,eAAe,KAAK,CAAC,YAAY,IAAI,KAAK,gBAAgB,QAAQ,CAAC,QAAQ,IAAI,IAAI,EAAE;QAC9I,CAAC,CAAC,EAAE,CAAC;IAEP,MAAM,YAAY,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAQ,EAAE,EAAE,CAAC,CAAC;QACzD,GAAG,EAAE,GAAG,CAAC,OAAO,IAAI,KAAK;QACzB,GAAG,EAAE,GAAG,CAAC,GAAG,IAAI,EAAE;KACnB,CAAC,CAAC,CAAC;IAEJ,MAAM,WAAW,GAA0C;QACzD,EAAE,IAAI,EAAE,cAAc,EAAE,IAAI,EAAE,+BAA+B,IAAI,EAAE,EAAE;KACtE,CAAC;IAEF,qBAAqB;IACrB,YAAY,CAAC,OAAO,CAAC,CAAC,CAAM,EAAE,EAAE;QAC9B,IAAI,CAAC,CAAC,GAAG,EAAE,CAAC;YACV,WAAW,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,IAAI,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;QAC/D,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO;QACL,GAAG;QACH,KAAK;QACL,MAAM,EAAE,QAAQ,CAAC,UAAU,IAAI,IAAI;QACnC,WAAW,EAAE,IAAI,CAAC,WAAW,IAAI,IAAI;QACrC,OAAO,EAAE,gBAAgB,GAAG,aAAa,GAAG,SAAS;QACrD,KAAK,EAAE,WAAW;QAClB,MAAM,EAAE,YAAY;QACpB,QAAQ,EAAE;YACR,IAAI,EAAE,SAAS;YACf,WAAW,EAAE,QAAQ,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,YAAY,GAAG,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI;YAChG,MAAM,EAAE,gBAAgB;YACxB,SAAS,EAAE,+BAA+B,IAAI,EAAE;YAChD,QAAQ,EAAE,YAAY;SACvB;QACD,IAAI,EAAE,EAAE,MAAM,EAAE,cAAc,CAAC,gBAAgB,GAAG,aAAa,GAAG,SAAS,CAAC,EAAE;KAC/E,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,YAAY,CACzB,OAAe,EACf,KAAa,EACb,GAAW,EACX,OAAmD;IAEnD,MAAM,MAAM,GAAG,GAAG,OAAO,2BAA2B,kBAAkB,CAAC,KAAK,CAAC,WAAW,CAAC;IACzF,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,MAAM,EAAE,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;IACvE,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,KAAK,GAAG;QAAE,OAAO,IAAI,CAAC;IAElD,IAAI,IAAS,CAAC;IACd,IAAI,CAAC;QACH,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACjC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,EAAE,OAAO,IAAI,EAAE,CAAC;IACpC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAEtC,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,CAAS,EAAE,EAAE;QAChD,MAAM,KAAK,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,cAAc,EAAE,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC;QACpF,MAAM,OAAO,GAAG,CAAC,CAAC,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QACvF,OAAO,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,KAAK,iCAAiC,CAAC,CAAC,IAAI,MAAM,KAAK,QAAQ,OAAO,EAAE,CAAC;IACpG,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAEhB,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC;QACrC,IAAI,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,IAAI;QACvB,IAAI,EAAE,+BAA+B,CAAC,CAAC,IAAI,EAAE;KAC9C,CAAC,CAAC,CAAC;IAEJ,OAAO;QACL,GAAG;QACH,KAAK,EAAE,uBAAuB,KAAK,GAAG;QACtC,MAAM,EAAE,IAAI;QACZ,WAAW,EAAE,GAAG,OAAO,CAAC,MAAM,iBAAiB,KAAK,GAAG;QACvD,OAAO;QACP,KAAK;QACL,MAAM,EAAE,EAAE;QACV,QAAQ,EAAE;YACR,IAAI,EAAE,gBAAgB;YACtB,WAAW,EAAE,IAAI;YACjB,MAAM,EAAE,gBAAgB;YACxB,SAAS,EAAE,IAAI;YACf,QAAQ,EAAE,YAAY;SACvB;QACD,IAAI,EAAE,EAAE,MAAM,EAAE,cAAc,CAAC,OAAO,CAAC,EAAE;KAC1C,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,cAAc,CAC3B,OAAe,EACf,GAAW,EACX,OAAmD;IAEnD,cAAc;IACd,MAAM,WAAW,GAAG,MAAM,SAAS,CAAC,GAAG,OAAO,QAAQ,EAAE,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;IACxF,IAAI,SAAS,GAAQ,EAAE,CAAC;IACxB,IAAI,WAAW,EAAE,MAAM,KAAK,GAAG,EAAE,CAAC;QAChC,IAAI,CAAC;YAAC,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC,CAAA,CAAC;IAC5D,CAAC;IAED,qBAAqB;IACrB,MAAM,WAAW,GAAG,MAAM,SAAS,CAAC,GAAG,OAAO,6BAA6B,EAAE,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC7G,IAAI,SAAS,GAAQ,EAAE,CAAC;IACxB,IAAI,WAAW,EAAE,MAAM,KAAK,GAAG,EAAE,CAAC;QAChC,IAAI,CAAC;YAAC,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC,CAAA,CAAC;IAC5D,CAAC;IAED,MAAM,UAAU,GAAG,MAAM,CAAC,SAAS,CAAC,UAAU,IAAI,CAAC,CAAC,CAAC,cAAc,EAAE,CAAC;IACtE,MAAM,OAAO,GAAG,CAAC,MAAM,CAAC,SAAS,CAAC,cAAc,IAAI,CAAC,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAEjF,IAAI,OAAO,GAAG,gFAAgF,CAAC;IAC/F,OAAO,IAAI,KAAK,UAAU,mBAAmB,OAAO,iBAAiB,CAAC;IAEtE,MAAM,KAAK,GAAG,SAAS,CAAC,YAAY,IAAI,EAAE,CAAC;IAC3C,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrB,OAAO,IAAI,sBAAsB,CAAC;QAClC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,SAAS,CAAC;YAC3D,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,WAAW,CAAC;YAC1C,OAAO,IAAI,OAAO,OAAO,kBAAkB,MAAM,KAAK,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,oBAAoB,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,KAAK,CAAC;QAC3I,CAAC;IACH,CAAC;IAED,OAAO;QACL,GAAG;QACH,KAAK,EAAE,YAAY;QACnB,MAAM,EAAE,IAAI;QACZ,WAAW,EAAE,4BAA4B,UAAU,WAAW;QAC9D,OAAO;QACP,KAAK,EAAE,EAAE;QACT,MAAM,EAAE,EAAE;QACV,QAAQ,EAAE;YACR,IAAI,EAAE,SAAS;YACf,WAAW,EAAE,IAAI;YACjB,MAAM,EAAE,gBAAgB;YACxB,SAAS,EAAE,wBAAwB;YACnC,QAAQ,EAAE,YAAY;SACvB;QACD,IAAI,EAAE,EAAE,MAAM,EAAE,cAAc,CAAC,OAAO,CAAC,EAAE;KAC1C,CAAC;AACJ,CAAC"}

package/dist/read/decoders/hackernews.d.ts

@@ -0,0 +1,2 @@
+import type { Decoder } from '../types.js';
+export declare const hackernewsDecoder: Decoder;

package/dist/read/decoders/hackernews.js

@@ -0,0 +1,168 @@
+import { safeFetch } from '../../discovery/fetch.js';
+const DEFAULT_API_BASE = 'https://hacker-news.firebaseio.com';
+function estimateTokens(text) {
+    return Math.ceil(text.length / 4);
+}
+export const hackernewsDecoder = {
+    name: 'hackernews',
+    patterns: [
+        /news\.ycombinator\.com\/item\?id=\d+/,
+        /news\.ycombinator\.com\/?(?:\?|$)/,
+    ],
+    async decode(url, options = {}) {
+        try {
+            const apiBase = options._apiBaseUrl || DEFAULT_API_BASE;
+            const fetchOpts = { skipSsrf: options.skipSsrf };
+            // Check if this is an item page or front page
+            const itemMatch = url.match(/item\?id=(\d+)/);
+            if (itemMatch) {
+                return decodeItem(url, itemMatch[1], apiBase, fetchOpts);
+            }
+            return decodeFrontPage(url, apiBase, fetchOpts);
+        }
+        catch {
+            return null;
+        }
+    },
+};
+async function decodeItem(url, id, apiBase, fetchOpts) {
+    try {
+        const result = await safeFetch(`${apiBase}/v0/item/${id}.json`, fetchOpts);
+        if (!result || result.status !== 200)
+            return null;
+        let item;
+        try {
+            item = JSON.parse(result.body);
+        }
+        catch {
+            return null;
+        }
+        if (!item)
+            return null;
+        const title = item.title || null;
+        const author = item.by || null;
+        const score = item.score ?? 0;
+        const itemUrl = item.url || null;
+        const text = item.text || '';
+        // Fetch top 10 comments
+        const kids = item.kids || [];
+        const commentIds = kids.slice(0, 10);
+        const comments = await fetchComments(commentIds, apiBase, fetchOpts);
+        const commentText = comments
+            .map((c) => `${c.by || '[deleted]'}: ${c.text || '[deleted]'}`)
+            .join('\n\n');
+        const contentParts = [];
+        if (text)
+            contentParts.push(text);
+        contentParts.push(`Score: ${score} | ${kids.length} comments`);
+        if (commentText)
+            contentParts.push(`---\n${commentText}`);
+        const content = contentParts.join('\n\n');
+        const links = [];
+        if (itemUrl) {
+            links.push({ text: title || 'Link', href: itemUrl });
+        }
+        return {
+            url,
+            title,
+            author,
+            description: `HN ${item.type || 'story'} by ${author} (${score} points)`,
+            content,
+            links,
+            images: [],
+            metadata: {
+                type: item.type || 'story',
+                publishedAt: item.time ? new Date(item.time * 1000).toISOString() : null,
+                source: 'hackernews-firebase',
+                canonical: `https://news.ycombinator.com/item?id=${id}`,
+                siteName: 'Hacker News',
+            },
+            cost: { tokens: estimateTokens(content) },
+        };
+    }
+    catch {
+        return null;
+    }
+}
+async function decodeFrontPage(url, apiBase, fetchOpts) {
+    try {
+        const result = await safeFetch(`${apiBase}/v0/topstories.json`, fetchOpts);
+        if (!result || result.status !== 200)
+            return null;
+        let storyIds;
+        try {
+            storyIds = JSON.parse(result.body);
+        }
+        catch {
+            return null;
+        }
+        if (!Array.isArray(storyIds))
+            return null;
+        // Fetch first 10 stories
+        const topIds = storyIds.slice(0, 10);
+        const stories = await fetchStories(topIds, apiBase, fetchOpts);
+        const content = stories
+            .map((s, i) => `${i + 1}. ${s.title || '[untitled]'} (${s.score ?? 0} pts, ${(s.descendants ?? 0)} comments) by ${s.by || '[deleted]'}`)
+            .join('\n');
+        const links = stories
+            .filter((s) => s.url)
+            .map((s) => ({ text: s.title || 'Link', href: s.url }));
+        return {
+            url,
+            title: 'Hacker News — Top Stories',
+            author: null,
+            description: `Top ${stories.length} stories`,
+            content,
+            links,
+            images: [],
+            metadata: {
+                type: 'listing',
+                publishedAt: null,
+                source: 'hackernews-firebase',
+                canonical: 'https://news.ycombinator.com/',
+                siteName: 'Hacker News',
+            },
+            cost: { tokens: estimateTokens(content) },
+        };
+    }
+    catch {
+        return null;
+    }
+}
+async function fetchComments(ids, apiBase, fetchOpts) {
+    const comments = [];
+    for (const id of ids) {
+        try {
+            const result = await safeFetch(`${apiBase}/v0/item/${id}.json`, fetchOpts);
+            if (result && result.status === 200) {
+                const comment = JSON.parse(result.body);
+                if (comment && !comment.deleted) {
+                    comments.push(comment);
+                }
+            }
+        }
+        catch {
+            // skip failed comments
+        }
+    }
+    return comments;
+}
+async function fetchStories(ids, apiBase, fetchOpts) {
+    const stories = [];
+    for (const id of ids) {
+        try {
+            const result = await safeFetch(`${apiBase}/v0/item/${id}.json`, fetchOpts);
+            if (result && result.status === 200) {
+                const story = JSON.parse(result.body);
+                if (story) {
+                    stories.push(story);
+                }
+            }
+        }
+        catch {
+            // skip failed stories
+        }
+    }
+    return stories;
+}
+//# sourceMappingURL=hackernews.js.map

package/dist/read/decoders/hackernews.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"hackernews.js","sourceRoot":"","sources":["../../../src/read/decoders/hackernews.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AAErD,MAAM,gBAAgB,GAAG,oCAAoC,CAAC;AAE9D,SAAS,cAAc,CAAC,IAAY;IAClC,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACpC,CAAC;AAED,MAAM,CAAC,MAAM,iBAAiB,GAAY;IACxC,IAAI,EAAE,YAAY;IAClB,QAAQ,EAAE;QACR,sCAAsC;QACtC,mCAAmC;KACpC;IAED,KAAK,CAAC,MAAM,CAAC,GAAW,EAAE,UAAsD,EAAE;QAChF,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,OAAO,CAAC,WAAW,IAAI,gBAAgB,CAAC;YACxD,MAAM,SAAS,GAAG,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC;YAEjD,8CAA8C;YAC9C,MAAM,SAAS,GAAG,GAAG,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;YAE9C,IAAI,SAAS,EAAE,CAAC;gBACd,OAAO,UAAU,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;YAC3D,CAAC;YAED,OAAO,eAAe,CAAC,GAAG,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;QAClD,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;CACF,CAAC;AAEF,KAAK,UAAU,UAAU,CACvB,GAAW,EACX,EAAU,EACV,OAAe,EACf,SAAiC;IAEjC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,GAAG,OAAO,YAAY,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;QAC3E,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,KAAK,GAAG;YAAE,OAAO,IAAI,CAAC;QAElD,IAAI,IAAS,CAAC;QACd,IAAI,CAAC;YACH,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACjC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;QAED,IAAI,CAAC,IAAI;YAAE,OAAO,IAAI,CAAC;QAEvB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC;QACjC,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,IAAI,IAAI,CAAC;QAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC;QAC9B,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC;QACjC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QAE7B,wBAAwB;QACxB,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QAC7B,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACrC,MAAM,QAAQ,GAAG,MAAM,aAAa,CAAC,UAAU,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;QAErE,MAAM,WAAW,GAAG,QAAQ;aACzB,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,WAAW,KAAK,CAAC,CAAC,IAAI,IAAI,WAAW,EAAE,CAAC;aACnE,IAAI,CAAC,MAAM,CAAC,CAAC;QAEhB,MAAM,YAAY,GAAa,EAAE,CAAC;QAClC,IAAI,IAAI;YAAE,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClC,YAAY,CAAC,IAAI,CAAC,UAAU,KAAK,MAAM,IAAI,CAAC,MAAM,WAAW,CAAC,CAAC;QAC/D,IAAI,WAAW;YAAE,YAAY,CAAC,IAAI,CAAC,QAAQ,WAAW,EAAE,CAAC,CAAC;QAC1D,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAE1C,MAAM,KAAK,GAA0C,EAAE,CAAC;QACxD,IAAI,OAAO,EAAE,CAAC;YACZ,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,IAAI,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;QACvD,CAAC;QAED,OAAO;YACL,GAAG;YACH,KAAK;YACL,MAAM;YACN,WAAW,EAAE,MAAM,IAAI,CAAC,IAAI,IAAI,OAAO,OAAO,MAAM,KAAK,KAAK,UAAU;YACxE,OAAO;YACP,KAAK;YACL,MAAM,EAAE,EAAE;YACV,QAAQ,EAAE;gBACR,IAAI,EAAE,IAAI,CAAC,IAAI,IAAI,OAAO;gBAC1B,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI;gBACxE,MAAM,EAAE,qBAAqB;gBAC7B,SAAS,EAAE,wCAAwC,EAAE,EAAE;gBACvD,QAAQ,EAAE,aAAa;aACxB;YACD,IAAI,EAAE,EAAE,MAAM,EAAE,cAAc,CAAC,OAAO,CAAC,EAAE;SAC1C,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,KAAK,UAAU,eAAe,CAC5B,GAAW,EACX,OAAe,EACf,SAAiC;IAEjC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,GAAG,OAAO,qBAAqB,EAAE,SAAS,CAAC,CAAC;QAC3E,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,KAAK,GAAG;YAAE,OAAO,IAAI,CAAC;QAElD,IAAI,QAAkB,CAAC;QACvB,IAAI,CAAC;YACH,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACrC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;QAED,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC;YAAE,OAAO,IAAI,CAAC;QAE1C,yBAAyB;QACzB,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACrC,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,MAAM,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;QAE/D,MAAM,OAAO,GAAG,OAAO;aACpB,GAAG,CAAC,CAAC,CAAM,EAAE,CAAS,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,KAAK,IAAI,YAAY,KAAK,CAAC,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,IAAI,CAAC,CAAC,iBAAiB,CAAC,CAAC,EAAE,IAAI,WAAW,EAAE,CAAC;aACpJ,IAAI,CAAC,IAAI,CAAC,CAAC;QAEd,MAAM,KAAK,GAAG,OAAO;aAClB,MAAM,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;aACzB,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,IAAI,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QAE/D,OAAO;YACL,GAAG;YACH,KAAK,EAAE,2BAA2B;YAClC,MAAM,EAAE,IAAI;YACZ,WAAW,EAAE,OAAO,OAAO,CAAC,MAAM,UAAU;YAC5C,OAAO;YACP,KAAK;YACL,MAAM,EAAE,EAAE;YACV,QAAQ,EAAE;gBACR,IAAI,EAAE,SAAS;gBACf,WAAW,EAAE,IAAI;gBACjB,MAAM,EAAE,qBAAqB;gBAC7B,SAAS,EAAE,+BAA+B;gBAC1C,QAAQ,EAAE,aAAa;aACxB;YACD,IAAI,EAAE,EAAE,MAAM,EAAE,cAAc,CAAC,OAAO,CAAC,EAAE;SAC1C,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,KAAK,UAAU,aAAa,CAC1B,GAAa,EACb,OAAe,EACf,SAAiC;IAEjC,MAAM,QAAQ,GAAU,EAAE,CAAC;IAC3B,KAAK,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;QACrB,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,GAAG,OAAO,YAAY,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;YAC3E,IAAI,MAAM,IAAI,MAAM,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;gBACpC,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBACxC,IAAI,OAAO,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;oBAChC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACzB,CAAC;YACH,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,uBAAuB;QACzB,CAAC;IACH,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,KAAK,UAAU,YAAY,CACzB,GAAa,EACb,OAAe,EACf,SAAiC;IAEjC,MAAM,OAAO,GAAU,EAAE,CAAC;IAC1B,KAAK,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;QACrB,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,GAAG,OAAO,YAAY,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;YAC3E,IAAI,MAAM,IAAI,MAAM,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;gBACpC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBACtC,IAAI,KAAK,EAAE,CAAC;oBACV,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACtB,CAAC;YACH,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,sBAAsB;QACxB,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC"}

package/dist/read/decoders/index.d.ts

@@ -0,0 +1,2 @@
+import type { Decoder } from '../types.js';
+export declare function findDecoder(url: string): Decoder | null;

package/dist/read/decoders/index.js

@@ -0,0 +1,12 @@
+import { redditDecoder } from './reddit.js';
+import { youtubeDecoder } from './youtube.js';
+import { wikipediaDecoder } from './wikipedia.js';
+import { hackernewsDecoder } from './hackernews.js';
+import { grokipediaDecoder } from './grokipedia.js';
+import { twitterDecoder } from './twitter.js';
+import { deepwikiDecoder } from './deepwiki.js';
+const decoders = [redditDecoder, youtubeDecoder, wikipediaDecoder, hackernewsDecoder, grokipediaDecoder, twitterDecoder, deepwikiDecoder];
+export function findDecoder(url) {
+    return decoders.find(d => d.patterns.some(p => p.test(url))) ?? null;
+}
+//# sourceMappingURL=index.js.map

package/dist/read/decoders/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/read/decoders/index.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC9C,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAClD,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACpD,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACpD,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC9C,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAEhD,MAAM,QAAQ,GAAc,CAAC,aAAa,EAAE,cAAc,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,cAAc,EAAE,eAAe,CAAC,CAAC;AAErJ,MAAM,UAAU,WAAW,CAAC,GAAW;IACrC,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;AACvE,CAAC"}

package/dist/read/decoders/reddit.d.ts

@@ -0,0 +1,2 @@
+import type { Decoder } from '../types.js';
+export declare const redditDecoder: Decoder;

package/dist/read/decoders/reddit.js

@@ -0,0 +1,142 @@
+import { safeFetch } from '../../discovery/fetch.js';
+function estimateTokens(text) {
+    return Math.ceil(text.length / 4);
+}
+export const redditDecoder = {
+    name: 'reddit',
+    patterns: [
+        /reddit\.com\/r\/[^/]+\/comments\//,
+        /reddit\.com\/r\/[^/]+\/?$/,
+        /reddit\.com\/r\/[^/]+\/?(?:\?|$)/,
+        /reddit\.com\/user\/[^/]+/,
+    ],
+    async decode(url, options = {}) {
+        try {
+            // Append .json to the URL to get JSON response
+            const jsonUrl = url.replace(/\/?(\?|$)/, '.json$1');
+            const result = await safeFetch(jsonUrl, { skipSsrf: options.skipSsrf });
+            if (!result || result.status !== 200)
+                return null;
+            let data;
+            try {
+                data = JSON.parse(result.body);
+            }
+            catch {
+                return null;
+            }
+            // Post page: response is an array [post, comments]
+            if (Array.isArray(data) && data.length >= 1) {
+                return decodePostPage(url, data);
+            }
+            // Subreddit/user listing: response has data.children
+            if (data && data.data && Array.isArray(data.data.children)) {
+                return decodeListingPage(url, data);
+            }
+            return null;
+        }
+        catch {
+            return null;
+        }
+    },
+};
+function decodePostPage(url, data) {
+    try {
+        const postData = data[0]?.data?.children?.[0]?.data;
+        if (!postData)
+            return null;
+        const title = postData.title || null;
+        const author = postData.author || null;
+        const selftext = postData.selftext || '';
+        const score = postData.score ?? 0;
+        const subreddit = postData.subreddit || '';
+        // Extract comments
+        const commentChildren = data[1]?.data?.children || [];
+        const comments = commentChildren
+            .filter((c) => c.kind === 't1' && c.data)
+            .slice(0, 25)
+            .map((c) => ({
+            author: c.data.author || '[deleted]',
+            body: c.data.body || '',
+            score: c.data.score ?? 0,
+        }));
+        const commentText = comments
+            .map((c) => `${c.author} (${c.score} pts): ${c.body}`)
+            .join('\n\n');
+        const content = selftext
+            ? `${selftext}\n\n---\nScore: ${score} | ${comments.length} comments\n\n${commentText}`
+            : `Score: ${score} | ${comments.length} comments\n\n${commentText}`;
+        const links = [];
+        if (postData.url && postData.url !== postData.permalink) {
+            links.push({ text: 'Link', href: postData.url });
+        }
+        return {
+            url,
+            title,
+            author,
+            description: `r/${subreddit} post by u/${author} (${score} points)`,
+            content,
+            links,
+            images: [],
+            metadata: {
+                type: 'discussion',
+                publishedAt: postData.created_utc ? new Date(postData.created_utc * 1000).toISOString() : null,
+                source: 'reddit-json',
+                canonical: postData.permalink ? `https://www.reddit.com${postData.permalink}` : null,
+                siteName: 'Reddit',
+            },
+            cost: { tokens: estimateTokens(content) },
+        };
+    }
+    catch {
+        return null;
+    }
+}
+function decodeListingPage(url, data) {
+    try {
+        const children = data.data.children || [];
+        const posts = children
+            .filter((c) => c.data)
+            .slice(0, 25)
+            .map((c) => ({
+            title: c.data.title || c.data.link_title || '',
+            author: c.data.author || '[deleted]',
+            score: c.data.score ?? 0,
+            numComments: c.data.num_comments ?? 0,
+            permalink: c.data.permalink || '',
+            subreddit: c.data.subreddit || '',
+        }));
+        const content = posts
+            .map((p, i) => `${i + 1}. ${p.title} (${p.score} pts, ${p.numComments} comments) by u/${p.author}`)
+            .join('\n');
+        const links = posts
+            .filter((p) => p.permalink)
+            .map((p) => ({ text: p.title, href: `https://www.reddit.com${p.permalink}` }));
+        // Try to determine subreddit name from URL
+        const subMatch = url.match(/\/r\/([^/]+)/);
+        const subreddit = subMatch ? subMatch[1] : null;
+        const userMatch = url.match(/\/user\/([^/]+)/);
+        const user = userMatch ? userMatch[1] : null;
+        const title = subreddit ? `r/${subreddit}` : user ? `u/${user}` : 'Reddit listing';
+        return {
+            url,
+            title,
+            author: null,
+            description: `${posts.length} posts`,
+            content,
+            links,
+            images: [],
+            metadata: {
+                type: 'listing',
+                publishedAt: null,
+                source: 'reddit-json',
+                canonical: null,
+                siteName: 'Reddit',
+            },
+            cost: { tokens: estimateTokens(content) },
+        };
+    }
+    catch {
+        return null;
+    }
+}
+//# sourceMappingURL=reddit.js.map

package/dist/read/decoders/reddit.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"reddit.js","sourceRoot":"","sources":["../../../src/read/decoders/reddit.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AAErD,SAAS,cAAc,CAAC,IAAY;IAClC,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACpC,CAAC;AAED,MAAM,CAAC,MAAM,aAAa,GAAY;IACpC,IAAI,EAAE,QAAQ;IACd,QAAQ,EAAE;QACR,mCAAmC;QACnC,2BAA2B;QAC3B,kCAAkC;QAClC,0BAA0B;KAC3B;IAED,KAAK,CAAC,MAAM,CAAC,GAAW,EAAE,UAAsD,EAAE;QAChF,IAAI,CAAC;YACH,+CAA+C;YAC/C,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;YAEpD,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;YACxE,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,KAAK,GAAG;gBAAE,OAAO,IAAI,CAAC;YAElD,IAAI,IAAS,CAAC;YACd,IAAI,CAAC;gBACH,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YACjC,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,IAAI,CAAC;YACd,CAAC;YAED,mDAAmD;YACnD,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;gBAC5C,OAAO,cAAc,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;YACnC,CAAC;YAED,qDAAqD;YACrD,IAAI,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC3D,OAAO,iBAAiB,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;YACtC,CAAC;YAED,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;CACF,CAAC;AAEF,SAAS,cAAc,CAAC,GAAW,EAAE,IAAW;IAC9C,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC;QACpD,IAAI,CAAC,QAAQ;YAAE,OAAO,IAAI,CAAC;QAE3B,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,IAAI,IAAI,CAAC;QACrC,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,IAAI,IAAI,CAAC;QACvC,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,IAAI,EAAE,CAAC;QACzC,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,IAAI,CAAC,CAAC;QAClC,MAAM,SAAS,GAAG,QAAQ,CAAC,SAAS,IAAI,EAAE,CAAC;QAE3C,mBAAmB;QACnB,MAAM,eAAe,GAAG,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,QAAQ,IAAI,EAAE,CAAC;QACtD,MAAM,QAAQ,GAAG,eAAe;aAC7B,MAAM,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,IAAI,CAAC,CAAC,IAAI,CAAC;aAC7C,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;aACZ,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC;YAChB,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,IAAI,WAAW;YACpC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE;YACvB,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC;SACzB,CAAC,CAAC,CAAC;QAEN,MAAM,WAAW,GAAG,QAAQ;aACzB,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,KAAK,UAAU,CAAC,CAAC,IAAI,EAAE,CAAC;aAC1D,IAAI,CAAC,MAAM,CAAC,CAAC;QAEhB,MAAM,OAAO,GAAG,QAAQ;YACtB,CAAC,CAAC,GAAG,QAAQ,mBAAmB,KAAK,MAAM,QAAQ,CAAC,MAAM,gBAAgB,WAAW,EAAE;YACvF,CAAC,CAAC,UAAU,KAAK,MAAM,QAAQ,CAAC,MAAM,gBAAgB,WAAW,EAAE,CAAC;QAEtE,MAAM,KAAK,GAA0C,EAAE,CAAC;QACxD,IAAI,QAAQ,CAAC,GAAG,IAAI,QAAQ,CAAC,GAAG,KAAK,QAAQ,CAAC,SAAS,EAAE,CAAC;YACxD,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,CAAC,GAAG,EAAE,CAAC,CAAC;QACnD,CAAC;QAED,OAAO;YACL,GAAG;YACH,KAAK;YACL,MAAM;YACN,WAAW,EAAE,KAAK,SAAS,cAAc,MAAM,KAAK,KAAK,UAAU;YACnE,OAAO;YACP,KAAK;YACL,MAAM,EAAE,EAAE;YACV,QAAQ,EAAE;gBACR,IAAI,EAAE,YAAY;gBAClB,WAAW,EAAE,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,WAAW,GAAG,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI;gBAC9F,MAAM,EAAE,aAAa;gBACrB,SAAS,EAAE,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,yBAAyB,QAAQ,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,IAAI;gBACpF,QAAQ,EAAE,QAAQ;aACnB;YACD,IAAI,EAAE,EAAE,MAAM,EAAE,cAAc,CAAC,OAAO,CAAC,EAAE;SAC1C,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,iBAAiB,CAAC,GAAW,EAAE,IAAS;IAC/C,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC;QAC1C,MAAM,KAAK,GAAG,QAAQ;aACnB,MAAM,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;aAC1B,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;aACZ,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC;YAChB,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,IAAI,CAAC,UAAU,IAAI,EAAE;YAC9C,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,IAAI,WAAW;YACpC,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC;YACxB,WAAW,EAAE,CAAC,CAAC,IAAI,CAAC,YAAY,IAAI,CAAC;YACrC,SAAS,EAAE,CAAC,CAAC,IAAI,CAAC,SAAS,IAAI,EAAE;YACjC,SAAS,EAAE,CAAC,CAAC,IAAI,CAAC,SAAS,IAAI,EAAE;SAClC,CAAC,CAAC,CAAC;QAEN,MAAM,OAAO,GAAG,KAAK;aAClB,GAAG,CAAC,CAAC,CAAM,EAAE,CAAS,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC,WAAW,mBAAmB,CAAC,CAAC,MAAM,EAAE,CAAC;aAC/G,IAAI,CAAC,IAAI,CAAC,CAAC;QAEd,MAAM,KAAK,GAAG,KAAK;aAChB,MAAM,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;aAC/B,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,EAAE,IAAI,EAAE,yBAAyB,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,CAAC;QAEtF,2CAA2C;QAC3C,MAAM,QAAQ,GAAG,GAAG,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;QAC3C,MAAM,SAAS,GAAG,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAChD,MAAM,SAAS,GAAG,GAAG,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;QAC/C,MAAM,IAAI,GAAG,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAE7C,MAAM,KAAK,GAAG,SAAS,CAAC,CAAC,CAAC,KAAK,SAAS,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC,gBAAgB,CAAC;QAEnF,OAAO;YACL,GAAG;YACH,KAAK;YACL,MAAM,EAAE,IAAI;YACZ,WAAW,EAAE,GAAG,KAAK,CAAC,MAAM,QAAQ;YACpC,OAAO;YACP,KAAK;YACL,MAAM,EAAE,EAAE;YACV,QAAQ,EAAE;gBACR,IAAI,EAAE,SAAS;gBACf,WAAW,EAAE,IAAI;gBACjB,MAAM,EAAE,aAAa;gBACrB,SAAS,EAAE,IAAI;gBACf,QAAQ,EAAE,QAAQ;aACnB;YACD,IAAI,EAAE,EAAE,MAAM,EAAE,cAAc,CAAC,OAAO,CAAC,EAAE;SAC1C,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC"}

package/dist/read/decoders/twitter.d.ts

@@ -0,0 +1,12 @@
+import type { Decoder } from '../types.js';
+/**
+ * Twitter/X decoder — uses fxtwitter.com public API as side channel.
+ *
+ * Handles:
+ *   - Individual tweets/posts (with text, media, quotes, articles)
+ *   - Profile URLs (basic profile info)
+ *
+ * fxtwitter API returns full tweet JSON including embedded articles,
+ * media URLs, quote tweets, and engagement metrics — all without auth.
+ */
+export declare const twitterDecoder: Decoder;