@apitap/core 1.0.0

package/dist/read/extract.js

@@ -0,0 +1,320 @@
+// src/read/extract.ts
+// ---- HTML entity decoding ----
+const ENTITY_MAP = {
+    '&': '&',
+    '&lt;': '<',
+    '&gt;': '>',
+    '&quot;': '"',
+    '&#39;': "'",
+    '&apos;': "'",
+    '&nbsp;': ' ',
+};
+function decodeEntities(text) {
+    return text.replace(/&(?:amp|lt|gt|quot|apos|nbsp|#39);/g, (m) => ENTITY_MAP[m] ?? m);
+}
+// ---- parseHead ----
+function extractMetaContent(html, attrName, attrValue) {
+    // Handle both orders: property="X" content="Y" and content="Y" property="X"
+    // Also handle name="X" content="Y" for author etc.
+    const patterns = [
+        new RegExp(`<meta\\s+${attrName}=["']${escapeRegex(attrValue)}["']\\s+content=["']([^"']*)["']`, 'i'),
+        new RegExp(`<meta\\s+content=["']([^"']*)["']\\s+${attrName}=["']${escapeRegex(attrValue)}["']`, 'i'),
+    ];
+    for (const re of patterns) {
+        const m = html.match(re);
+        if (m)
+            return decodeEntities(m[1]);
+    }
+    return null;
+}
+function escapeRegex(s) {
+    return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+export function parseHead(html) {
+    // Extract <title>
+    const titleMatch = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
+    const title = titleMatch ? decodeEntities(titleMatch[1].trim()) : null;
+    // Extract canonical
+    const canonicalMatch = html.match(/<link\s+[^>]*rel=["']canonical["'][^>]*href=["']([^"']*)["'][^>]*\/?>/i)
+        ?? html.match(/<link\s+[^>]*href=["']([^"']*)["'][^>]*rel=["']canonical["'][^>]*\/?>/i);
+    const canonical = canonicalMatch ? decodeEntities(canonicalMatch[1]) : null;
+    return {
+        title,
+        ogTitle: extractMetaContent(html, 'property', 'og:title'),
+        ogDescription: extractMetaContent(html, 'property', 'og:description'),
+        ogImage: extractMetaContent(html, 'property', 'og:image'),
+        ogType: extractMetaContent(html, 'property', 'og:type'),
+        ogSiteName: extractMetaContent(html, 'property', 'og:site_name'),
+        canonical,
+        author: extractMetaContent(html, 'name', 'author'),
+        publishedTime: extractMetaContent(html, 'property', 'article:published_time'),
+    };
+}
+// ---- extractContent ----
+/** Tags whose entire content (including children) should be removed */
+const NOISE_TAGS = ['script', 'style', 'noscript', 'svg', 'iframe', 'nav', 'header', 'footer', 'aside'];
+/** SPA shell markers */
+const SPA_MARKERS = [
+    '<div id="root"',
+    '<div id="app"',
+    '<div id="__next"',
+    'bundle.js',
+    'main.js',
+    'app.js',
+    '__NEXT_DATA__',
+    'window.__INITIAL_STATE__',
+    'window.__NUXT__',
+];
+function stripTags(html, tags) {
+    let result = html;
+    for (const tag of tags) {
+        // Use non-greedy match with dotAll behavior via [\s\S]
+        const re = new RegExp(`<${tag}[\\s>][\\s\\S]*?<\\/${tag}>`, 'gi');
+        result = result.replace(re, '');
+        // Also strip self-closing variants (e.g. <iframe ... />)
+        const selfClose = new RegExp(`<${tag}[^>]*/?>`, 'gi');
+        result = result.replace(selfClose, '');
+    }
+    return result;
+}
+function findContentRoot(html) {
+    // Priority order for content root
+    const selectors = [
+        { re: /<article[^>]*>([\s\S]*?)<\/article>/i },
+        { re: /<main[^>]*>([\s\S]*?)<\/main>/i },
+        { re: /<[^>]+role=["']main["'][^>]*>([\s\S]*?)<\/div>/i },
+    ];
+    for (const { re } of selectors) {
+        const m = html.match(re);
+        if (m)
+            return m[1];
+    }
+    // Class-based selectors
+    const classPatterns = [
+        /class=["'][^"']*\bpost-content\b/i,
+        /class=["'][^"']*\barticle-body\b/i,
+        /class=["'][^"']*\bentry-content\b/i,
+    ];
+    for (const cp of classPatterns) {
+        const m = html.match(cp);
+        if (m) {
+            // Find the enclosing tag and extract its content
+            const idx = m.index;
+            const extracted = extractTagContent(html, idx);
+            if (extracted)
+                return extracted;
+        }
+    }
+    // id="content"
+    const contentId = html.match(/id=["']content["']/i);
+    if (contentId) {
+        const extracted = extractTagContent(html, contentId.index);
+        if (extracted)
+            return extracted;
+    }
+    // Fallback: <body>
+    const bodyMatch = html.match(/<body[^>]*>([\s\S]*?)<\/body>/i);
+    if (bodyMatch)
+        return bodyMatch[1];
+    return html;
+}
+function extractTagContent(html, attrIndex) {
+    // Walk backwards to find the opening < of the tag
+    let tagStart = attrIndex;
+    while (tagStart > 0 && html[tagStart] !== '<')
+        tagStart--;
+    // Find the tag name
+    const tagNameMatch = html.slice(tagStart).match(/^<(\w+)/);
+    if (!tagNameMatch)
+        return null;
+    const tagName = tagNameMatch[1];
+    // Find matching close tag accounting for nesting
+    let depth = 1;
+    const openRe = new RegExp(`<${tagName}[\\s>]`, 'gi');
+    const closeRe = new RegExp(`</${tagName}>`, 'gi');
+    // Find where the opening tag ends (the first > after tagStart)
+    const openTagEnd = html.indexOf('>', tagStart);
+    if (openTagEnd === -1)
+        return null;
+    let pos = openTagEnd + 1;
+    const contentStart = pos;
+    while (depth > 0 && pos < html.length) {
+        openRe.lastIndex = pos;
+        closeRe.lastIndex = pos;
+        const nextOpen = openRe.exec(html);
+        const nextClose = closeRe.exec(html);
+        if (!nextClose)
+            break; // no more close tags
+        if (nextOpen && nextOpen.index < nextClose.index) {
+            depth++;
+            pos = nextOpen.index + nextOpen[0].length;
+        }
+        else {
+            depth--;
+            if (depth === 0) {
+                return html.slice(contentStart, nextClose.index);
+            }
+            pos = nextClose.index + nextClose[0].length;
+        }
+    }
+    return null;
+}
+function htmlToMarkdown(html, links, images) {
+    let md = html;
+    // Remove HTML comments
+    md = md.replace(/<!--[\s\S]*?-->/g, '');
+    // Convert headings
+    for (let level = 1; level <= 6; level++) {
+        const prefix = '#'.repeat(level);
+        const re = new RegExp(`<h${level}[^>]*>([\\s\\S]*?)<\\/h${level}>`, 'gi');
+        md = md.replace(re, (_m, content) => {
+            const text = stripAllTags(content).trim();
+            return `\n\n${prefix} ${text}\n\n`;
+        });
+    }
+    // Convert blockquotes (before paragraphs)
+    md = md.replace(/<blockquote[^>]*>([\s\S]*?)<\/blockquote>/gi, (_m, content) => {
+        const text = stripAllTags(content).trim();
+        const quoted = text.split('\n').map((l) => `> ${l}`).join('\n');
+        return `\n\n${quoted}\n\n`;
+    });
+    // Convert code blocks: <pre><code>...</code></pre>
+    md = md.replace(/<pre[^>]*>\s*<code[^>]*>([\s\S]*?)<\/code>\s*<\/pre>/gi, (_m, content) => {
+        const decoded = decodeEntities(content.trim());
+        return `\n\n\`\`\`\n${decoded}\n\`\`\`\n\n`;
+    });
+    // Convert standalone <pre> (without <code>)
+    md = md.replace(/<pre[^>]*>([\s\S]*?)<\/pre>/gi, (_m, content) => {
+        const decoded = decodeEntities(stripAllTags(content).trim());
+        return `\n\n\`\`\`\n${decoded}\n\`\`\`\n\n`;
+    });
+    // Convert inline code
+    md = md.replace(/<code[^>]*>([\s\S]*?)<\/code>/gi, (_m, content) => {
+        return `\`${decodeEntities(content)}\``;
+    });
+    // Convert tables
+    md = md.replace(/<table[^>]*>([\s\S]*?)<\/table>/gi, (_m, tableContent) => {
+        return convertTable(tableContent);
+    });
+    // Convert images (before stripping tags, so we can extract src/alt)
+    md = md.replace(/<img\s+[^>]*>/gi, (tag) => {
+        const srcMatch = tag.match(/src=["']([^"']*)["']/i);
+        const altMatch = tag.match(/alt=["']([^"']*)["']/i);
+        const src = srcMatch ? decodeEntities(srcMatch[1]) : '';
+        const alt = altMatch ? decodeEntities(altMatch[1]) : '';
+        if (src) {
+            images.push({ alt, src });
+            return `![${alt}](${src})`;
+        }
+        return '';
+    });
+    // Convert links
+    md = md.replace(/<a\s+[^>]*href=["']([^"']*)["'][^>]*>([\s\S]*?)<\/a>/gi, (_m, href, content) => {
+        const text = stripAllTags(content).trim();
+        const decodedHref = decodeEntities(href);
+        if (text && decodedHref) {
+            links.push({ text, href: decodedHref });
+            return `[${text}](${decodedHref})`;
+        }
+        return text;
+    });
+    // Convert bold
+    md = md.replace(/<(?:strong|b)[^>]*>([\s\S]*?)<\/(?:strong|b)>/gi, (_m, content) => {
+        return `**${stripAllTags(content)}**`;
+    });
+    // Convert italic
+    md = md.replace(/<(?:em|i)(?:\s[^>]*)?>(?!mg)([\s\S]*?)<\/(?:em|i)>/gi, (_m, content) => {
+        return `*${stripAllTags(content)}*`;
+    });
+    // Convert ordered lists
+    md = md.replace(/<ol[^>]*>([\s\S]*?)<\/ol>/gi, (_m, listContent) => {
+        let counter = 0;
+        const items = listContent.replace(/<li[^>]*>([\s\S]*?)<\/li>/gi, (_lm, item) => {
+            counter++;
+            return `${counter}. ${stripAllTags(item).trim()}\n`;
+        });
+        return `\n\n${stripAllTags(items).trim()}\n\n`;
+    });
+    // Convert unordered lists
+    md = md.replace(/<ul[^>]*>([\s\S]*?)<\/ul>/gi, (_m, listContent) => {
+        const items = listContent.replace(/<li[^>]*>([\s\S]*?)<\/li>/gi, (_lm, item) => {
+            return `- ${stripAllTags(item).trim()}\n`;
+        });
+        return `\n\n${stripAllTags(items).trim()}\n\n`;
+    });
+    // Convert paragraphs
+    md = md.replace(/<p[^>]*>([\s\S]*?)<\/p>/gi, (_m, content) => {
+        return `\n\n${content.trim()}\n\n`;
+    });
+    // Convert <br> tags
+    md = md.replace(/<br\s*\/?>/gi, '\n');
+    // Strip remaining HTML tags
+    md = stripAllTags(md);
+    // Decode entities
+    md = decodeEntities(md);
+    // Collapse whitespace: no more than 2 consecutive newlines
+    md = md.replace(/\n{3,}/g, '\n\n');
+    // Trim leading/trailing whitespace
+    md = md.trim();
+    return md;
+}
+function stripAllTags(html) {
+    return html.replace(/<[^>]*>/g, '');
+}
+function convertTable(tableHtml) {
+    const rows = [];
+    // Extract rows
+    const rowMatches = tableHtml.match(/<tr[^>]*>[\s\S]*?<\/tr>/gi) || [];
+    for (const row of rowMatches) {
+        const cells = [];
+        const cellMatches = row.match(/<(?:td|th)[^>]*>[\s\S]*?<\/(?:td|th)>/gi) || [];
+        for (const cell of cellMatches) {
+            const content = cell.replace(/<\/?(?:td|th)[^>]*>/gi, '');
+            cells.push(stripAllTags(content).trim());
+        }
+        if (cells.length > 0)
+            rows.push(cells);
+    }
+    if (rows.length === 0)
+        return '';
+    // Normalize column count
+    const maxCols = Math.max(...rows.map((r) => r.length));
+    const normalized = rows.map((r) => {
+        while (r.length < maxCols)
+            r.push('');
+        return r;
+    });
+    // Build markdown table
+    const lines = [];
+    const header = normalized[0];
+    lines.push('| ' + header.join(' | ') + ' |');
+    lines.push('| ' + header.map(() => '---').join(' | ') + ' |');
+    for (let i = 1; i < normalized.length; i++) {
+        lines.push('| ' + normalized[i].join(' | ') + ' |');
+    }
+    return '\n\n' + lines.join('\n') + '\n\n';
+}
+function getTextContent(html) {
+    return stripAllTags(html).replace(/\s+/g, ' ').trim();
+}
+export function extractContent(html) {
+    const links = [];
+    const images = [];
+    // Strip noise tags first
+    const cleaned = stripTags(html, NOISE_TAGS);
+    // Find content root
+    const contentHtml = findContentRoot(cleaned);
+    // Convert to markdown
+    const content = htmlToMarkdown(contentHtml, links, images);
+    // Detect SPA shell
+    const textContent = getTextContent(contentHtml);
+    const hasSpaMarker = SPA_MARKERS.some((marker) => html.includes(marker));
+    const isSpaShell = textContent.length < 200 && hasSpaMarker;
+    return {
+        content,
+        links,
+        images,
+        isSpaShell,
+    };
+}
+//# sourceMappingURL=extract.js.map

package/dist/read/extract.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"extract.js","sourceRoot":"","sources":["../../src/read/extract.ts"],"names":[],"mappings":"AAAA,sBAAsB;AAqBtB,iCAAiC;AAEjC,MAAM,UAAU,GAA2B;IACzC,OAAO,EAAE,GAAG;IACZ,MAAM,EAAE,GAAG;IACX,MAAM,EAAE,GAAG;IACX,QAAQ,EAAE,GAAG;IACb,OAAO,EAAE,GAAG;IACZ,QAAQ,EAAE,GAAG;IACb,QAAQ,EAAE,GAAG;CACd,CAAC;AAEF,SAAS,cAAc,CAAC,IAAY;IAClC,OAAO,IAAI,CAAC,OAAO,CAAC,qCAAqC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;AACxF,CAAC;AAED,sBAAsB;AAEtB,SAAS,kBAAkB,CAAC,IAAY,EAAE,QAAgB,EAAE,SAAiB;IAC3E,4EAA4E;IAC5E,mDAAmD;IACnD,MAAM,QAAQ,GAAG;QACf,IAAI,MAAM,CAAC,YAAY,QAAQ,QAAQ,WAAW,CAAC,SAAS,CAAC,kCAAkC,EAAE,GAAG,CAAC;QACrG,IAAI,MAAM,CAAC,wCAAwC,QAAQ,QAAQ,WAAW,CAAC,SAAS,CAAC,MAAM,EAAE,GAAG,CAAC;KACtG,CAAC;IACF,KAAK,MAAM,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1B,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACzB,IAAI,CAAC;YAAE,OAAO,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACrC,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,WAAW,CAAC,CAAS;IAC5B,OAAO,CAAC,CAAC,OAAO,CAAC,qBAAqB,EAAE,MAAM,CAAC,CAAC;AAClD,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,IAAY;IACpC,kBAAkB;IAClB,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAC;IAClE,MAAM,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAEvE,oBAAoB;IACpB,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,wEAAwE,CAAC;WACtG,IAAI,CAAC,KAAK,CAAC,wEAAwE,CAAC,CAAC;IAC1F,MAAM,SAAS,GAAG,cAAc,CAAC,CAAC,CAAC,cAAc,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAE5E,OAAO;QACL,KAAK;QACL,OAAO,EAAE,kBAAkB,CAAC,IAAI,EAAE,UAAU,EAAE,UAAU,CAAC;QACzD,aAAa,EAAE,kBAAkB,CAAC,IAAI,EAAE,UAAU,EAAE,gBAAgB,CAAC;QACrE,OAAO,EAAE,kBAAkB,CAAC,IAAI,EAAE,UAAU,EAAE,UAAU,CAAC;QACzD,MAAM,EAAE,kBAAkB,CAAC,IAAI,EAAE,UAAU,EAAE,SAAS,CAAC;QACvD,UAAU,EAAE,kBAAkB,CAAC,IAAI,EAAE,UAAU,EAAE,cAAc,CAAC;QAChE,SAAS;QACT,MAAM,EAAE,kBAAkB,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC;QAClD,aAAa,EAAE,kBAAkB,CAAC,IAAI,EAAE,UAAU,EAAE,wBAAwB,CAAC;KAC9E,CAAC;AACJ,CAAC;AAED,2BAA2B;AAE3B,uEAAuE;AACvE,MAAM,UAAU,GAAG,CAAC,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;AAExG,wBAAwB;AACxB,MAAM,WAAW,GAAG;IAClB,gBAAgB;IAChB,eAAe;IACf,kBAAkB;IAClB,WAAW;IACX,SAAS;IACT,QAAQ;IACR,eAAe;IACf,0BAA0B;IAC1B,iBAAiB;CAClB,CAAC;AAEF,SAAS,SAAS,CAAC,IAAY,EAAE,IAAc;IAC7C,IAAI,MAAM,GAAG,IAAI,CAAC;IAClB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,uDAAuD;QACvD,MAAM,EAAE,GAAG,IAAI,MAAM,CAAC,IAAI,GAAG,uBAAuB,GAAG,GAAG,EAAE,IAAI,CAAC,CAAC;QAClE,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;QAChC,yDAAyD;QACzD,MAAM,SAAS,GAAG,IAAI,MAAM,CAAC,IAAI,GAAG,UAAU,EAAE,IAAI,CAAC,CAAC;QACtD,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IACzC,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,eAAe,CAAC,IAAY;IACnC,kCAAkC;IAClC,MAAM,SAAS,GAA0B;QACvC,EAAE,EAAE,EAAE,sCAAsC,EAAE;QAC9C,EAAE,EAAE,EAAE,gCAAgC,EAAE;QACxC,EAAE,EAAE,EAAE,iDAAiD,EAAE;KAC1D,CAAC;IAEF,KAAK,MAAM,EAAE,EAAE,EAAE,IAAI,SAAS,EAAE,CAAC;QAC/B,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACzB,IAAI,CAAC;YAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;IACrB,CAAC;IAED,wBAAwB;IACxB,MAAM,aAAa,GAAG;QACpB,mCAAmC;QACnC,mCAAmC;QACnC,oCAAoC;KACrC,CAAC;IAEF,KAAK,MAAM,EAAE,IAAI,aAAa,EAAE,CAAC;QAC/B,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACzB,IAAI,CAAC,EAAE,CAAC;YACN,iDAAiD;YACjD,MAAM,GAAG,GAAG,CAAC,CAAC,KAAM,CAAC;YACrB,MAAM,SAAS,GAAG,iBAAiB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YAC/C,IAAI,SAAS;gBAAE,OAAO,SAAS,CAAC;QAClC,CAAC;IACH,CAAC;IAED,eAAe;IACf,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;IACpD,IAAI,SAAS,EAAE,CAAC;QACd,MAAM,SAAS,GAAG,iBAAiB,CAAC,IAAI,EAAE,SAAS,CAAC,KAAM,CAAC,CAAC;QAC5D,IAAI,SAAS;YAAE,OAAO,SAAS,CAAC;IAClC,CAAC;IAED,mBAAmB;IACnB,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;IAC/D,IAAI,SAAS;QAAE,OAAO,SAAS,CAAC,CAAC,CAAC,CAAC;IAEnC,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,iBAAiB,CAAC,IAAY,EAAE,SAAiB;IACxD,kDAAkD;IAClD,IAAI,QAAQ,GAAG,SAAS,CAAC;IACzB,OAAO,QAAQ,GAAG,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,KAAK,GAAG;QAAE,QAAQ,EAAE,CAAC;IAE1D,oBAAoB;IACpB,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IAC3D,IAAI,CAAC,YAAY;QAAE,OAAO,IAAI,CAAC;IAE/B,MAAM,OAAO,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;IAEhC,iDAAiD;IACjD,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,OAAO,QAAQ,EAAE,IAAI,CAAC,CAAC;IACrD,MAAM,OAAO,GAAG,IAAI,MAAM,CAAC,KAAK,OAAO,GAAG,EAAE,IAAI,CAAC,CAAC;IAElD,+DAA+D;IAC/D,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAC/C,IAAI,UAAU,KAAK,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IAEnC,IAAI,GAAG,GAAG,UAAU,GAAG,CAAC,CAAC;IACzB,MAAM,YAAY,GAAG,GAAG,CAAC;IAEzB,OAAO,KAAK,GAAG,CAAC,IAAI,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QACtC,MAAM,CAAC,SAAS,GAAG,GAAG,CAAC;QACvB,OAAO,CAAC,SAAS,GAAG,GAAG,CAAC;QAExB,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnC,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAErC,IAAI,CAAC,SAAS;YAAE,MAAM,CAAC,qBAAqB;QAE5C,IAAI,QAAQ,IAAI,QAAQ,CAAC,KAAK,GAAG,SAAS,CAAC,KAAK,EAAE,CAAC;YACjD,KAAK,EAAE,CAAC;YACR,GAAG,GAAG,QAAQ,CAAC,KAAK,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QAC5C,CAAC;aAAM,CAAC;YACN,KAAK,EAAE,CAAC;YACR,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;gBAChB,OAAO,IAAI,CAAC,KAAK,CAAC,YAAY,EAAE,SAAS,CAAC,KAAK,CAAC,CAAC;YACnD,CAAC;YACD,GAAG,GAAG,SAAS,CAAC,KAAK,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QAC9C,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,cAAc,CACrB,IAAY,EACZ,KAA4C,EAC5C,MAA2C;IAE3C,IAAI,EAAE,GAAG,IAAI,CAAC;IAEd,uBAAuB;IACvB,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC;IAExC,mBAAmB;IACnB,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC;QACxC,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACjC,MAAM,EAAE,GAAG,IAAI,MAAM,CAAC,KAAK,KAAK,0BAA0B,KAAK,GAAG,EAAE,IAAI,CAAC,CAAC;QAC1E,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,EAAE,EAAE,CAAC,EAAE,EAAE,OAAO,EAAE,EAAE;YAClC,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;YAC1C,OAAO,OAAO,MAAM,IAAI,IAAI,MAAM,CAAC;QACrC,CAAC,CAAC,CAAC;IACL,CAAC;IAED,0CAA0C;IAC1C,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,6CAA6C,EAAE,CAAC,EAAE,EAAE,OAAO,EAAE,EAAE;QAC7E,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;QAC1C,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxE,OAAO,OAAO,MAAM,MAAM,CAAC;IAC7B,CAAC,CAAC,CAAC;IAEH,mDAAmD;IACnD,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,wDAAwD,EAAE,CAAC,EAAE,EAAE,OAAO,EAAE,EAAE;QACxF,MAAM,OAAO,GAAG,cAAc,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;QAC/C,OAAO,eAAe,OAAO,cAAc,CAAC;IAC9C,CAAC,CAAC,CAAC;IAEH,4CAA4C;IAC5C,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,+BAA+B,EAAE,CAAC,EAAE,EAAE,OAAO,EAAE,EAAE;QAC/D,MAAM,OAAO,GAAG,cAAc,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAC7D,OAAO,eAAe,OAAO,cAAc,CAAC;IAC9C,CAAC,CAAC,CAAC;IAEH,sBAAsB;IACtB,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,iCAAiC,EAAE,CAAC,EAAE,EAAE,OAAO,EAAE,EAAE;QACjE,OAAO,KAAK,cAAc,CAAC,OAAO,CAAC,IAAI,CAAC;IAC1C,CAAC,CAAC,CAAC;IAEH,iBAAiB;IACjB,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,mCAAmC,EAAE,CAAC,EAAE,EAAE,YAAY,EAAE,EAAE;QACxE,OAAO,YAAY,CAAC,YAAY,CAAC,CAAC;IACpC,CAAC,CAAC,CAAC;IAEH,oEAAoE;IACpE,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,iBAAiB,EAAE,CAAC,GAAG,EAAE,EAAE;QACzC,MAAM,QAAQ,GAAG,GAAG,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAC;QACpD,MAAM,QAAQ,GAAG,GAAG,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAC;QACpD,MAAM,GAAG,GAAG,QAAQ,CAAC,CAAC,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACxD,MAAM,GAAG,GAAG,QAAQ,CAAC,CAAC,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACxD,IAAI,GAAG,EAAE,CAAC;YACR,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;YAC1B,OAAO,KAAK,GAAG,KAAK,GAAG,GAAG,CAAC;QAC7B,CAAC;QACD,OAAO,EAAE,CAAC;IACZ,CAAC,CAAC,CAAC;IAEH,gBAAgB;IAChB,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,wDAAwD,EAAE,CAAC,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE;QAC9F,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;QAC1C,MAAM,WAAW,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;QACzC,IAAI,IAAI,IAAI,WAAW,EAAE,CAAC;YACxB,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC;YACxC,OAAO,IAAI,IAAI,KAAK,WAAW,GAAG,CAAC;QACrC,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,eAAe;IACf,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,iDAAiD,EAAE,CAAC,EAAE,EAAE,OAAO,EAAE,EAAE;QACjF,OAAO,KAAK,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC;IACxC,CAAC,CAAC,CAAC;IAEH,iBAAiB;IACjB,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,sDAAsD,EAAE,CAAC,EAAE,EAAE,OAAO,EAAE,EAAE;QACtF,OAAO,IAAI,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC;IACtC,CAAC,CAAC,CAAC;IAEH,wBAAwB;IACxB,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,6BAA6B,EAAE,CAAC,EAAE,EAAE,WAAW,EAAE,EAAE;QACjE,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,MAAM,KAAK,GAAG,WAAW,CAAC,OAAO,CAAC,6BAA6B,EAAE,CAAC,GAAW,EAAE,IAAY,EAAE,EAAE;YAC7F,OAAO,EAAE,CAAC;YACV,OAAO,GAAG,OAAO,KAAK,YAAY,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC;QACtD,CAAC,CAAC,CAAC;QACH,OAAO,OAAO,YAAY,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC;IACjD,CAAC,CAAC,CAAC;IAEH,0BAA0B;IAC1B,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,6BAA6B,EAAE,CAAC,EAAE,EAAE,WAAW,EAAE,EAAE;QACjE,MAAM,KAAK,GAAG,WAAW,CAAC,OAAO,CAAC,6BAA6B,EAAE,CAAC,GAAW,EAAE,IAAY,EAAE,EAAE;YAC7F,OAAO,KAAK,YAAY,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC;QAC5C,CAAC,CAAC,CAAC;QACH,OAAO,OAAO,YAAY,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC;IACjD,CAAC,CAAC,CAAC;IAEH,qBAAqB;IACrB,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,2BAA2B,EAAE,CAAC,EAAE,EAAE,OAAO,EAAE,EAAE;QAC3D,OAAO,OAAO,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC;IACrC,CAAC,CAAC,CAAC;IAEH,oBAAoB;IACpB,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,cAAc,EAAE,IAAI,CAAC,CAAC;IAEtC,4BAA4B;IAC5B,EAAE,GAAG,YAAY,CAAC,EAAE,CAAC,CAAC;IAEtB,kBAAkB;IAClB,EAAE,GAAG,cAAc,CAAC,EAAE,CAAC,CAAC;IAExB,2DAA2D;IAC3D,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAEnC,mCAAmC;IACnC,EAAE,GAAG,EAAE,CAAC,IAAI,EAAE,CAAC;IAEf,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,YAAY,CAAC,IAAY;IAChC,OAAO,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;AACtC,CAAC;AAED,SAAS,YAAY,CAAC,SAAiB;IACrC,MAAM,IAAI,GAAe,EAAE,CAAC;IAE5B,eAAe;IACf,MAAM,UAAU,GAAG,SAAS,CAAC,KAAK,CAAC,2BAA2B,CAAC,IAAI,EAAE,CAAC;IACtE,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;QAC7B,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,MAAM,WAAW,GAAG,GAAG,CAAC,KAAK,CAAC,yCAAyC,CAAC,IAAI,EAAE,CAAC;QAC/E,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;YAC/B,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,uBAAuB,EAAE,EAAE,CAAC,CAAC;YAC1D,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAC3C,CAAC;QACD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;YAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACzC,CAAC;IAED,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAEjC,yBAAyB;IACzB,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;IACvD,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QAChC,OAAO,CAAC,CAAC,MAAM,GAAG,OAAO;YAAE,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACtC,OAAO,CAAC,CAAC;IACX,CAAC,CAAC,CAAC;IAEH,uBAAuB;IACvB,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,MAAM,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;IAC7B,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,CAAC;IAC7C,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,CAAC;IAE9D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3C,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,CAAC;IACtD,CAAC;IAED,OAAO,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC;AAC5C,CAAC;AAED,SAAS,cAAc,CAAC,IAAY;IAClC,OAAO,YAAY,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;AACxD,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,MAAM,KAAK,GAA0C,EAAE,CAAC;IACxD,MAAM,MAAM,GAAwC,EAAE,CAAC;IAEvD,yBAAyB;IACzB,MAAM,OAAO,GAAG,SAAS,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;IAE5C,oBAAoB;IACpB,MAAM,WAAW,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;IAE7C,sBAAsB;IACtB,MAAM,OAAO,GAAG,cAAc,CAAC,WAAW,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;IAE3D,mBAAmB;IACnB,MAAM,WAAW,GAAG,cAAc,CAAC,WAAW,CAAC,CAAC;IAChD,MAAM,YAAY,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;IACzE,MAAM,UAAU,GAAG,WAAW,CAAC,MAAM,GAAG,GAAG,IAAI,YAAY,CAAC;IAE5D,OAAO;QACL,OAAO;QACP,KAAK;QACL,MAAM;QACN,UAAU;KACX,CAAC;AACJ,CAAC"}

package/dist/read/index.d.ts

@@ -0,0 +1,14 @@
+export { peek } from './peek.js';
+export type { PeekOptions } from './peek.js';
+export type { PeekResult, ReadResult, Decoder } from './types.js';
+import type { ReadResult } from './types.js';
+export interface ReadOptions {
+    skipSsrf?: boolean;
+    maxBytes?: number;
+}
+/**
+ * Universal content decoder. Routes to site-specific decoders for known sites
+ * (Reddit, YouTube, Wikipedia, HN), falls back to generic HTML extraction.
+ * Returns null if content cannot be extracted.
+ */
+export declare function read(url: string, options?: ReadOptions): Promise<ReadResult | null>;

package/dist/read/index.js

@@ -0,0 +1,66 @@
+// src/read/index.ts
+export { peek } from './peek.js';
+import { safeFetch } from '../discovery/fetch.js';
+import { findDecoder } from './decoders/index.js';
+import { parseHead, extractContent } from './extract.js';
+/**
+ * Universal content decoder. Routes to site-specific decoders for known sites
+ * (Reddit, YouTube, Wikipedia, HN), falls back to generic HTML extraction.
+ * Returns null if content cannot be extracted.
+ */
+export async function read(url, options = {}) {
+    // Try site-specific decoder first
+    const decoder = findDecoder(url);
+    if (decoder) {
+        const result = await decoder.decode(url, { skipSsrf: options.skipSsrf });
+        if (result) {
+            if (options.maxBytes && result.content.length > options.maxBytes) {
+                result.content = result.content.slice(0, options.maxBytes);
+                result.cost.tokens = Math.ceil(result.content.length / 4);
+            }
+            return result;
+        }
+        // Decoder returned null -- fall through to generic
+    }
+    // Generic pipeline: fetch HTML -> parse head -> extract body
+    const fetchResult = await safeFetch(url, { skipSsrf: options.skipSsrf });
+    if (!fetchResult || fetchResult.status !== 200)
+        return null;
+    const html = fetchResult.body;
+    const head = parseHead(html);
+    const body = extractContent(html);
+    // Determine source
+    let source;
+    if (body.isSpaShell) {
+        source = 'spa-shell';
+    }
+    else if (body.content.trim().length === 0) {
+        source = 'og-tags-only';
+    }
+    else {
+        source = 'readability';
+    }
+    let content = body.content;
+    if (options.maxBytes && content.length > options.maxBytes) {
+        content = content.slice(0, options.maxBytes);
+    }
+    const title = head.ogTitle || head.title || null;
+    return {
+        url,
+        title,
+        author: head.author || null,
+        description: head.ogDescription || null,
+        content,
+        links: body.links,
+        images: body.images,
+        metadata: {
+            type: head.ogType || 'unknown',
+            publishedAt: head.publishedTime || null,
+            source,
+            canonical: head.canonical || null,
+            siteName: head.ogSiteName || null,
+        },
+        cost: { tokens: Math.ceil(content.length / 4) },
+    };
+}
+//# sourceMappingURL=index.js.map

package/dist/read/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/read/index.ts"],"names":[],"mappings":"AAAA,oBAAoB;AACpB,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAKjC,OAAO,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAClD,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAClD,OAAO,EAAE,SAAS,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAOzD;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,IAAI,CAAC,GAAW,EAAE,UAAuB,EAAE;IAC/D,kCAAkC;IAClC,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC;IACjC,IAAI,OAAO,EAAE,CAAC;QACZ,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,MAAM,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;QACzE,IAAI,MAAM,EAAE,CAAC;YACX,IAAI,OAAO,CAAC,QAAQ,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;gBACjE,MAAM,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;gBAC3D,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAC5D,CAAC;YACD,OAAO,MAAM,CAAC;QAChB,CAAC;QACD,mDAAmD;IACrD,CAAC;IAED,6DAA6D;IAC7D,MAAM,WAAW,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;IACzE,IAAI,CAAC,WAAW,IAAI,WAAW,CAAC,MAAM,KAAK,GAAG;QAAE,OAAO,IAAI,CAAC;IAE5D,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC;IAC9B,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IAC7B,MAAM,IAAI,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;IAElC,mBAAmB;IACnB,IAAI,MAAc,CAAC;IACnB,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;QACpB,MAAM,GAAG,WAAW,CAAC;IACvB,CAAC;SAAM,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5C,MAAM,GAAG,cAAc,CAAC;IAC1B,CAAC;SAAM,CAAC;QACN,MAAM,GAAG,aAAa,CAAC;IACzB,CAAC;IAED,IAAI,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;IAC3B,IAAI,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC,MAAM,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;QAC1D,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC/C,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC;IAEjD,OAAO;QACL,GAAG;QACH,KAAK;QACL,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,IAAI;QAC3B,WAAW,EAAE,IAAI,CAAC,aAAa,IAAI,IAAI;QACvC,OAAO;QACP,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,QAAQ,EAAE;YACR,IAAI,EAAE,IAAI,CAAC,MAAM,IAAI,SAAS;YAC9B,WAAW,EAAE,IAAI,CAAC,aAAa,IAAI,IAAI;YACvC,MAAM;YACN,SAAS,EAAE,IAAI,CAAC,SAAS,IAAI,IAAI;YACjC,QAAQ,EAAE,IAAI,CAAC,UAAU,IAAI,IAAI;SAClC;QACD,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE;KAChD,CAAC;AACJ,CAAC"}

package/dist/read/peek.d.ts

@@ -0,0 +1,9 @@
+import type { PeekResult } from './types.js';
+export interface PeekOptions {
+    skipSsrf?: boolean;
+}
+/**
+ * HTTP HEAD-only triage: checks accessibility, detects bot protection and frameworks.
+ * Falls back to GET if HEAD fails.
+ */
+export declare function peek(url: string, options?: PeekOptions): Promise<PeekResult>;

package/dist/read/peek.js

@@ -0,0 +1,137 @@
+import { safeFetch } from '../discovery/fetch.js';
+/**
+ * HTTP HEAD-only triage: checks accessibility, detects bot protection and frameworks.
+ * Falls back to GET if HEAD fails.
+ */
+export async function peek(url, options = {}) {
+    const signals = [];
+    // Try HEAD first
+    let result = await safeFetch(url, {
+        method: 'HEAD',
+        skipSsrf: options.skipSsrf,
+    });
+    // Fall back to GET if HEAD fails (null = network/SSRF error)
+    if (!result) {
+        result = await safeFetch(url, {
+            method: 'GET',
+            skipSsrf: options.skipSsrf,
+        });
+    }
+    // Both HEAD and GET failed
+    if (!result) {
+        return {
+            url,
+            status: 0,
+            accessible: false,
+            contentType: null,
+            server: null,
+            framework: null,
+            botProtection: null,
+            signals: ['fetch failed'],
+            recommendation: 'blocked',
+        };
+    }
+    const { status, headers } = result;
+    // Extract basic metadata
+    const contentType = headers['content-type'] || null;
+    const server = headers['server'] || null;
+    // Detect bot protection
+    const botProtection = detectBotProtection(headers, signals);
+    // Detect framework
+    const framework = detectFramework(headers, signals);
+    // Determine accessibility and recommendation
+    const accessible = status >= 200 && status < 400 && !botProtection;
+    const recommendation = computeRecommendation(status, botProtection);
+    return {
+        url,
+        status,
+        accessible,
+        contentType,
+        server,
+        framework,
+        botProtection,
+        signals,
+        recommendation,
+    };
+}
+function detectBotProtection(headers, signals) {
+    // Cloudflare: cf-ray or cf-cache-status
+    if (headers['cf-ray']) {
+        signals.push('cf-ray header');
+        return 'cloudflare';
+    }
+    if (headers['cf-cache-status']) {
+        signals.push('cf-cache-status header');
+        return 'cloudflare';
+    }
+    // PerimeterX: x-px-* headers
+    for (const key of Object.keys(headers)) {
+        if (key.startsWith('x-px-')) {
+            signals.push(`${key} header`);
+            return 'perimeterx';
+        }
+    }
+    // DataDome: x-datadome* headers
+    for (const key of Object.keys(headers)) {
+        if (key.startsWith('x-datadome')) {
+            signals.push(`${key} header`);
+            return 'datadome';
+        }
+    }
+    return null;
+}
+function detectFramework(headers, signals) {
+    // Next.js: x-powered-by: Next.js
+    const poweredBy = headers['x-powered-by'];
+    if (poweredBy && /next\.js/i.test(poweredBy)) {
+        signals.push('x-powered-by: Next.js');
+        return 'next.js';
+    }
+    // Express: x-powered-by: Express
+    if (poweredBy && /express/i.test(poweredBy)) {
+        signals.push('x-powered-by: Express');
+        return 'express';
+    }
+    // PHP: x-powered-by: PHP/*
+    if (poweredBy && /php/i.test(poweredBy)) {
+        signals.push('x-powered-by: PHP');
+        return 'php';
+    }
+    // WordPress: link header containing api.w.org
+    const link = headers['link'];
+    if (link && link.includes('api.w.org')) {
+        signals.push('link: api.w.org');
+        return 'wordpress';
+    }
+    // Shopify: x-shopify-stage header
+    if (headers['x-shopify-stage']) {
+        signals.push('x-shopify-stage header');
+        return 'shopify';
+    }
+    // Drupal: x-drupal-* headers
+    for (const key of Object.keys(headers)) {
+        if (key.startsWith('x-drupal-')) {
+            signals.push(`${key} header`);
+            return 'drupal';
+        }
+    }
+    return null;
+}
+function computeRecommendation(status, botProtection) {
+    // Auth required
+    if (status === 401 || status === 407) {
+        return 'auth_required';
+    }
+    // Blocked: bot protection, 403, 429, or 5xx
+    if (botProtection) {
+        return 'blocked';
+    }
+    if (status === 403 || status === 429) {
+        return 'blocked';
+    }
+    if (status >= 500) {
+        return 'blocked';
+    }
+    return 'read';
+}
+//# sourceMappingURL=peek.js.map

package/dist/read/peek.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"peek.js","sourceRoot":"","sources":["../../src/read/peek.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAMlD;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,IAAI,CAAC,GAAW,EAAE,UAAuB,EAAE;IAC/D,MAAM,OAAO,GAAa,EAAE,CAAC;IAE7B,iBAAiB;IACjB,IAAI,MAAM,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE;QAChC,MAAM,EAAE,MAAM;QACd,QAAQ,EAAE,OAAO,CAAC,QAAQ;KAC3B,CAAC,CAAC;IAEH,6DAA6D;IAC7D,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE;YAC5B,MAAM,EAAE,KAAK;YACb,QAAQ,EAAE,OAAO,CAAC,QAAQ;SAC3B,CAAC,CAAC;IACL,CAAC;IAED,2BAA2B;IAC3B,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,OAAO;YACL,GAAG;YACH,MAAM,EAAE,CAAC;YACT,UAAU,EAAE,KAAK;YACjB,WAAW,EAAE,IAAI;YACjB,MAAM,EAAE,IAAI;YACZ,SAAS,EAAE,IAAI;YACf,aAAa,EAAE,IAAI;YACnB,OAAO,EAAE,CAAC,cAAc,CAAC;YACzB,cAAc,EAAE,SAAS;SAC1B,CAAC;IACJ,CAAC;IAED,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,CAAC;IAEnC,yBAAyB;IACzB,MAAM,WAAW,GAAG,OAAO,CAAC,cAAc,CAAC,IAAI,IAAI,CAAC;IACpD,MAAM,MAAM,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC;IAEzC,wBAAwB;IACxB,MAAM,aAAa,GAAG,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAE5D,mBAAmB;IACnB,MAAM,SAAS,GAAG,eAAe,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAEpD,6CAA6C;IAC7C,MAAM,UAAU,GAAG,MAAM,IAAI,GAAG,IAAI,MAAM,GAAG,GAAG,IAAI,CAAC,aAAa,CAAC;IACnE,MAAM,cAAc,GAAG,qBAAqB,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;IAEpE,OAAO;QACL,GAAG;QACH,MAAM;QACN,UAAU;QACV,WAAW;QACX,MAAM;QACN,SAAS;QACT,aAAa;QACb,OAAO;QACP,cAAc;KACf,CAAC;AACJ,CAAC;AAED,SAAS,mBAAmB,CAC1B,OAA+B,EAC/B,OAAiB;IAEjB,wCAAwC;IACxC,IAAI,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;QACtB,OAAO,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAC9B,OAAO,YAAY,CAAC;IACtB,CAAC;IACD,IAAI,OAAO,CAAC,iBAAiB,CAAC,EAAE,CAAC;QAC/B,OAAO,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;QACvC,OAAO,YAAY,CAAC;IACtB,CAAC;IAED,6BAA6B;IAC7B,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QACvC,IAAI,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YAC5B,OAAO,CAAC,IAAI,CAAC,GAAG,GAAG,SAAS,CAAC,CAAC;YAC9B,OAAO,YAAY,CAAC;QACtB,CAAC;IACH,CAAC;IAED,gCAAgC;IAChC,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QACvC,IAAI,GAAG,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;YACjC,OAAO,CAAC,IAAI,CAAC,GAAG,GAAG,SAAS,CAAC,CAAC;YAC9B,OAAO,UAAU,CAAC;QACpB,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,eAAe,CACtB,OAA+B,EAC/B,OAAiB;IAEjB,iCAAiC;IACjC,MAAM,SAAS,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;IAC1C,IAAI,SAAS,IAAI,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;QAC7C,OAAO,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;QACtC,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,iCAAiC;IACjC,IAAI,SAAS,IAAI,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;QAC5C,OAAO,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;QACtC,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,2BAA2B;IAC3B,IAAI,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;QACxC,OAAO,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;QAClC,OAAO,KAAK,CAAC;IACf,CAAC;IAED,8CAA8C;IAC9C,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAC7B,IAAI,IAAI,IAAI,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;QACvC,OAAO,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QAChC,OAAO,WAAW,CAAC;IACrB,CAAC;IAED,kCAAkC;IAClC,IAAI,OAAO,CAAC,iBAAiB,CAAC,EAAE,CAAC;QAC/B,OAAO,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;QACvC,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,6BAA6B;IAC7B,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QACvC,IAAI,GAAG,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;YAChC,OAAO,CAAC,IAAI,CAAC,GAAG,GAAG,SAAS,CAAC,CAAC;YAC9B,OAAO,QAAQ,CAAC;QAClB,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,qBAAqB,CAC5B,MAAc,EACd,aAA4B;IAE5B,gBAAgB;IAChB,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;QACrC,OAAO,eAAe,CAAC;IACzB,CAAC;IAED,4CAA4C;IAC5C,IAAI,aAAa,EAAE,CAAC;QAClB,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;QACrC,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,IAAI,MAAM,IAAI,GAAG,EAAE,CAAC;QAClB,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/read/types.d.ts

@@ -0,0 +1,44 @@
+export interface PeekResult {
+    url: string;
+    status: number;
+    accessible: boolean;
+    contentType: string | null;
+    server: string | null;
+    framework: string | null;
+    botProtection: string | null;
+    signals: string[];
+    recommendation: 'read' | 'capture' | 'auth_required' | 'blocked';
+}
+export interface ReadResult {
+    url: string;
+    title: string | null;
+    author: string | null;
+    description: string | null;
+    content: string;
+    links: Array<{
+        text: string;
+        href: string;
+    }>;
+    images: Array<{
+        alt: string;
+        src: string;
+    }>;
+    metadata: {
+        type: string;
+        publishedAt: string | null;
+        source: string;
+        canonical: string | null;
+        siteName: string | null;
+    };
+    cost: {
+        tokens: number;
+    };
+}
+export interface Decoder {
+    name: string;
+    patterns: RegExp[];
+    decode(url: string, options?: {
+        skipSsrf?: boolean;
+        [key: string]: any;
+    }): Promise<ReadResult | null>;
+}

package/dist/read/types.js

@@ -0,0 +1,3 @@
+// src/read/types.ts
+export {};
+//# sourceMappingURL=types.js.map

package/dist/read/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/read/types.ts"],"names":[],"mappings":"AAAA,oBAAoB"}