@staticn0va/wigolo 0.1.0

package/dist/extraction/site-extractors/stackoverflow.js

@@ -0,0 +1,88 @@
+import { parseHTML } from 'linkedom';
+import TurndownService from 'turndown';
+const turndown = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced' });
+function parseVotes(el) {
+    if (!el)
+        return 0;
+    const voteEl = el.querySelector('.js-vote-count');
+    const val = voteEl?.getAttribute('data-value') ?? voteEl?.textContent?.trim() ?? '0';
+    return parseInt(val, 10) || 0;
+}
+function parseAnswers(document) {
+    const answerEls = document.querySelectorAll('#answers .answer');
+    const answers = [];
+    for (const el of Array.from(answerEls)) {
+        const accepted = el.classList.contains('accepted-answer');
+        const votes = parseVotes(el);
+        const bodyEl = el.querySelector('.s-prose, .js-post-body, .post-text');
+        const bodyHtml = bodyEl ? bodyEl.innerHTML : '';
+        answers.push({ accepted, votes, bodyHtml });
+    }
+    return answers;
+}
+function buildMarkdown(title, tags, votes, questionHtml, answers) {
+    const tagLine = `Tags: ${tags.join(', ')} | Votes: ${votes}`;
+    const questionMd = turndown.turndown(questionHtml).trim();
+    const sections = [
+        `# ${title}`,
+        tagLine,
+        '',
+        questionMd,
+    ];
+    const accepted = answers.filter((a) => a.accepted);
+    const others = answers.filter((a) => !a.accepted).sort((a, b) => b.votes - a.votes);
+    const ordered = [...accepted, ...others];
+    for (const answer of ordered) {
+        const heading = answer.accepted
+            ? `## Accepted Answer (Votes: ${answer.votes})`
+            : `## Answer (Votes: ${answer.votes})`;
+        const bodyMd = turndown.turndown(answer.bodyHtml).trim();
+        sections.push('---', '', heading, '', bodyMd);
+    }
+    return sections.join('\n\n');
+}
+export const stackoverflowExtractor = {
+    name: 'stackoverflow',
+    canHandle(url) {
+        try {
+            const hostname = new URL(url).hostname;
+            return hostname === 'stackoverflow.com' ||
+                hostname.endsWith('.stackoverflow.com') ||
+                hostname === 'stackexchange.com' ||
+                hostname.endsWith('.stackexchange.com');
+        }
+        catch {
+            return false;
+        }
+    },
+    extract(html, url) {
+        if (!html)
+            return null;
+        const { document } = parseHTML(html);
+        const titleEl = document.querySelector('.question-hyperlink');
+        if (!titleEl)
+            return null;
+        const title = titleEl.textContent?.trim() ?? '';
+        if (!title)
+            return null;
+        const questionBodyEl = document.querySelector('#question .s-prose, #question .js-post-body, #question .post-text');
+        if (!questionBodyEl)
+            return null;
+        const questionHtml = questionBodyEl.innerHTML;
+        const tagEls = document.querySelectorAll('.js-post-tag-list-wrapper .post-tag, .post-taglist .post-tag');
+        const tags = Array.from(tagEls).map((el) => el.textContent?.trim() ?? '').filter(Boolean);
+        const questionEl = document.querySelector('#question');
+        const votes = parseVotes(questionEl);
+        const answers = parseAnswers(document);
+        const markdown = buildMarkdown(title, tags, votes, questionHtml, answers);
+        return {
+            title,
+            markdown,
+            metadata: {},
+            links: [],
+            images: [],
+            extractor: 'site-specific',
+        };
+    },
+};
+//# sourceMappingURL=stackoverflow.js.map

package/dist/extraction/site-extractors/stackoverflow.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"stackoverflow.js","sourceRoot":"","sources":["../../../src/extraction/site-extractors/stackoverflow.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AACrC,OAAO,eAAe,MAAM,UAAU,CAAC;AAGvC,MAAM,QAAQ,GAAG,IAAI,eAAe,CAAC,EAAE,YAAY,EAAE,KAAK,EAAE,cAAc,EAAE,QAAQ,EAAE,CAAC,CAAC;AAQxF,SAAS,UAAU,CAAC,EAAkB;IACpC,IAAI,CAAC,EAAE;QAAE,OAAO,CAAC,CAAC;IAClB,MAAM,MAAM,GAAG,EAAE,CAAC,aAAa,CAAC,gBAAgB,CAAC,CAAC;IAClD,MAAM,GAAG,GAAG,MAAM,EAAE,YAAY,CAAC,YAAY,CAAC,IAAI,MAAM,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,GAAG,CAAC;IACrF,OAAO,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC;AAChC,CAAC;AAED,SAAS,YAAY,CAAC,QAAkB;IACtC,MAAM,SAAS,GAAG,QAAQ,CAAC,gBAAgB,CAAC,kBAAkB,CAAC,CAAC;IAChE,MAAM,OAAO,GAAa,EAAE,CAAC;IAE7B,KAAK,MAAM,EAAE,IAAI,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;QACvC,MAAM,QAAQ,GAAG,EAAE,CAAC,SAAS,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC;QAC1D,MAAM,KAAK,GAAG,UAAU,CAAC,EAAa,CAAC,CAAC;QACxC,MAAM,MAAM,GAAG,EAAE,CAAC,aAAa,CAAC,qCAAqC,CAAC,CAAC;QACvE,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,CAAE,MAAkB,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7D,OAAO,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC;IAC9C,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,aAAa,CACpB,KAAa,EACb,IAAc,EACd,KAAa,EACb,YAAoB,EACpB,OAAiB;IAEjB,MAAM,OAAO,GAAG,SAAS,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,KAAK,EAAE,CAAC;IAC7D,MAAM,UAAU,GAAG,QAAQ,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,IAAI,EAAE,CAAC;IAE1D,MAAM,QAAQ,GAAa;QACzB,KAAK,KAAK,EAAE;QACZ,OAAO;QACP,EAAE;QACF,UAAU;KACX,CAAC;IAEF,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;IACnD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IACpF,MAAM,OAAO,GAAG,CAAC,GAAG,QAAQ,EAAE,GAAG,MAAM,CAAC,CAAC;IAEzC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,OAAO,GAAG,MAAM,CAAC,QAAQ;YAC7B,CAAC,CAAC,8BAA8B,MAAM,CAAC,KAAK,GAAG;YAC/C,CAAC,CAAC,qBAAqB,MAAM,CAAC,KAAK,GAAG,CAAC;QACzC,MAAM,MAAM,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC;QACzD,QAAQ,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,CAAC,CAAC;IAChD,CAAC;IAED,OAAO,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAC/B,CAAC;AAED,MAAM,CAAC,MAAM,sBAAsB,GAAc;IAC/C,IAAI,EAAE,eAAe;IAErB,SAAS,CAAC,GAAW;QACnB,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;YACvC,OAAO,QAAQ,KAAK,mBAAmB;gBACrC,QAAQ,CAAC,QAAQ,CAAC,oBAAoB,CAAC;gBACvC,QAAQ,KAAK,mBAAmB;gBAChC,QAAQ,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAC;QAC5C,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED,OAAO,CAAC,IAAY,EAAE,GAAW;QAC/B,IAAI,CAAC,IAAI;YAAE,OAAO,IAAI,CAAC;QAEvB,MAAM,EAAE,QAAQ,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;QAErC,MAAM,OAAO,GAAG,QAAQ,CAAC,aAAa,CAAC,qBAAqB,CAAC,CAAC;QAC9D,IAAI,CAAC,OAAO;YAAE,OAAO,IAAI,CAAC;QAE1B,MAAM,KAAK,GAAG,OAAO,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QAChD,IAAI,CAAC,KAAK;YAAE,OAAO,IAAI,CAAC;QAExB,MAAM,cAAc,GAAG,QAAQ,CAAC,aAAa,CAAC,mEAAmE,CAAC,CAAC;QACnH,IAAI,CAAC,cAAc;YAAE,OAAO,IAAI,CAAC;QAEjC,MAAM,YAAY,GAAI,cAA0B,CAAC,SAAS,CAAC;QAE3D,MAAM,MAAM,GAAG,QAAQ,CAAC,gBAAgB,CAAC,8DAA8D,CAAC,CAAC;QACzG,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAE1F,MAAM,UAAU,GAAG,QAAQ,CAAC,aAAa,CAAC,WAAW,CAAC,CAAC;QACvD,MAAM,KAAK,GAAG,UAAU,CAAC,UAA4B,CAAC,CAAC;QAEvD,MAAM,OAAO,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;QAEvC,MAAM,QAAQ,GAAG,aAAa,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,YAAY,EAAE,OAAO,CAAC,CAAC;QAE1E,OAAO;YACL,KAAK;YACL,QAAQ;YACR,QAAQ,EAAE,EAAE;YACZ,KAAK,EAAE,EAAE;YACT,MAAM,EAAE,EAAE;YACV,SAAS,EAAE,eAAe;SAC3B,CAAC;IACJ,CAAC;CACF,CAAC"}

package/dist/extraction/trafilatura.d.ts

@@ -0,0 +1,6 @@
+import type { ExtractionResult } from '../types.js';
+export declare function isTrafilaturaAvailable(): Promise<boolean>;
+export declare function resetAvailabilityCache(): void;
+export declare function runPythonWithStdin(script: string, stdin: string, timeoutMs: number): Promise<string>;
+export declare function trafilaturaExtract(html: string, url: string): Promise<ExtractionResult | null>;
+//# sourceMappingURL=trafilatura.d.ts.map

package/dist/extraction/trafilatura.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"trafilatura.d.ts","sourceRoot":"","sources":["../../src/extraction/trafilatura.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAoBpD,wBAAsB,sBAAsB,IAAI,OAAO,CAAC,OAAO,CAAC,CAW/D;AAED,wBAAgB,sBAAsB,IAAI,IAAI,CAE7C;AAED,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,MAAM,CAAC,CAsCjB;AAED,wBAAsB,kBAAkB,CACtC,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,GACV,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAuClC"}

package/dist/extraction/trafilatura.js

@@ -0,0 +1,105 @@
+// src/extraction/trafilatura.ts
+import { spawn, execFile as execFileCb } from 'node:child_process';
+import { promisify } from 'node:util';
+import { createLogger } from '../logger.js';
+const execFileAsync = promisify(execFileCb);
+const log = createLogger('extract');
+const MIN_CONTENT_THRESHOLD = 100;
+const SUBPROCESS_TIMEOUT_MS = 15000;
+const AVAILABILITY_CHECK_TIMEOUT_MS = 5000;
+const TRAFILATURA_SCRIPT = `
+import sys, json
+from trafilatura import extract
+html = sys.stdin.read()
+result = extract(html, output_format='json', include_links=True, include_images=True, favor_precision=True)
+print(result or '{}')
+`.trim();
+let availableCache = null;
+export async function isTrafilaturaAvailable() {
+    if (availableCache !== null)
+        return availableCache;
+    try {
+        await execFileAsync('python3', ['-c', 'import trafilatura'], {
+            timeout: AVAILABILITY_CHECK_TIMEOUT_MS,
+        });
+        availableCache = true;
+    }
+    catch {
+        availableCache = false;
+    }
+    return availableCache;
+}
+export function resetAvailabilityCache() {
+    availableCache = null;
+}
+export function runPythonWithStdin(script, stdin, timeoutMs) {
+    const procRef = { current: null };
+    const procPromise = new Promise((resolve, reject) => {
+        const proc = spawn('python3', ['-c', script], { timeout: timeoutMs });
+        procRef.current = proc;
+        let stdout = '';
+        let stderr = '';
+        proc.stdout.on('data', (d) => {
+            stdout += d.toString();
+        });
+        proc.stderr.on('data', (d) => {
+            stderr += d.toString();
+        });
+        proc.on('close', (code, signal) => {
+            if (signal) {
+                reject(new Error(`Python killed by signal ${signal}: ${stderr}`));
+            }
+            else if (code === 0) {
+                resolve(stdout);
+            }
+            else {
+                reject(new Error(`Python exited ${code}: ${stderr}`));
+            }
+        });
+        proc.on('error', reject);
+        proc.stdin.write(stdin);
+        proc.stdin.end();
+    });
+    const timeoutPromise = new Promise((_, reject) => setTimeout(() => {
+        procRef.current?.kill();
+        reject(new Error(`Python timed out after ${timeoutMs}ms`));
+    }, timeoutMs));
+    return Promise.race([procPromise, timeoutPromise]);
+}
+export async function trafilaturaExtract(html, url) {
+    try {
+        const stdout = await runPythonWithStdin(TRAFILATURA_SCRIPT, html, SUBPROCESS_TIMEOUT_MS);
+        const trimmed = stdout.trim();
+        if (!trimmed || trimmed === 'null')
+            return null;
+        let parsed;
+        try {
+            parsed = JSON.parse(trimmed);
+        }
+        catch {
+            log.debug('Trafilatura output was not valid JSON', { url });
+            return null;
+        }
+        if (!parsed || typeof parsed !== 'object')
+            return null;
+        const text = typeof parsed.text === 'string' ? parsed.text : '';
+        if (text.length < MIN_CONTENT_THRESHOLD)
+            return null;
+        return {
+            title: typeof parsed.title === 'string' ? parsed.title : '',
+            markdown: text,
+            metadata: {
+                author: typeof parsed.author === 'string' ? parsed.author : undefined,
+                date: typeof parsed.date === 'string' ? parsed.date : undefined,
+            },
+            links: [],
+            images: [],
+            extractor: 'trafilatura',
+        };
+    }
+    catch (err) {
+        log.debug('Trafilatura extraction failed', { url, error: String(err) });
+        return null;
+    }
+}
+//# sourceMappingURL=trafilatura.js.map

package/dist/extraction/trafilatura.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"trafilatura.js","sourceRoot":"","sources":["../../src/extraction/trafilatura.ts"],"names":[],"mappings":"AAAA,gCAAgC;AAChC,OAAO,EAAE,KAAK,EAAE,QAAQ,IAAI,UAAU,EAAE,MAAM,oBAAoB,CAAC;AACnE,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAEtC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAE5C,MAAM,aAAa,GAAG,SAAS,CAAC,UAAU,CAAC,CAAC;AAC5C,MAAM,GAAG,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC;AAEpC,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAClC,MAAM,qBAAqB,GAAG,KAAK,CAAC;AACpC,MAAM,6BAA6B,GAAG,IAAI,CAAC;AAE3C,MAAM,kBAAkB,GAAG;;;;;;CAM1B,CAAC,IAAI,EAAE,CAAC;AAET,IAAI,cAAc,GAAmB,IAAI,CAAC;AAE1C,MAAM,CAAC,KAAK,UAAU,sBAAsB;IAC1C,IAAI,cAAc,KAAK,IAAI;QAAE,OAAO,cAAc,CAAC;IACnD,IAAI,CAAC;QACH,MAAM,aAAa,CAAC,SAAS,EAAE,CAAC,IAAI,EAAE,oBAAoB,CAAC,EAAE;YAC3D,OAAO,EAAE,6BAA6B;SACvC,CAAC,CAAC;QACH,cAAc,GAAG,IAAI,CAAC;IACxB,CAAC;IAAC,MAAM,CAAC;QACP,cAAc,GAAG,KAAK,CAAC;IACzB,CAAC;IACD,OAAO,cAAc,CAAC;AACxB,CAAC;AAED,MAAM,UAAU,sBAAsB;IACpC,cAAc,GAAG,IAAI,CAAC;AACxB,CAAC;AAED,MAAM,UAAU,kBAAkB,CAChC,MAAc,EACd,KAAa,EACb,SAAiB;IAEjB,MAAM,OAAO,GAAG,EAAE,OAAO,EAAE,IAAuC,EAAE,CAAC;IAErE,MAAM,WAAW,GAAG,IAAI,OAAO,CAAS,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC1D,MAAM,IAAI,GAAG,KAAK,CAAC,SAAS,EAAE,CAAC,IAAI,EAAE,MAAM,CAAC,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,CAAC;QACtE,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;QACvB,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,IAAI,MAAM,GAAG,EAAE,CAAC;QAEhB,IAAI,CAAC,MAAO,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAS,EAAE,EAAE;YACpC,MAAM,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC;QACzB,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,MAAO,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAS,EAAE,EAAE;YACpC,MAAM,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC;QACzB,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE;YAChC,IAAI,MAAM,EAAE,CAAC;gBACX,MAAM,CAAC,IAAI,KAAK,CAAC,2BAA2B,MAAM,KAAK,MAAM,EAAE,CAAC,CAAC,CAAC;YACpE,CAAC;iBAAM,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;gBACtB,OAAO,CAAC,MAAM,CAAC,CAAC;YAClB,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,IAAI,KAAK,CAAC,iBAAiB,IAAI,KAAK,MAAM,EAAE,CAAC,CAAC,CAAC;YACxD,CAAC;QACH,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAEzB,IAAI,CAAC,KAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACzB,IAAI,CAAC,KAAM,CAAC,GAAG,EAAE,CAAC;IACpB,CAAC,CAAC,CAAC;IAEH,MAAM,cAAc,GAAG,IAAI,OAAO,CAAS,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,CACvD,UAAU,CAAC,GAAG,EAAE;QACd,OAAO,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC;QACxB,MAAM,CAAC,IAAI,KAAK,CAAC,0BAA0B,SAAS,IAAI,CAAC,CAAC,CAAC;IAC7D,CAAC,EAAE,SAAS,CAAC,CACd,CAAC;IAEF,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC,CAAC;AACrD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,IAAY,EACZ,GAAW;IAEX,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,kBAAkB,CACrC,kBAAkB,EAClB,IAAI,EACJ,qBAAqB,CACtB,CAAC;QAEF,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC;QAC9B,IAAI,CAAC,OAAO,IAAI,OAAO,KAAK,MAAM;YAAE,OAAO,IAAI,CAAC;QAEhD,IAAI,MAA+B,CAAC;QACpC,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC/B,CAAC;QAAC,MAAM,CAAC;YACP,GAAG,CAAC,KAAK,CAAC,uCAAuC,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;YAC5D,OAAO,IAAI,CAAC;QACd,CAAC;QAED,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ;YAAE,OAAO,IAAI,CAAC;QAEvD,MAAM,IAAI,GAAG,OAAO,MAAM,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;QAChE,IAAI,IAAI,CAAC,MAAM,GAAG,qBAAqB;YAAE,OAAO,IAAI,CAAC;QAErD,OAAO;YACL,KAAK,EAAE,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;YAC3D,QAAQ,EAAE,IAAI;YACd,QAAQ,EAAE;gBACR,MAAM,EAAE,OAAO,MAAM,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;gBACrE,IAAI,EAAE,OAAO,MAAM,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS;aAChE;YACD,KAAK,EAAE,EAAE;YACT,MAAM,EAAE,EAAE;YACV,SAAS,EAAE,aAAa;SACzB,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,KAAK,CAAC,+BAA+B,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACxE,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC"}

package/dist/fetch/auth.d.ts

@@ -0,0 +1,8 @@
+import type { CDPSession } from '../types.js';
+export interface AuthOptions {
+    storageStatePath?: string;
+    userDataDir?: string;
+}
+export declare function getAuthOptions(): AuthOptions | null;
+export declare function listSessions(): Promise<CDPSession[]>;
+//# sourceMappingURL=auth.d.ts.map

package/dist/fetch/auth.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"auth.d.ts","sourceRoot":"","sources":["../../src/fetch/auth.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAE9C,MAAM,WAAW,WAAW;IAC1B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,wBAAgB,cAAc,IAAI,WAAW,GAAG,IAAI,CAyBnD;AAED,wBAAsB,YAAY,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC,CAE1D"}

package/dist/fetch/auth.js

@@ -0,0 +1,32 @@
+import { existsSync, cpSync, mkdtempSync } from 'node:fs';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+import { getConfig } from '../config.js';
+import { createLogger } from '../logger.js';
+export function getAuthOptions() {
+    const config = getConfig();
+    const logger = createLogger('fetch');
+    if (config.authStatePath) {
+        if (!existsSync(config.authStatePath)) {
+            throw new Error(`Auth state file not found: ${config.authStatePath}`);
+        }
+        return { storageStatePath: config.authStatePath };
+    }
+    if (config.chromeProfilePath) {
+        const lockFile = join(config.chromeProfilePath, 'SingletonLock');
+        if (existsSync(lockFile)) {
+            logger.warn('Chrome appears to be running (SingletonLock found) — close Chrome before using its profile', {
+                profilePath: config.chromeProfilePath,
+            });
+        }
+        const tempDir = mkdtempSync(join(tmpdir(), 'wigolo-chrome-'));
+        cpSync(config.chromeProfilePath, tempDir, { recursive: true });
+        logger.debug('copied Chrome profile to temp directory', { from: config.chromeProfilePath, to: tempDir });
+        return { userDataDir: tempDir };
+    }
+    return null;
+}
+export async function listSessions() {
+    return [];
+}
+//# sourceMappingURL=auth.js.map

package/dist/fetch/auth.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"auth.js","sourceRoot":"","sources":["../../src/fetch/auth.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAC1D,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAQ5C,MAAM,UAAU,cAAc;IAC5B,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,MAAM,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;IAErC,IAAI,MAAM,CAAC,aAAa,EAAE,CAAC;QACzB,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC;YACtC,MAAM,IAAI,KAAK,CAAC,8BAA8B,MAAM,CAAC,aAAa,EAAE,CAAC,CAAC;QACxE,CAAC;QACD,OAAO,EAAE,gBAAgB,EAAE,MAAM,CAAC,aAAa,EAAE,CAAC;IACpD,CAAC;IAED,IAAI,MAAM,CAAC,iBAAiB,EAAE,CAAC;QAC7B,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,iBAAiB,EAAE,eAAe,CAAC,CAAC;QACjE,IAAI,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;YACzB,MAAM,CAAC,IAAI,CAAC,4FAA4F,EAAE;gBACxG,WAAW,EAAE,MAAM,CAAC,iBAAiB;aACtC,CAAC,CAAC;QACL,CAAC;QACD,MAAM,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,gBAAgB,CAAC,CAAC,CAAC;QAC9D,MAAM,CAAC,MAAM,CAAC,iBAAiB,EAAE,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC/D,MAAM,CAAC,KAAK,CAAC,yCAAyC,EAAE,EAAE,IAAI,EAAE,MAAM,CAAC,iBAAiB,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;QACzG,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,CAAC;IAClC,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY;IAChC,OAAO,EAAE,CAAC;AACZ,CAAC"}

package/dist/fetch/browser-pool.d.ts

@@ -0,0 +1,28 @@
+import { type BrowserContext } from 'playwright';
+import type { RawFetchResult, BrowserType } from '../types.js';
+export interface BrowserFetchOptions {
+    timeoutMs?: number;
+    storageStatePath?: string;
+    userDataDir?: string;
+    headers?: Record<string, string>;
+    screenshot?: boolean;
+}
+export interface BrowserPoolOptions {
+    browserType?: BrowserType;
+}
+export declare class BrowserPool {
+    private browser;
+    private pool;
+    private activeCount;
+    private waitQueue;
+    private idleTimers;
+    private shutdownCalled;
+    private readonly browserType;
+    constructor(options?: BrowserPoolOptions);
+    private launchBrowser;
+    acquire(): Promise<BrowserContext>;
+    release(ctx: BrowserContext): void;
+    fetchWithBrowser(url: string, options?: BrowserFetchOptions): Promise<RawFetchResult>;
+    shutdown(): Promise<void>;
+}
+//# sourceMappingURL=browser-pool.d.ts.map

package/dist/fetch/browser-pool.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"browser-pool.d.ts","sourceRoot":"","sources":["../../src/fetch/browser-pool.ts"],"names":[],"mappings":"AAAA,OAAO,EAA2C,KAAK,cAAc,EAAE,MAAM,YAAY,CAAC;AAG1F,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAE/D,MAAM,WAAW,mBAAmB;IAClC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,WAAW,kBAAkB;IACjC,WAAW,CAAC,EAAE,WAAW,CAAC;CAC3B;AAED,qBAAa,WAAW;IACtB,OAAO,CAAC,OAAO,CAAwB;IACvC,OAAO,CAAC,IAAI,CAAwB;IACpC,OAAO,CAAC,WAAW,CAAK;IACxB,OAAO,CAAC,SAAS,CAA4C;IAC7D,OAAO,CAAC,UAAU,CAA4D;IAC9E,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAc;gBAE9B,OAAO,CAAC,EAAE,kBAAkB;YAI1B,aAAa;IAUrB,OAAO,IAAI,OAAO,CAAC,cAAc,CAAC;IAyBxC,OAAO,CAAC,GAAG,EAAE,cAAc,GAAG,IAAI;IAyB5B,gBAAgB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,mBAAwB,GAAG,OAAO,CAAC,cAAc,CAAC;IA+DzF,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;CAoBhC"}

package/dist/fetch/browser-pool.js

@@ -0,0 +1,138 @@
+import { chromium, firefox, webkit } from 'playwright';
+import { getConfig } from '../config.js';
+import { createLogger } from '../logger.js';
+export class BrowserPool {
+    browser = null;
+    pool = [];
+    activeCount = 0;
+    waitQueue = [];
+    idleTimers = new Map();
+    shutdownCalled = false;
+    browserType;
+    constructor(options) {
+        this.browserType = options?.browserType ?? 'chromium';
+    }
+    async launchBrowser() {
+        if (!this.browser) {
+            const launcher = this.browserType === 'firefox' ? firefox
+                : this.browserType === 'webkit' ? webkit
+                    : chromium;
+            this.browser = await launcher.launch({ headless: true });
+        }
+        return this.browser;
+    }
+    async acquire() {
+        const config = getConfig();
+        const maxBrowsers = config.maxBrowsers;
+        if (this.pool.length > 0) {
+            const ctx = this.pool.pop();
+            const timer = this.idleTimers.get(ctx);
+            if (timer !== undefined) {
+                clearTimeout(timer);
+                this.idleTimers.delete(ctx);
+            }
+            return ctx;
+        }
+        if (this.activeCount < maxBrowsers) {
+            this.activeCount++;
+            const browser = await this.launchBrowser();
+            return browser.newContext();
+        }
+        return new Promise((resolve) => {
+            this.waitQueue.push(resolve);
+        });
+    }
+    release(ctx) {
+        const config = getConfig();
+        const idleTimeoutMs = config.browserIdleTimeoutMs;
+        if (this.waitQueue.length > 0) {
+            const resolve = this.waitQueue.shift();
+            resolve(ctx);
+            return;
+        }
+        this.pool.push(ctx);
+        const timer = setTimeout(() => {
+            const idx = this.pool.indexOf(ctx);
+            if (idx !== -1) {
+                this.pool.splice(idx, 1);
+                this.idleTimers.delete(ctx);
+                this.activeCount = Math.max(0, this.activeCount - 1);
+                ctx.close().catch(() => { });
+            }
+        }, idleTimeoutMs);
+        this.idleTimers.set(ctx, timer);
+    }
+    async fetchWithBrowser(url, options = {}) {
+        const config = getConfig();
+        const logger = createLogger('fetch');
+        const navTimeoutMs = options.timeoutMs ?? config.playwrightNavTimeoutMs;
+        const loadTimeoutMs = config.playwrightLoadTimeoutMs;
+        const ctx = await this.acquire();
+        const page = await ctx.newPage();
+        if (options.headers) {
+            await page.setExtraHTTPHeaders(options.headers);
+        }
+        let statusCode = 200;
+        let contentType = '';
+        let responseHeaders = {};
+        let finalUrl = url;
+        try {
+            const response = await page.goto(url, {
+                timeout: navTimeoutMs,
+                waitUntil: 'domcontentloaded',
+            });
+            if (response) {
+                statusCode = response.status();
+                finalUrl = response.url();
+                const rawHeaders = response.headers();
+                responseHeaders = rawHeaders;
+                contentType = rawHeaders['content-type'] ?? '';
+            }
+            try {
+                await page.waitForLoadState('networkidle', { timeout: loadTimeoutMs });
+            }
+            catch {
+                // networkidle timeout is non-fatal — page content is still usable
+                logger.debug('networkidle timeout, using page content as-is', { url });
+            }
+            const html = await page.content();
+            let screenshotBase64;
+            if (options.screenshot) {
+                const buf = await page.screenshot({ fullPage: true });
+                screenshotBase64 = buf.toString('base64');
+            }
+            return {
+                url,
+                finalUrl,
+                html,
+                contentType,
+                statusCode,
+                method: 'playwright',
+                headers: responseHeaders,
+                screenshot: screenshotBase64,
+            };
+        }
+        finally {
+            await page.close();
+            this.release(ctx);
+        }
+    }
+    async shutdown() {
+        if (this.shutdownCalled)
+            return;
+        this.shutdownCalled = true;
+        for (const [, timer] of this.idleTimers) {
+            clearTimeout(timer);
+        }
+        this.idleTimers.clear();
+        const closePromises = this.pool.map((ctx) => ctx.close().catch(() => { }));
+        this.pool = [];
+        await Promise.all(closePromises);
+        if (this.browser) {
+            await this.browser.close().catch(() => { });
+            this.browser = null;
+        }
+        this.activeCount = 0;
+    }
+}
+//# sourceMappingURL=browser-pool.js.map

package/dist/fetch/browser-pool.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"browser-pool.js","sourceRoot":"","sources":["../../src/fetch/browser-pool.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAqC,MAAM,YAAY,CAAC;AAC1F,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAe5C,MAAM,OAAO,WAAW;IACd,OAAO,GAAmB,IAAI,CAAC;IAC/B,IAAI,GAAqB,EAAE,CAAC;IAC5B,WAAW,GAAG,CAAC,CAAC;IAChB,SAAS,GAAyC,EAAE,CAAC;IACrD,UAAU,GAAG,IAAI,GAAG,EAAiD,CAAC;IACtE,cAAc,GAAG,KAAK,CAAC;IACd,WAAW,CAAc;IAE1C,YAAY,OAA4B;QACtC,IAAI,CAAC,WAAW,GAAG,OAAO,EAAE,WAAW,IAAI,UAAU,CAAC;IACxD,CAAC;IAEO,KAAK,CAAC,aAAa;QACzB,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAClB,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,OAAO;gBACvD,CAAC,CAAC,IAAI,CAAC,WAAW,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM;oBACxC,CAAC,CAAC,QAAQ,CAAC;YACb,IAAI,CAAC,OAAO,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;QAC3D,CAAC;QACD,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED,KAAK,CAAC,OAAO;QACX,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;QAC3B,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;QAEvC,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzB,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAG,CAAC;YAC7B,MAAM,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACvC,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;gBACxB,YAAY,CAAC,KAAK,CAAC,CAAC;gBACpB,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC9B,CAAC;YACD,OAAO,GAAG,CAAC;QACb,CAAC;QAED,IAAI,IAAI,CAAC,WAAW,GAAG,WAAW,EAAE,CAAC;YACnC,IAAI,CAAC,WAAW,EAAE,CAAC;YACnB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;YAC3C,OAAO,OAAO,CAAC,UAAU,EAAE,CAAC;QAC9B,CAAC;QAED,OAAO,IAAI,OAAO,CAAiB,CAAC,OAAO,EAAE,EAAE;YAC7C,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC/B,CAAC,CAAC,CAAC;IACL,CAAC;IAED,OAAO,CAAC,GAAmB;QACzB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;QAC3B,MAAM,aAAa,GAAG,MAAM,CAAC,oBAAoB,CAAC;QAElD,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC9B,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,EAAG,CAAC;YACxC,OAAO,CAAC,GAAG,CAAC,CAAC;YACb,OAAO;QACT,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAEpB,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YAC5B,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YACnC,IAAI,GAAG,KAAK,CAAC,CAAC,EAAE,CAAC;gBACf,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;gBACzB,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBAC5B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC;gBACrD,GAAG,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;YAC9B,CAAC;QACH,CAAC,EAAE,aAAa,CAAC,CAAC;QAElB,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IAClC,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,GAAW,EAAE,UAA+B,EAAE;QACnE,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;QAC3B,MAAM,MAAM,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;QACrC,MAAM,YAAY,GAAG,OAAO,CAAC,SAAS,IAAI,MAAM,CAAC,sBAAsB,CAAC;QACxE,MAAM,aAAa,GAAG,MAAM,CAAC,uBAAuB,CAAC;QAErD,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;QACjC,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,OAAO,EAAE,CAAC;QAEjC,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAClD,CAAC;QAED,IAAI,UAAU,GAAG,GAAG,CAAC;QACrB,IAAI,WAAW,GAAG,EAAE,CAAC;QACrB,IAAI,eAAe,GAA2B,EAAE,CAAC;QACjD,IAAI,QAAQ,GAAG,GAAG,CAAC;QAEnB,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;gBACpC,OAAO,EAAE,YAAY;gBACrB,SAAS,EAAE,kBAAkB;aAC9B,CAAC,CAAC;YAEH,IAAI,QAAQ,EAAE,CAAC;gBACb,UAAU,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC;gBAC/B,QAAQ,GAAG,QAAQ,CAAC,GAAG,EAAE,CAAC;gBAC1B,MAAM,UAAU,GAAG,QAAQ,CAAC,OAAO,EAAE,CAAC;gBACtC,eAAe,GAAG,UAAU,CAAC;gBAC7B,WAAW,GAAG,UAAU,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;YACjD,CAAC;YAED,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,gBAAgB,CAAC,aAAa,EAAE,EAAE,OAAO,EAAE,aAAa,EAAE,CAAC,CAAC;YACzE,CAAC;YAAC,MAAM,CAAC;gBACP,kEAAkE;gBAClE,MAAM,CAAC,KAAK,CAAC,+CAA+C,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;YACzE,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;YAElC,IAAI,gBAAoC,CAAC;YACzC,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;gBACvB,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;gBACtD,gBAAgB,GAAG,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;YAC5C,CAAC;YAED,OAAO;gBACL,GAAG;gBACH,QAAQ;gBACR,IAAI;gBACJ,WAAW;gBACX,UAAU;gBACV,MAAM,EAAE,YAAY;gBACpB,OAAO,EAAE,eAAe;gBACxB,UAAU,EAAE,gBAAgB;aAC7B,CAAC;QACJ,CAAC;gBAAS,CAAC;YACT,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;YACnB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACpB,CAAC;IACH,CAAC;IAED,KAAK,CAAC,QAAQ;QACZ,IAAI,IAAI,CAAC,cAAc;YAAE,OAAO;QAChC,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;QAE3B,KAAK,MAAM,CAAC,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACxC,YAAY,CAAC,KAAK,CAAC,CAAC;QACtB,CAAC;QACD,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;QAExB,MAAM,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC,CAAC;QAC1E,IAAI,CAAC,IAAI,GAAG,EAAE,CAAC;QACf,MAAM,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;QAEjC,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;YAC3C,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACtB,CAAC;QAED,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC;IACvB,CAAC;CACF"}

package/dist/fetch/content-check.d.ts

@@ -0,0 +1,2 @@
+export declare function contentAppearsEmpty(html: string): boolean;
+//# sourceMappingURL=content-check.d.ts.map

package/dist/fetch/content-check.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"content-check.d.ts","sourceRoot":"","sources":["../../src/fetch/content-check.ts"],"names":[],"mappings":"AAsDA,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAUzD"}

package/dist/fetch/content-check.js

@@ -0,0 +1,62 @@
+const VISIBLE_TEXT_THRESHOLD = 200;
+const SCRIPT_RATIO_THRESHOLD = 0.8;
+function stripScriptsAndStyles(html) {
+    return html
+        .replace(/<script[\s\S]*?<\/script>/gi, '')
+        .replace(/<style[\s\S]*?<\/style>/gi, '');
+}
+function extractVisibleText(html) {
+    const stripped = stripScriptsAndStyles(html);
+    const noTags = stripped.replace(/<[^>]+>/g, ' ');
+    return noTags.replace(/\s+/g, ' ').trim();
+}
+function hasSpaShellIndicator(html) {
+    const spaPatterns = [
+        /<div[^>]+id=["']root["'][^>]*>\s*<\/div>/i,
+        /<div[^>]+id=["']app["'][^>]*>\s*<\/div>/i,
+        /<div[^>]+id=["']__next["'][^>]*>\s*<\/div>/i,
+    ];
+    return spaPatterns.some((pattern) => pattern.test(html));
+}
+function hasNextData(html) {
+    if (!/__NEXT_DATA__/.test(html))
+        return false;
+    const withoutScripts = stripScriptsAndStyles(html);
+    const visibleText = withoutScripts.replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim();
+    return visibleText.length < VISIBLE_TEXT_THRESHOLD;
+}
+function hasNoscriptRequired(html) {
+    const noscriptMatches = html.match(/<noscript[^>]*>([\s\S]*?)<\/noscript>/gi);
+    if (!noscriptMatches)
+        return false;
+    return noscriptMatches.some((tag) => {
+        const inner = tag.replace(/<[^>]+>/g, '').toLowerCase();
+        return inner.includes('javascript') || inner.includes('enable');
+    });
+}
+function hasHighScriptRatio(html) {
+    const bodyMatch = html.match(/<body[^>]*>([\s\S]*?)<\/body>/i);
+    const bodyContent = bodyMatch ? bodyMatch[1] : html;
+    const scriptMatches = bodyContent.match(/<script[\s\S]*?<\/script>/gi) ?? [];
+    const scriptText = scriptMatches.join('');
+    const scriptLen = scriptText.length;
+    const totalLen = bodyContent.length;
+    if (totalLen === 0)
+        return false;
+    return scriptLen / totalLen > SCRIPT_RATIO_THRESHOLD;
+}
+export function contentAppearsEmpty(html) {
+    const visibleText = extractVisibleText(html);
+    if (visibleText.length < VISIBLE_TEXT_THRESHOLD)
+        return true;
+    if (hasSpaShellIndicator(html))
+        return true;
+    if (hasNextData(html))
+        return true;
+    if (hasNoscriptRequired(html))
+        return true;
+    if (hasHighScriptRatio(html))
+        return true;
+    return false;
+}
+//# sourceMappingURL=content-check.js.map

package/dist/fetch/content-check.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"content-check.js","sourceRoot":"","sources":["../../src/fetch/content-check.ts"],"names":[],"mappings":"AAAA,MAAM,sBAAsB,GAAG,GAAG,CAAC;AACnC,MAAM,sBAAsB,GAAG,GAAG,CAAC;AAEnC,SAAS,qBAAqB,CAAC,IAAY;IACzC,OAAO,IAAI;SACR,OAAO,CAAC,6BAA6B,EAAE,EAAE,CAAC;SAC1C,OAAO,CAAC,2BAA2B,EAAE,EAAE,CAAC,CAAC;AAC9C,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAY;IACtC,MAAM,QAAQ,GAAG,qBAAqB,CAAC,IAAI,CAAC,CAAC;IAC7C,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC;IACjD,OAAO,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;AAC5C,CAAC;AAED,SAAS,oBAAoB,CAAC,IAAY;IACxC,MAAM,WAAW,GAAG;QAClB,2CAA2C;QAC3C,0CAA0C;QAC1C,6CAA6C;KAC9C,CAAC;IACF,OAAO,WAAW,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED,SAAS,WAAW,CAAC,IAAY;IAC/B,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAC9C,MAAM,cAAc,GAAG,qBAAqB,CAAC,IAAI,CAAC,CAAC;IACnD,MAAM,WAAW,GAAG,cAAc,CAAC,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IACxF,OAAO,WAAW,CAAC,MAAM,GAAG,sBAAsB,CAAC;AACrD,CAAC;AAED,SAAS,mBAAmB,CAAC,IAAY;IACvC,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,yCAAyC,CAAC,CAAC;IAC9E,IAAI,CAAC,eAAe;QAAE,OAAO,KAAK,CAAC;IACnC,OAAO,eAAe,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE;QAClC,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;QACxD,OAAO,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAClE,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAY;IACtC,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;IAC/D,MAAM,WAAW,GAAG,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAEpD,MAAM,aAAa,GAAG,WAAW,CAAC,KAAK,CAAC,6BAA6B,CAAC,IAAI,EAAE,CAAC;IAC7E,MAAM,UAAU,GAAG,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAE1C,MAAM,SAAS,GAAG,UAAU,CAAC,MAAM,CAAC;IACpC,MAAM,QAAQ,GAAG,WAAW,CAAC,MAAM,CAAC;IAEpC,IAAI,QAAQ,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACjC,OAAO,SAAS,GAAG,QAAQ,GAAG,sBAAsB,CAAC;AACvD,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,IAAY;IAC9C,MAAM,WAAW,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;IAC7C,IAAI,WAAW,CAAC,MAAM,GAAG,sBAAsB;QAAE,OAAO,IAAI,CAAC;IAE7D,IAAI,oBAAoB,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IAC5C,IAAI,WAAW,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IACnC,IAAI,mBAAmB,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IAC3C,IAAI,kBAAkB,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IAE1C,OAAO,KAAK,CAAC;AACf,CAAC"}

package/dist/fetch/http-client.d.ts

@@ -0,0 +1,15 @@
+export interface HttpFetchOptions {
+    headers?: Record<string, string>;
+    timeoutMs?: number;
+}
+export interface HttpFetchResult {
+    url: string;
+    finalUrl: string;
+    html: string;
+    contentType: string;
+    statusCode: number;
+    headers: Record<string, string>;
+    rawBuffer?: Buffer;
+}
+export declare function httpFetch(url: string, options?: HttpFetchOptions): Promise<HttpFetchResult>;
+//# sourceMappingURL=http-client.d.ts.map

package/dist/fetch/http-client.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"http-client.d.ts","sourceRoot":"","sources":["../../src/fetch/http-client.ts"],"names":[],"mappings":"AAGA,MAAM,WAAW,gBAAgB;IAC/B,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,eAAe;IAC9B,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAmCD,wBAAsB,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,gBAAqB,GAAG,OAAO,CAAC,eAAe,CAAC,CAyCrG"}