@agentrhq/webcmd 0.1.1 → 0.2.0

package/dist/src/browser/analyze.test.js

@@ -0,0 +1,209 @@
+import { describe, it, expect } from 'vitest';
+import { analyzeSite, detectAntiBot, classifyPattern, findNearestAdapter, scoreEndpointEvidence, } from './analyze.js';
+function mkSignals(overrides = {}) {
+    return {
+        requestedUrl: 'https://example.com/',
+        finalUrl: 'https://example.com/',
+        cookieNames: [],
+        networkEntries: [],
+        initialState: {
+            __INITIAL_STATE__: false,
+            __NUXT__: false,
+            __NEXT_DATA__: false,
+            __APOLLO_STATE__: false,
+        },
+        title: 'Example',
+        ...overrides,
+    };
+}
+function mkCmd(site, name, domain) {
+    return {
+        site,
+        name,
+        access: 'read',
+        description: '',
+        domain,
+        browser: false,
+        args: [],
+    };
+}
+describe('detectAntiBot', () => {
+    it('flags Aliyun WAF from cookie', () => {
+        const v = detectAntiBot(mkSignals({ cookieNames: ['JSESSIONID', 'acw_sc__v2'] }));
+        expect(v.detected).toBe(true);
+        expect(v.vendor).toBe('aliyun_waf');
+        expect(v.evidence).toContain('cookie:acw_sc__v2');
+        expect(v.implication).toMatch(/browser context/i);
+    });
+    it('flags Aliyun WAF from challenge HTML body', () => {
+        const v = detectAntiBot(mkSignals({
+            networkEntries: [
+                {
+                    url: 'https://x.com/',
+                    status: 200,
+                    contentType: 'text/html',
+                    bodyPreview: "var arg1 = 'A1B2C3D4E5F6A7B8C9D0E1F2A3B4C5D6';",
+                },
+            ],
+        }));
+        expect(v.detected).toBe(true);
+        expect(v.vendor).toBe('aliyun_waf');
+    });
+    it('flags Cloudflare from cf_clearance cookie', () => {
+        const v = detectAntiBot(mkSignals({ cookieNames: ['cf_clearance'] }));
+        expect(v.vendor).toBe('cloudflare');
+        expect(v.implication).toMatch(/Cloudflare/i);
+    });
+    it('flags Akamai from _abck cookie', () => {
+        const v = detectAntiBot(mkSignals({ cookieNames: ['_abck', 'bm_sz'] }));
+        expect(v.vendor).toBe('akamai');
+    });
+    it('returns no-match verdict with actionable fallback advice', () => {
+        const v = detectAntiBot(mkSignals());
+        expect(v.detected).toBe(false);
+        expect(v.vendor).toBeNull();
+        expect(v.implication).toMatch(/Node-side COOKIE fetch first/);
+    });
+});
+describe('classifyPattern', () => {
+    it('returns A for JSON-heavy pages without SSR state', () => {
+        const v = classifyPattern(mkSignals({
+            networkEntries: [
+                { url: 'https://x.com/api/a', status: 200, contentType: 'application/json', bodyPreview: '{"items":[{"title":"A","id":"1"}]}' },
+                { url: 'https://x.com/api/b', status: 200, contentType: 'application/json;charset=utf-8', bodyPreview: '{"data":{"results":[{"name":"B","url":"/b"}]}}' },
+            ],
+        }));
+        expect(v.pattern).toBe('A');
+        expect(v.json_responses).toBe(2);
+        expect(v.real_data_candidates).toBe(2);
+    });
+    it('does not call analytics JSON a real API pattern', () => {
+        const v = classifyPattern(mkSignals({
+            networkEntries: [
+                { url: 'https://x.com/analytics/collect', status: 200, contentType: 'application/json', bodyPreview: '{"event":"view","clientId":"abc","experiment":"A"}' },
+                { url: 'https://x.com/personalization', status: 200, contentType: 'application/json', bodyPreview: '{"sessionId":"s1","metrics":{"latency":12}}' },
+            ],
+        }));
+        expect(v.pattern).toBe('C');
+        expect(v.json_responses).toBe(2);
+        expect(v.real_data_candidates).toBe(0);
+        expect(v.reason).toMatch(/telemetry|side-channel/);
+    });
+    it('returns B when __INITIAL_STATE__ is present, beating JSON signals', () => {
+        const v = classifyPattern(mkSignals({
+            initialState: { __INITIAL_STATE__: true, __NUXT__: false, __NEXT_DATA__: false, __APOLLO_STATE__: false },
+            networkEntries: [
+                { url: 'https://x.com/api/a', status: 200, contentType: 'application/json', bodyPreview: '{}' },
+            ],
+        }));
+        expect(v.pattern).toBe('B');
+    });
+    it('returns D when auth failures dominate', () => {
+        const v = classifyPattern(mkSignals({
+            networkEntries: [
+                { url: 'https://x.com/api/a', status: 401, contentType: 'application/json', bodyPreview: '' },
+                { url: 'https://x.com/api/b', status: 403, contentType: 'application/json', bodyPreview: '' },
+            ],
+        }));
+        expect(v.pattern).toBe('D');
+        expect(v.auth_failures).toBe(2);
+    });
+    it('returns C by default for static pages', () => {
+        const v = classifyPattern(mkSignals());
+        expect(v.pattern).toBe('C');
+    });
+});
+describe('scoreEndpointEvidence', () => {
+    it('scores non-empty business JSON above telemetry side-channel JSON', () => {
+        const data = scoreEndpointEvidence({
+            url: 'https://x.com/api/search',
+            status: 200,
+            contentType: 'application/json',
+            bodyPreview: '{"data":{"items":[{"title":"A","price":12,"url":"/a"}],"total":1}}',
+        });
+        const telemetry = scoreEndpointEvidence({
+            url: 'https://x.com/analytics/collect',
+            status: 200,
+            contentType: 'application/json',
+            bodyPreview: '{"event":"view","clientId":"abc"}',
+        });
+        expect(data.verdict).toBe('likely_data');
+        expect(data.real_data_score).toBeGreaterThan(telemetry.real_data_score);
+        expect(data.sample_paths).toContain('$.data.items:array(1)');
+        expect(telemetry.verdict).toBe('noise');
+    });
+    it('marks auth-gated JSON as blocked rather than data', () => {
+        const evidence = scoreEndpointEvidence({
+            url: 'https://x.com/api/private',
+            status: 403,
+            contentType: 'application/json',
+            bodyPreview: '{"error":"forbidden"}',
+        });
+        expect(evidence.verdict).toBe('blocked');
+        expect(evidence.real_data_score).toBeLessThan(0.1);
+    });
+});
+describe('findNearestAdapter', () => {
+    it('matches by domain suffix', () => {
+        const reg = new Map([
+            ['github search', mkCmd('github', 'search', 'github.com')],
+            ['github auth', mkCmd('github', 'auth', 'github.com')],
+            ['linkedin search', mkCmd('linkedin', 'search', 'linkedin.com')],
+        ]);
+        const v = findNearestAdapter('https://docs.github.com/', reg);
+        expect(v?.site).toBe('github');
+        expect(v?.example_commands).toContain('github search');
+    });
+    it('falls back to site-name containment when no domain is registered', () => {
+        const reg = new Map([
+            ['github search', mkCmd('github', 'search')],
+        ]);
+        const v = findNearestAdapter('https://gist.github.com/', reg);
+        expect(v?.site).toBe('github');
+    });
+    it('returns null when no adapter matches', () => {
+        const reg = new Map([
+            ['github search', mkCmd('github', 'search', 'github.com')],
+        ]);
+        const v = findNearestAdapter('https://random-site.io/', reg);
+        expect(v).toBeNull();
+    });
+    it('prefers the site with the most commands', () => {
+        const reg = new Map([
+            ['a search', mkCmd('a', 'search', 'a.com')],
+            ['b search', mkCmd('b', 'search', 'a.com')],
+            ['b detail', mkCmd('b', 'detail', 'a.com')],
+            ['b company', mkCmd('b', 'company', 'a.com')],
+        ]);
+        const v = findNearestAdapter('https://jobs.a.com/', reg);
+        expect(v?.site).toBe('b');
+    });
+});
+describe('analyzeSite', () => {
+    it('recommends browser-context fetch when WAF is detected', () => {
+        const report = analyzeSite(mkSignals({ cookieNames: ['acw_sc__v2'] }), new Map());
+        expect(report.anti_bot.vendor).toBe('aliyun_waf');
+        expect(report.recommended_next_step).toMatch(/browser context/i);
+    });
+    it('recommends reading SSR state when Pattern B fires', () => {
+        const report = analyzeSite(mkSignals({
+            initialState: { __INITIAL_STATE__: false, __NUXT__: true, __NEXT_DATA__: false, __APOLLO_STATE__: false },
+        }), new Map());
+        expect(report.pattern.pattern).toBe('B');
+        expect(report.recommended_next_step).toMatch(/__NUXT__|__INITIAL_STATE__|__NEXT_DATA__/);
+    });
+    it('includes __APOLLO_STATE__ in Pattern B next-step guidance', () => {
+        const report = analyzeSite(mkSignals({
+            initialState: { __INITIAL_STATE__: false, __NUXT__: false, __NEXT_DATA__: false, __APOLLO_STATE__: true },
+        }), new Map());
+        expect(report.pattern.pattern).toBe('B');
+        expect(report.recommended_next_step).toMatch(/__APOLLO_STATE__/);
+    });
+    it('includes nearest_adapter when the registry has a match', () => {
+        const reg = new Map([
+            ['github search', mkCmd('github', 'search', 'github.com')],
+        ]);
+        const report = analyzeSite(mkSignals({ finalUrl: 'https://github.com/features/actions' }), reg);
+        expect(report.nearest_adapter?.site).toBe('github');
+    });
+});

package/dist/src/browser/article-extract.d.ts

@@ -0,0 +1,57 @@
+/**
+ * Article extraction via Readability — generic `page → article HTML` pipeline.
+ *
+ * Complements `src/browser/extract.ts`: that one takes a caller-supplied
+ * selector. This one works with zero configuration on arbitrary article pages
+ * (blogs, news, docs) by running `@mozilla/readability` inside the page
+ * context via CDP evaluate.
+ *
+ * Pipeline:
+ *   1. Short-circuit non-HTML documents (`text/plain`, JSON, XML) — a page
+ *      renderer wrapping a plain-text file would pollute the DOM pipeline.
+ *   2. Short-circuit the "body is a single <pre>" case, which browsers use
+ *      when loading *.txt / *.md over file:// or raw.githubusercontent.com.
+ *   3. Deep-clone the document, apply caller-supplied `cleanSelectors` to the
+ *      clone (preserves live page state for subsequent snapshot/click).
+ *   4. Inject Readability + isProbablyReaderable sources into the page,
+ *      parse on the clone. `isProbablyReaderable` gates the parse unless
+ *      `force: true`.
+ *   5. On Readability miss, walk a fallback selector chain
+ *      (main → [role="main"] → #main-content → … → body) and return the
+ *      first root with >80 characters of text.
+ *
+ * Readability runs in the page's own window because it needs real DOM APIs
+ * (getComputedStyle, treeWalker). Running it Node-side would require jsdom —
+ * a heavy dep the rest of Webcmd doesn't need.
+ */
+export interface ExtractArticleOptions {
+    /** CSS selectors removed from the cloned document before Readability runs. */
+    cleanSelectors?: string[];
+    /** Fallback chain when Readability fails. Defaults to the common structural ids. */
+    fallbackSelectors?: string[];
+    /** Bypass `isProbablyReaderable` and always attempt a parse. */
+    force?: boolean;
+}
+export type ExtractSource = 'readability' | 'fallback' | 'raw-text' | 'pre';
+export interface ExtractedArticle {
+    html: string;
+    title: string;
+    byline?: string;
+    publishedTime?: string;
+    siteName?: string;
+    source: ExtractSource;
+}
+export declare const DEFAULT_FALLBACK_SELECTORS: string[];
+/**
+ * Build the JS expression evaluated in-page to extract the article. Exported
+ * for testability — callers on the host side should use `extractArticle`.
+ */
+export declare function buildExtractArticleJs(options?: ExtractArticleOptions): string;
+export interface PageLike {
+    evaluate(js: string): Promise<unknown>;
+}
+/**
+ * Run the extract pipeline on the given page. Returns `null` when no usable
+ * content is found (Readability miss + empty fallback chain).
+ */
+export declare function extractArticle(page: PageLike, options?: ExtractArticleOptions): Promise<ExtractedArticle | null>;

package/dist/src/browser/article-extract.e2e.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/src/browser/article-extract.e2e.test.js

@@ -0,0 +1,105 @@
+import { afterEach, describe, expect, it } from 'vitest';
+import { JSDOM } from 'jsdom';
+import * as fs from 'node:fs';
+import * as os from 'node:os';
+import * as path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { buildExtractArticleJs } from './article-extract.js';
+import { downloadArticle } from '../download/article-download.js';
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const fixturesDir = path.join(__dirname, '__fixtures__', 'article-extract');
+const tempDirs = [];
+afterEach(() => {
+    for (const dir of tempDirs)
+        fs.rmSync(dir, { recursive: true, force: true });
+    tempDirs.length = 0;
+});
+function loadFixture(name) {
+    return fs.readFileSync(path.join(fixturesDir, name), 'utf8');
+}
+function escapeHtml(text) {
+    return text.replace(/[&<>]/g, ch => ({ '&': '&amp;', '<': '&lt;', '>': '&gt;' }[ch]));
+}
+function runExtract(html, url, options = {}, contentType) {
+    const dom = new JSDOM(html, {
+        url,
+        contentType: 'text/html',
+        pretendToBeVisual: true,
+        runScripts: 'outside-only',
+    });
+    if (contentType) {
+        Object.defineProperty(dom.window.document, 'contentType', {
+            value: contentType,
+            configurable: true,
+        });
+    }
+    return dom.window.eval(buildExtractArticleJs(options));
+}
+async function renderMarkdown(article, url, options = {}) {
+    const tempDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'webcmd-article-e2e-'));
+    tempDirs.push(tempDir);
+    const result = await downloadArticle({
+        title: article.title || 'untitled',
+        contentHtml: article.html,
+        sourceUrl: url,
+    }, {
+        output: tempDir,
+        downloadImages: false,
+        cleanSelectors: options.cleanSelectors,
+    });
+    expect(result[0].status).toBe('success');
+    return fs.readFileSync(result[0].saved, 'utf8');
+}
+describe('article extract → markdown e2e fixtures', () => {
+    it('extracts a Wikipedia article fixture and keeps infobox/reference noise out of markdown', async () => {
+        const url = 'https://en.wikipedia.org/wiki/Markdown';
+        const cleanSelectors = ['.infobox', '.navbox', '.reference', '.mw-editsection', '.metadata'];
+        const article = runExtract(loadFixture('wikipedia-markdown.html'), url, { cleanSelectors });
+        expect(article?.source).toBe('readability');
+        expect(article?.title).toBe('Markdown');
+        if (!article)
+            throw new Error('expected extracted article');
+        const md = await renderMarkdown(article, url, { cleanSelectors });
+        expect(md).toContain('lightweight markup language');
+        expect(md).toContain('John Gruber');
+        expect(md).not.toContain('Syntax description');
+        expect(md).not.toContain('Standard file extension');
+    });
+    it('extracts a Deno blog fixture, preserves embedded iframes as markdown links, and drops page chrome', async () => {
+        const url = 'https://deno.com/blog/v2.0';
+        const article = runExtract(loadFixture('deno-v2.html'), url);
+        expect(article?.source).toBe('readability');
+        expect(article?.title).toBe('Announcing Deno 2 | Deno');
+        if (!article)
+            throw new Error('expected extracted article');
+        const md = await renderMarkdown(article, url);
+        expect(md).toContain('## Announcing Deno 2');
+        expect(md).toContain('The web is humanity’s largest software platform');
+        expect(md).toMatch(/\]\(https:\/\/www\.youtube(?:-nocookie)?\.com\/embed\/[^)]+\)/);
+        expect(md).not.toContain('Skip to main content');
+    });
+    it('short-circuits non-HTML raw text pages end-to-end', async () => {
+        const url = 'https://raw.githubusercontent.com/openai/openai-cookbook/main/README.md';
+        const text = loadFixture('openai-cookbook-readme.txt');
+        const html = `<html><head><title>OpenAI Cookbook README</title></head><body><pre>${escapeHtml(text)}</pre></body></html>`;
+        const article = runExtract(html, url, {}, 'text/plain');
+        expect(article?.source).toBe('raw-text');
+        if (!article)
+            throw new Error('expected extracted article');
+        const md = await renderMarkdown(article, url);
+        expect(md).toContain('OPENAI\\_API\\_KEY');
+        expect(md).toContain('Example code and guides for accomplishing common tasks');
+    });
+    it('short-circuits a single-pre document end-to-end', async () => {
+        const url = 'https://raw.githubusercontent.com/openai/openai-cookbook/main/README.md';
+        const text = loadFixture('openai-cookbook-readme.txt');
+        const html = `<html><head><title>OpenAI Cookbook README</title></head><body><pre>${escapeHtml(text)}</pre></body></html>`;
+        const article = runExtract(html, url);
+        expect(article?.source).toBe('pre');
+        if (!article)
+            throw new Error('expected extracted article');
+        const md = await renderMarkdown(article, url);
+        expect(md).toContain('OPENAI\\_API\\_KEY');
+        expect(md).toContain('Most code examples are written in Python');
+    });
+});

package/dist/src/browser/article-extract.js

@@ -0,0 +1,169 @@
+/**
+ * Article extraction via Readability — generic `page → article HTML` pipeline.
+ *
+ * Complements `src/browser/extract.ts`: that one takes a caller-supplied
+ * selector. This one works with zero configuration on arbitrary article pages
+ * (blogs, news, docs) by running `@mozilla/readability` inside the page
+ * context via CDP evaluate.
+ *
+ * Pipeline:
+ *   1. Short-circuit non-HTML documents (`text/plain`, JSON, XML) — a page
+ *      renderer wrapping a plain-text file would pollute the DOM pipeline.
+ *   2. Short-circuit the "body is a single <pre>" case, which browsers use
+ *      when loading *.txt / *.md over file:// or raw.githubusercontent.com.
+ *   3. Deep-clone the document, apply caller-supplied `cleanSelectors` to the
+ *      clone (preserves live page state for subsequent snapshot/click).
+ *   4. Inject Readability + isProbablyReaderable sources into the page,
+ *      parse on the clone. `isProbablyReaderable` gates the parse unless
+ *      `force: true`.
+ *   5. On Readability miss, walk a fallback selector chain
+ *      (main → [role="main"] → #main-content → … → body) and return the
+ *      first root with >80 characters of text.
+ *
+ * Readability runs in the page's own window because it needs real DOM APIs
+ * (getComputedStyle, treeWalker). Running it Node-side would require jsdom —
+ * a heavy dep the rest of Webcmd doesn't need.
+ */
+import * as fs from 'node:fs';
+import { createRequire } from 'node:module';
+const requireFromHere = createRequire(import.meta.url);
+let cachedSources = null;
+function readabilitySources() {
+    if (cachedSources)
+        return cachedSources;
+    const readabilityPath = requireFromHere.resolve('@mozilla/readability/Readability.js');
+    const readerablePath = requireFromHere.resolve('@mozilla/readability/Readability-readerable.js');
+    cachedSources = {
+        readability: fs.readFileSync(readabilityPath, 'utf8'),
+        readerable: fs.readFileSync(readerablePath, 'utf8'),
+    };
+    return cachedSources;
+}
+export const DEFAULT_FALLBACK_SELECTORS = [
+    'main',
+    '[role="main"]',
+    '#main-content',
+    '#main',
+    '#content',
+    '.content',
+    'article',
+    'body',
+];
+const MIN_FALLBACK_TEXT_LENGTH = 80;
+/**
+ * Build the JS expression evaluated in-page to extract the article. Exported
+ * for testability — callers on the host side should use `extractArticle`.
+ */
+export function buildExtractArticleJs(options = {}) {
+    const { readability, readerable } = readabilitySources();
+    const cleanSelectors = options.cleanSelectors ?? [];
+    const fallbackSelectors = options.fallbackSelectors ?? DEFAULT_FALLBACK_SELECTORS;
+    const force = !!options.force;
+    // Library sources contain backticks and ${...} fragments, so we embed them
+    // as JSON-encoded string literals and eval them inside a Function() scope.
+    // This isolates their var declarations from the outer IIFE without polluting
+    // window globals.
+    const readabilityLit = JSON.stringify(readability);
+    const readerableLit = JSON.stringify(readerable);
+    const cleanLit = JSON.stringify(cleanSelectors);
+    const fallbackLit = JSON.stringify(fallbackSelectors);
+    const forceLit = JSON.stringify(force);
+    return [
+        '(() => {',
+        '  const cleanSelectors = ' + cleanLit + ';',
+        '  const fallbackSelectors = ' + fallbackLit + ';',
+        '  const force = ' + forceLit + ';',
+        '  const minFallbackText = ' + MIN_FALLBACK_TEXT_LENGTH + ';',
+        '  const readabilitySrc = ' + readabilityLit + ';',
+        '  const readerableSrc = ' + readerableLit + ';',
+        '',
+        '  function escapeHtml(s) {',
+        '    return String(s).replace(/[&<>]/g, c => ({ "&": "&amp;", "<": "&lt;", ">": "&gt;" }[c]));',
+        '  }',
+        '',
+        '  // Short-circuit 1: non-HTML document',
+        '  const ct = document.contentType || "";',
+        '  if (ct && ct !== "text/html" && ct !== "application/xhtml+xml") {',
+        '    const body = document.body ? (document.body.textContent || "") : "";',
+        '    return { source: "raw-text", html: "<pre>" + escapeHtml(body) + "</pre>", title: document.title || "" };',
+        '  }',
+        '',
+        '  // Short-circuit 2: body is a single <pre>',
+        '  if (document.body) {',
+        '    const kids = document.body.children;',
+        '    if (kids.length === 1 && kids[0] && kids[0].tagName === "PRE") {',
+        '      return { source: "pre", html: document.body.outerHTML, title: document.title || "" };',
+        '    }',
+        '  }',
+        '',
+        '  // Deep-clone + adapter-supplied dirty-node removal',
+        '  const cloneDoc = document.cloneNode(true);',
+        '  for (const sel of cleanSelectors) {',
+        '    try { for (const n of cloneDoc.querySelectorAll(sel)) n.remove(); }',
+        '    catch (e) { /* ignore invalid selector */ }',
+        '  }',
+        '',
+        '  // Inject Readability into an isolated Function scope and extract the',
+        '  // constructors we need. Library sources use their own module.exports',
+        '  // guard (if typeof module === "object"), which is falsy here.',
+        '  const libs = (new Function(',
+        '    readabilitySrc + "\\n" + readerableSrc + "\\nreturn {" +',
+        '    " Readability: typeof Readability !== \\"undefined\\" ? Readability : null," +',
+        '    " isProbablyReaderable: typeof isProbablyReaderable !== \\"undefined\\" ? isProbablyReaderable : null" +',
+        '    " };"',
+        '  ))();',
+        '  const Readability = libs.Readability;',
+        '  const isProbablyReaderable = libs.isProbablyReaderable;',
+        '',
+        '  const readerableOk = force || (typeof isProbablyReaderable === "function" ? isProbablyReaderable(cloneDoc) : true);',
+        '  let article = null;',
+        '  if (readerableOk && typeof Readability === "function") {',
+        '    try { article = new Readability(cloneDoc).parse(); } catch (e) { article = null; }',
+        '  }',
+        '  if (article && article.content) {',
+        '    return {',
+        '      source: "readability",',
+        '      html: article.content,',
+        '      title: article.title || document.title || "",',
+        '      byline: article.byline || undefined,',
+        '      publishedTime: article.publishedTime || undefined,',
+        '      siteName: article.siteName || undefined,',
+        '    };',
+        '  }',
+        '',
+        '  // Fallback chain',
+        '  for (const sel of fallbackSelectors) {',
+        '    let el = null;',
+        '    try { el = cloneDoc.querySelector(sel); } catch (e) { continue; }',
+        '    if (!el) continue;',
+        '    const text = (el.textContent || "").trim();',
+        '    if (text.length < minFallbackText) continue;',
+        '    return { source: "fallback", html: el.outerHTML, title: document.title || "" };',
+        '  }',
+        '',
+        '  return null;',
+        '})()',
+    ].join('\n');
+}
+/**
+ * Run the extract pipeline on the given page. Returns `null` when no usable
+ * content is found (Readability miss + empty fallback chain).
+ */
+export async function extractArticle(page, options = {}) {
+    const js = buildExtractArticleJs(options);
+    const raw = await page.evaluate(js);
+    if (raw == null || typeof raw !== 'object')
+        return null;
+    const r = raw;
+    if (typeof r.html !== 'string' || typeof r.source !== 'string')
+        return null;
+    const source = r.source;
+    return {
+        html: r.html,
+        title: typeof r.title === 'string' ? r.title : '',
+        ...(r.byline && { byline: r.byline }),
+        ...(r.publishedTime && { publishedTime: r.publishedTime }),
+        ...(r.siteName && { siteName: r.siteName }),
+        source,
+    };
+}

package/dist/src/browser/article-extract.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/src/browser/article-extract.test.js

@@ -0,0 +1,94 @@
+import { describe, expect, it } from 'vitest';
+import { buildExtractArticleJs, extractArticle, DEFAULT_FALLBACK_SELECTORS, } from './article-extract.js';
+function fakePage(response) {
+    const state = { lastJs: null };
+    return {
+        lastJs: null,
+        async evaluate(js) {
+            state.lastJs = js;
+            Object.assign(this, state);
+            return response;
+        },
+    };
+}
+describe('buildExtractArticleJs', () => {
+    it('embeds Readability + Readerable sources once per evaluation', () => {
+        const js = buildExtractArticleJs();
+        // Both libs should be inlined (matched by identifying strings from the
+        // upstream @mozilla/readability sources).
+        expect(js).toContain('function Readability(doc, options)');
+        expect(js).toContain('function isProbablyReaderable');
+    });
+    it('serializes caller-supplied options into the evaluated JS', () => {
+        const js = buildExtractArticleJs({
+            cleanSelectors: ['.ads', '#banner'],
+            fallbackSelectors: ['article', 'body'],
+            force: true,
+        });
+        expect(js).toContain('[".ads","#banner"]');
+        expect(js).toContain('["article","body"]');
+        expect(js).toContain('const force = true;');
+    });
+    it('uses the default fallback chain when none is supplied', () => {
+        const js = buildExtractArticleJs();
+        for (const sel of DEFAULT_FALLBACK_SELECTORS) {
+            expect(js).toContain(JSON.stringify(sel));
+        }
+    });
+    it('runs fallback selection against the cleaned clone', () => {
+        const js = buildExtractArticleJs({ cleanSelectors: ['.noise'] });
+        expect(js).toContain('el = cloneDoc.querySelector(sel);');
+        expect(js).not.toContain('el = document.querySelector(sel);');
+    });
+    it('produces syntactically valid JavaScript', () => {
+        // Parsing via the Function constructor rejects any syntax error in the
+        // generated code — including accidental template-literal break-outs from
+        // the embedded Readability sources.
+        expect(() => new Function(buildExtractArticleJs())).not.toThrow();
+        expect(() => new Function(buildExtractArticleJs({ force: true }))).not.toThrow();
+        expect(() => new Function(buildExtractArticleJs({
+            cleanSelectors: ['.a', '.b'],
+            fallbackSelectors: ['main', 'body'],
+        }))).not.toThrow();
+    });
+});
+describe('extractArticle (host-side)', () => {
+    it('returns a normalized ExtractedArticle when the page responds with one', async () => {
+        const page = fakePage({
+            source: 'readability',
+            html: '<p>hello</p>',
+            title: 'Hello',
+            byline: 'Alice',
+            publishedTime: '2026-04-22',
+            siteName: 'Example',
+        });
+        const res = await extractArticle(page);
+        expect(res).toEqual({
+            source: 'readability',
+            html: '<p>hello</p>',
+            title: 'Hello',
+            byline: 'Alice',
+            publishedTime: '2026-04-22',
+            siteName: 'Example',
+        });
+    });
+    it('drops undefined optional fields cleanly', async () => {
+        const page = fakePage({ source: 'fallback', html: '<main>x</main>', title: 't' });
+        const res = await extractArticle(page);
+        expect(res).toEqual({ source: 'fallback', html: '<main>x</main>', title: 't' });
+        expect(res).not.toHaveProperty('byline');
+        expect(res).not.toHaveProperty('publishedTime');
+    });
+    it('returns null on a missing body or malformed payload', async () => {
+        expect(await extractArticle(fakePage(null))).toBeNull();
+        expect(await extractArticle(fakePage('oops'))).toBeNull();
+        expect(await extractArticle(fakePage({ source: 'readability' }))).toBeNull();
+        expect(await extractArticle(fakePage({ html: '<p>x</p>' }))).toBeNull();
+    });
+    it('defaults title to empty string when the page omits it', async () => {
+        const page = fakePage({ source: 'pre', html: '<body><pre>x</pre></body>' });
+        const res = await extractArticle(page);
+        expect(res?.title).toBe('');
+        expect(res?.source).toBe('pre');
+    });
+});