@jackwener/opencli 1.7.6 → 1.7.8

package/dist/src/browser/analyze.js

@@ -0,0 +1,230 @@
+/**
+ * `browser analyze <url>` — turn site-recon guesswork into deterministic CLI output.
+ *
+ * When an agent starts a new adapter, the first question is "which pattern am
+ * I looking at?" (A/B/C/D/E from site-recon docs) and "will Node-side fetch
+ * work, or will anti-bot middleware block me?". Today the agent has to open
+ * the page, poke `network`, try cURL, fail, guess again. This module condenses
+ * that into one call that returns a classification + evidence.
+ *
+ * Kept pure (no page imports) so the bulk is unit-testable; the CLI wrapper
+ * drives a real page, feeds the resulting signals here, and prints the verdict.
+ */
+/**
+ * WAF vendors we can reliably detect from cookies + response body markers
+ * alone. Signals are orthogonal per vendor — so when two vendors match
+ * simultaneously (rare), we keep all evidence and report the higher-signal
+ * vendor first.
+ */
+const WAF_SIGNATURES = [
+    {
+        vendor: 'aliyun_waf',
+        cookiePatterns: [/^acw_sc__v2$/, /^acw_tc$/, /^ssxmod_itna/],
+        bodyPatterns: [/arg1\s*=\s*['"][0-9A-F]{30,}/, /\/ntc_captcha\//i],
+        implication: 'Direct Node-side fetch/curl will return the slider HTML. Validate the endpoint in browser context first; HTML COOKIE adapters still finish with Node-side fetch + page.getCookies.',
+    },
+    {
+        vendor: 'cloudflare',
+        cookiePatterns: [/^__cf_bm$/, /^cf_clearance$/, /^__cfduid$/],
+        bodyPatterns: [/Cloudflare Ray ID/i, /Checking your browser before accessing/i, /cf-chl-/i],
+        implication: 'Cloudflare bot check. Start from a real browser session; probe in browser context first. HTML COOKIE adapters still finish with Node-side fetch + page.getCookies.',
+    },
+    {
+        vendor: 'akamai',
+        cookiePatterns: [/^_abck$/, /^bm_sz$/, /^bm_sv$/],
+        bodyPatterns: [/akamai/i],
+        implication: 'Akamai Bot Manager. Probe in browser context first; keep final HTML COOKIE adapters on Node-side fetch + page.getCookies.',
+    },
+    {
+        vendor: 'geetest',
+        cookiePatterns: [],
+        bodyPatterns: [/geetest/i, /gt_captcha/i],
+        implication: 'Geetest slider/puzzle captcha. Agent cannot bypass programmatically — requires UI strategy or human-in-loop.',
+    },
+];
+export function detectAntiBot(signals) {
+    const evidence = [];
+    let match = null;
+    for (const sig of WAF_SIGNATURES) {
+        const hits = [];
+        for (const pat of sig.cookiePatterns) {
+            const hit = signals.cookieNames.find((c) => pat.test(c));
+            if (hit)
+                hits.push(`cookie:${hit}`);
+        }
+        for (const pat of sig.bodyPatterns) {
+            for (const entry of signals.networkEntries) {
+                if (entry.bodyPreview && pat.test(entry.bodyPreview)) {
+                    hits.push(`body:${entry.url}`);
+                    break;
+                }
+            }
+        }
+        if (hits.length > 0 && !match) {
+            match = sig;
+            evidence.push(...hits);
+        }
+    }
+    if (!match) {
+        return {
+            detected: false,
+            vendor: null,
+            evidence: [],
+            implication: 'No known anti-bot signatures. Try Node-side COOKIE fetch first; if endpoint validation is blocked, retry from browser context.',
+        };
+    }
+    return {
+        detected: true,
+        vendor: match.vendor,
+        evidence,
+        implication: match.implication,
+    };
+}
+/**
+ * Apply the decision tree from `site-recon.md` mechanically.
+ *
+ * B beats A when initial-state globals are present: even if the page fetches
+ * more data via XHR afterwards, the SSR payload is the highest-leverage source.
+ * D (token-gated) dominates when we see 401/403 on what looks like API
+ * endpoints — without that, an authenticated route looks identical to A.
+ */
+export function classifyPattern(signals) {
+    const jsonEntries = signals.networkEntries.filter((e) => /json/i.test(e.contentType));
+    const authFailures = signals.networkEntries.filter((e) => e.status === 401 || e.status === 403).length;
+    const hasInitialState = signals.initialState.__INITIAL_STATE__ ||
+        signals.initialState.__NUXT__ ||
+        signals.initialState.__NEXT_DATA__ ||
+        signals.initialState.__APOLLO_STATE__;
+    if (authFailures >= 2 && jsonEntries.length >= 1) {
+        return {
+            pattern: 'D',
+            reason: `${authFailures} auth-failing API responses seen — endpoint is token-gated`,
+            json_responses: jsonEntries.length,
+            auth_failures: authFailures,
+        };
+    }
+    if (hasInitialState) {
+        const which = Object.entries(signals.initialState)
+            .filter(([, v]) => v)
+            .map(([k]) => k);
+        return {
+            pattern: 'B',
+            reason: `SSR state global present: ${which.join(', ')}`,
+            json_responses: jsonEntries.length,
+            auth_failures: authFailures,
+        };
+    }
+    if (jsonEntries.length >= 1) {
+        return {
+            pattern: 'A',
+            reason: `${jsonEntries.length} JSON XHR/fetch responses observed — classic API pattern`,
+            json_responses: jsonEntries.length,
+            auth_failures: authFailures,
+        };
+    }
+    // No API, no SSR state — probably static HTML or a bundled SPA that lazy-loads.
+    // Pattern C (HTML scrape) is the default fallback; E (streaming) we can't
+    // reliably detect without watching WebSocket frames, so we label 'C' and
+    // leave the agent to upgrade to E manually if they see WS traffic.
+    return {
+        pattern: 'C',
+        reason: 'No JSON XHR and no SSR state — HTML scrape (Pattern C); escalate to E manually if WebSocket traffic appears',
+        json_responses: jsonEntries.length,
+        auth_failures: authFailures,
+    };
+}
+/**
+ * Find existing adapters that target the same site.
+ *
+ * Keep the hostname match simple — agents extend naming conventions
+ * differently per site, so we match on the registered `domain` field and fall
+ * back to site-name containment. Returning `null` is fine; agents can always
+ * read site-memory docs.
+ */
+export function findNearestAdapter(finalUrl, registry) {
+    let host;
+    try {
+        host = new URL(finalUrl).hostname;
+    }
+    catch {
+        return null;
+    }
+    // Strip leading www.; 'www' as a site identifier is never what an adapter uses.
+    const cleanedHost = host.replace(/^www\./, '');
+    // Extract apex (xx.com) and registrable parts for fuzzy match.
+    const parts = cleanedHost.split('.');
+    const apex = parts.slice(-2).join('.');
+    const siteKey = parts.length > 1 ? parts[parts.length - 2] : cleanedHost;
+    const hits = new Map();
+    for (const cmd of registry.values()) {
+        const domain = cmd.domain?.toLowerCase();
+        const siteMatches = (domain && (cleanedHost.endsWith(domain) || domain.endsWith(apex))) ||
+            cmd.site.toLowerCase() === siteKey?.toLowerCase() ||
+            cleanedHost.includes(cmd.site.toLowerCase());
+        if (siteMatches) {
+            const list = hits.get(cmd.site) ?? [];
+            list.push(cmd);
+            hits.set(cmd.site, list);
+        }
+    }
+    if (hits.size === 0)
+        return null;
+    // Pick the site with the most commands — likely the most-developed adapter,
+    // and the best reference for a new command on the same host.
+    let best = null;
+    for (const entry of hits) {
+        if (!best || entry[1].length > best[1].length)
+            best = entry;
+    }
+    if (!best)
+        return null;
+    return {
+        site: best[0],
+        example_commands: best[1].slice(0, 5).map((c) => `${c.site} ${c.name}`),
+        reason: `${best[1].length} existing adapter${best[1].length === 1 ? '' : 's'} target this site — reuse strategy/cookie config`,
+    };
+}
+/**
+ * Synthesize the verdict from collected signals + registry.
+ *
+ * The `recommended_next_step` is deliberately a single imperative
+ * sentence — agents act on it directly instead of re-deriving advice from
+ * the structured fields.
+ */
+export function analyzeSite(signals, registry) {
+    const pattern = classifyPattern(signals);
+    const antiBot = detectAntiBot(signals);
+    const nearest = findNearestAdapter(signals.finalUrl, registry);
+    let next;
+    if (antiBot.detected) {
+        next = antiBot.implication;
+    }
+    else if (pattern.pattern === 'A') {
+        next = 'Pick the most specific JSON endpoint from `opencli browser network` and try a bare Node fetch with cookies; escalate to browser-context fetch only if blocked.';
+    }
+    else if (pattern.pattern === 'B') {
+        next = 'Read the SSR global via `opencli browser eval "JSON.stringify(window.__INITIAL_STATE__ ?? window.__NUXT__ ?? window.__NEXT_DATA__ ?? window.__APOLLO_STATE__)"` — no API needed.';
+    }
+    else if (pattern.pattern === 'C') {
+        next = 'No API visible — use SSR HTML scrape (e.g. `opencli browser extract`) against the rendered page.';
+    }
+    else if (pattern.pattern === 'D') {
+        next = 'Endpoints need auth. Re-open the page from a signed-in session, then retry analyze; see `field-decode-playbook` §4 for token tracing.';
+    }
+    else if (pattern.pattern === 'E') {
+        next = 'WebSocket stream detected — find the underlying HTTP poll/long-poll endpoint; raw WS is not supported.';
+    }
+    else {
+        next = 'No strong signal. Manually inspect `opencli browser network --all` and pick a pattern.';
+    }
+    return {
+        requested_url: signals.requestedUrl,
+        final_url: signals.finalUrl,
+        title: signals.title,
+        pattern,
+        anti_bot: antiBot,
+        initial_state: signals.initialState,
+        nearest_adapter: nearest,
+        recommended_next_step: next,
+    };
+}

package/dist/src/browser/analyze.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/src/browser/analyze.test.js

@@ -0,0 +1,164 @@
+import { describe, it, expect } from 'vitest';
+import { analyzeSite, detectAntiBot, classifyPattern, findNearestAdapter, } from './analyze.js';
+function mkSignals(overrides = {}) {
+    return {
+        requestedUrl: 'https://example.com/',
+        finalUrl: 'https://example.com/',
+        cookieNames: [],
+        networkEntries: [],
+        initialState: {
+            __INITIAL_STATE__: false,
+            __NUXT__: false,
+            __NEXT_DATA__: false,
+            __APOLLO_STATE__: false,
+        },
+        title: 'Example',
+        ...overrides,
+    };
+}
+function mkCmd(site, name, domain) {
+    return {
+        site,
+        name,
+        description: '',
+        domain,
+        args: [],
+    };
+}
+describe('detectAntiBot', () => {
+    it('flags Aliyun WAF from cookie', () => {
+        const v = detectAntiBot(mkSignals({ cookieNames: ['JSESSIONID', 'acw_sc__v2'] }));
+        expect(v.detected).toBe(true);
+        expect(v.vendor).toBe('aliyun_waf');
+        expect(v.evidence).toContain('cookie:acw_sc__v2');
+        expect(v.implication).toMatch(/browser context/i);
+    });
+    it('flags Aliyun WAF from challenge HTML body', () => {
+        const v = detectAntiBot(mkSignals({
+            networkEntries: [
+                {
+                    url: 'https://x.com/',
+                    status: 200,
+                    contentType: 'text/html',
+                    bodyPreview: "var arg1 = 'A1B2C3D4E5F6A7B8C9D0E1F2A3B4C5D6';",
+                },
+            ],
+        }));
+        expect(v.detected).toBe(true);
+        expect(v.vendor).toBe('aliyun_waf');
+    });
+    it('flags Cloudflare from cf_clearance cookie', () => {
+        const v = detectAntiBot(mkSignals({ cookieNames: ['cf_clearance'] }));
+        expect(v.vendor).toBe('cloudflare');
+        expect(v.implication).toMatch(/Cloudflare/i);
+    });
+    it('flags Akamai from _abck cookie', () => {
+        const v = detectAntiBot(mkSignals({ cookieNames: ['_abck', 'bm_sz'] }));
+        expect(v.vendor).toBe('akamai');
+    });
+    it('returns no-match verdict with actionable fallback advice', () => {
+        const v = detectAntiBot(mkSignals());
+        expect(v.detected).toBe(false);
+        expect(v.vendor).toBeNull();
+        expect(v.implication).toMatch(/Node-side COOKIE fetch first/);
+    });
+});
+describe('classifyPattern', () => {
+    it('returns A for JSON-heavy pages without SSR state', () => {
+        const v = classifyPattern(mkSignals({
+            networkEntries: [
+                { url: 'https://x.com/api/a', status: 200, contentType: 'application/json', bodyPreview: '{}' },
+                { url: 'https://x.com/api/b', status: 200, contentType: 'application/json;charset=utf-8', bodyPreview: '{}' },
+            ],
+        }));
+        expect(v.pattern).toBe('A');
+        expect(v.json_responses).toBe(2);
+    });
+    it('returns B when __INITIAL_STATE__ is present, beating JSON signals', () => {
+        const v = classifyPattern(mkSignals({
+            initialState: { __INITIAL_STATE__: true, __NUXT__: false, __NEXT_DATA__: false, __APOLLO_STATE__: false },
+            networkEntries: [
+                { url: 'https://x.com/api/a', status: 200, contentType: 'application/json', bodyPreview: '{}' },
+            ],
+        }));
+        expect(v.pattern).toBe('B');
+    });
+    it('returns D when auth failures dominate', () => {
+        const v = classifyPattern(mkSignals({
+            networkEntries: [
+                { url: 'https://x.com/api/a', status: 401, contentType: 'application/json', bodyPreview: '' },
+                { url: 'https://x.com/api/b', status: 403, contentType: 'application/json', bodyPreview: '' },
+            ],
+        }));
+        expect(v.pattern).toBe('D');
+        expect(v.auth_failures).toBe(2);
+    });
+    it('returns C by default for static pages', () => {
+        const v = classifyPattern(mkSignals());
+        expect(v.pattern).toBe('C');
+    });
+});
+describe('findNearestAdapter', () => {
+    it('matches by domain suffix', () => {
+        const reg = new Map([
+            ['51job search', mkCmd('51job', 'search', '51job.com')],
+            ['51job detail', mkCmd('51job', 'detail', '51job.com')],
+            ['xueqiu search', mkCmd('xueqiu', 'search', 'xueqiu.com')],
+        ]);
+        const v = findNearestAdapter('https://jobs.51job.com/', reg);
+        expect(v?.site).toBe('51job');
+        expect(v?.example_commands).toContain('51job search');
+    });
+    it('falls back to site-name containment when no domain is registered', () => {
+        const reg = new Map([
+            ['51job search', mkCmd('51job', 'search')],
+        ]);
+        const v = findNearestAdapter('https://we.51job.com/', reg);
+        expect(v?.site).toBe('51job');
+    });
+    it('returns null when no adapter matches', () => {
+        const reg = new Map([
+            ['xueqiu search', mkCmd('xueqiu', 'search', 'xueqiu.com')],
+        ]);
+        const v = findNearestAdapter('https://random-site.io/', reg);
+        expect(v).toBeNull();
+    });
+    it('prefers the site with the most commands', () => {
+        const reg = new Map([
+            ['a search', mkCmd('a', 'search', 'a.com')],
+            ['b search', mkCmd('b', 'search', 'a.com')],
+            ['b detail', mkCmd('b', 'detail', 'a.com')],
+            ['b company', mkCmd('b', 'company', 'a.com')],
+        ]);
+        const v = findNearestAdapter('https://jobs.a.com/', reg);
+        expect(v?.site).toBe('b');
+    });
+});
+describe('analyzeSite', () => {
+    it('recommends browser-context fetch when WAF is detected', () => {
+        const report = analyzeSite(mkSignals({ cookieNames: ['acw_sc__v2'] }), new Map());
+        expect(report.anti_bot.vendor).toBe('aliyun_waf');
+        expect(report.recommended_next_step).toMatch(/browser context/i);
+    });
+    it('recommends reading SSR state when Pattern B fires', () => {
+        const report = analyzeSite(mkSignals({
+            initialState: { __INITIAL_STATE__: false, __NUXT__: true, __NEXT_DATA__: false, __APOLLO_STATE__: false },
+        }), new Map());
+        expect(report.pattern.pattern).toBe('B');
+        expect(report.recommended_next_step).toMatch(/__NUXT__|__INITIAL_STATE__|__NEXT_DATA__/);
+    });
+    it('includes __APOLLO_STATE__ in Pattern B next-step guidance', () => {
+        const report = analyzeSite(mkSignals({
+            initialState: { __INITIAL_STATE__: false, __NUXT__: false, __NEXT_DATA__: false, __APOLLO_STATE__: true },
+        }), new Map());
+        expect(report.pattern.pattern).toBe('B');
+        expect(report.recommended_next_step).toMatch(/__APOLLO_STATE__/);
+    });
+    it('includes nearest_adapter when the registry has a match', () => {
+        const reg = new Map([
+            ['51job search', mkCmd('51job', 'search', '51job.com')],
+        ]);
+        const report = analyzeSite(mkSignals({ finalUrl: 'https://we.51job.com/' }), reg);
+        expect(report.nearest_adapter?.site).toBe('51job');
+    });
+});

package/dist/src/browser/article-extract.d.ts

@@ -0,0 +1,57 @@
+/**
+ * Article extraction via Readability — generic `page → article HTML` pipeline.
+ *
+ * Complements `src/browser/extract.ts`: that one takes a caller-supplied
+ * selector. This one works with zero configuration on arbitrary article pages
+ * (blogs, news, docs) by running `@mozilla/readability` inside the page
+ * context via CDP evaluate.
+ *
+ * Pipeline:
+ *   1. Short-circuit non-HTML documents (`text/plain`, JSON, XML) — a page
+ *      renderer wrapping a plain-text file would pollute the DOM pipeline.
+ *   2. Short-circuit the "body is a single <pre>" case, which browsers use
+ *      when loading *.txt / *.md over file:// or raw.githubusercontent.com.
+ *   3. Deep-clone the document, apply caller-supplied `cleanSelectors` to the
+ *      clone (preserves live page state for subsequent snapshot/click).
+ *   4. Inject Readability + isProbablyReaderable sources into the page,
+ *      parse on the clone. `isProbablyReaderable` gates the parse unless
+ *      `force: true`.
+ *   5. On Readability miss, walk a fallback selector chain
+ *      (main → [role="main"] → #main-content → … → body) and return the
+ *      first root with >80 characters of text.
+ *
+ * Readability runs in the page's own window because it needs real DOM APIs
+ * (getComputedStyle, treeWalker). Running it Node-side would require jsdom —
+ * a heavy dep the rest of OpenCLI doesn't need.
+ */
+export interface ExtractArticleOptions {
+    /** CSS selectors removed from the cloned document before Readability runs. */
+    cleanSelectors?: string[];
+    /** Fallback chain when Readability fails. Defaults to the common structural ids. */
+    fallbackSelectors?: string[];
+    /** Bypass `isProbablyReaderable` and always attempt a parse. */
+    force?: boolean;
+}
+export type ExtractSource = 'readability' | 'fallback' | 'raw-text' | 'pre';
+export interface ExtractedArticle {
+    html: string;
+    title: string;
+    byline?: string;
+    publishedTime?: string;
+    siteName?: string;
+    source: ExtractSource;
+}
+export declare const DEFAULT_FALLBACK_SELECTORS: string[];
+/**
+ * Build the JS expression evaluated in-page to extract the article. Exported
+ * for testability — callers on the host side should use `extractArticle`.
+ */
+export declare function buildExtractArticleJs(options?: ExtractArticleOptions): string;
+export interface PageLike {
+    evaluate(js: string): Promise<unknown>;
+}
+/**
+ * Run the extract pipeline on the given page. Returns `null` when no usable
+ * content is found (Readability miss + empty fallback chain).
+ */
+export declare function extractArticle(page: PageLike, options?: ExtractArticleOptions): Promise<ExtractedArticle | null>;

package/dist/src/browser/article-extract.e2e.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/src/browser/article-extract.e2e.test.js

@@ -0,0 +1,105 @@
+import { afterEach, describe, expect, it } from 'vitest';
+import { JSDOM } from 'jsdom';
+import * as fs from 'node:fs';
+import * as os from 'node:os';
+import * as path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { buildExtractArticleJs } from './article-extract.js';
+import { downloadArticle } from '../download/article-download.js';
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const fixturesDir = path.join(__dirname, '__fixtures__', 'article-extract');
+const tempDirs = [];
+afterEach(() => {
+    for (const dir of tempDirs)
+        fs.rmSync(dir, { recursive: true, force: true });
+    tempDirs.length = 0;
+});
+function loadFixture(name) {
+    return fs.readFileSync(path.join(fixturesDir, name), 'utf8');
+}
+function escapeHtml(text) {
+    return text.replace(/[&<>]/g, ch => ({ '&': '&amp;', '<': '&lt;', '>': '&gt;' }[ch]));
+}
+function runExtract(html, url, options = {}, contentType) {
+    const dom = new JSDOM(html, {
+        url,
+        contentType: 'text/html',
+        pretendToBeVisual: true,
+        runScripts: 'outside-only',
+    });
+    if (contentType) {
+        Object.defineProperty(dom.window.document, 'contentType', {
+            value: contentType,
+            configurable: true,
+        });
+    }
+    return dom.window.eval(buildExtractArticleJs(options));
+}
+async function renderMarkdown(article, url, options = {}) {
+    const tempDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'opencli-article-e2e-'));
+    tempDirs.push(tempDir);
+    const result = await downloadArticle({
+        title: article.title || 'untitled',
+        contentHtml: article.html,
+        sourceUrl: url,
+    }, {
+        output: tempDir,
+        downloadImages: false,
+        cleanSelectors: options.cleanSelectors,
+    });
+    expect(result[0].status).toBe('success');
+    return fs.readFileSync(result[0].saved, 'utf8');
+}
+describe('article extract → markdown e2e fixtures', () => {
+    it('extracts a Wikipedia article fixture and keeps infobox/reference noise out of markdown', async () => {
+        const url = 'https://en.wikipedia.org/wiki/Markdown';
+        const cleanSelectors = ['.infobox', '.navbox', '.reference', '.mw-editsection', '.metadata'];
+        const article = runExtract(loadFixture('wikipedia-markdown.html'), url, { cleanSelectors });
+        expect(article?.source).toBe('readability');
+        expect(article?.title).toBe('Markdown');
+        if (!article)
+            throw new Error('expected extracted article');
+        const md = await renderMarkdown(article, url, { cleanSelectors });
+        expect(md).toContain('lightweight markup language');
+        expect(md).toContain('John Gruber');
+        expect(md).not.toContain('Syntax description');
+        expect(md).not.toContain('Standard file extension');
+    });
+    it('extracts a Deno blog fixture, preserves embedded iframes as markdown links, and drops page chrome', async () => {
+        const url = 'https://deno.com/blog/v2.0';
+        const article = runExtract(loadFixture('deno-v2.html'), url);
+        expect(article?.source).toBe('readability');
+        expect(article?.title).toBe('Announcing Deno 2 | Deno');
+        if (!article)
+            throw new Error('expected extracted article');
+        const md = await renderMarkdown(article, url);
+        expect(md).toContain('## Announcing Deno 2');
+        expect(md).toContain('The web is humanity’s largest software platform');
+        expect(md).toMatch(/\]\(https:\/\/www\.youtube(?:-nocookie)?\.com\/embed\/[^)]+\)/);
+        expect(md).not.toContain('Skip to main content');
+    });
+    it('short-circuits non-HTML raw text pages end-to-end', async () => {
+        const url = 'https://raw.githubusercontent.com/openai/openai-cookbook/main/README.md';
+        const text = loadFixture('openai-cookbook-readme.txt');
+        const html = `<html><head><title>OpenAI Cookbook README</title></head><body><pre>${escapeHtml(text)}</pre></body></html>`;
+        const article = runExtract(html, url, {}, 'text/plain');
+        expect(article?.source).toBe('raw-text');
+        if (!article)
+            throw new Error('expected extracted article');
+        const md = await renderMarkdown(article, url);
+        expect(md).toContain('OPENAI\\_API\\_KEY');
+        expect(md).toContain('Example code and guides for accomplishing common tasks');
+    });
+    it('short-circuits a single-pre document end-to-end', async () => {
+        const url = 'https://raw.githubusercontent.com/openai/openai-cookbook/main/README.md';
+        const text = loadFixture('openai-cookbook-readme.txt');
+        const html = `<html><head><title>OpenAI Cookbook README</title></head><body><pre>${escapeHtml(text)}</pre></body></html>`;
+        const article = runExtract(html, url);
+        expect(article?.source).toBe('pre');
+        if (!article)
+            throw new Error('expected extracted article');
+        const md = await renderMarkdown(article, url);
+        expect(md).toContain('OPENAI\\_API\\_KEY');
+        expect(md).toContain('Most code examples are written in Python');
+    });
+});