@jackwener/opencli 1.7.6 → 1.7.8

package/dist/src/browser/article-extract.js

@@ -0,0 +1,169 @@
+/**
+ * Article extraction via Readability — generic `page → article HTML` pipeline.
+ *
+ * Complements `src/browser/extract.ts`: that one takes a caller-supplied
+ * selector. This one works with zero configuration on arbitrary article pages
+ * (blogs, news, docs) by running `@mozilla/readability` inside the page
+ * context via CDP evaluate.
+ *
+ * Pipeline:
+ *   1. Short-circuit non-HTML documents (`text/plain`, JSON, XML) — a page
+ *      renderer wrapping a plain-text file would pollute the DOM pipeline.
+ *   2. Short-circuit the "body is a single <pre>" case, which browsers use
+ *      when loading *.txt / *.md over file:// or raw.githubusercontent.com.
+ *   3. Deep-clone the document, apply caller-supplied `cleanSelectors` to the
+ *      clone (preserves live page state for subsequent snapshot/click).
+ *   4. Inject Readability + isProbablyReaderable sources into the page,
+ *      parse on the clone. `isProbablyReaderable` gates the parse unless
+ *      `force: true`.
+ *   5. On Readability miss, walk a fallback selector chain
+ *      (main → [role="main"] → #main-content → … → body) and return the
+ *      first root with >80 characters of text.
+ *
+ * Readability runs in the page's own window because it needs real DOM APIs
+ * (getComputedStyle, treeWalker). Running it Node-side would require jsdom —
+ * a heavy dep the rest of OpenCLI doesn't need.
+ */
+import * as fs from 'node:fs';
+import { createRequire } from 'node:module';
+const requireFromHere = createRequire(import.meta.url);
+let cachedSources = null;
+function readabilitySources() {
+    if (cachedSources)
+        return cachedSources;
+    const readabilityPath = requireFromHere.resolve('@mozilla/readability/Readability.js');
+    const readerablePath = requireFromHere.resolve('@mozilla/readability/Readability-readerable.js');
+    cachedSources = {
+        readability: fs.readFileSync(readabilityPath, 'utf8'),
+        readerable: fs.readFileSync(readerablePath, 'utf8'),
+    };
+    return cachedSources;
+}
+export const DEFAULT_FALLBACK_SELECTORS = [
+    'main',
+    '[role="main"]',
+    '#main-content',
+    '#main',
+    '#content',
+    '.content',
+    'article',
+    'body',
+];
+const MIN_FALLBACK_TEXT_LENGTH = 80;
+/**
+ * Build the JS expression evaluated in-page to extract the article. Exported
+ * for testability — callers on the host side should use `extractArticle`.
+ */
+export function buildExtractArticleJs(options = {}) {
+    const { readability, readerable } = readabilitySources();
+    const cleanSelectors = options.cleanSelectors ?? [];
+    const fallbackSelectors = options.fallbackSelectors ?? DEFAULT_FALLBACK_SELECTORS;
+    const force = !!options.force;
+    // Library sources contain backticks and ${...} fragments, so we embed them
+    // as JSON-encoded string literals and eval them inside a Function() scope.
+    // This isolates their var declarations from the outer IIFE without polluting
+    // window globals.
+    const readabilityLit = JSON.stringify(readability);
+    const readerableLit = JSON.stringify(readerable);
+    const cleanLit = JSON.stringify(cleanSelectors);
+    const fallbackLit = JSON.stringify(fallbackSelectors);
+    const forceLit = JSON.stringify(force);
+    return [
+        '(() => {',
+        '  const cleanSelectors = ' + cleanLit + ';',
+        '  const fallbackSelectors = ' + fallbackLit + ';',
+        '  const force = ' + forceLit + ';',
+        '  const minFallbackText = ' + MIN_FALLBACK_TEXT_LENGTH + ';',
+        '  const readabilitySrc = ' + readabilityLit + ';',
+        '  const readerableSrc = ' + readerableLit + ';',
+        '',
+        '  function escapeHtml(s) {',
+        '    return String(s).replace(/[&<>]/g, c => ({ "&": "&amp;", "<": "&lt;", ">": "&gt;" }[c]));',
+        '  }',
+        '',
+        '  // Short-circuit 1: non-HTML document',
+        '  const ct = document.contentType || "";',
+        '  if (ct && ct !== "text/html" && ct !== "application/xhtml+xml") {',
+        '    const body = document.body ? (document.body.textContent || "") : "";',
+        '    return { source: "raw-text", html: "<pre>" + escapeHtml(body) + "</pre>", title: document.title || "" };',
+        '  }',
+        '',
+        '  // Short-circuit 2: body is a single <pre>',
+        '  if (document.body) {',
+        '    const kids = document.body.children;',
+        '    if (kids.length === 1 && kids[0] && kids[0].tagName === "PRE") {',
+        '      return { source: "pre", html: document.body.outerHTML, title: document.title || "" };',
+        '    }',
+        '  }',
+        '',
+        '  // Deep-clone + adapter-supplied dirty-node removal',
+        '  const cloneDoc = document.cloneNode(true);',
+        '  for (const sel of cleanSelectors) {',
+        '    try { for (const n of cloneDoc.querySelectorAll(sel)) n.remove(); }',
+        '    catch (e) { /* ignore invalid selector */ }',
+        '  }',
+        '',
+        '  // Inject Readability into an isolated Function scope and extract the',
+        '  // constructors we need. Library sources use their own module.exports',
+        '  // guard (if typeof module === "object"), which is falsy here.',
+        '  const libs = (new Function(',
+        '    readabilitySrc + "\\n" + readerableSrc + "\\nreturn {" +',
+        '    " Readability: typeof Readability !== \\"undefined\\" ? Readability : null," +',
+        '    " isProbablyReaderable: typeof isProbablyReaderable !== \\"undefined\\" ? isProbablyReaderable : null" +',
+        '    " };"',
+        '  ))();',
+        '  const Readability = libs.Readability;',
+        '  const isProbablyReaderable = libs.isProbablyReaderable;',
+        '',
+        '  const readerableOk = force || (typeof isProbablyReaderable === "function" ? isProbablyReaderable(cloneDoc) : true);',
+        '  let article = null;',
+        '  if (readerableOk && typeof Readability === "function") {',
+        '    try { article = new Readability(cloneDoc).parse(); } catch (e) { article = null; }',
+        '  }',
+        '  if (article && article.content) {',
+        '    return {',
+        '      source: "readability",',
+        '      html: article.content,',
+        '      title: article.title || document.title || "",',
+        '      byline: article.byline || undefined,',
+        '      publishedTime: article.publishedTime || undefined,',
+        '      siteName: article.siteName || undefined,',
+        '    };',
+        '  }',
+        '',
+        '  // Fallback chain',
+        '  for (const sel of fallbackSelectors) {',
+        '    let el = null;',
+        '    try { el = cloneDoc.querySelector(sel); } catch (e) { continue; }',
+        '    if (!el) continue;',
+        '    const text = (el.textContent || "").trim();',
+        '    if (text.length < minFallbackText) continue;',
+        '    return { source: "fallback", html: el.outerHTML, title: document.title || "" };',
+        '  }',
+        '',
+        '  return null;',
+        '})()',
+    ].join('\n');
+}
+/**
+ * Run the extract pipeline on the given page. Returns `null` when no usable
+ * content is found (Readability miss + empty fallback chain).
+ */
+export async function extractArticle(page, options = {}) {
+    const js = buildExtractArticleJs(options);
+    const raw = await page.evaluate(js);
+    if (raw == null || typeof raw !== 'object')
+        return null;
+    const r = raw;
+    if (typeof r.html !== 'string' || typeof r.source !== 'string')
+        return null;
+    const source = r.source;
+    return {
+        html: r.html,
+        title: typeof r.title === 'string' ? r.title : '',
+        ...(r.byline && { byline: r.byline }),
+        ...(r.publishedTime && { publishedTime: r.publishedTime }),
+        ...(r.siteName && { siteName: r.siteName }),
+        source,
+    };
+}

package/dist/src/browser/article-extract.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/src/browser/article-extract.test.js

@@ -0,0 +1,94 @@
+import { describe, expect, it } from 'vitest';
+import { buildExtractArticleJs, extractArticle, DEFAULT_FALLBACK_SELECTORS, } from './article-extract.js';
+function fakePage(response) {
+    const state = { lastJs: null };
+    return {
+        lastJs: null,
+        async evaluate(js) {
+            state.lastJs = js;
+            Object.assign(this, state);
+            return response;
+        },
+    };
+}
+describe('buildExtractArticleJs', () => {
+    it('embeds Readability + Readerable sources once per evaluation', () => {
+        const js = buildExtractArticleJs();
+        // Both libs should be inlined (matched by identifying strings from the
+        // upstream @mozilla/readability sources).
+        expect(js).toContain('function Readability(doc, options)');
+        expect(js).toContain('function isProbablyReaderable');
+    });
+    it('serializes caller-supplied options into the evaluated JS', () => {
+        const js = buildExtractArticleJs({
+            cleanSelectors: ['.ads', '#banner'],
+            fallbackSelectors: ['article', 'body'],
+            force: true,
+        });
+        expect(js).toContain('[".ads","#banner"]');
+        expect(js).toContain('["article","body"]');
+        expect(js).toContain('const force = true;');
+    });
+    it('uses the default fallback chain when none is supplied', () => {
+        const js = buildExtractArticleJs();
+        for (const sel of DEFAULT_FALLBACK_SELECTORS) {
+            expect(js).toContain(JSON.stringify(sel));
+        }
+    });
+    it('runs fallback selection against the cleaned clone', () => {
+        const js = buildExtractArticleJs({ cleanSelectors: ['.noise'] });
+        expect(js).toContain('el = cloneDoc.querySelector(sel);');
+        expect(js).not.toContain('el = document.querySelector(sel);');
+    });
+    it('produces syntactically valid JavaScript', () => {
+        // Parsing via the Function constructor rejects any syntax error in the
+        // generated code — including accidental template-literal break-outs from
+        // the embedded Readability sources.
+        expect(() => new Function(buildExtractArticleJs())).not.toThrow();
+        expect(() => new Function(buildExtractArticleJs({ force: true }))).not.toThrow();
+        expect(() => new Function(buildExtractArticleJs({
+            cleanSelectors: ['.a', '.b'],
+            fallbackSelectors: ['main', 'body'],
+        }))).not.toThrow();
+    });
+});
+describe('extractArticle (host-side)', () => {
+    it('returns a normalized ExtractedArticle when the page responds with one', async () => {
+        const page = fakePage({
+            source: 'readability',
+            html: '<p>hello</p>',
+            title: 'Hello',
+            byline: 'Alice',
+            publishedTime: '2026-04-22',
+            siteName: 'Example',
+        });
+        const res = await extractArticle(page);
+        expect(res).toEqual({
+            source: 'readability',
+            html: '<p>hello</p>',
+            title: 'Hello',
+            byline: 'Alice',
+            publishedTime: '2026-04-22',
+            siteName: 'Example',
+        });
+    });
+    it('drops undefined optional fields cleanly', async () => {
+        const page = fakePage({ source: 'fallback', html: '<main>x</main>', title: 't' });
+        const res = await extractArticle(page);
+        expect(res).toEqual({ source: 'fallback', html: '<main>x</main>', title: 't' });
+        expect(res).not.toHaveProperty('byline');
+        expect(res).not.toHaveProperty('publishedTime');
+    });
+    it('returns null on a missing body or malformed payload', async () => {
+        expect(await extractArticle(fakePage(null))).toBeNull();
+        expect(await extractArticle(fakePage('oops'))).toBeNull();
+        expect(await extractArticle(fakePage({ source: 'readability' }))).toBeNull();
+        expect(await extractArticle(fakePage({ html: '<p>x</p>' }))).toBeNull();
+    });
+    it('defaults title to empty string when the page omits it', async () => {
+        const page = fakePage({ source: 'pre', html: '<body><pre>x</pre></body>' });
+        const res = await extractArticle(page);
+        expect(res?.title).toBe('');
+        expect(res?.source).toBe('pre');
+    });
+});

package/dist/src/browser/cdp.js

@@ -91,7 +91,12 @@ export class CDPBridge {
                         }
                     }
                 }
-                catch { }
+                catch (err) {
+                    if (process.env.OPENCLI_VERBOSE) {
+                        // eslint-disable-next-line no-console
+                        console.error('[cdp] Failed to parse WebSocket message:', err instanceof Error ? err.message : err);
+                    }
+                }
             });
         });
     }
@@ -253,8 +258,12 @@ class CDPPage extends BasePage {
                             this._networkEntries[idx].responseBodyFullSize = fullSize;
                             this._networkEntries[idx].responseBodyTruncated = truncated;
                         }
-                    }).catch(() => {
+                    }).catch((err) => {
                         // Body unavailable for some requests (e.g. uploads) — non-fatal
+                        if (process.env.OPENCLI_VERBOSE) {
+                            // eslint-disable-next-line no-console
+                            console.error(`[cdp] getResponseBody failed for ${p.requestId}:`, err instanceof Error ? err.message : err);
+                        }
                     }).finally(() => {
                         this._pendingBodyFetches.delete(bodyFetch);
                     });

package/dist/src/browser/verify-fixture.d.ts

@@ -0,0 +1,59 @@
+export type FixtureExpect = {
+    rowCount?: {
+        min?: number;
+        max?: number;
+    };
+    columns?: string[];
+    types?: Record<string, string>;
+    patterns?: Record<string, string>;
+    notEmpty?: string[];
+    /**
+     * Substrings/regex fragments that MUST NOT appear in the column value.
+     *
+     * Catches silent content contamination that `notEmpty` alone misses —
+     * e.g. a `description` field that accidentally carries "address: ..." /
+     * "category: ..." fragments from sibling DOM nodes, or a `title` that
+     * bled in a navigation-breadcrumb prefix. Each entry is matched as a
+     * plain substring against the stringified column value.
+     */
+    mustNotContain?: Record<string, string[]>;
+    /**
+     * Columns whose values must be truthy. Complements `notEmpty` (which
+     * only rejects empty-string/null/undefined) by also catching silent
+     * `|| 0` / `|| false` fallbacks in numeric/boolean fields. Fires when
+     * the value coerces to `false` in JS.
+     */
+    mustBeTruthy?: string[];
+};
+export type FixtureArgs = Record<string, unknown> | unknown[];
+export type Fixture = {
+    args?: FixtureArgs;
+    expect?: FixtureExpect;
+};
+export type ValidationFailure = {
+    rule: 'rowCount' | 'column' | 'type' | 'pattern' | 'notEmpty' | 'mustNotContain' | 'mustBeTruthy';
+    detail: string;
+    rowIndex?: number;
+};
+export type Row = Record<string, unknown>;
+export declare function fixturePath(site: string, command: string): string;
+export declare function loadFixture(site: string, command: string): Fixture | null;
+export declare function writeFixture(site: string, command: string, fixture: Fixture): string;
+/**
+ * Derive a reasonable fixture from sample output. Used by `--write-fixture`
+ * to seed a first draft the author can hand-tune.
+ *
+ * Heuristics:
+ * - rowCount.min = 1 if rows non-empty, else 0
+ * - columns = keys from the first row
+ * - types = typeof of the first row's values, with "number|string" for mixed
+ * - no auto patterns / notEmpty — author should add those deliberately
+ */
+export declare function deriveFixture(rows: Row[], args?: FixtureArgs): Fixture;
+export declare function validateRows(rows: Row[], fixture: Fixture): ValidationFailure[];
+/**
+ * Convert fixture args into argv tokens appended after the command name.
+ * - Array form is passed through verbatim (stringified), supporting positional subjects.
+ * - Object form is expanded to `--key value` pairs.
+ */
+export declare function expandFixtureArgs(args: FixtureArgs | undefined): string[];

package/dist/src/browser/verify-fixture.js

@@ -0,0 +1,213 @@
+/**
+ * Verify fixture: structural expectations for `opencli browser verify` output.
+ *
+ * The adapter-author skill runbook says every published adapter must write a
+ * fixture under `~/.opencli/sites/<site>/verify/<command>.json` so later verify
+ * runs can catch shape regressions (missing columns, wrong types, bleeding
+ * values) without relying on exact content match — BBS / news / market data is
+ * too volatile for value equality.
+ *
+ * Schema:
+ *   {
+ *     // args can be either:
+ *     //   - an object of named flags: { "limit": 3 }  → expands to `--limit 3`
+ *     //   - a raw argv array:         ["123", "--limit", "3"]  → passed verbatim
+ *     // Use the array form for adapters that take positional subjects (e.g. <tid>, <url>, <query>).
+ *     "args": { "limit": 3 },
+ *     "expect": {
+ *       "rowCount": { "min": 1, "max": 10 },  // inclusive bounds
+ *       "columns":  ["a", "b"],                // every row must have these keys
+ *       "types":    { "a": "string", "b": "number|string" },
+ *       "patterns": { "url": "^https?://" },
+ *       "notEmpty": ["title", "url"],          // trimmed string must be non-empty
+ *       "mustNotContain": {                     // catch content-contamination bleed
+ *         "description": ["address:", "category:"]
+ *       },
+ *       "mustBeTruthy": ["count"]               // catch silent `|| 0` fallbacks
+ *     }
+ *   }
+ */
+import * as fs from 'node:fs';
+import * as os from 'node:os';
+import * as path from 'node:path';
+export function fixturePath(site, command) {
+    return path.join(os.homedir(), '.opencli', 'sites', site, 'verify', `${command}.json`);
+}
+export function loadFixture(site, command) {
+    const p = fixturePath(site, command);
+    if (!fs.existsSync(p))
+        return null;
+    try {
+        const raw = fs.readFileSync(p, 'utf-8');
+        const parsed = JSON.parse(raw);
+        return parsed;
+    }
+    catch (err) {
+        throw new Error(`Failed to parse fixture ${p}: ${err instanceof Error ? err.message : String(err)}`);
+    }
+}
+export function writeFixture(site, command, fixture) {
+    const p = fixturePath(site, command);
+    fs.mkdirSync(path.dirname(p), { recursive: true });
+    fs.writeFileSync(p, `${JSON.stringify(fixture, null, 2)}\n`, 'utf-8');
+    return p;
+}
+/**
+ * Derive a reasonable fixture from sample output. Used by `--write-fixture`
+ * to seed a first draft the author can hand-tune.
+ *
+ * Heuristics:
+ * - rowCount.min = 1 if rows non-empty, else 0
+ * - columns = keys from the first row
+ * - types = typeof of the first row's values, with "number|string" for mixed
+ * - no auto patterns / notEmpty — author should add those deliberately
+ */
+export function deriveFixture(rows, args) {
+    const expect = {};
+    if (rows.length === 0) {
+        expect.rowCount = { min: 0 };
+        return { ...(args ? { args } : {}), expect };
+    }
+    expect.rowCount = { min: 1 };
+    const first = rows[0];
+    const columns = Object.keys(first);
+    expect.columns = columns;
+    const types = {};
+    for (const col of columns) {
+        const observed = new Set();
+        for (const row of rows) {
+            const v = row[col];
+            observed.add(jsType(v));
+        }
+        types[col] = [...observed].sort().join('|');
+    }
+    expect.types = types;
+    return { ...(args ? { args } : {}), expect };
+}
+export function validateRows(rows, fixture) {
+    const failures = [];
+    const expect = fixture.expect;
+    if (!expect)
+        return failures;
+    if (expect.rowCount) {
+        const { min, max } = expect.rowCount;
+        if (typeof min === 'number' && rows.length < min) {
+            failures.push({ rule: 'rowCount', detail: `got ${rows.length} rows, expected at least ${min}` });
+        }
+        if (typeof max === 'number' && rows.length > max) {
+            failures.push({ rule: 'rowCount', detail: `got ${rows.length} rows, expected at most ${max}` });
+        }
+    }
+    const columns = expect.columns ?? [];
+    const types = expect.types ?? {};
+    const patterns = expect.patterns ?? {};
+    const notEmpty = expect.notEmpty ?? [];
+    const compiledPatterns = {};
+    for (const [col, src] of Object.entries(patterns)) {
+        try {
+            compiledPatterns[col] = new RegExp(src);
+        }
+        catch (err) {
+            failures.push({ rule: 'pattern', detail: `pattern for "${col}" invalid: ${err instanceof Error ? err.message : String(err)}` });
+        }
+    }
+    rows.forEach((row, i) => {
+        for (const col of columns) {
+            if (!(col in row)) {
+                failures.push({ rule: 'column', detail: `missing column "${col}"`, rowIndex: i });
+            }
+        }
+        for (const [col, declared] of Object.entries(types)) {
+            if (!(col in row))
+                continue;
+            const actual = jsType(row[col]);
+            if (!typeMatches(actual, declared)) {
+                failures.push({
+                    rule: 'type',
+                    detail: `"${col}" is ${actual}, expected ${declared}`,
+                    rowIndex: i,
+                });
+            }
+        }
+        for (const [col, re] of Object.entries(compiledPatterns)) {
+            if (!(col in row))
+                continue;
+            const v = row[col];
+            if (v === null || v === undefined)
+                continue;
+            if (!re.test(String(v))) {
+                failures.push({
+                    rule: 'pattern',
+                    detail: `"${col}"=${JSON.stringify(String(v).slice(0, 60))} does not match /${re.source}/`,
+                    rowIndex: i,
+                });
+            }
+        }
+        for (const col of notEmpty) {
+            const v = row[col];
+            if (v === null || v === undefined || String(v).trim() === '') {
+                failures.push({ rule: 'notEmpty', detail: `"${col}" is empty`, rowIndex: i });
+            }
+        }
+        for (const [col, needles] of Object.entries(expect.mustNotContain ?? {})) {
+            if (!(col in row))
+                continue;
+            const v = row[col];
+            if (v === null || v === undefined)
+                continue;
+            const haystack = String(v);
+            for (const needle of needles) {
+                if (haystack.includes(needle)) {
+                    failures.push({
+                        rule: 'mustNotContain',
+                        detail: `"${col}" contains forbidden substring ${JSON.stringify(needle)}`,
+                        rowIndex: i,
+                    });
+                }
+            }
+        }
+        for (const col of expect.mustBeTruthy ?? []) {
+            if (!(col in row))
+                continue;
+            if (!row[col]) {
+                failures.push({
+                    rule: 'mustBeTruthy',
+                    detail: `"${col}" is falsy (${JSON.stringify(row[col])}) — likely silent fallback`,
+                    rowIndex: i,
+                });
+            }
+        }
+    });
+    return failures;
+}
+/**
+ * Convert fixture args into argv tokens appended after the command name.
+ * - Array form is passed through verbatim (stringified), supporting positional subjects.
+ * - Object form is expanded to `--key value` pairs.
+ */
+export function expandFixtureArgs(args) {
+    if (!args)
+        return [];
+    if (Array.isArray(args))
+        return args.map((v) => String(v));
+    const out = [];
+    for (const [k, v] of Object.entries(args)) {
+        out.push(`--${k}`, String(v));
+    }
+    return out;
+}
+function jsType(v) {
+    if (v === null)
+        return 'null';
+    if (Array.isArray(v))
+        return 'array';
+    return typeof v;
+}
+function typeMatches(actual, declared) {
+    const allowed = declared.split('|').map((s) => s.trim()).filter(Boolean);
+    if (allowed.length === 0)
+        return true;
+    if (allowed.includes('any'))
+        return true;
+    return allowed.includes(actual);
+}

package/dist/src/browser/verify-fixture.test.d.ts

@@ -0,0 +1 @@
+export {};