@jackwener/opencli 1.7.8 → 1.7.10

package/clis/web/read.js

@@ -15,63 +15,204 @@
  */
 import { cli, Strategy } from '@jackwener/opencli/registry';
 import { downloadArticle } from '@jackwener/opencli/download/article-download';
-const command = cli({
-    site: 'web',
-    name: 'read',
-    description: 'Fetch any web page and export as Markdown',
-    strategy: Strategy.COOKIE,
-    navigateBefore: false, // we handle navigation ourselves
-    args: [
-        { name: 'url', required: true, help: 'Any web page URL' },
-        { name: 'output', default: './web-articles', help: 'Output directory' },
-        { name: 'download-images', type: 'boolean', default: true, help: 'Download images locally' },
-        { name: 'wait', type: 'int', default: 3, help: 'Seconds to wait after page load' },
-        { name: 'stdout', type: 'boolean', default: false, help: 'Print markdown to stdout instead of saving to a file' },
-    ],
-    columns: ['title', 'author', 'publish_time', 'status', 'size', 'saved'],
-    func: async (page, kwargs) => {
-        const url = kwargs.url;
-        const waitSeconds = kwargs.wait ?? 3;
-        // Navigate to the target URL
-        await page.goto(url);
-        await page.wait(waitSeconds);
-        // Extract article content using browser-side heuristics
-        const data = await page.evaluate(`
+
+const NETWORK_IDLE_QUIET_MS = 1000;
+const NETWORK_IDLE_POLL_MS = 500;
+
+function sleep(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+function boolish(value) {
+    if (value === true) return true;
+    if (typeof value === 'string') return ['1', 'true', 'yes', 'on'].includes(value.toLowerCase());
+    return false;
+}
+
+function normalizeFrameMode(value) {
+    const mode = String(value || 'same-origin').toLowerCase();
+    if (['same-origin', 'none'].includes(mode)) return mode;
+    return 'same-origin';
+}
+
+function normalizeWaitUntil(value) {
+    const waitUntil = String(value || 'domstable').toLowerCase();
+    if (['domstable', 'networkidle'].includes(waitUntil)) return waitUntil;
+    return 'domstable';
+}
+
+function normalizeNetworkEntry(entry) {
+    const preview = typeof entry?.responsePreview === 'string' ? entry.responsePreview : '';
+    return {
+        method: typeof entry?.method === 'string' ? entry.method : 'GET',
+        url: typeof entry?.url === 'string' ? entry.url : '',
+        status: typeof entry?.responseStatus === 'number' ? entry.responseStatus : 0,
+        contentType: typeof entry?.responseContentType === 'string' ? entry.responseContentType : '',
+        size: typeof entry?.responseBodyFullSize === 'number' ? entry.responseBodyFullSize : preview.length,
+        bodyTruncated: entry?.responseBodyTruncated === true,
+    };
+}
+
+function isInterestingNetworkEntry(entry) {
+    const ct = (entry.contentType || '').toLowerCase();
+    const url = entry.url || '';
+    const method = (entry.method || 'GET').toUpperCase();
+    const staticAsset = /\.(js|css|png|jpg|jpeg|gif|svg|woff|woff2|ico|map)(\?|$)/i.test(url);
+    const noisy = /analytics|tracking|telemetry|beacon|pixel|gtag|fbevents/i.test(url);
+    const apiLikeUrl = /\/(api|ajax|graphql|rest|service|handler)(\/|[?._-]|$)|\.(ashx|aspx|asmx|php)(\?|$)/i.test(url);
+    const dataLikeContent = ct.includes('json')
+        || ct.includes('xml')
+        || ct.includes('text/plain')
+        || ct.includes('javascript')
+        || (apiLikeUrl && ct.includes('text/html'));
+    return (
+        !staticAsset
+        && !noisy
+        && (dataLikeContent || apiLikeUrl || method !== 'GET')
+    );
+}
+
+async function drainNetworkCapture(page, sink) {
+    if (!page.readNetworkCapture) return [];
+    const raw = await page.readNetworkCapture().catch(() => []);
+    const entries = Array.isArray(raw) ? raw.map(normalizeNetworkEntry).filter(entry => entry.url) : [];
+    sink.push(...entries);
+    return entries;
+}
+
+async function maybeStartNetworkCapture(page) {
+    if (!page.startNetworkCapture) return false;
+    try {
+        return await page.startNetworkCapture('');
+    } catch {
+        return false;
+    }
+}
+
+async function waitForNetworkIdle(page, maxSeconds, sink) {
+    const timeoutMs = Math.max(1, Number(maxSeconds) || 1) * 1000;
+    const deadline = Date.now() + timeoutMs;
+    let quietSince = Date.now();
+    while (Date.now() < deadline) {
+        const entries = await drainNetworkCapture(page, sink);
+        if (entries.length > 0) quietSince = Date.now();
+        if (Date.now() - quietSince >= NETWORK_IDLE_QUIET_MS) return { ok: true };
+        await sleep(NETWORK_IDLE_POLL_MS);
+    }
+    return { ok: false, timedOut: true };
+}
+
+function buildWaitForSelectorAcrossFramesJs(selector, timeoutMs) {
+    return `
+      (async () => {
+        const selector = ${JSON.stringify(selector)};
+        const timeoutAt = Date.now() + ${Number(timeoutMs) || 10000};
+        const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));
+        const sameOriginFrameDocs = () => Array.from(document.querySelectorAll('iframe')).map((frame) => {
+          try {
+            const href = new URL(frame.getAttribute('src') || frame.src || '', window.location.href).href;
+            if (new URL(href).origin !== window.location.origin) return null;
+            return { href, doc: frame.contentDocument };
+          } catch {
+            return null;
+          }
+        }).filter(Boolean);
+        const findMatch = () => {
+          try {
+            if (document.querySelector(selector)) return { ok: true, scope: 'main', url: window.location.href };
+          } catch (err) {
+            return { ok: false, invalidSelector: true, error: String(err && err.message || err) };
+          }
+          for (const frame of sameOriginFrameDocs()) {
+            try {
+              if (frame.doc?.querySelector(selector)) return { ok: true, scope: 'iframe', url: frame.href };
+            } catch {}
+          }
+          return { ok: false };
+        };
+        while (Date.now() < timeoutAt) {
+          const found = findMatch();
+          if (found.ok || found.invalidSelector) return found;
+          await sleep(100);
+        }
+        return { ok: false, timedOut: true, selector };
+      })()
+    `;
+}
+
+function buildRenderAwareExtractorJs(options) {
+    return `
       (() => {
+        const frameMode = ${JSON.stringify(options.frames)};
         const result = {
           title: '',
           author: '',
           publishTime: '',
           contentHtml: '',
-          imageUrls: []
+          imageUrls: [],
+          diagnostics: {
+            url: window.location.href,
+            frames: [],
+            emptyContainers: [],
+            includedFrameCount: 0
+          }
+        };
+
+        const absolutize = (value, base) => {
+          if (!value || value.startsWith('data:') || value.startsWith('javascript:') || value.startsWith('#')) return value || '';
+          try { return new URL(value, base).href; } catch { return value; }
+        };
+        const absolutizeTree = (root, base) => {
+          root.querySelectorAll?.('[href]').forEach(el => el.setAttribute('href', absolutize(el.getAttribute('href'), base)));
+          root.querySelectorAll?.('[src]').forEach(el => el.setAttribute('src', absolutize(el.getAttribute('src'), base)));
+          root.querySelectorAll?.('[poster]').forEach(el => el.setAttribute('poster', absolutize(el.getAttribute('poster'), base)));
+          root.querySelectorAll?.('[action]').forEach(el => el.setAttribute('action', absolutize(el.getAttribute('action'), base)));
+        };
+        const textLen = (node) => (node?.textContent || '').replace(/\\s+/g, ' ').trim().length;
+        const describeFrame = (frame, index) => {
+          const rawSrc = frame.getAttribute('src') || frame.src || '';
+          let href = '';
+          try { href = new URL(rawSrc, window.location.href).href; } catch { href = rawSrc; }
+          let sameOrigin = false;
+          try { sameOrigin = href ? new URL(href).origin === window.location.origin : false; } catch {}
+          let accessible = false;
+          let title = frame.getAttribute('title') || frame.getAttribute('name') || frame.id || '';
+          let length = 0;
+          try {
+            accessible = !!frame.contentDocument;
+            title = title || frame.contentDocument?.title || '';
+            length = textLen(frame.contentDocument?.body);
+          } catch {}
+          return { index, src: href, title, sameOrigin, accessible, textLength: length };
+        };
+        const collectEmptyContainers = (root, scope, baseUrl) => {
+          const likely = 'table, tbody, ul[id], ol[id], div[id], section[id], [class*="grid"], [class*="data"], [class*="list"], [id*="grid"], [id*="data"], [id*="list"]';
+          root.querySelectorAll?.(likely).forEach((el) => {
+            const id = el.getAttribute('id') || '';
+            const cls = el.getAttribute('class') || '';
+            const name = [id, cls].join(' ').toLowerCase();
+            if (!/(grid|data|list|table|content|result)/.test(name) && !['TABLE', 'TBODY', 'UL', 'OL'].includes(el.nodeName)) return;
+            if (textLen(el) > 20) return;
+            result.diagnostics.emptyContainers.push({
+              scope,
+              url: baseUrl,
+              tag: el.tagName.toLowerCase(),
+              id,
+              className: cls,
+            });
+          });
         };
 
-        // --- Title extraction ---
-        // Priority: og:title > <title> > first <h1>
         const ogTitle = document.querySelector('meta[property="og:title"]');
-        if (ogTitle) {
-          result.title = ogTitle.getAttribute('content')?.trim() || '';
-        }
-        if (!result.title) {
-          result.title = document.title?.trim() || '';
-        }
-        if (!result.title) {
-          const h1 = document.querySelector('h1');
-          result.title = h1?.textContent?.trim() || 'untitled';
-        }
-        // Strip site suffix (e.g. " | Anthropic", " - Blog")
+        if (ogTitle) result.title = ogTitle.getAttribute('content')?.trim() || '';
+        if (!result.title) result.title = document.title?.trim() || '';
+        if (!result.title) result.title = document.querySelector('h1')?.textContent?.trim() || 'untitled';
         result.title = result.title.replace(/\\s*[|\\-–—]\\s*[^|\\-–—]{1,30}$/, '').trim();
 
-        // --- Author extraction ---
-        const authorMeta = document.querySelector(
-          'meta[name="author"], meta[property="article:author"], meta[name="twitter:creator"]'
-        );
+        const authorMeta = document.querySelector('meta[name="author"], meta[property="article:author"], meta[name="twitter:creator"]');
         result.author = authorMeta?.getAttribute('content')?.trim() || '';
 
-        // --- Publish time extraction ---
-        const timeMeta = document.querySelector(
-          'meta[property="article:published_time"], meta[name="date"], meta[name="publishdate"], time[datetime]'
-        );
+        const timeMeta = document.querySelector('meta[property="article:published_time"], meta[name="date"], meta[name="publishdate"], time[datetime]');
         if (timeMeta) {
           result.publishTime = timeMeta.getAttribute('content')
             || timeMeta.getAttribute('datetime')
@@ -79,34 +220,19 @@ const command = cli({
             || '';
         }
 
-        // --- Content extraction ---
-        // Strategy: try semantic elements first, then fall back to largest text block
         let contentEl = null;
-
-        // 1. <article>
         const articles = document.querySelectorAll('article');
         if (articles.length === 1) {
           contentEl = articles[0];
         } else if (articles.length > 1) {
-          // Pick the largest article by text length
           let maxLen = 0;
           articles.forEach(a => {
-            const len = a.textContent?.length || 0;
+            const len = textLen(a);
             if (len > maxLen) { maxLen = len; contentEl = a; }
           });
         }
-
-        // 2. [role="main"]
-        if (!contentEl) {
-          contentEl = document.querySelector('[role="main"]');
-        }
-
-        // 3. <main>
-        if (!contentEl) {
-          contentEl = document.querySelector('main');
-        }
-
-        // 4. Largest text-dense block fallback
+        if (!contentEl) contentEl = document.querySelector('[role="main"]');
+        if (!contentEl) contentEl = document.querySelector('main');
         if (!contentEl) {
           const candidates = document.querySelectorAll(
             'div[class*="content"], div[class*="article"], div[class*="post"], ' +
@@ -115,26 +241,51 @@ const command = cli({
           );
           let maxLen = 0;
           candidates.forEach(c => {
-            const len = c.textContent?.length || 0;
+            const len = textLen(c);
             if (len > maxLen) { maxLen = len; contentEl = c; }
           });
         }
+        if (!contentEl || textLen(contentEl) < 200) contentEl = document.body;
+
+        const clone = contentEl.cloneNode(true);
+        absolutizeTree(clone, window.location.href);
+
+        const originalFrames = Array.from(contentEl.querySelectorAll('iframe'));
+        const clonedFrames = Array.from(clone.querySelectorAll('iframe'));
+        const allFrames = Array.from(document.querySelectorAll('iframe'));
+        result.diagnostics.frames = allFrames.map(describeFrame);
 
-        // 5. Last resort: document.body
-        if (!contentEl || (contentEl.textContent?.length || 0) < 200) {
-          contentEl = document.body;
+        if (frameMode === 'same-origin') {
+          originalFrames.forEach((frame, index) => {
+            const cloned = clonedFrames[index];
+            if (!cloned) return;
+            const desc = describeFrame(frame, index);
+            if (!desc.sameOrigin || !desc.accessible) return;
+            try {
+              const doc = frame.contentDocument;
+              if (!doc?.body) return;
+              const frameBody = doc.body.cloneNode(true);
+              absolutizeTree(frameBody, desc.src || window.location.href);
+              collectEmptyContainers(frameBody, 'iframe', desc.src);
+              const section = document.createElement('section');
+              section.setAttribute('data-opencli-iframe-source', desc.src);
+              const heading = document.createElement('h2');
+              heading.textContent = '来自 iframe: ' + (desc.src || frame.getAttribute('src') || ('#' + index));
+              section.appendChild(heading);
+              Array.from(frameBody.childNodes).forEach(node => section.appendChild(node));
+              cloned.replaceWith(section);
+              result.diagnostics.includedFrameCount += 1;
+            } catch {}
+          });
         }
 
-        // Clean up noise elements before extraction
-        const clone = contentEl.cloneNode(true);
+        collectEmptyContainers(clone, 'main', window.location.href);
+
         const noise = 'nav, header, footer, aside, .sidebar, .nav, .menu, .footer, ' +
           '.header, .comments, .comment, .ad, .ads, .advertisement, .social-share, ' +
           '.related-posts, .newsletter, .cookie-banner, script, style, noscript, iframe';
         clone.querySelectorAll(noise).forEach(el => el.remove());
 
-        // Deduplicate: some sites (e.g. Anthropic) render each paragraph twice
-        // (a visible version + a line-broken animation version with missing spaces).
-        // Compare by stripping ALL whitespace so "Hello world" matches "Helloworld".
         const stripWS = (s) => (s || '').replace(/\\s+/g, '');
         const dedup = (parent) => {
           const children = Array.from(parent.children || []);
@@ -144,9 +295,7 @@ const command = cli({
             const cur = stripWS(curRaw);
             const prev = stripWS(prevRaw);
             if (cur.length < 20 || prev.length < 20) continue;
-            // Exact match after whitespace strip, or >90% overlap
             if (cur === prev) {
-              // Keep the one with more proper spacing (more spaces = better formatted)
               const curSpaces = (curRaw.match(/ /g) || []).length;
               const prevSpaces = (prevRaw.match(/ /g) || []).length;
               if (curSpaces >= prevSpaces) children[i - 1].remove();
@@ -163,10 +312,6 @@ const command = cli({
           if (el.children && el.children.length > 2) dedup(el);
         });
 
-        // --- Lazy-load image src rewrite ---
-        // Many sites render <img src="placeholder.gif" data-src="real.jpg">.
-        // Promote the real URL onto src so both the markdown body and the
-        // image download list reference the same URL.
         clone.querySelectorAll('img').forEach(img => {
           const srcset = img.getAttribute('data-srcset') || '';
           const srcsetFirst = srcset.split(',')[0]?.trim().split(' ')[0] || '';
@@ -174,12 +319,11 @@ const command = cli({
             || img.getAttribute('data-original')
             || img.getAttribute('data-lazy-src')
             || srcsetFirst;
-          if (real) img.setAttribute('src', real);
+          if (real) img.setAttribute('src', absolutize(real, window.location.href));
         });
 
         result.contentHtml = clone.innerHTML;
 
-        // --- Image extraction ---
         const seen = new Set();
         clone.querySelectorAll('img').forEach(img => {
           const src = img.getAttribute('src') || '';
@@ -191,7 +335,87 @@ const command = cli({
 
         return result;
       })()
-    `);
+    `;
+}
+
+function formatDiagnostics(data, networkEntries, captureSupported) {
+    const lines = [];
+    const diag = data?.diagnostics || {};
+    lines.push('[web-read diagnose]');
+    lines.push(`url: ${diag.url || '-'}`);
+    lines.push(`frames: ${Array.isArray(diag.frames) ? diag.frames.length : 0}, included_same_origin: ${diag.includedFrameCount || 0}`);
+    for (const frame of (diag.frames || []).slice(0, 20)) {
+        lines.push(`  [frame ${frame.index}] ${frame.sameOrigin ? 'same-origin' : 'cross-origin'} ${frame.accessible ? 'accessible' : 'blocked'} text=${frame.textLength || 0} ${frame.src || '-'}`);
+    }
+    if (Array.isArray(diag.emptyContainers) && diag.emptyContainers.length > 0) {
+        lines.push(`empty_containers: ${diag.emptyContainers.length}`);
+        for (const item of diag.emptyContainers.slice(0, 12)) {
+            const selector = `${item.tag}${item.id ? `#${item.id}` : ''}${item.className ? `.${String(item.className).trim().split(/\\s+/).filter(Boolean).join('.')}` : ''}`;
+            lines.push(`  ${item.scope}: ${selector} (${item.url || '-'})`);
+        }
+    }
+    const interesting = networkEntries.filter(isInterestingNetworkEntry);
+    lines.push(`network_capture: ${captureSupported ? 'enabled' : 'unavailable'}, entries=${networkEntries.length}, api_like=${interesting.length}`);
+    for (const entry of interesting.slice(0, 20)) {
+        lines.push(`  ${entry.method} ${entry.status || '-'} ${entry.contentType || '-'} ${entry.url}`);
+    }
+    return `${lines.join('\n')}\n`;
+}
+
+const command = cli({
+    site: 'web',
+    name: 'read',
+    description: 'Fetch any web page and export as Markdown',
+    strategy: Strategy.COOKIE,
+    navigateBefore: false, // we handle navigation ourselves
+    args: [
+        { name: 'url', required: true, help: 'Any web page URL' },
+        { name: 'output', default: './web-articles', help: 'Output directory' },
+        { name: 'download-images', type: 'boolean', default: true, help: 'Download images locally' },
+        { name: 'wait', type: 'int', default: 3, help: 'Seconds to wait after page load' },
+        { name: 'wait-for', valueRequired: true, help: 'CSS selector to wait for in the main document or same-origin iframes' },
+        { name: 'wait-until', default: 'domstable', choices: ['domstable', 'networkidle'], help: 'Readiness policy after navigation: domstable or networkidle' },
+        { name: 'frames', default: 'same-origin', choices: ['same-origin', 'none'], help: 'Iframe handling mode: same-origin or none' },
+        { name: 'diagnose', type: 'boolean', default: false, help: 'Print render diagnostics (frames, empty containers, XHR/API-like requests) to stderr' },
+        { name: 'stdout', type: 'boolean', default: false, help: 'Print markdown to stdout instead of saving to a file' },
+    ],
+    columns: ['title', 'author', 'publish_time', 'status', 'size', 'saved'],
+    func: async (page, kwargs, debug = false) => {
+        const url = kwargs.url;
+        const waitSeconds = kwargs.wait ?? 3;
+        const waitUntil = normalizeWaitUntil(kwargs['wait-until']);
+        const frameMode = normalizeFrameMode(kwargs.frames);
+        const shouldDiagnose = boolish(kwargs.diagnose) || debug || !!process.env.OPENCLI_VERBOSE;
+        const networkEntries = [];
+        const captureSupported = (waitUntil === 'networkidle' || shouldDiagnose)
+            ? await maybeStartNetworkCapture(page)
+            : false;
+        // Navigate to the target URL
+        await page.goto(url);
+        if (kwargs['wait-for']) {
+            const waitResult = await page.evaluate(buildWaitForSelectorAcrossFramesJs(String(kwargs['wait-for']), waitSeconds * 1000));
+            if (waitResult?.invalidSelector) {
+                throw new Error(`Invalid --wait-for selector "${kwargs['wait-for']}": ${waitResult.error || 'querySelector failed'}`);
+            }
+            if (!waitResult?.ok) {
+                throw new Error(`Timed out waiting for selector "${kwargs['wait-for']}" in main document or same-origin iframes`);
+            }
+        } else if (waitUntil !== 'networkidle') {
+            await page.wait(waitSeconds);
+        }
+        if (waitUntil === 'networkidle') {
+            if (!captureSupported) {
+                throw new Error('Network capture is unavailable, so --wait-until networkidle cannot be satisfied');
+            }
+            const idle = await waitForNetworkIdle(page, waitSeconds, networkEntries);
+            if (!idle?.ok) {
+                throw new Error(`Timed out waiting for network idle after ${waitSeconds}s`);
+            }
+        }
+        // Extract article content using browser-side heuristics
+        const data = await page.evaluate(buildRenderAwareExtractorJs({ frames: frameMode }));
+        if (captureSupported) await drainNetworkCapture(page, networkEntries);
+        if (shouldDiagnose) process.stderr.write(formatDiagnostics(data, networkEntries, captureSupported));
         // Determine Referer from URL for image downloads
         let referer = '';
         try {
@@ -211,6 +435,12 @@ const command = cli({
             downloadImages: kwargs['download-images'],
             imageHeaders: referer ? { Referer: referer } : undefined,
             stdout: kwargs.stdout,
+            configureTurndown: (td) => {
+                td.addRule('preserveButtons', {
+                    filter: (node) => node.nodeName === 'BUTTON',
+                    replacement: (content) => content,
+                });
+            },
         });
         // `--stdout` is a content-streaming mode. The markdown body already went
         // to process.stdout inside downloadArticle(), so returning rows here
@@ -219,4 +449,12 @@ const command = cli({
         return kwargs.stdout ? null : result;
     },
 });
-export const __test__ = { command };
+export const __test__ = {
+    command,
+    buildRenderAwareExtractorJs,
+    buildWaitForSelectorAcrossFramesJs,
+    formatDiagnostics,
+    isInterestingNetworkEntry,
+    normalizeFrameMode,
+    normalizeWaitUntil,
+};