dsh-scrape 0.0.1 → 0.1.0

package/lib/scrape.mjs

@@ -0,0 +1,560 @@
+/**
+ * 编排层：URL / HTML → `{markdown, title, provenance, removed, warnings}`。
+ *
+ * 一次抓取按固定顺序做八件事，任何一步都能单独关掉或说清：
+ *   1) 校验 URL；
+ *   2) 读 robots.txt 并判定（robots:false 才跳过，且必须警告）；
+ *   3) 同域限速排队，抓页面（手动跟随重定向、记 provenance）；
+ *   4) 认字符集解码（GBK/Big5 不再乱码）；
+ *   5) 按行/元素去噪（script/style/nav/footer/cookie 条/分享按钮）；
+ *   6) 密度法挑正文并给 confidence；
+ *   7) 渲染成带 `<!-- L42 -->` 行号锚点的 Markdown；
+ *   8) **不执行 JS**：内容不足且看到 SPA 骨架时显式警告，而不是假装成功。
+ */
+
+import { ScrapeError, describeError } from './errors.mjs'
+import { DEFAULT_USER_AGENT, fetchOptional, fetchPage } from './net.mjs'
+import { createThrottle, hostKey, DEFAULT_MIN_INTERVAL_MS } from './limit.mjs'
+import { parseRobots, checkAllowance, originOf, pathOf, describeAllowance } from './robots.mjs'
+import { parseHtml } from './html.mjs'
+import { selectMain, blockLength, EXTRACT_VERSION, MIN_MAIN_CHARS } from './extract.mjs'
+import { renderMarkdown, textLinesToMarkdown } from './markdown.mjs'
+
+/** 包版本（包名/工具名/版本号只在 package.json 与本常量各出现一次）。 */
+export const VERSION = '0.1.0'
+
+/** 工具名（DSH 里模型看到的名字）。 */
+export const TOOL_NAME = 'dsh_scrape'
+
+/** 默认参数（可被工具参数或 CLI 覆盖）。 */
+export const DEFAULTS = {
+  timeoutMs: 15000,
+  maxBytes: 2_000_000,
+  maxRedirects: 5,
+  maxChars: 200_000,
+  maxUrls: 20,
+  minIntervalMs: DEFAULT_MIN_INTERVAL_MS,
+  minChars: MIN_MAIN_CHARS,
+  robots: true,
+  lineRefs: true,
+}
+
+/** 固定声明的能力边界（每次都随结果返回，别让调用方以为它什么都能干）。 */
+export const LIMITATIONS = [
+  '不执行 JavaScript：客户端渲染的页面只能拿到服务端 HTML，内容可能不全（会显式警告）。',
+  '正文抽取是「结构化密度」启发式，不认 CSS 隐藏、不认付费墙，误判时可以用 lineRefs:false 看原文。',
+  '只处理 HTML / 纯文本 / JSON / XML；PDF、图片、Office 文档里的文字不解析。',
+  '不携带登录态与 Cookie，需要登录的页面会拿到登录页。',
+  'robots.txt 只按 `dsh-scrape` 与 `*` 两组判定（长匹配优先、Allow 同长胜出），不实现非标准扩展。',
+]
+
+/** removed 里固定出现的分类（缺省 0，调用方不必判空）。 */
+const REMOVED_KEYS = [
+  'script', 'style', 'noscript', 'template', 'svg', 'canvas', 'iframe', 'nav', 'footer',
+  'aside', 'form', 'dialog', 'button', 'consent', 'share', 'comment', 'ad',
+]
+
+/**
+ * 造一条 warning。
+ * @param {string} code 稳定错误码
+ * @param {string} message 中文说明
+ * @param {'info'|'warn'|'error'} [severity] 严重程度
+ * @returns {{code:string,message:string,severity:string}} warning
+ */
+export function warning(code, message, severity = 'warn') {
+  return { code, message, severity }
+}
+
+/**
+ * 把字符统计补齐成固定键（缺省 0）。
+ * @param {object} chars 各分类字数
+ * @param {object} elements 各分类元素数
+ * @returns {object} 补齐后的 removed
+ */
+function normalizeRemoved(chars = {}, elements = {}) {
+  const out = {}
+  let total = 0
+  for (const key of REMOVED_KEYS) {
+    const value = Math.round(chars[key] ?? 0)
+    out[key] = value
+    total += value
+  }
+  out.boilerplate = 0
+  out.total = total
+  out.elements = { ...elements }
+  return out
+}
+
+/**
+ * robots 信息的对外形状（blocked 结果与正常结果必须一致，否则调用方要写两套判断）。
+ * @param {object|null} info resolveRobots 结果
+ * @returns {object|null} 规范化 robots 信息
+ */
+function normalizeRobotsInfo(info) {
+  if (!info) return null
+  return {
+    url: info.url,
+    available: info.available,
+    status: info.status,
+    allowed: info.allowance ? info.allowance.allowed : null,
+    usedGroup: info.allowance?.usedGroup ?? null,
+    matchedRule: info.allowance?.matchedRule ?? null,
+    crawlDelayMs: info.crawlDelayMs ?? null,
+    error: info.error ?? null,
+  }
+}
+
+/**
+ * 判断页面是否疑似客户端渲染（SPA），并给出理由。
+ * @param {object} signals parseHtml 的 SPA 信号
+ * @param {number} textChars 可见正文字数
+ * @returns {{suspected:boolean,reasons:string[]}} 判定
+ */
+export function detectSpa(signals, textChars) {
+  const reasons = []
+  if (signals.nextData) reasons.push('页面含 __NEXT_DATA__（Next.js 数据脚本）')
+  if (signals.nuxt) reasons.push('页面含 __NUXT__（Nuxt 数据脚本）')
+  if (signals.initialState) reasons.push('页面含 __INITIAL_STATE__ / __APOLLO_STATE__（前端状态脚本）')
+  if (signals.rootEmpty) reasons.push('#root / #app 等挂载点为空壳')
+  if (signals.noscriptJs) reasons.push('<noscript> 里写着「需要启用 JavaScript」')
+  const shell = reasons.length > 0
+  const suspected = (textChars < 200 && shell) || (textChars < 120 && signals.scriptCount >= 5 && !signals.hasMain)
+  if (!shell && textChars < 120 && signals.scriptCount >= 5 && !signals.hasMain) {
+    reasons.push(`正文只有 ${textChars} 字，但页面有 ${signals.scriptCount} 个脚本且没有 <main>/<article>`)
+  }
+  return { suspected, reasons }
+}
+
+/**
+ * 没抓到正文时的兜底 Markdown：宁可给一段说明，也不返回空字符串。
+ * @param {{title:string,finalUrl:string,spa:object}} info 上下文
+ * @returns {string} Markdown
+ */
+function emptyBodyMarkdown(info) {
+  const lines = ['<!-- L1 -->', '> **未提取到可见正文。**']
+  if (info.spa?.suspected) {
+    lines.push(`> 该页疑似客户端渲染（SPA）：${info.spa.reasons.join('；')}。`)
+    lines.push('> 本工具不执行 JavaScript，所以拿到的是页面骨架。可以改用预渲染版本、站点的静态副本，或 `robots:false` 后再试一次。')
+  } else {
+    lines.push('> 去掉脚本、导航、页脚与弹层后没有剩下文字：可能是空页面、纯图片页、或被反爬拦下的占位页。')
+  }
+  if (info.finalUrl) lines.push(`> 页面地址：${info.finalUrl}`)
+  return lines.join('\n')
+}
+
+/**
+ * 解析并判定 robots.txt（失败不抛错，交给调用方决定口径）。
+ * @param {string} url 页面 URL
+ * @param {object} options 选项（throttle/timeoutMs/signal/userAgent）
+ * @returns {Promise<object>} robots 信息
+ */
+export async function resolveRobots(url, options = {}) {
+  const robotsUrl = `${originOf(url)}/robots.txt`
+  const fetched = await fetchOptional(robotsUrl, {
+    timeoutMs: options.timeoutMs ?? DEFAULTS.timeoutMs,
+    signal: options.signal,
+    throttle: options.throttle,
+    userAgent: options.userAgent ?? DEFAULT_USER_AGENT,
+  })
+
+  if (!fetched) {
+    return { url: robotsUrl, available: false, status: null, allowance: null, error: '抓不到 robots.txt（网络错误）' }
+  }
+  if (fetched.status !== 200) {
+    return {
+      url: robotsUrl, available: false, status: fetched.status, allowance: null,
+      error: `robots.txt 返回 HTTP ${fetched.status}`,
+    }
+  }
+  const parsed = parseRobots(fetched.text)
+  const allowance = checkAllowance(parsed, pathOf(url), { userAgent: options.userAgent ?? DEFAULT_USER_AGENT })
+  return {
+    url: robotsUrl,
+    available: true,
+    status: fetched.status,
+    allowance,
+    crawlDelayMs: allowance.crawlDelayMs,
+    sitemaps: parsed.sitemaps.slice(0, 10),
+    warnings: parsed.warnings.slice(0, 5),
+  }
+}
+
+/**
+ * 把一段 HTML 变成结果（URL 路径与本地 HTML 路径共用）。
+ * @param {string} html HTML 原文
+ * @param {object} options 选项
+ * @returns {object} 正文与统计
+ */
+function extractFromHtml(html, options = {}) {
+  const doc = parseHtml(html, { baseUrl: options.baseUrl ?? '' })
+  const totalVisible = doc.blocks.reduce((sum, block) => sum + blockLength(block), 0)
+  const spa = detectSpa(doc.signals, totalVisible)
+  const main = selectMain(doc.blocks, { spa: spa.suspected, minChars: options.minChars ?? DEFAULTS.minChars })
+
+  let rendered = main.blocks.length > 0
+    ? renderMarkdown(main.blocks, { lineRefs: options.lineRefs !== false, maxChars: options.maxChars ?? DEFAULTS.maxChars })
+    : textLinesToMarkdown(doc.textLines, { lineRefs: options.lineRefs !== false })
+
+  const warnings = []
+  if (spa.suspected) {
+    warnings.push(warning(
+      'spa_suspected',
+      `疑似 SPA（客户端渲染）：正文仅 ${totalVisible} 字，${spa.reasons.join('；')}。本工具不执行 JS，内容可能不全——请改用静态版本或预渲染地址。`,
+      'warn',
+    ))
+  }
+  if (totalVisible === 0) {
+    warnings.push(warning('no_visible_text', '去掉脚本/导航/页脚后没有任何可见文字。', 'error'))
+  } else if (main.chars.kept < (options.minChars ?? DEFAULTS.minChars)) {
+    warnings.push(warning('thin_content', `可信正文只有 ${main.chars.kept} 字（低于 ${options.minChars ?? DEFAULTS.minChars} 字），可能是列表页/首页。`, 'info'))
+  }
+  if (rendered.markdown.trim().length === 0) {
+    rendered = { ...rendered, markdown: emptyBodyMarkdown({ title: doc.title, finalUrl: options.baseUrl ?? '', spa }) }
+  }
+
+  return { doc, main, rendered, spa, totalVisible, warnings }
+}
+
+/**
+ * 抓一个 URL 并转成可引用的 Markdown。
+ * @param {{url?:string,urls?:string[],html?:string}} input 输入（url / urls / html 三选一）
+ * @param {object} [options] 选项
+ * @returns {Promise<object>} 结果
+ */
+export async function scrape(input = {}, options = {}) {
+  const urls = Array.isArray(input.urls) ? input.urls.filter(Boolean) : []
+  if (urls.length > 0 && !input.url) return scrapeMany(urls, options)
+  if (!input.url && typeof input.html === 'string') {
+    // 本地 HTML 也可以带一个 baseUrl（CLI --file 会给 file:// 地址），相对链接才能补绝对
+    return scrapeHtml(input.html, { ...options, baseUrl: options.baseUrl ?? input.baseUrl })
+  }
+  if (!input.url) {
+    throw new ScrapeError('invalid_input', '缺少输入：需要 url、urls 或 html 之一', {
+      hint: '例：{ "url": "https://example.com" }；本地 HTML 用 { "html": "<html>…" }，CLI 用 --file',
+    })
+  }
+
+  const settings = {
+    ...DEFAULTS,
+    ...options,
+    maxUrls: options.maxUrls ?? DEFAULTS.maxUrls,
+  }
+  const throttle = options.throttle ?? createThrottle({ minIntervalMs: settings.minIntervalMs })
+  const fetchedAt = new Date().toISOString()
+  const warnings = []
+  const notes = []
+  const target = String(input.url)
+
+  let robotsInfo = null
+  if (settings.robots !== false) {
+    robotsInfo = await resolveRobots(target, { ...settings, throttle })
+    if (robotsInfo.crawlDelayMs && robotsInfo.crawlDelayMs > 0) {
+      throttle.setHostInterval(hostKey(target), robotsInfo.crawlDelayMs)
+      notes.push(`robots.txt 声明 Crawl-delay=${robotsInfo.crawlDelayMs / 1000}s，已按此间隔限速`)
+    }
+    if (robotsInfo.allowance && !robotsInfo.allowance.allowed) {
+      // 遵守 robots：不发页面请求，但要给出可读结论与证据（命中了第几行）
+      return {
+        tool: TOOL_NAME,
+        version: VERSION,
+        kind: 'url',
+        blocked: 'robots',
+        title: '',
+        markdown: '',
+        provenance: {
+          requestedUrl: target,
+          finalUrl: null,
+          redirects: [],
+          fetchedAt,
+          httpStatus: null,
+          statusText: '',
+          bytes: 0,
+          contentType: '',
+          charset: '',
+          truncated: false,
+          elapsedMs: 0,
+          robots: normalizeRobotsInfo(robotsInfo),
+        },
+        removed: normalizeRemoved({}, {}),
+        content: {
+          chars: 0, blocks: 0, headings: 0, tables: 0, codeBlocks: 0, links: 0,
+          confidence: 0, method: 'blocked', reasons: ['robotstxt 禁止抓取'], extractor: EXTRACT_VERSION,
+          spaSuspected: false, rawTextChars: 0,
+        },
+        warnings: [warning(
+          'robots_disallowed',
+          `${describeAllowance(robotsInfo.allowance, pathOf(target))}。已按 robots.txt 跳过抓取；如确有权抓，传 robots:false（会记一条警告）。`,
+          'error',
+        )],
+        notes: robotsInfo.warnings ?? [],
+        limitations: LIMITATIONS,
+      }
+    }
+    if (!robotsInfo.available) {
+      warnings.push(warning('robots_unavailable', `${robotsInfo.error}，按「允许」处理（这不算站点的禁止声明）。`, 'info'))
+    } else if (robotsInfo.allowance) {
+      notes.push(describeAllowance(robotsInfo.allowance, pathOf(target)))
+    }
+  } else {
+    warnings.push(warning(
+      'robots_disabled',
+      'robots:false 已关闭 robots.txt 检查：本次抓取没有遵守目标站点的 robots 规则，请确认你有权抓这个页面。',
+      'warn',
+    ))
+  }
+
+  const page = await fetchPage(target, {
+    timeoutMs: settings.timeoutMs,
+    maxBytes: settings.maxBytes,
+    maxRedirects: settings.maxRedirects,
+    signal: settings.signal,
+    throttle,
+    userAgent: settings.userAgent ?? DEFAULT_USER_AGENT,
+  })
+
+  if (page.truncated) {
+    warnings.push(warning('truncated', `响应超过 maxBytes=${settings.maxBytes} 字节，已截断；后半部分内容没有解析。`, 'warn'))
+  }
+  if (page.redirects.length > 0) {
+    const crossOrigin = page.redirects.some((hop) => originOf(hop.from) !== originOf(hop.to))
+    if (crossOrigin) {
+      warnings.push(warning('cross_origin_redirect', `抓取过程中发生了跨站重定向（${page.redirects.length} 跳），引用时请以最终地址为准。`, 'info'))
+    }
+    notes.push(`跟随 ${page.redirects.length} 次重定向，最终地址 ${page.finalUrl}`)
+  }
+  if (page.charsetFallback) {
+    warnings.push(warning('charset_fallback', `声明的字符集「${page.charsetDeclared}」无法解码，已退回 UTF-8；中文可能乱码。`, 'warn'))
+  }
+
+  const contentType = page.contentType.toLowerCase()
+  const isHtml = /html|xhtml\+xml|<unknown>/.test(contentType) || contentType === '' || /<\s*(?:!doctype|html|head|body)/i.test(page.text.slice(0, 512))
+
+  // 二进制 / 非文本：不假装解析
+  if (!page.isText && !isHtml) {
+    const markdown = [
+      '<!-- L1 -->',
+      `> **这不是 HTML 文本**：content-type = \`${page.contentType || '(缺失)'}\`，${page.bytes} 字节。`,
+      '> 本工具只把 HTML / 纯文本 / JSON / XML 变成 Markdown；PDF、图片、Office 文档请走文档解析工具。',
+    ].join('\n')
+    return buildResult({
+      kind: 'url', title: '', markdown, page, robotsInfo, fetchedAt,
+      removed: normalizeRemoved({}, {}),
+      content: { chars: 0, blocks: 0, headings: 0, tables: 0, codeBlocks: 0, links: 0, confidence: 0, method: 'binary', reasons: ['二进制内容'], extractor: EXTRACT_VERSION, spaSuspected: false, rawTextChars: 0 },
+      warnings: [...warnings, warning('binary_content', `content-type 是 ${page.contentType || '(缺失)'}，不是可解析的文本。`, 'error')],
+      notes, limitations: LIMITATIONS, target,
+    })
+  }
+
+  // 非 HTML 的文本（纯文本 / JSON / XML / Markdown）：整段包成代码块，行号照旧从原文件数
+  if (!isHtml) {
+    const rendered = textLinesToMarkdown(page.text.replace(/\r\n?/g, '\n').split('\n'), { lineRefs: settings.lineRefs !== false })
+    const markdown = ['```' + fenceLanguage(contentType), page.text.replace(/\r\n?/g, '\n').trimEnd(), '```'].join('\n')
+    return buildResult({
+      kind: 'url', title: page.finalUrl, markdown, rendered: { ...rendered, markdown }, page, robotsInfo, fetchedAt,
+      removed: normalizeRemoved({}, {}),
+      content: {
+        chars: page.text.length, blocks: 1, headings: 0, tables: 0, codeBlocks: 1, links: 0,
+        confidence: 0.9, method: 'plain-text', reasons: ['content-type 不是 HTML，按纯文本整段保留'], extractor: EXTRACT_VERSION,
+        spaSuspected: false, rawTextChars: page.text.length,
+      },
+      warnings: [...warnings, warning('not_html', `content-type 是 ${page.contentType || '(缺失)'}，按纯文本处理（未做正文抽取）。`, 'info')],
+      notes, limitations: LIMITATIONS, target,
+    })
+  }
+
+  const parsed = extractFromHtml(page.text, {
+    baseUrl: page.finalUrl,
+    lineRefs: settings.lineRefs,
+    maxChars: settings.maxChars,
+    minChars: settings.minChars,
+  })
+
+  const removed = normalizeRemoved(parsed.doc.removed.chars, parsed.doc.removed.elements)
+  removed.boilerplate = Math.round(parsed.main.chars.droppedBoilerplate)
+  removed.total += removed.boilerplate
+
+  const content = {
+    chars: parsed.main.chars.kept,
+    markdownChars: parsed.rendered.chars,
+    blocks: parsed.rendered.blockCount,
+    headings: parsed.doc.stats.headings,
+    tables: parsed.doc.stats.tables,
+    codeBlocks: parsed.doc.stats.codeBlocks,
+    links: parsed.doc.stats.links,
+    confidence: parsed.main.confidence,
+    method: parsed.main.method,
+    reasons: parsed.main.reasons,
+    extractor: parsed.main.version,
+    spaSuspected: parsed.spa.suspected,
+    spaReasons: parsed.spa.reasons,
+    rawTextChars: parsed.totalVisible,
+  }
+
+  return buildResult({
+    kind: 'url',
+    title: parsed.doc.title,
+    markdown: parsed.rendered.markdown,
+    rendered: parsed.rendered,
+    page, robotsInfo, fetchedAt,
+    removed,
+    content,
+    warnings: [...warnings, ...parsed.warnings],
+    notes: [...notes, ...(parsed.doc.signals.nextData ? ['页面带 __NEXT_DATA__：这是 Next.js 的数据脚本，不执行它拿不到组件渲染后的文字'] : [])],
+    limitations: LIMITATIONS,
+    target,
+  })
+}
+
+/**
+ * 只给一段 HTML（离线/测试/本地文件）。不做网络、不判 robots。
+ * @param {string} html HTML 原文
+ * @param {object} [options] 选项
+ * @returns {object} 结果
+ */
+export function scrapeHtml(html, options = {}) {
+  const settings = { ...DEFAULTS, ...options }
+  const parsed = extractFromHtml(html, {
+    baseUrl: options.baseUrl ?? '',
+    lineRefs: settings.lineRefs,
+    maxChars: settings.maxChars,
+    minChars: settings.minChars,
+  })
+  const removed = normalizeRemoved(parsed.doc.removed.chars, parsed.doc.removed.elements)
+  removed.boilerplate = Math.round(parsed.main.chars.droppedBoilerplate)
+  removed.total += removed.boilerplate
+
+  return {
+    tool: TOOL_NAME,
+    version: VERSION,
+    kind: 'html',
+    title: parsed.doc.title,
+    markdown: parsed.rendered.markdown,
+    provenance: {
+      requestedUrl: null,
+      finalUrl: options.baseUrl ? String(options.baseUrl) : null,
+      redirects: [],
+      fetchedAt: new Date().toISOString(),
+      httpStatus: null,
+      statusText: '',
+      bytes: Buffer.byteLength(String(html), 'utf8'),
+      contentType: 'text/html',
+      charset: 'utf-8',
+      truncated: false,
+      elapsedMs: 0,
+      robots: null,
+    },
+    removed,
+    content: {
+      chars: parsed.main.chars.kept,
+      markdownChars: parsed.rendered.chars,
+      blocks: parsed.rendered.blockCount,
+      headings: parsed.doc.stats.headings,
+      tables: parsed.doc.stats.tables,
+      codeBlocks: parsed.doc.stats.codeBlocks,
+      links: parsed.doc.stats.links,
+      confidence: parsed.main.confidence,
+      method: parsed.main.method,
+      reasons: parsed.main.reasons,
+      extractor: parsed.main.version,
+      spaSuspected: parsed.spa.suspected,
+      spaReasons: parsed.spa.reasons,
+      rawTextChars: parsed.totalVisible,
+    },
+    warnings: parsed.warnings,
+    notes: [],
+    lineMap: parsed.rendered.lines ?? [],
+    limitations: LIMITATIONS,
+  }
+}
+
+/**
+ * 批量抓（同域自动串行限速；单个失败不影响其它）。
+ * @param {string[]} urls URL 列表
+ * @param {object} [options] 选项
+ * @returns {Promise<object>} 多页结果
+ */
+export async function scrapeMany(urls, options = {}) {
+  const settings = { ...DEFAULTS, ...options }
+  const list = urls.slice(0, settings.maxUrls)
+  const throttle = options.throttle ?? createThrottle({ minIntervalMs: settings.minIntervalMs })
+  const pages = []
+  const failures = []
+
+  for (const url of list) {
+    try {
+      pages.push(await scrape({ url }, { ...settings, throttle, maxUrls: settings.maxUrls }))
+    } catch (error) {
+      failures.push({ url: String(url), error: describeError(error) })
+    }
+  }
+
+  return {
+    tool: TOOL_NAME,
+    version: VERSION,
+    kind: 'multi',
+    ok: failures.length === 0 && pages.every((page) => page.blocked === undefined),
+    count: pages.length,
+    requested: list.length,
+    truncatedList: urls.length > list.length,
+    pages,
+    failures,
+    warnings: urls.length > list.length
+      ? [warning('too_many_urls', `一次最多抓 ${settings.maxUrls} 个 URL，后面的 ${urls.length - list.length} 个没抓。`, 'warn')]
+      : [],
+    limitations: LIMITATIONS,
+  }
+}
+
+/**
+ * 组装单页结果（URL 路径统一出口）。
+ * @param {object} parts 部件
+ * @returns {object} 结果
+ */
+function buildResult(parts) {
+  const { page, robotsInfo, fetchedAt, target } = parts
+  const warnings = parts.warnings ?? []
+  if (page.elapsedMs > 8000) {
+    warnings.push(warning('slow_response', `服务器响应用了 ${page.elapsedMs}ms，接近默认超时。`, 'info'))
+  }
+  return {
+    tool: TOOL_NAME,
+    version: VERSION,
+    kind: parts.kind,
+    title: parts.title ?? '',
+    markdown: parts.markdown ?? '',
+    provenance: {
+      requestedUrl: target,
+      finalUrl: page.finalUrl,
+      redirects: page.redirects,
+      fetchedAt,
+      httpStatus: page.status,
+      statusText: page.statusText,
+      bytes: page.bytes,
+      contentType: page.contentType,
+      charset: page.charset,
+      charsetDeclared: page.charsetDeclared,
+      truncated: page.truncated,
+      elapsedMs: page.elapsedMs,
+      robots: normalizeRobotsInfo(robotsInfo),
+    },
+    removed: parts.removed,
+    content: parts.content,
+    warnings,
+    notes: parts.notes ?? [],
+    lineMap: parts.rendered?.lines ?? [],
+    limitations: parts.limitations ?? LIMITATIONS,
+  }
+}
+
+/**
+ * 按 content-type 猜代码围栏语言。
+ * @param {string} contentType 响应类型
+ * @returns {string} 语言名
+ */
+function fenceLanguage(contentType) {
+  if (/json/.test(contentType)) return 'json'
+  if (/xml|rss|atom/.test(contentType)) return 'xml'
+  if (/markdown/.test(contentType)) return 'markdown'
+  if (/javascript/.test(contentType)) return 'js'
+  return 'text'
+}

package/package.json

@@ -1,26 +1,62 @@
 {
   "name": "dsh-scrape",
-  "version": "0.0.1",
-  "description": "Turn any page, feed or repo into clean Markdown/text with per-site rules, retries and JS-page fallbacks, so an agent can cite what it actually fetched. | 把网页、RSS、仓库抓成干净的 Markdown/文本，带站点规则、重试与 JS 页面兜底，让 agent 引用的内容确实来自它抓到的东西。",
+  "version": "0.1.0",
+  "description": "把网页变成干净、可引用、带行号的 Markdown：遵守 robots.txt、同域限速、跟随重定向、自动认字符集、按行去噪、密度法挑正文并给 confidence，每条内容带 `<!-- L42 -->` 源行号锚点，不执行 JS 且对疑似 SPA 显式警告。零运行时依赖，含 CLI。 | Turns any web page into clean, citable Markdown with per-line provenance: robots-aware, host-throttled, redirect-following, charset-aware (GBK/Big5), boilerplate-stripped, density-based main-content extraction with a confidence score, and `<!-- L42 -->` line anchors. Never executes JavaScript; warns explicitly when a page looks like a client-rendered SPA.",
   "keywords": [
     "dsh",
     "dsh-plugin",
     "deepseek-harness",
     "scrape",
-    "crawl",
-    "fetch",
+    "scraper",
+    "web-scraping",
     "markdown",
-    "browser",
-    "geo",
+    "html-to-markdown",
+    "readability",
+    "robots-txt",
+    "citation",
+    "provenance",
+    "crawl",
+    "web-reader",
     "抓取",
-    "爬虫"
+    "网页转markdown"
   ],
   "license": "MIT",
+  "author": "he fangsheng (https://github.com/dongsheng123132)",
   "type": "module",
+  "main": "./plugin.mjs",
+  "bin": {
+    "dsh-scrape": "./lib/cli.js"
+  },
+  "exports": {
+    ".": "./plugin.mjs",
+    "./cli": "./lib/cli.js",
+    "./lib/*": "./lib/*",
+    "./package.json": "./package.json"
+  },
   "files": [
-    "README.md"
+    "plugin.mjs",
+    "lib/",
+    "scripts/",
+    "cordis.patch.yml",
+    "dsh-plugin.naming.json",
+    "README.md",
+    "LICENSE"
   ],
+  "dsh": {
+    "bundle": {
+      "patch": "./cordis.patch.yml"
+    }
+  },
   "engines": {
-    "node": ">=22"
+    "node": ">=22",
+    "dsh": ">=0.2.2-rc.1 <0.3.0"
+  },
+  "scripts": {
+    "test": "node --test test/*.test.mjs",
+    "test:dir": "node --test test/",
+    "check": "node --check plugin.mjs && node scripts/check.mjs",
+    "verify:install": "node scripts/verify-install.mjs",
+    "scrape": "node lib/cli.js",
+    "audit:readme": "node ../dsh-geo-audit/lib/cli.js --file README.md"
   }
 }