dsh-scrape 0.0.1 → 0.1.0

package/lib/robots.mjs

@@ -0,0 +1,171 @@
+/**
+ * robots.txt：解析 + 「我这个爬虫能不能抓这条路径」的判定。
+ *
+ * 判定口径照 RFC 9309（与 Google 的公开实现一致）：
+ * 1) 有精确匹配我这条 UA 的组时只用精确组，忽略 `*`；
+ * 2) 组内按**最长路径匹配**胜出；长度相同时 `Allow` 胜；
+ * 3) 没有任何规则命中 = 允许；
+ * 4) `Crawl-delay` 非标准但常见，读出来当同域最低间隔。
+ *
+ * 只解析、不推测：看不懂的行进 warnings，不当成结论。
+ */
+
+/** 本工具的 UA 产品令牌：robots.txt 里写 `User-agent: dsh-scrape` 就能精确指名它。 */
+export const BOT_TOKEN = 'dsh-scrape'
+
+/**
+ * 解析 robots.txt。
+ * @param {string} text robots.txt 原文
+ * @returns {{groups:Array<object>,sitemaps:string[],warnings:string[],lineCount:number}} 解析结果
+ */
+export function parseRobots(text) {
+  const lines = String(text ?? '').replace(/\r\n?/g, '\n').split('\n')
+  const groups = []
+  const sitemaps = []
+  const warnings = []
+  let current = null
+  let lastWasAgent = false
+
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i].replace(/#.*$/, '').trim()
+    if (line.length === 0) continue
+    const colon = line.indexOf(':')
+    if (colon === -1) {
+      warnings.push(`第 ${i + 1} 行不是 key:value，已跳过`)
+      continue
+    }
+    const key = line.slice(0, colon).trim().toLowerCase()
+    const value = line.slice(colon + 1).trim()
+
+    if (key === 'user-agent' || key === 'useragent') {
+      if (current === null || !lastWasAgent) {
+        current = { agents: [], rules: [], crawlDelay: null, line: i + 1 }
+        groups.push(current)
+      }
+      if (value.length > 0) current.agents.push(value.toLowerCase())
+      lastWasAgent = true
+      continue
+    }
+
+    lastWasAgent = false
+    if (key === 'sitemap') { sitemaps.push(value); continue }
+    if (key === 'crawl-delay') {
+      const seconds = Number(value)
+      if (current && Number.isFinite(seconds) && seconds >= 0) current.crawlDelay = seconds
+      else warnings.push(`第 ${i + 1} 行的 Crawl-delay 不是数字，已忽略`)
+      continue
+    }
+    if (key === 'allow' || key === 'disallow') {
+      if (current === null) {
+        warnings.push(`第 ${i + 1} 行的 ${key} 出现在任何 User-agent 之前，已跳过`)
+        continue
+      }
+      if (value.length === 0) continue // `Disallow:` 空值 = 不限制
+      current.rules.push({ type: key, path: value, line: i + 1 })
+      continue
+    }
+    if (key === 'host' || key === 'clean-param') continue
+    warnings.push(`第 ${i + 1} 行的指令「${key}」不受支持，已忽略`)
+  }
+
+  return { groups, sitemaps, warnings, lineCount: lines.length }
+}
+
+/**
+ * 一条 robots 路径规则是否命中目标路径（支持 `*` 与结尾 `$`）。
+ * @param {string} pattern 规则路径
+ * @param {string} target 目标路径（以 / 开头）
+ * @returns {boolean} 是否命中
+ */
+export function pathMatches(pattern, target) {
+  if (!pattern) return false
+  let body = pattern
+  let anchored = false
+  if (body.endsWith('$')) { anchored = true; body = body.slice(0, -1) }
+  const escaped = body.replace(/[.+?^${}()|[\]\\]/g, '\\$&').replace(/\*/g, '.*')
+  try {
+    return new RegExp(`^${escaped}${anchored ? '$' : ''}`).test(target)
+  } catch {
+    return false
+  }
+}
+
+/**
+ * 找出适用于我这个 UA 的组（精确优先，否则 `*`）。
+ * @param {object} robots parseRobots 结果
+ * @param {string} [userAgent] 完整 UA 或产品令牌
+ * @returns {{groups:Array<object>,usedGroup:'exact'|'*'|'none',declared:boolean}} 命中的组
+ */
+export function matchingGroups(robots, userAgent = BOT_TOKEN) {
+  const ua = String(userAgent).toLowerCase()
+  const exact = robots.groups.filter((group) => group.agents.some((agent) => agent !== '*' && ua.includes(agent)))
+  if (exact.length > 0) return { groups: exact, usedGroup: 'exact', declared: true }
+  const wildcard = robots.groups.filter((group) => group.agents.includes('*'))
+  return { groups: wildcard, usedGroup: wildcard.length > 0 ? '*' : 'none', declared: false }
+}
+
+/**
+ * 判定我这个爬虫对某条路径的结论。
+ * @param {object} robots parseRobots 结果
+ * @param {string} [target='/'] 目标路径（含查询串也无妨，规则只匹配路径）
+ * @param {{userAgent?:string}} [options] 选项
+ * @returns {{allowed:boolean,usedGroup:string,declared:boolean,matchedRule:object|null,crawlDelayMs:number|null}} 结论
+ */
+export function checkAllowance(robots, target = '/', options = {}) {
+  const matched = matchingGroups(robots, options.userAgent ?? BOT_TOKEN)
+  const rules = matched.groups.flatMap((group) => group.rules).filter((rule) => pathMatches(rule.path, target))
+  const delays = matched.groups.map((group) => group.crawlDelay).filter((value) => typeof value === 'number')
+  const crawlDelayMs = delays.length > 0 ? Math.round(Math.max(...delays) * 1000) : null
+
+  let best = null
+  for (const rule of rules) {
+    if (best === null || rule.path.length > best.path.length || (rule.path.length === best.path.length && rule.type === 'allow')) {
+      best = rule
+    }
+  }
+
+  return {
+    allowed: best === null ? true : best.type === 'allow',
+    usedGroup: matched.usedGroup,
+    declared: matched.declared,
+    matchedRule: best,
+    crawlDelayMs,
+  }
+}
+
+/**
+ * 取 URL 的路径（给 robots 匹配用）。
+ * @param {string} url 页面 URL
+ * @returns {string} 以 / 开头的路径
+ */
+export function pathOf(url) {
+  try {
+    return new URL(url).pathname || '/'
+  } catch {
+    return '/'
+  }
+}
+
+/**
+ * 取 URL 的 origin（拼 robots.txt 地址用）。
+ * @param {string} url 页面 URL
+ * @returns {string} 协议 + 主机 + 端口
+ */
+export function originOf(url) {
+  return new URL(url).origin
+}
+
+/**
+ * 一行结论文案（报告与 warnings 共用，避免两处口径不一致）。
+ * @param {object} result checkAllowance 结果
+ * @param {string} target 路径
+ * @returns {string} 中文说明
+ */
+export function describeAllowance(result, target) {
+  if (result.allowed) {
+    return result.matchedRule
+      ? `robots.txt 允许抓 ${target}（命中 ${result.matchedRule.type}: ${result.matchedRule.path}，第 ${result.matchedRule.line} 行）`
+      : `robots.txt 没有禁止抓 ${target}（${result.usedGroup === 'exact' ? '按 dsh-scrape 组' : result.usedGroup === '*' ? '按 * 组' : '无相关组'}，无命中规则 = 允许）`
+  }
+  return `robots.txt 禁止抓 ${target}（命中 ${result.matchedRule.type}: ${result.matchedRule.path}，第 ${result.matchedRule.line} 行）`
+}