@opendesign-plus/geo-scripts 0.0.1-rc.1

package/dist/generate-llms-full.d.ts

@@ -0,0 +1,76 @@
+/**
+ * generate-llms-full.ts - 生成 llms-full.txt 文件
+ *
+ * 从网站sitemap或本地HTML目录抓取页面内容，转换为Markdown格式，
+ * 生成适合AI大语言模型读取的 llms-full.txt 文件。
+ *
+ * 支持两种调用方式：
+ *
+ * 1. 命令行调用：
+ *    npx tsx generate-llms-full.ts [options]
+ *
+ *    选项：
+ *      -s, --site <url>              网站根目录地址
+ *      --html-dir <dirpath>          本地HTML构建产物目录（从本地文件读取，不传的话则遍历网站sitemap）
+ *      --prefix <string>             生成文件的自定义开头内容
+ *      --spa                         是否为SPA应用，使用Playwright渲染
+ *      --selector <selector>         HTML元素选择器，默认 'main'
+ *      --rmc --remove-class <...>    转md时移除的HTML元素class选择器（逗号分隔或传数组）
+ *      --rme --remove-elements <...> 转md时移除的HTML元素标签（逗号分隔或传数组）
+ *      -o --output <file>            输出文件路径，默认 './llms-full.txt'
+ *      -e, --exclude <patterns...>   排除URL的正则表达式模式
+ *      -c, --concurrency <number>    并发数，默认 '5'
+ *
+ *    示例：
+ *      # 从网站sitemap抓取
+ *      npx tsx generate-llms-full.ts -s https://example.com --spa -c 10
+ *
+ *      # 从本地HTML目录读取
+ *      npx tsx generate-llms-full.ts --html-dir ./dist -s https://example.com
+ *
+ *      # 排除特定URL
+ *      npx tsx generate-llms-full.ts -s https://example.com -e "/admin" -e "/login"
+ *
+ * 2. 模块导入调用：
+ *    import generateLLMsFull from '@opendesign-plus/geo-scripts/generate-llms-full';
+ *
+ *    generateLLMsFull({
+ *      site: 'https://example.com',
+ *      spa: true,
+ *      selector: 'main',
+ *      concurrency: '10',
+ *      output: './llms-full.txt'
+ *    });
+ *
+ *    // 从本地HTML目录读取
+ *    generateLLMsFull({
+ *      htmlDir: './dist',
+ *      site: 'https://example.com',  // 用于生成完整URL
+ *      selector: 'article',
+ *      output: './llms-full.txt'
+ *    });
+ */
+export interface GenerateLLMsFullOptions {
+    /** 网站根目录地址 */
+    site?: string;
+    /** 生成文件的自定义开头内容 */
+    prefix?: string;
+    /** 是否为SPA应用，使用Playwright渲染 */
+    spa?: boolean;
+    /** HTML元素选择器，默认 'main' */
+    selector?: string;
+    /** 转md时移除的HTML元素class选择器（逗号分隔或传数组） */
+    removeClass?: string | string[];
+    /** 转md时移除的HTML元素标签（逗号分隔或传数组） */
+    removeElements?: string | string[];
+    /** 本地HTML构建产物目录（从本地文件读取，不传的话则遍历网站sitemap） */
+    htmlDir?: string;
+    /** 输出文件路径，默认 './llms-full.txt' */
+    output?: string;
+    /** 排除URL的正则表达式模式 */
+    exclude?: (string | RegExp)[];
+    /** 并发数，默认 '5' */
+    concurrency?: string | number;
+}
+declare function generateLLMsFull(options: GenerateLLMsFullOptions): Promise<void>;
+export default generateLLMsFull;

package/dist/generate-llms-full.js

@@ -0,0 +1,238 @@
+import { chromium as x } from "playwright-core";
+import $ from "turndown";
+import b from "p-limit";
+import { createWriteStream as v, readdirSync as C, existsSync as L } from "node:fs";
+import { resolve as S, join as d, relative as A } from "node:path";
+import { fileURLToPath as P } from "node:url";
+import { pipeline as T } from "node:stream/promises";
+import w from "node:process";
+import { Command as k } from "commander";
+import { JSDOM as R } from "jsdom";
+import { readFile as M } from "node:fs/promises";
+const h = new $({
+  headingStyle: "atx",
+  bulletListMarker: "-"
+});
+h.addRule("remove-imgs", {
+  filter: ["img"],
+  replacement: () => ""
+});
+h.addRule("links", {
+  filter: ["a"],
+  replacement: (e, s) => {
+    var c;
+    const o = (c = s.querySelector(".o-card-title")) == null ? void 0 : c.textContent;
+    return o ? `(${o})[${s.getAttribute("href")}] ` : e;
+  }
+});
+h.addRule("table", {
+  filter: "table",
+  replacement: (e, s) => {
+    let o = "";
+    return Array.from(s.querySelectorAll("tr")).forEach((l, a) => {
+      const p = l.querySelectorAll("th, td"), f = Array.from(p).map((t) => {
+        var n;
+        return ((n = t.textContent) == null ? void 0 : n.trim()) || "";
+      }).join(" | ");
+      if (o += "| " + f + ` |
+`, a === 0) {
+        const t = Array.from(p).map(() => "---").join(" | ");
+        o += "| " + t + ` |
+`;
+      }
+    }), o;
+  }
+});
+async function E() {
+  const e = [
+    "C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe",
+    "C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe",
+    "/usr/bin/chromium-browser",
+    "/usr/bin/chromium",
+    "/usr/bin/google-chrome",
+    "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"
+  ];
+  let s;
+  for (const o of e)
+    if (L(o)) {
+      s = o, console.log(`使用浏览器：${o}`);
+      break;
+    }
+  return s || (console.log("⚠  未找到系统 Chrome，尝试使用 playwright 自带 Chromium..."), console.log("   如果失败，请运行：npx playwright install chromium")), await x.launch({
+    headless: !0,
+    executablePath: s,
+    args: ["--no-sandbox", "--disable-setuid-sandbox"]
+  });
+}
+async function F(e) {
+  const s = [];
+  try {
+    const l = await fetch(`${e}robots.txt`);
+    if (!l.ok)
+      console.log(`Fetch robots.txt failed ${l.status}`), s.push(`${e}sitemap.xml`);
+    else {
+      const p = (await l.text()).split(`
+`);
+      for (const f of p) {
+        const t = f.trim();
+        if (/^sitemap:/i.test(t)) {
+          const n = t.replace(/^sitemap:\s*/i, "");
+          n && s.push(n);
+        }
+      }
+      s.length === 0 && s.push(`${e}sitemap.xml`);
+    }
+  } catch {
+    s.push(`${e}sitemap.xml`);
+  }
+  let o = "";
+  if (o = (await Promise.all(
+    s.map(async (l) => {
+      try {
+        const a = await fetch(l);
+        return a.ok ? await a.text() : "";
+      } catch {
+        return "";
+      }
+    })
+  )).join(""), !o)
+    return [];
+  const c = (l) => [...l.matchAll(/<loc>([^<]+)<\/loc>/g)].map((a) => a[1]);
+  for (; o.includes("sitemapindex"); ) {
+    const l = c(o);
+    o = (await Promise.all(
+      l.map(async (p) => {
+        try {
+          const f = await fetch(p);
+          return f.ok ? await f.text() : "";
+        } catch {
+          return "";
+        }
+      })
+    )).join("");
+  }
+  return c(o);
+}
+async function H(e) {
+  var f;
+  if (e.removeElements) {
+    const t = Array.isArray(e.removeElements) ? e.removeElements : e.removeElements.split(",").map((r) => r.trim()), n = new Set(t);
+    h.remove((r) => n.has(r.tagName.toLowerCase()));
+  }
+  if (e.removeClass) {
+    const t = Array.isArray(e.removeClass) ? e.removeClass : e.removeClass.split(",").map((n) => n.trim());
+    h.remove((n) => {
+      for (const r of t)
+        if (n.classList.contains(r))
+          return !0;
+      return !1;
+    });
+  }
+  const s = (f = e.exclude) != null && f.length ? e.exclude.map((t) => t instanceof RegExp ? t : new RegExp(t)) : null, o = e.htmlDir ? S(e.htmlDir) : "", c = e.site ? (() => {
+    try {
+      return new URL(e.site).origin;
+    } catch {
+      return "";
+    }
+  })() : "";
+  if (!c && !o) {
+    console.log("请指定 --site 或 --html-dir");
+    return;
+  }
+  let l = [];
+  if (o) {
+    const t = (n, r = []) => {
+      const i = C(n, { withFileTypes: !0 }), u = [];
+      e: for (const m of i) {
+        const g = d(n, m.name);
+        if (s) {
+          for (const y of s)
+            if (y.test(g.replace(/\\/g, "/")))
+              continue e;
+        }
+        m.isFile() && m.name.endsWith(".html") && r.push(g), m.isDirectory() && u.push(g);
+      }
+      for (const m of u)
+        t(m, r);
+      return r;
+    };
+    l = t(o);
+  } else {
+    let t = e.site;
+    if (t = t.endsWith("/") ? t : t + "/", console.log(`站点: ${t}`), console.log(`SPA模式: ${e.spa ? "是" : "否"}`), console.log(`选择器: ${e.selector}`), console.log(`并发数: ${e.concurrency}`), l = await F(t), s && (l = l.filter((n) => !s.some((r) => r.test(n.replace(/\\/g, "/"))))), !l.length) {
+      console.log("sitemap empty");
+      return;
+    }
+  }
+  console.log(`共需处理 ${l.length} 个URL`);
+  const a = !o && e.spa ? await E() : null;
+  async function p(t) {
+    try {
+      console.log(`Processing: ${t}`);
+      let n;
+      if (a) {
+        const r = await a.newPage();
+        await r.goto(t, { waitUntil: "networkidle" }), n = await r.evaluate((i) => {
+          const u = document.querySelector(i);
+          return u ? u.innerHTML : null;
+        }, e.selector || "main"), await r.close();
+      } else {
+        let r = "";
+        if (o)
+          r = await M(t, "utf-8");
+        else {
+          const m = await fetch(t);
+          if (!m.ok) {
+            console.error(`Failed to fetch ${t}: ${m.status}`);
+            return;
+          }
+          r = await m.text();
+        }
+        const u = new R(r).window.document.querySelector(e.selector || "main");
+        n = u ? u.innerHTML : null;
+      }
+      if (n) {
+        const r = h.turndown(n);
+        if (console.log(`✓ ${t}`), o) {
+          let i = A(o, t).replace(/\\/g, "/").replace(/\.html$/, "");
+          return i === "index" ? i = "/" : i.endsWith("/index") ? i = i.slice(0, -5) : i = "/" + i, i = i.startsWith("/") ? i : "/" + i, `
+---
+${c ? c + i : i}
+---
+
+` + r + `
+`;
+        }
+        return `
+---
+${t}
+---
+
+` + r + `
+`;
+      } else
+        console.log(`No <${e.selector || "main"}> element found on ${t}`);
+    } catch (n) {
+      const r = n;
+      console.error(`Error processing ${t}:`, r.message);
+    }
+  }
+  await T(
+    async function* () {
+      e.prefix && (yield e.prefix);
+      const t = b(typeof e.concurrency == "number" ? e.concurrency : parseInt(e.concurrency || "5", 10)), n = l.map((r) => t(() => p(r)));
+      for await (const r of n)
+        r && (yield r);
+    },
+    v(e.output || d(w.cwd(), "llms-full.txt"))
+  ), a && await a.close();
+}
+if (w.argv[1] === P(import.meta.url)) {
+  const e = new k();
+  e.option("-s, --site <url>", "网站URL").option("--prefix <string>", "生成文件的自定义开头").option("--spa", "是否为SPA应用，使用Playwright渲染", !1).option("--selector <selector>", "HTML元素选择器", "main").option("--rmc --remove-class <selector, selector, ...>", "转md时移除的HTML元素class选择器").option("--rme --remove-elements <tag, tag, ...>", "转md时移除的HTML元素选择器").option("--html-dir <dirpath>", "html构建产物目录地址").option("-o --output <file>", "输出文件路径").option("-e, --exclude <patterns...>", "排除URL的正则表达式模式").option("-c, --concurrency <number>", "并发数", "5").parse();
+  const s = e.opts();
+  H(s);
+}
+export {
+  H as default
+};

package/dist/generate-robots-txt.d.ts

@@ -0,0 +1,74 @@
+/**
+ * generate-robots-txt.ts - 生成 robots.txt 文件
+ *
+ * 支持两种调用方式：
+ *
+ * 1. 命令行调用：
+ *    node generate-robots-txt.ts [options]
+ *
+ *    选项：
+ *      -u, --user-agent <string>   User-agent，默认 '*'
+ *      -a, --allow <paths...>      允许的路径，可多次使用
+ *      -d, --disallow <paths...>   禁止的路径，可多次使用
+ *      -s, --sitemap <url>         Sitemap URL
+ *      --crawl-delay <seconds>     爬取延迟秒数
+ *      --host <host>               主机地址
+ *      -o, --output <file>         输出文件路径，默认 './robots.txt'
+ *      -r, --rule <json>           多规则模式，JSON字符串，可多次使用
+ *
+ *    示例：
+ *      # 简单模式
+ *      node generate-robots-txt.ts -d /admin -d /private -s https://example.com/sitemap.xml
+ *
+ *      # 多规则模式
+ *      node generate-robots-txt.ts \
+ *        -r '{"userAgent":"Googlebot","allow":["/"],"disallow":["/private"]}' \
+ *        -r '{"userAgent":"Bingbot","disallow":["/admin"],"crawlDelay":5}' \
+ *        -s https://example.com/sitemap.xml
+ *
+ * 2. 模块导入调用：
+ *    import generateRobotsTxt from './generate-robots-txt.ts';
+ *
+ *    // 简单模式
+ *    generateRobotsTxt({
+ *      userAgent: '*',
+ *      allow: ['/public'],
+ *      disallow: ['/admin', '/private'],
+ *      sitemap: 'https://example.com/sitemap.xml',
+ *      crawlDelay: 10,
+ *      host: 'example.com',
+ *      output: './robots.txt'
+ *    });
+ *
+ *    // 多规则模式
+ *    generateRobotsTxt({
+ *      rules: [
+ *        { userAgent: 'Googlebot', allow: ['/'], disallow: ['/private'] },
+ *        { userAgent: 'Bingbot', disallow: ['/admin'], crawlDelay: 5 }
+ *      ],
+ *      sitemap: 'https://example.com/sitemap.xml',
+ *      output: './robots.txt'
+ *    });
+ */
+export interface RobotsTxtRule {
+    userAgent?: string;
+    allow?: string[];
+    disallow?: string[];
+    crawlDelay?: number;
+}
+export interface RobotsTxtConfig {
+    userAgent?: string;
+    allow?: string[];
+    disallow?: string[];
+    sitemap?: string;
+    crawlDelay?: number;
+    host?: string;
+    rules?: RobotsTxtRule[];
+    output?: string;
+}
+export interface RobotsTxtResult {
+    output: string;
+    content: string;
+}
+declare function generateRobotsTxt(config: RobotsTxtConfig): RobotsTxtResult;
+export default generateRobotsTxt;

package/dist/generate-robots-txt.js

@@ -0,0 +1,68 @@
+import { writeFileSync as m } from "node:fs";
+import { resolve as w } from "node:path";
+import { fileURLToPath as g } from "node:url";
+import c from "node:process";
+import { Command as d } from "commander";
+function $(p) {
+  const {
+    userAgent: n = "*",
+    allow: l = [],
+    disallow: s = [],
+    sitemap: a,
+    crawlDelay: e,
+    host: r,
+    rules: i = [],
+    output: f = w(c.cwd(), "robots.txt")
+  } = p, o = [];
+  if (i.length > 0)
+    for (const t of i) {
+      if (t.userAgent && o.push(`User-agent: ${t.userAgent}`), t.allow && t.allow.length > 0)
+        for (const u of t.allow)
+          o.push(`Allow: ${u}`);
+      if (t.disallow && t.disallow.length > 0)
+        for (const u of t.disallow)
+          o.push(`Disallow: ${u}`);
+      t.crawlDelay !== void 0 && o.push(`Crawl-delay: ${t.crawlDelay}`), o.push("");
+    }
+  else {
+    if (o.push(`User-agent: ${n}`), o.push(""), l.length > 0)
+      for (const t of l)
+        o.push(`Allow: ${t}`);
+    if (s.length > 0)
+      for (const t of s)
+        o.push(`Disallow: ${t}`);
+    e !== void 0 && o.push(`Crawl-delay: ${e}`), o.push("");
+  }
+  a && o.push(`Sitemap: ${a}`), r && o.push(`Host: ${r}`);
+  const h = o.join(`
+`).trim() + `
+`;
+  return m(f, h, "utf-8"), { output: f, content: h };
+}
+if (c.argv[1] === g(import.meta.url)) {
+  let p = function(e, r) {
+    try {
+      const i = JSON.parse(e);
+      return r.concat(i);
+    } catch {
+      return console.error(`无效的JSON规则: ${e}`), r;
+    }
+  };
+  const n = new d();
+  n.option("-u, --user-agent <string>", "User-agent", "*").option("-a, --allow <paths...>", "允许的路径").option("-d, --disallow <paths...>", "禁止的路径").option("-s, --sitemap <url>", "Sitemap URL").option("--crawl-delay <seconds>", "爬取延迟秒数", parseInt).option("--host <host>", "主机地址").option("-o, --output <file>", "输出文件路径", w(c.cwd(), "robots.txt")).option("-r, --rule <json>", "多规则模式，传入JSON字符串，可多次使用", p, []).parse();
+  const l = n.opts(), s = {
+    userAgent: l.userAgent,
+    allow: l.allow || [],
+    disallow: l.disallow || [],
+    sitemap: l.sitemap,
+    crawlDelay: l.crawlDelay,
+    host: l.host,
+    output: l.output
+  };
+  l.rule && l.rule.length > 0 && (s.rules = l.rule);
+  const a = $(s);
+  console.log(`robots.txt 已生成: ${a.output}`);
+}
+export {
+  $ as default
+};

package/dist/vitepress-sitemap-transformer.d.ts

@@ -0,0 +1,32 @@
+type Awaitable<T> = T | Promise<T>;
+/**
+ * 用于Vitepress项目的config.sitemap.transformItems
+ *
+ * ```js
+ * export default defineConfig({
+ *   // ...
+ *   sitemap: {
+ *     hostname: 'https://example.com'
+ *     transformItems: sitemapItemTransformer(
+ *       (items) => {
+ *         // 处理
+ *         return items;
+ *       },
+ *       {
+ *         'zh/': {
+ *           priority: 1
+ *         },
+ *         '^zh/blog/.*': { // 以^开头的键将被解析为正则表达式来匹配url
+ *           priority: 0.5
+ *         }
+ *       }
+ *     )
+ *   }
+ * })
+ * ```
+ *
+ * @param steps 变长参数，值可以是函数，或者一个对象
+ * @returns
+ */
+export default function sitemapItemTransformer(...steps: (Record<string, any> | ((sitemapItems: any[]) => Awaitable<any[]>))[]): (items: any[]) => Promise<any[]>;
+export {};

package/dist/vitepress-sitemap-transformer.js

@@ -0,0 +1,29 @@
+function u(...n) {
+  return async (f) => {
+    if (!Array.isArray(n)) return f;
+    let e = f;
+    for (const t of n)
+      if (typeof t == "function")
+        e = await t(e);
+      else if (typeof t == "object") {
+        const o = Object.entries(t).filter(([r]) => r.startsWith("^")).map((r) => [new RegExp(r[0]), r[1]]);
+        e = e.map((r) => {
+          for (const [a, s] of o)
+            if (a.test(r.url))
+              return {
+                ...r,
+                ...s,
+                ...t[r.url]
+              };
+          return {
+            ...r,
+            ...t[r.url]
+          };
+        });
+      }
+    return e;
+  };
+}
+export {
+  u as default
+};

package/package.json

@@ -0,0 +1,40 @@
+{
+  "name": "@opendesign-plus/geo-scripts",
+  "version": "0.0.1-rc.1",
+  "description": "",
+  "type": "module",
+  "publishConfig": {
+    "access": "public",
+    "registry": "https://registry.npmjs.org/"
+  },
+  "files": [
+    "dist"
+  ],
+  "exports": {
+    "./*": {
+      "types": "./dist/*.d.ts",
+      "import": "./dist/*.js"
+    }
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "p-limit": "^7.3.0",
+    "playwright-core": "latest",
+    "turndown": "latest",
+    "commander": "^13.1.0",
+    "jsdom": "^24.0.0"
+  },
+  "devDependencies": {
+    "@types/turndown": "latest",
+    "@types/node": "^22.13.1",
+    "@types/jsdom": "latest",
+    "vite": "^6.2.3",
+    "vite-plugin-dts": "^4.5.3",
+    "typescript": "~5.8.2"
+  },
+  "scripts": {
+    "build": "vite build"
+  }
+}