nolo-cli 0.1.39 → 0.1.41

package/ai/tools/cloudflareCrawlTool.ts

@@ -0,0 +1,289 @@
+// packages/ai/tools/cloudflareCrawlTool.ts
+// Cloudflare Browser Rendering /crawl 工具
+// Docs: https://developers.cloudflare.com/browser-rendering/rest-api/crawl-endpoint/
+
+import { callToolApi } from "./toolApiClient";
+import { assertFetchableDocsUrl, discoverCanonicalDocsUrl } from "./fetchWebpageSupport";
+
+// ──────────────────────────────────────────────────────────
+// Schema
+// ──────────────────────────────────────────────────────────
+export const cloudflareCrawlFunctionSchema = {
+  name: "cloudflareCrawl",
+  description:
+    "使用 Cloudflare Browser Rendering 的 /crawl 接口，从指定 URL 出发自动发现并爬取整个站点（或指定深度/页数），" +
+    "支持 JS 渲染，返回 Markdown 格式内容，适合需要抓取多页且页面依赖 JavaScript 渲染的场景。" +
+    "如果目标是单页静态内容，优先使用 fetchWebpage；此工具适合多页爬取或需要完整站点索引的任务。",
+  parameters: {
+    type: "object",
+    properties: {
+      url: {
+        type: "string",
+        description: "爬取起始 URL，必须是完整的 http/https 地址。",
+      },
+      limit: {
+        type: "integer",
+        description: "最多爬取的页面数量（默认 10，最大 100000）。建议先从小值开始测试。",
+        default: 10,
+      },
+      depth: {
+        type: "integer",
+        description: "从起始 URL 开始的最大链接层深度（默认不限）。",
+      },
+      formats: {
+        type: "array",
+        items: { type: "string", enum: ["markdown", "html"] },
+        description: "返回格式，支持 markdown（推荐）和 html，默认 [\"markdown\"]。",
+        default: ["markdown"],
+      },
+      render: {
+        type: "boolean",
+        description:
+          "是否启动无头浏览器执行 JavaScript（默认 true）。" +
+          "静态网站可设为 false 以加快速度。",
+        default: true,
+      },
+      source: {
+        type: "string",
+        enum: ["all", "sitemaps", "links"],
+        description: "URL 发现来源：all（默认）、sitemaps 或 links。",
+      },
+      includeSubdomains: {
+        type: "boolean",
+        description: "是否跟踪子域名链接（默认 false）。",
+      },
+      includePatterns: {
+        type: "array",
+        items: { type: "string" },
+        description: "只访问匹配这些通配符的 URL（* 匹配单段，** 匹配多段）。",
+      },
+      excludePatterns: {
+        type: "array",
+        items: { type: "string" },
+        description: "跳过匹配这些通配符的 URL（优先级高于 includePatterns）。",
+      },
+      wait: {
+        type: "boolean",
+        description:
+          "是否等待爬取完成后再返回结果（默认 true，服务端最多等 60s）。" +
+          "设为 false 时立即返回 jobId，可用 cloudflareCrawlStatus 工具轮询结果。",
+        default: true,
+      },
+    },
+    required: ["url"],
+  },
+};
+
+// ──────────────────────────────────────────────────────────
+// Types
+// ──────────────────────────────────────────────────────────
+interface CrawlRecord {
+  url: string;
+  status: string;
+  markdown?: string;
+  html?: string;
+  metadata?: {
+    status: number;
+    title: string;
+    url: string;
+  };
+}
+
+interface CrawlResult {
+  jobId: string;
+  status?: string;
+  total?: number;
+  finished?: number;
+  browserSecondsUsed?: number;
+  records?: CrawlRecord[];
+  cursor?: number;
+}
+
+// ──────────────────────────────────────────────────────────
+// Executor: 启动爬取（可选等待结果）
+// ──────────────────────────────────────────────────────────
+export async function cloudflareCrawlFunc(
+  args: {
+    url: string;
+    limit?: number;
+    depth?: number;
+    formats?: string[];
+    render?: boolean;
+    source?: string;
+    includeSubdomains?: boolean;
+    includePatterns?: string[];
+    excludePatterns?: string[];
+    wait?: boolean;
+  },
+  thunkApi: any
+): Promise<{ rawData: any; displayData: string }> {
+  const {
+    url,
+    limit = 10,
+    depth,
+    formats = ["markdown"],
+    render = true,
+    source,
+    includeSubdomains,
+    includePatterns,
+    excludePatterns,
+    wait = true,
+  } = args;
+
+  if (!url || !url.startsWith("http")) {
+    throw new Error("必须提供有效的 http/https URL");
+  }
+
+  const resolution = await discoverCanonicalDocsUrl(url);
+  const targetUrl = resolution.resolvedUrl;
+  if (targetUrl !== url) {
+    await assertFetchableDocsUrl(targetUrl, fetch, url);
+  }
+  const normalizationNote =
+    targetUrl !== url ? `\n🧭 文档地址已规范化: ${url} → ${targetUrl}` : "";
+
+  const options: Record<string, any> = {};
+  if (includeSubdomains !== undefined) options.includeSubdomains = includeSubdomains;
+  if (includePatterns?.length) options.includePatterns = includePatterns;
+  if (excludePatterns?.length) options.excludePatterns = excludePatterns;
+
+  const body: Record<string, any> = { url: targetUrl, limit, formats, render, wait };
+  if (depth !== undefined) body.depth = depth;
+  if (source !== undefined) body.source = source;
+  if (Object.keys(options).length > 0) body.options = options;
+
+  const data = await callToolApi<CrawlResult>(
+    thunkApi,
+    "/api/cloudflare-crawl",
+    body,
+    { withAuth: true }
+  );
+
+  // 仅返回 jobId（未等待）
+  if (!wait || data.status === "running") {
+    const rawData = targetUrl === url ? data : { ...data, originalUrl: url, resolvedUrl: targetUrl };
+    return {
+      rawData,
+      displayData:
+        `🚀 爬取任务已启动 (jobId: ${data.jobId})\n目标: ${targetUrl}` +
+        normalizationNote +
+        `\n使用 cloudflareCrawlStatus 工具查询结果。`,
+    };
+  }
+
+  // 已完成：格式化结果
+  const records = data.records ?? [];
+  const completedCount = records.filter((r) => r.status === "completed").length;
+  const mdParts = records
+    .filter((r) => r.status === "completed" && r.markdown)
+    .map((r) => `## ${r.metadata?.title || r.url}\n> ${r.url}\n\n${r.markdown}`)
+    .join("\n\n---\n\n");
+
+  const rawData = {
+    jobId: data.jobId,
+    status: data.status,
+    total: data.total,
+    finished: data.finished,
+    browserSecondsUsed: data.browserSecondsUsed,
+    pages: records.map((r) => ({
+      url: r.url,
+      status: r.status,
+      title: r.metadata?.title,
+      content: r.markdown || r.html || "",
+    })),
+    ...(targetUrl === url ? {} : { originalUrl: url, resolvedUrl: targetUrl }),
+  };
+
+  const statusIcon = data.status === "completed" ? "✅" : "⚠️";
+  const displayData =
+    `${statusIcon} Cloudflare 爬取完成\n` +
+    `- 目标: ${targetUrl}\n` +
+    `- 状态: ${data.status}\n` +
+    `- 已完成页面: ${completedCount} / ${data.total ?? "?"}\n` +
+    `- 浏览器用时: ${data.browserSecondsUsed?.toFixed(1) ?? "?"}s` +
+    normalizationNote +
+    `\n\n` +
+    (mdParts ? `**内容摘要（前 3 页）：**\n\n${records.slice(0, 3).map((r) => `- [${r.metadata?.title || r.url}](${r.url})`).join("\n")}` : "无可用内容");
+
+  return { rawData, displayData };
+}
+
+// ──────────────────────────────────────────────────────────
+// Schema: 查询已有任务状态
+// ──────────────────────────────────────────────────────────
+export const cloudflareCrawlStatusFunctionSchema = {
+  name: "cloudflareCrawlStatus",
+  description:
+    "查询由 cloudflareCrawl 工具创建的爬取任务的当前状态和结果。" +
+    "当 cloudflareCrawl 以 wait=false 模式启动时，使用此工具轮询结果。",
+  parameters: {
+    type: "object",
+    properties: {
+      jobId: {
+        type: "string",
+        description: "由 cloudflareCrawl 返回的任务 ID。",
+      },
+      limit: {
+        type: "integer",
+        description: "最多返回多少条记录（用于分页）。",
+      },
+      status: {
+        type: "string",
+        enum: ["queued", "completed", "disallowed", "skipped", "errored", "cancelled"],
+        description: "按 URL 状态过滤结果。",
+      },
+      cursor: {
+        type: "integer",
+        description: "分页游标，从上一页响应的 cursor 字段获取。",
+      },
+    },
+    required: ["jobId"],
+  },
+};
+
+// ──────────────────────────────────────────────────────────
+// Executor: 查询任务状态
+// ──────────────────────────────────────────────────────────
+export async function cloudflareCrawlStatusFunc(
+  args: { jobId: string; limit?: number; status?: string; cursor?: number },
+  thunkApi: any
+): Promise<{ rawData: any; displayData: string }> {
+  const { jobId, limit, status, cursor } = args;
+
+  if (!jobId) throw new Error("必须提供 jobId");
+
+  // 通过 GET /api/cloudflare-crawl/:jobId 查询
+  const { getToolRequestContext } = await import("./toolApiClient");
+  const { baseUrl, token } = getToolRequestContext(thunkApi);
+
+  const params = new URLSearchParams();
+  if (limit !== undefined) params.set("limit", String(limit));
+  if (status) params.set("status", status);
+  if (cursor !== undefined) params.set("cursor", String(cursor));
+  const qs = params.toString() ? `?${params}` : "";
+
+  const headers: Record<string, string> = { "Content-Type": "application/json" };
+  if (token) headers["Authorization"] = `Bearer ${token}`;
+
+  const resp = await fetch(`${baseUrl}/api/cloudflare-crawl/${jobId}${qs}`, { headers });
+  if (!resp.ok) {
+    const err = await resp.json().catch(() => ({})) as any;
+    throw new Error(`查询爬取任务失败: ${err?.error?.message ?? resp.status}`);
+  }
+
+  const data = await resp.json() as CrawlResult & { status: string };
+
+  const isRunning = data.status === "running";
+  const statusIcon = isRunning ? "🔄" : data.status === "completed" ? "✅" : "⚠️";
+  const records = (data as any).records ?? [];
+
+  return {
+    rawData: data,
+    displayData:
+      `${statusIcon} 任务 ${jobId} 状态: ${data.status}\n` +
+      `- 总页面: ${(data as any).total ?? "?"}, 已完成: ${(data as any).finished ?? "?"}\n` +
+      (records.length > 0
+        ? `\n已抓取页面:\n${records.slice(0, 5).map((r: CrawlRecord) => `  - ${r.metadata?.title || r.url} (${r.status})`).join("\n")}`
+        : ""),
+  };
+}

package/ai/tools/codeSearchTool.ts

@@ -0,0 +1,123 @@
+import { buildToolRequestHeaders, getToolBaseUrl } from "./toolApiClient";
+
+export type CodeSearchArgs = {
+    query?: string;
+    path?: string;
+    pathScope?: string;
+    glob?: string;
+    mode?: "content" | "files";
+    maxResults?: number;
+    maxFiles?: number;
+    caseSensitive?: boolean;
+};
+
+export const codeSearchFunctionSchema = {
+    name: "codeSearch",
+    description: [
+        "使用 ripgrep (rg) 在项目代码中搜索文本，或按 glob 列出文件。",
+        "",
+        "推荐场景：",
+        "- mode=content：按关键词搜索代码内容，替代旧的 searchRepo。",
+        "- mode=files：按路径范围和 glob 列出文件，替代旧的 listFiles。",
+        "",
+        "参数说明：",
+        "- query：搜索文本；mode=content 时必填，mode=files 时可省略。",
+        "- path/pathScope：限制搜索目录，例如 'packages/server'。",
+        "- glob：可选文件匹配模式，例如 '**/*.ts'。",
+        "- maxResults：最大返回数量；兼容旧参数 maxFiles。",
+        "- caseSensitive：是否区分大小写，默认 false。",
+    ].join("\n"),
+    parameters: {
+        type: "object",
+        properties: {
+            query: {
+                type: "string",
+                description: "可选：要搜索的文本。mode=content 时必须提供。",
+            },
+            path: {
+                type: "string",
+                description: "可选：限制搜索目录，例如 'packages/ai'。",
+            },
+            pathScope: {
+                type: "string",
+                description: "兼容旧参数：限制搜索目录，例如 'packages/ai'。",
+            },
+            glob: {
+                type: "string",
+                description: "可选：文件 glob，例如 '**/*.ts'。",
+            },
+            mode: {
+                type: "string",
+                enum: ["content", "files"],
+                description: "搜索模式：content=搜内容，files=列文件。",
+            },
+            maxResults: {
+                type: "number",
+                description: "最大返回数量。兼容旧参数 maxFiles。",
+            },
+            maxFiles: {
+                type: "number",
+                description: "兼容旧参数：等价于 maxResults。",
+            },
+            caseSensitive: {
+                type: "boolean",
+                description: "是否区分大小写，默认 false。",
+            },
+        },
+    },
+};
+
+export async function codeSearchFunc(
+    args: CodeSearchArgs,
+    thunkApi: any,
+    context?: { signal?: AbortSignal; agentKey?: string },
+): Promise<{ rawData: any; displayData?: string }> {
+    const baseUrl = getToolBaseUrl(thunkApi);
+    const apiUrl = `${baseUrl}/api/code-search`;
+    const mode = args.mode ?? "content";
+
+    const response = await fetch(apiUrl, {
+        method: "POST",
+        headers: buildToolRequestHeaders(thunkApi, {
+            withAuth: true,
+            agentKey: context?.agentKey,
+        }),
+        signal: context?.signal,
+        body: JSON.stringify({
+            query: args.query,
+            path: args.path ?? args.pathScope,
+            glob: args.glob,
+            mode,
+            maxResults: args.maxResults ?? args.maxFiles,
+            caseSensitive: args.caseSensitive,
+        }),
+    });
+
+    const responseText = await response.text();
+    let data: any = null;
+    try {
+        data = responseText ? JSON.parse(responseText) : {};
+    } catch {
+        data = null;
+    }
+    if (!response.ok || data?.error) {
+        throw new Error(
+            data?.error ||
+            `codeSearch 请求失败: ${response.status}${responseText ? `. 响应: ${responseText}` : ""}`
+        );
+    }
+
+    if (mode === "files") {
+        const files = Array.isArray(data.files) ? data.files : [];
+        return {
+            rawData: data,
+            displayData: `📁 codeSearch 列出 ${files.length} 个文件${data.truncated ? "（已截断）" : ""}`,
+        };
+    }
+
+    const hits = Array.isArray(data.hits) ? data.hits : [];
+    return {
+        rawData: data,
+        displayData: `🔍 codeSearch 找到 ${hits.length} 条匹配${data.truncated ? "（已截断）" : ""}`,
+    };
+}

package/ai/tools/codeTools.ts

@@ -0,0 +1,101 @@
+// 文件路径: packages/ai/tools/codeTools.ts
+
+import {
+    readFileFunctionSchema,
+    readFileFunc,
+    readFilePreviewFunc,
+} from "./readFileTool";
+import {
+    writeFileFunctionSchema,
+    writeFileFunc,
+    writeFilePreviewFunc,
+} from "./writeFileTool";
+
+import {
+    codeSearchFunctionSchema,
+    codeSearchFunc,
+} from "./codeSearchTool";
+import {
+    applyEditFunctionSchema,
+    applyEditFunc,
+} from "./applyEditTool";
+import {
+    applyLineEditsFunctionSchema,
+    applyLineEditsFunc,
+} from "./applyLineEditsTool";
+
+
+import type { ToolDefinition } from "./index";
+
+export const codeToolDefinitions: ToolDefinition[] = [
+    {
+        id: "readFile",
+        schema: readFileFunctionSchema,
+        executor: readFileFunc,
+        previewExecutor: readFilePreviewFunc,
+        description: {
+            name: "readFile",
+            description:
+                "读取项目中的指定文本文件的完整内容，常用于在修改前先查看代码。",
+            category: "代码 / 文件操作",
+        },
+        behavior: "data",
+        uiGroup: "data",
+    },
+    {
+        id: "writeFile",
+        schema: writeFileFunctionSchema,
+        executor: writeFileFunc,
+        previewExecutor: writeFilePreviewFunc,
+        description: {
+            name: "writeFile",
+            description:
+                "在项目中创建或覆盖一个文件的完整内容，适合新建文件或整文件重写。",
+            category: "代码编辑",
+        },
+        behavior: "action",
+        cancelable: true,
+        uiGroup: "data",
+    },
+    {
+        id: "applyEdit",
+        schema: applyEditFunctionSchema,
+        executor: applyEditFunc,
+        description: {
+            name: "applyEdit",
+            description:
+                "基于精确文本片段执行局部代码修改，默认唯一匹配，适合作为首选编辑工具。",
+            category: "代码编辑",
+        },
+        behavior: "action",
+        cancelable: true,
+        uiGroup: "data",
+    },
+
+    {
+        id: "codeSearch",
+        schema: codeSearchFunctionSchema,
+        executor: codeSearchFunc,
+        description: {
+            name: "codeSearch",
+            description: "基于 rg 搜索代码内容或列出文件，统一替代旧的 searchRepo/listFiles。",
+            category: "代码分析",
+        },
+        behavior: "data",
+        uiGroup: "data",
+    },
+    {
+        id: "applyLineEdits",
+        schema: applyLineEditsFunctionSchema,
+        executor: applyLineEditsFunc,
+        description: {
+            name: "applyLineEdits",
+            description:
+                "对指定代码文件按行号执行精确的文本编辑操作，适合 applyEdit 不方便表达的局部修改。",
+            category: "代码编辑",
+        },
+        behavior: "action",
+        cancelable: true,
+        uiGroup: "data",
+    },
+];

package/ai/tools/convertMarxistsBookTool.ts

@@ -0,0 +1,103 @@
+import { convertMarxistsBookToOfflineHtml } from "./marxistsOfflineBook";
+
+export const convertMarxistsBookToOfflineHtmlFunctionSchema = {
+  name: "convertMarxistsBookToOfflineHtml",
+  description:
+    "将 Marxists.org 中文旧式书籍页面转换成离线单 HTML。会从章节 URL 推导目录页，下载同目录章节、CSS 和背景图，保留原始排版并内嵌资源，适合 iPad/移动设备离线阅读。",
+  parameters: {
+    type: "object",
+    properties: {
+      startUrl: {
+        type: "string",
+        description:
+          "任意一个 Marxists.org 中文书籍章节 URL，例如 https://www.marxists.org/chinese/fromm/1973/01.htm。",
+      },
+      encoding: {
+        type: "string",
+        description: "源站 HTML/CSS 编码，老中文页面通常是 gb2312。默认 gb2312。",
+        default: "gb2312",
+      },
+      outputMode: {
+        type: "string",
+        enum: ["summary", "html"],
+        description:
+          "summary 只返回元数据和文件引用；html 同时返回完整 HTML 字符串。需要直接拿到文件内容时用 html。",
+        default: "summary",
+      },
+      singleFileName: {
+        type: "string",
+        description:
+          "生成文件建议名称，默认根据书名或 URL 生成；server-side agent 可用它作为保存文件名。",
+      },
+      maxPages: {
+        type: "number",
+        description: "最多允许抓取的同目录章节页数量，默认 80，防止误抓整个站点。",
+        default: 80,
+      },
+      saveAsFile: {
+        type: "boolean",
+        description:
+          "server-side agent 可设为 true，将 HTML 保存为 Nolo 文件并返回下载 URL；如果只需要 HTML 字符串，可设为 false。",
+        default: true,
+      },
+    },
+    required: ["startUrl"],
+  },
+};
+
+export type ConvertMarxistsBookToolArgs = {
+  startUrl: string;
+  encoding?: string;
+  outputMode?: "summary" | "html";
+  singleFileName?: string;
+  maxPages?: number;
+  saveAsFile?: boolean;
+};
+
+export async function convertMarxistsBookToOfflineHtmlFunc(
+  args: ConvertMarxistsBookToolArgs,
+): Promise<{ rawData: Record<string, unknown>; displayData: string }> {
+  const result = await convertMarxistsBookToOfflineHtml({
+    startUrl: args.startUrl,
+    encoding: args.encoding,
+    maxPages: args.maxPages,
+  });
+
+  const rawData: Record<string, unknown> = {
+    title: result.title,
+    startUrl: result.startUrl,
+    indexUrl: result.indexUrl,
+    stylesheetUrl: result.stylesheetUrl,
+    pages: result.pages,
+    validation: result.validation,
+    fileName: args.singleFileName,
+  };
+  if (args.outputMode === "html") {
+    rawData.html = result.html;
+  }
+
+  return {
+    rawData,
+    displayData: formatMarxistsOfflineBookDisplayData({
+      ...rawData,
+      htmlIncluded: args.outputMode === "html",
+    }),
+  };
+}
+
+export function formatMarxistsOfflineBookDisplayData(data: Record<string, unknown>): string {
+  const validation = (data.validation ?? {}) as any;
+  const pages = Array.isArray(data.pages) ? data.pages : [];
+  return [
+    "✅ 已生成离线 HTML",
+    `书名：${data.title || "未知"}`,
+    `章节数：${validation.pageCount ?? pages.length}`,
+    `大小：${validation.byteLength ?? "未知"} bytes`,
+    `资源内嵌：${validation.hasEmbeddedAssets ? "是" : "否"}`,
+    `残留网络 URL：${validation.hasNetworkUrls ? "是" : "否"}`,
+    data.fileUrl ? `文件：${data.fileUrl}` : undefined,
+    data.htmlIncluded ? "已在 rawData.html 返回完整 HTML。" : undefined,
+  ]
+    .filter(Boolean)
+    .join("\n");
+}