nolo-cli 0.1.31 → 0.1.32

package/ai/tools/cloudflareCrawlTool.ts

@@ -1,289 +0,0 @@
-// packages/ai/tools/cloudflareCrawlTool.ts
-// Cloudflare Browser Rendering /crawl 工具
-// Docs: https://developers.cloudflare.com/browser-rendering/rest-api/crawl-endpoint/
-
-import { callToolApi } from "./toolApiClient";
-import { assertFetchableDocsUrl, discoverCanonicalDocsUrl } from "./fetchWebpageSupport";
-
-// ──────────────────────────────────────────────────────────
-// Schema
-// ──────────────────────────────────────────────────────────
-export const cloudflareCrawlFunctionSchema = {
-  name: "cloudflareCrawl",
-  description:
-    "使用 Cloudflare Browser Rendering 的 /crawl 接口，从指定 URL 出发自动发现并爬取整个站点（或指定深度/页数），" +
-    "支持 JS 渲染，返回 Markdown 格式内容，适合需要抓取多页且页面依赖 JavaScript 渲染的场景。" +
-    "如果目标是单页静态内容，优先使用 fetchWebpage；此工具适合多页爬取或需要完整站点索引的任务。",
-  parameters: {
-    type: "object",
-    properties: {
-      url: {
-        type: "string",
-        description: "爬取起始 URL，必须是完整的 http/https 地址。",
-      },
-      limit: {
-        type: "integer",
-        description: "最多爬取的页面数量（默认 10，最大 100000）。建议先从小值开始测试。",
-        default: 10,
-      },
-      depth: {
-        type: "integer",
-        description: "从起始 URL 开始的最大链接层深度（默认不限）。",
-      },
-      formats: {
-        type: "array",
-        items: { type: "string", enum: ["markdown", "html"] },
-        description: "返回格式，支持 markdown（推荐）和 html，默认 [\"markdown\"]。",
-        default: ["markdown"],
-      },
-      render: {
-        type: "boolean",
-        description:
-          "是否启动无头浏览器执行 JavaScript（默认 true）。" +
-          "静态网站可设为 false 以加快速度。",
-        default: true,
-      },
-      source: {
-        type: "string",
-        enum: ["all", "sitemaps", "links"],
-        description: "URL 发现来源：all（默认）、sitemaps 或 links。",
-      },
-      includeSubdomains: {
-        type: "boolean",
-        description: "是否跟踪子域名链接（默认 false）。",
-      },
-      includePatterns: {
-        type: "array",
-        items: { type: "string" },
-        description: "只访问匹配这些通配符的 URL（* 匹配单段，** 匹配多段）。",
-      },
-      excludePatterns: {
-        type: "array",
-        items: { type: "string" },
-        description: "跳过匹配这些通配符的 URL（优先级高于 includePatterns）。",
-      },
-      wait: {
-        type: "boolean",
-        description:
-          "是否等待爬取完成后再返回结果（默认 true，服务端最多等 60s）。" +
-          "设为 false 时立即返回 jobId，可用 cloudflareCrawlStatus 工具轮询结果。",
-        default: true,
-      },
-    },
-    required: ["url"],
-  },
-};
-
-// ──────────────────────────────────────────────────────────
-// Types
-// ──────────────────────────────────────────────────────────
-interface CrawlRecord {
-  url: string;
-  status: string;
-  markdown?: string;
-  html?: string;
-  metadata?: {
-    status: number;
-    title: string;
-    url: string;
-  };
-}
-
-interface CrawlResult {
-  jobId: string;
-  status?: string;
-  total?: number;
-  finished?: number;
-  browserSecondsUsed?: number;
-  records?: CrawlRecord[];
-  cursor?: number;
-}
-
-// ──────────────────────────────────────────────────────────
-// Executor: 启动爬取（可选等待结果）
-// ──────────────────────────────────────────────────────────
-export async function cloudflareCrawlFunc(
-  args: {
-    url: string;
-    limit?: number;
-    depth?: number;
-    formats?: string[];
-    render?: boolean;
-    source?: string;
-    includeSubdomains?: boolean;
-    includePatterns?: string[];
-    excludePatterns?: string[];
-    wait?: boolean;
-  },
-  thunkApi: any
-): Promise<{ rawData: any; displayData: string }> {
-  const {
-    url,
-    limit = 10,
-    depth,
-    formats = ["markdown"],
-    render = true,
-    source,
-    includeSubdomains,
-    includePatterns,
-    excludePatterns,
-    wait = true,
-  } = args;
-
-  if (!url || !url.startsWith("http")) {
-    throw new Error("必须提供有效的 http/https URL");
-  }
-
-  const resolution = await discoverCanonicalDocsUrl(url);
-  const targetUrl = resolution.resolvedUrl;
-  if (targetUrl !== url) {
-    await assertFetchableDocsUrl(targetUrl, fetch, url);
-  }
-  const normalizationNote =
-    targetUrl !== url ? `\n🧭 文档地址已规范化: ${url} → ${targetUrl}` : "";
-
-  const options: Record<string, any> = {};
-  if (includeSubdomains !== undefined) options.includeSubdomains = includeSubdomains;
-  if (includePatterns?.length) options.includePatterns = includePatterns;
-  if (excludePatterns?.length) options.excludePatterns = excludePatterns;
-
-  const body: Record<string, any> = { url: targetUrl, limit, formats, render, wait };
-  if (depth !== undefined) body.depth = depth;
-  if (source !== undefined) body.source = source;
-  if (Object.keys(options).length > 0) body.options = options;
-
-  const data = await callToolApi<CrawlResult>(
-    thunkApi,
-    "/api/cloudflare-crawl",
-    body,
-    { withAuth: true }
-  );
-
-  // 仅返回 jobId（未等待）
-  if (!wait || data.status === "running") {
-    const rawData = targetUrl === url ? data : { ...data, originalUrl: url, resolvedUrl: targetUrl };
-    return {
-      rawData,
-      displayData:
-        `🚀 爬取任务已启动 (jobId: ${data.jobId})\n目标: ${targetUrl}` +
-        normalizationNote +
-        `\n使用 cloudflareCrawlStatus 工具查询结果。`,
-    };
-  }
-
-  // 已完成：格式化结果
-  const records = data.records ?? [];
-  const completedCount = records.filter((r) => r.status === "completed").length;
-  const mdParts = records
-    .filter((r) => r.status === "completed" && r.markdown)
-    .map((r) => `## ${r.metadata?.title || r.url}\n> ${r.url}\n\n${r.markdown}`)
-    .join("\n\n---\n\n");
-
-  const rawData = {
-    jobId: data.jobId,
-    status: data.status,
-    total: data.total,
-    finished: data.finished,
-    browserSecondsUsed: data.browserSecondsUsed,
-    pages: records.map((r) => ({
-      url: r.url,
-      status: r.status,
-      title: r.metadata?.title,
-      content: r.markdown || r.html || "",
-    })),
-    ...(targetUrl === url ? {} : { originalUrl: url, resolvedUrl: targetUrl }),
-  };
-
-  const statusIcon = data.status === "completed" ? "✅" : "⚠️";
-  const displayData =
-    `${statusIcon} Cloudflare 爬取完成\n` +
-    `- 目标: ${targetUrl}\n` +
-    `- 状态: ${data.status}\n` +
-    `- 已完成页面: ${completedCount} / ${data.total ?? "?"}\n` +
-    `- 浏览器用时: ${data.browserSecondsUsed?.toFixed(1) ?? "?"}s` +
-    normalizationNote +
-    `\n\n` +
-    (mdParts ? `**内容摘要（前 3 页）：**\n\n${records.slice(0, 3).map((r) => `- [${r.metadata?.title || r.url}](${r.url})`).join("\n")}` : "无可用内容");
-
-  return { rawData, displayData };
-}
-
-// ──────────────────────────────────────────────────────────
-// Schema: 查询已有任务状态
-// ──────────────────────────────────────────────────────────
-export const cloudflareCrawlStatusFunctionSchema = {
-  name: "cloudflareCrawlStatus",
-  description:
-    "查询由 cloudflareCrawl 工具创建的爬取任务的当前状态和结果。" +
-    "当 cloudflareCrawl 以 wait=false 模式启动时，使用此工具轮询结果。",
-  parameters: {
-    type: "object",
-    properties: {
-      jobId: {
-        type: "string",
-        description: "由 cloudflareCrawl 返回的任务 ID。",
-      },
-      limit: {
-        type: "integer",
-        description: "最多返回多少条记录（用于分页）。",
-      },
-      status: {
-        type: "string",
-        enum: ["queued", "completed", "disallowed", "skipped", "errored", "cancelled"],
-        description: "按 URL 状态过滤结果。",
-      },
-      cursor: {
-        type: "integer",
-        description: "分页游标，从上一页响应的 cursor 字段获取。",
-      },
-    },
-    required: ["jobId"],
-  },
-};
-
-// ──────────────────────────────────────────────────────────
-// Executor: 查询任务状态
-// ──────────────────────────────────────────────────────────
-export async function cloudflareCrawlStatusFunc(
-  args: { jobId: string; limit?: number; status?: string; cursor?: number },
-  thunkApi: any
-): Promise<{ rawData: any; displayData: string }> {
-  const { jobId, limit, status, cursor } = args;
-
-  if (!jobId) throw new Error("必须提供 jobId");
-
-  // 通过 GET /api/cloudflare-crawl/:jobId 查询
-  const { getToolRequestContext } = await import("./toolApiClient");
-  const { baseUrl, token } = getToolRequestContext(thunkApi);
-
-  const params = new URLSearchParams();
-  if (limit !== undefined) params.set("limit", String(limit));
-  if (status) params.set("status", status);
-  if (cursor !== undefined) params.set("cursor", String(cursor));
-  const qs = params.toString() ? `?${params}` : "";
-
-  const headers: Record<string, string> = { "Content-Type": "application/json" };
-  if (token) headers["Authorization"] = `Bearer ${token}`;
-
-  const resp = await fetch(`${baseUrl}/api/cloudflare-crawl/${jobId}${qs}`, { headers });
-  if (!resp.ok) {
-    const err = await resp.json().catch(() => ({})) as any;
-    throw new Error(`查询爬取任务失败: ${err?.error?.message ?? resp.status}`);
-  }
-
-  const data = await resp.json() as CrawlResult & { status: string };
-
-  const isRunning = data.status === "running";
-  const statusIcon = isRunning ? "🔄" : data.status === "completed" ? "✅" : "⚠️";
-  const records = (data as any).records ?? [];
-
-  return {
-    rawData: data,
-    displayData:
-      `${statusIcon} 任务 ${jobId} 状态: ${data.status}\n` +
-      `- 总页面: ${(data as any).total ?? "?"}, 已完成: ${(data as any).finished ?? "?"}\n` +
-      (records.length > 0
-        ? `\n已抓取页面:\n${records.slice(0, 5).map((r: CrawlRecord) => `  - ${r.metadata?.title || r.url} (${r.status})`).join("\n")}`
-        : ""),
-  };
-}

package/ai/tools/codeSearchTool.ts

@@ -1,123 +0,0 @@
-import { buildToolRequestHeaders, getToolBaseUrl } from "./toolApiClient";
-
-export type CodeSearchArgs = {
-    query?: string;
-    path?: string;
-    pathScope?: string;
-    glob?: string;
-    mode?: "content" | "files";
-    maxResults?: number;
-    maxFiles?: number;
-    caseSensitive?: boolean;
-};
-
-export const codeSearchFunctionSchema = {
-    name: "codeSearch",
-    description: [
-        "使用 ripgrep (rg) 在项目代码中搜索文本，或按 glob 列出文件。",
-        "",
-        "推荐场景：",
-        "- mode=content：按关键词搜索代码内容，替代旧的 searchRepo。",
-        "- mode=files：按路径范围和 glob 列出文件，替代旧的 listFiles。",
-        "",
-        "参数说明：",
-        "- query：搜索文本；mode=content 时必填，mode=files 时可省略。",
-        "- path/pathScope：限制搜索目录，例如 'packages/server'。",
-        "- glob：可选文件匹配模式，例如 '**/*.ts'。",
-        "- maxResults：最大返回数量；兼容旧参数 maxFiles。",
-        "- caseSensitive：是否区分大小写，默认 false。",
-    ].join("\n"),
-    parameters: {
-        type: "object",
-        properties: {
-            query: {
-                type: "string",
-                description: "可选：要搜索的文本。mode=content 时必须提供。",
-            },
-            path: {
-                type: "string",
-                description: "可选：限制搜索目录，例如 'packages/ai'。",
-            },
-            pathScope: {
-                type: "string",
-                description: "兼容旧参数：限制搜索目录，例如 'packages/ai'。",
-            },
-            glob: {
-                type: "string",
-                description: "可选：文件 glob，例如 '**/*.ts'。",
-            },
-            mode: {
-                type: "string",
-                enum: ["content", "files"],
-                description: "搜索模式：content=搜内容，files=列文件。",
-            },
-            maxResults: {
-                type: "number",
-                description: "最大返回数量。兼容旧参数 maxFiles。",
-            },
-            maxFiles: {
-                type: "number",
-                description: "兼容旧参数：等价于 maxResults。",
-            },
-            caseSensitive: {
-                type: "boolean",
-                description: "是否区分大小写，默认 false。",
-            },
-        },
-    },
-};
-
-export async function codeSearchFunc(
-    args: CodeSearchArgs,
-    thunkApi: any,
-    context?: { signal?: AbortSignal; agentKey?: string },
-): Promise<{ rawData: any; displayData?: string }> {
-    const baseUrl = getToolBaseUrl(thunkApi);
-    const apiUrl = `${baseUrl}/api/code-search`;
-    const mode = args.mode ?? "content";
-
-    const response = await fetch(apiUrl, {
-        method: "POST",
-        headers: buildToolRequestHeaders(thunkApi, {
-            withAuth: true,
-            agentKey: context?.agentKey,
-        }),
-        signal: context?.signal,
-        body: JSON.stringify({
-            query: args.query,
-            path: args.path ?? args.pathScope,
-            glob: args.glob,
-            mode,
-            maxResults: args.maxResults ?? args.maxFiles,
-            caseSensitive: args.caseSensitive,
-        }),
-    });
-
-    const responseText = await response.text();
-    let data: any = null;
-    try {
-        data = responseText ? JSON.parse(responseText) : {};
-    } catch {
-        data = null;
-    }
-    if (!response.ok || data?.error) {
-        throw new Error(
-            data?.error ||
-            `codeSearch 请求失败: ${response.status}${responseText ? `. 响应: ${responseText}` : ""}`
-        );
-    }
-
-    if (mode === "files") {
-        const files = Array.isArray(data.files) ? data.files : [];
-        return {
-            rawData: data,
-            displayData: `📁 codeSearch 列出 ${files.length} 个文件${data.truncated ? "（已截断）" : ""}`,
-        };
-    }
-
-    const hits = Array.isArray(data.hits) ? data.hits : [];
-    return {
-        rawData: data,
-        displayData: `🔍 codeSearch 找到 ${hits.length} 条匹配${data.truncated ? "（已截断）" : ""}`,
-    };
-}

package/ai/tools/codeTools.ts

@@ -1,101 +0,0 @@
-// 文件路径: packages/ai/tools/codeTools.ts
-
-import {
-    readFileFunctionSchema,
-    readFileFunc,
-    readFilePreviewFunc,
-} from "./readFileTool";
-import {
-    writeFileFunctionSchema,
-    writeFileFunc,
-    writeFilePreviewFunc,
-} from "./writeFileTool";
-
-import {
-    codeSearchFunctionSchema,
-    codeSearchFunc,
-} from "./codeSearchTool";
-import {
-    applyEditFunctionSchema,
-    applyEditFunc,
-} from "./applyEditTool";
-import {
-    applyLineEditsFunctionSchema,
-    applyLineEditsFunc,
-} from "./applyLineEditsTool";
-
-
-import type { ToolDefinition } from "./index";
-
-export const codeToolDefinitions: ToolDefinition[] = [
-    {
-        id: "readFile",
-        schema: readFileFunctionSchema,
-        executor: readFileFunc,
-        previewExecutor: readFilePreviewFunc,
-        description: {
-            name: "readFile",
-            description:
-                "读取项目中的指定文本文件的完整内容，常用于在修改前先查看代码。",
-            category: "代码 / 文件操作",
-        },
-        behavior: "data",
-        uiGroup: "data",
-    },
-    {
-        id: "writeFile",
-        schema: writeFileFunctionSchema,
-        executor: writeFileFunc,
-        previewExecutor: writeFilePreviewFunc,
-        description: {
-            name: "writeFile",
-            description:
-                "在项目中创建或覆盖一个文件的完整内容，适合新建文件或整文件重写。",
-            category: "代码编辑",
-        },
-        behavior: "action",
-        cancelable: true,
-        uiGroup: "data",
-    },
-    {
-        id: "applyEdit",
-        schema: applyEditFunctionSchema,
-        executor: applyEditFunc,
-        description: {
-            name: "applyEdit",
-            description:
-                "基于精确文本片段执行局部代码修改，默认唯一匹配，适合作为首选编辑工具。",
-            category: "代码编辑",
-        },
-        behavior: "action",
-        cancelable: true,
-        uiGroup: "data",
-    },
-
-    {
-        id: "codeSearch",
-        schema: codeSearchFunctionSchema,
-        executor: codeSearchFunc,
-        description: {
-            name: "codeSearch",
-            description: "基于 rg 搜索代码内容或列出文件，统一替代旧的 searchRepo/listFiles。",
-            category: "代码分析",
-        },
-        behavior: "data",
-        uiGroup: "data",
-    },
-    {
-        id: "applyLineEdits",
-        schema: applyLineEditsFunctionSchema,
-        executor: applyLineEditsFunc,
-        description: {
-            name: "applyLineEdits",
-            description:
-                "对指定代码文件按行号执行精确的文本编辑操作，适合 applyEdit 不方便表达的局部修改。",
-            category: "代码编辑",
-        },
-        behavior: "action",
-        cancelable: true,
-        uiGroup: "data",
-    },
-];

package/ai/tools/convertMarxistsBookTool.ts

@@ -1,103 +0,0 @@
-import { convertMarxistsBookToOfflineHtml } from "./marxistsOfflineBook";
-
-export const convertMarxistsBookToOfflineHtmlFunctionSchema = {
-  name: "convertMarxistsBookToOfflineHtml",
-  description:
-    "将 Marxists.org 中文旧式书籍页面转换成离线单 HTML。会从章节 URL 推导目录页，下载同目录章节、CSS 和背景图，保留原始排版并内嵌资源，适合 iPad/移动设备离线阅读。",
-  parameters: {
-    type: "object",
-    properties: {
-      startUrl: {
-        type: "string",
-        description:
-          "任意一个 Marxists.org 中文书籍章节 URL，例如 https://www.marxists.org/chinese/fromm/1973/01.htm。",
-      },
-      encoding: {
-        type: "string",
-        description: "源站 HTML/CSS 编码，老中文页面通常是 gb2312。默认 gb2312。",
-        default: "gb2312",
-      },
-      outputMode: {
-        type: "string",
-        enum: ["summary", "html"],
-        description:
-          "summary 只返回元数据和文件引用；html 同时返回完整 HTML 字符串。需要直接拿到文件内容时用 html。",
-        default: "summary",
-      },
-      singleFileName: {
-        type: "string",
-        description:
-          "生成文件建议名称，默认根据书名或 URL 生成；server-side agent 可用它作为保存文件名。",
-      },
-      maxPages: {
-        type: "number",
-        description: "最多允许抓取的同目录章节页数量，默认 80，防止误抓整个站点。",
-        default: 80,
-      },
-      saveAsFile: {
-        type: "boolean",
-        description:
-          "server-side agent 可设为 true，将 HTML 保存为 Nolo 文件并返回下载 URL；如果只需要 HTML 字符串，可设为 false。",
-        default: true,
-      },
-    },
-    required: ["startUrl"],
-  },
-};
-
-export type ConvertMarxistsBookToolArgs = {
-  startUrl: string;
-  encoding?: string;
-  outputMode?: "summary" | "html";
-  singleFileName?: string;
-  maxPages?: number;
-  saveAsFile?: boolean;
-};
-
-export async function convertMarxistsBookToOfflineHtmlFunc(
-  args: ConvertMarxistsBookToolArgs,
-): Promise<{ rawData: Record<string, unknown>; displayData: string }> {
-  const result = await convertMarxistsBookToOfflineHtml({
-    startUrl: args.startUrl,
-    encoding: args.encoding,
-    maxPages: args.maxPages,
-  });
-
-  const rawData: Record<string, unknown> = {
-    title: result.title,
-    startUrl: result.startUrl,
-    indexUrl: result.indexUrl,
-    stylesheetUrl: result.stylesheetUrl,
-    pages: result.pages,
-    validation: result.validation,
-    fileName: args.singleFileName,
-  };
-  if (args.outputMode === "html") {
-    rawData.html = result.html;
-  }
-
-  return {
-    rawData,
-    displayData: formatMarxistsOfflineBookDisplayData({
-      ...rawData,
-      htmlIncluded: args.outputMode === "html",
-    }),
-  };
-}
-
-export function formatMarxistsOfflineBookDisplayData(data: Record<string, unknown>): string {
-  const validation = (data.validation ?? {}) as any;
-  const pages = Array.isArray(data.pages) ? data.pages : [];
-  return [
-    "✅ 已生成离线 HTML",
-    `书名：${data.title || "未知"}`,
-    `章节数：${validation.pageCount ?? pages.length}`,
-    `大小：${validation.byteLength ?? "未知"} bytes`,
-    `资源内嵌：${validation.hasEmbeddedAssets ? "是" : "否"}`,
-    `残留网络 URL：${validation.hasNetworkUrls ? "是" : "否"}`,
-    data.fileUrl ? `文件：${data.fileUrl}` : undefined,
-    data.htmlIncluded ? "已在 rawData.html 返回完整 HTML。" : undefined,
-  ]
-    .filter(Boolean)
-    .join("\n");
-}