nolo-cli 0.1.31 → 0.1.33

package/ai/tools/fetchWebpageSupport.ts

@@ -1,333 +0,0 @@
-const DOCS_HOST_RE = /^docs\./i;
-
-export interface DocsIndexEntry {
-  title: string;
-  url: string;
-  source: "llms.txt" | "llms-full.txt";
-}
-
-export interface DocsResolution {
-  resolvedUrl: string;
-  source: "original" | "llms.txt" | "llms-full.txt";
-}
-
-export interface ExtractionIssue {
-  code: "EMPTY_EXTRACTION" | "HTML_SHELL";
-  message: string;
-}
-
-export function extractAdvertisedMarkdownUrl(markdown: string, sourceUrl: string) {
-  if (!markdown.trim()) return null;
-
-  let source: URL;
-  try {
-    source = new URL(sourceUrl);
-  } catch {
-    return null;
-  }
-
-  const hintMatch = markdown.match(/Get this page as Markdown:\s*(https?:\/\/[^\s)]+)/i);
-  const rawHint = hintMatch?.[1]?.trim();
-  if (!rawHint) return null;
-
-  try {
-    const hinted = new URL(rawHint);
-    if (hinted.origin !== source.origin) return null;
-    if (!/\.md$/i.test(hinted.pathname)) return null;
-    return hinted.toString();
-  } catch {
-    return null;
-  }
-}
-
-function normalizePathname(pathname: string) {
-  return pathname
-    .replace(/\/index\.md$/i, "/")
-    .replace(/\.md$/i, "")
-    .replace(/\/+/g, "/")
-    .replace(/\/$/, "") || "/";
-}
-
-function tokenize(value: string) {
-  return Array.from(
-    new Set(
-      value
-        .toLowerCase()
-        .split(/[^a-z0-9]+/i)
-        .map((part) => part.trim())
-        .filter(Boolean)
-        .filter((part) => !["md", "html", "htm", "docs", "doc"].includes(part))
-    )
-  );
-}
-
-function splitNormalizedSegments(pathname: string) {
-  const normalized = normalizePathname(pathname);
-  return normalized
-    .split("/")
-    .map((segment) => segment.trim())
-    .filter(Boolean);
-}
-
-function segmentSignature(segment: string) {
-  const tokens = tokenize(segment);
-  if (tokens.length > 0) return tokens.join("-");
-  return segment.trim().toLowerCase();
-}
-
-function hasCompatiblePathStructure(requestedPath: string, candidatePath: string) {
-  const requestedSegments = splitNormalizedSegments(requestedPath);
-  const candidateSegments = splitNormalizedSegments(candidatePath);
-
-  if (requestedSegments.length <= 1 || candidateSegments.length <= 1) {
-    return true;
-  }
-
-  if (candidateSegments.length > requestedSegments.length) {
-    return false;
-  }
-
-  for (let index = 1; index <= candidateSegments.length; index += 1) {
-    const requestedSegment = requestedSegments.at(-index);
-    const candidateSegment = candidateSegments.at(-index);
-    if (!requestedSegment || !candidateSegment) return false;
-    if (segmentSignature(requestedSegment) !== segmentSignature(candidateSegment)) {
-      return false;
-    }
-  }
-
-  return true;
-}
-
-function collectEntryTokens(entry: DocsIndexEntry) {
-  const parsed = new URL(entry.url);
-  return {
-    pathTokens: tokenize(parsed.pathname),
-    titleTokens: tokenize(entry.title),
-    normalizedPath: normalizePathname(parsed.pathname),
-  };
-}
-
-function normalizeCandidateUrl(rawUrl: string, baseUrl: string) {
-  try {
-    const parsed = new URL(rawUrl, baseUrl);
-    return parsed.toString();
-  } catch {
-    return null;
-  }
-}
-
-export function isDocsHost(rawUrl: string) {
-  try {
-    const parsed = new URL(rawUrl);
-    return DOCS_HOST_RE.test(parsed.hostname);
-  } catch {
-    return false;
-  }
-}
-
-export function parseLlmsIndex(text: string, baseUrl: string): DocsIndexEntry[] {
-  const entries: DocsIndexEntry[] = [];
-  const linkRe = /-\s+\[([^\]]+)\]\(([^)]+)\)/g;
-
-  for (const match of text.matchAll(linkRe)) {
-    const title = match[1]?.trim();
-    const rawUrl = match[2]?.trim();
-    if (!title || !rawUrl) continue;
-    const url = normalizeCandidateUrl(rawUrl, baseUrl);
-    if (!url) continue;
-    entries.push({ title, url, source: "llms.txt" });
-  }
-
-  return entries;
-}
-
-export function parseLlmsFullSources(text: string, baseUrl: string): DocsIndexEntry[] {
-  const entries: DocsIndexEntry[] = [];
-  const lines = text.split(/\r?\n/);
-  let currentTitle = "";
-
-  for (const rawLine of lines) {
-    const line = rawLine.trim();
-    if (!line) continue;
-
-    if (line.startsWith("# ")) {
-      currentTitle = line.slice(2).trim();
-      continue;
-    }
-
-    if (!line.toLowerCase().startsWith("source:")) continue;
-
-    const rawUrl = line.slice("source:".length).trim();
-    const url = normalizeCandidateUrl(rawUrl, baseUrl);
-    if (!url || !currentTitle) continue;
-    entries.push({ title: currentTitle, url, source: "llms-full.txt" });
-  }
-
-  return entries;
-}
-
-function scoreDocsEntry(requestedUrl: URL, entry: DocsIndexEntry) {
-  const requestedPath = normalizePathname(requestedUrl.pathname);
-  const requestedTokens = tokenize(requestedUrl.pathname);
-  const lastRequestedToken = requestedTokens.at(-1);
-  const { pathTokens, titleTokens, normalizedPath } = collectEntryTokens(entry);
-
-  if (requestedPath === normalizedPath) return 10_000;
-
-  let score = 0;
-  const unionTokens = new Set([...pathTokens, ...titleTokens]);
-
-  for (const token of requestedTokens) {
-    if (pathTokens.includes(token)) score += 3;
-    if (titleTokens.includes(token)) score += 4;
-  }
-
-  if (lastRequestedToken) {
-    if (pathTokens.at(-1) === lastRequestedToken) score += 10;
-    if (titleTokens.includes(lastRequestedToken)) score += 4;
-  }
-
-  if (requestedTokens.length > 0 && requestedTokens.every((token) => unionTokens.has(token))) {
-    score += 10;
-  }
-
-  score -= Math.abs(unionTokens.size - requestedTokens.length);
-  return score;
-}
-
-export function resolveDocsUrlFromEntries(
-  rawUrl: string,
-  entries: DocsIndexEntry[]
-): DocsResolution {
-  const requestedUrl = new URL(rawUrl);
-  const sameOriginEntries = entries.filter((entry) => {
-    try {
-      return new URL(entry.url).origin === requestedUrl.origin;
-    } catch {
-      return false;
-    }
-  });
-
-  if (sameOriginEntries.length === 0) {
-    return { resolvedUrl: rawUrl, source: "original" };
-  }
-
-  const ranked = sameOriginEntries
-    .filter((entry) => hasCompatiblePathStructure(requestedUrl.pathname, new URL(entry.url).pathname))
-    .map((entry) => ({
-      entry,
-      score: scoreDocsEntry(requestedUrl, entry),
-    }))
-    .sort((a, b) => b.score - a.score);
-
-  const best = ranked[0];
-  if (!best || best.score < 8) {
-    return { resolvedUrl: rawUrl, source: "original" };
-  }
-
-  return {
-    resolvedUrl: best.entry.url,
-    source: best.entry.source,
-  };
-}
-
-export async function discoverCanonicalDocsUrl(
-  rawUrl: string,
-  fetchImpl: typeof fetch = fetch
-): Promise<DocsResolution> {
-  if (!isDocsHost(rawUrl)) {
-    return { resolvedUrl: rawUrl, source: "original" };
-  }
-
-  const parsed = new URL(rawUrl);
-  const baseUrl = parsed.origin + "/";
-  const entries: DocsIndexEntry[] = [];
-
-  const llmsText = await fetchTextIfOk(new URL("/llms.txt", parsed.origin).toString(), fetchImpl);
-  if (llmsText) {
-    entries.push(...parseLlmsIndex(llmsText, baseUrl));
-  }
-
-  const fromLlms = resolveDocsUrlFromEntries(rawUrl, entries);
-  if (fromLlms.source !== "original") {
-    return fromLlms;
-  }
-
-  const llmsFullText = await fetchTextIfOk(
-    new URL("/llms-full.txt", parsed.origin).toString(),
-    fetchImpl
-  );
-  if (llmsFullText) {
-    entries.push(...parseLlmsFullSources(llmsFullText, baseUrl));
-  }
-
-  return resolveDocsUrlFromEntries(rawUrl, entries);
-}
-
-async function fetchTextIfOk(url: string, fetchImpl: typeof fetch) {
-  try {
-    const response = await fetchImpl(url, {
-      method: "GET",
-      redirect: "follow",
-      headers: { Accept: "text/plain, text/markdown, text/html;q=0.8, */*;q=0.1" },
-    });
-    if (!response.ok) return null;
-    return await response.text();
-  } catch {
-    return null;
-  }
-}
-
-export async function assertFetchableDocsUrl(
-  rawUrl: string,
-  fetchImpl: typeof fetch = fetch,
-  originalUrl?: string
-): Promise<void> {
-  if (!isDocsHost(rawUrl)) return;
-
-  try {
-    const response = await fetchImpl(rawUrl, {
-      method: "HEAD",
-      redirect: "follow",
-      headers: { Accept: "text/html,application/xhtml+xml" },
-    });
-
-    if (response.ok || response.status === 405 || response.status === 501) {
-      return;
-    }
-
-    throw new Error(`文档页面返回 ${response.status}`);
-  } catch (error) {
-    const message = error instanceof Error ? error.message : String(error);
-    const rewriteHint =
-      originalUrl && originalUrl !== rawUrl
-        ? `原始地址 ${originalUrl} 已规范化为 ${rawUrl}，但规范化地址不可用。`
-        : "";
-    throw new Error(`访问网页失败：文档地址不可用 (${rawUrl})。${rewriteHint}${message}`);
-  }
-}
-
-export function detectExtractionIssue(markdown: string, sourceUrl: string): ExtractionIssue | null {
-  const trimmed = markdown.trim();
-  if (!trimmed) {
-    return {
-      code: "EMPTY_EXTRACTION",
-      message: `访问网页失败：${sourceUrl} 未提取到正文内容。`,
-    };
-  }
-
-  const lower = trimmed.toLowerCase();
-  if (
-    lower.startsWith("<!doctype html") ||
-    lower.startsWith("<html") ||
-    lower.includes("__next_error__")
-  ) {
-    return {
-      code: "HTML_SHELL",
-      message: `访问网页失败：${sourceUrl} 返回了错误页或 HTML 壳，而不是正文内容。`,
-    };
-  }
-
-  return null;
-}

package/ai/tools/fetchWebpageTool.ts

@@ -1,97 +0,0 @@
-// /ai/tools/fetchWebpageTool.ts
-// 现在底层走 Cloudflare Browser Rendering，支持 JS 动态渲染
-
-import { callToolApi } from "./toolApiClient";
-import {
-  assertFetchableDocsUrl,
-  detectExtractionIssue,
-  discoverCanonicalDocsUrl,
-  extractAdvertisedMarkdownUrl,
-} from "./fetchWebpageSupport";
-
-/**
- * [Schema] 定义了 'fetchWebpage' 工具的结构，供 LLM 调用。
- */
-export const fetchWebpageFunctionSchema = {
-  name: "fetchWebpage",
-  description:
-    "访问指定的网页 URL，使用真实浏览器渲染后提取 Markdown 内容，支持 JS 动态渲染页面（SPA/React 等）。" +
-    "对于 docs.* 文档站，会自动通过 /llms.txt 和 /llms-full.txt 规范化 URL。" +
-    "适合文章阅读、内容总结、网页数据提取。" +
-    "如果用户明确给了 URL 并要求据此更新代码/文档，应优先直接抓取这些 URL，并把抓到的字段视为权威来源。",
-  parameters: {
-    type: "object",
-    properties: {
-      url: {
-        type: "string",
-        description:
-          "要抓取其内容的网页的完整 URL 地址（http/https）。对于 docs.* 文档站，可提供大致推测的页面路径，工具会先尝试规范化到权威文档 URL。",
-      },
-      waitForNetworkIdle: {
-        type: "boolean",
-        description: "是否等待网络请求结束再提取（适合 SPA/动态页面），默认 false。",
-        default: false,
-      },
-    },
-    required: ["url"],
-  },
-};
-
-/**
- * [Executor] 'fetchWebpage' 工具的执行函数。
- * 底层调用 Cloudflare Browser Rendering /api/cf-markdown，支持 JS 渲染。
- */
-export async function fetchWebpageFunc(
-  args: { url: string; waitForNetworkIdle?: boolean },
-  thunkApi: any
-): Promise<{ rawData: string; displayData: string }> {
-  const { url, waitForNetworkIdle = false } = args;
-  if (!url || typeof url !== "string" || !url.startsWith("http")) {
-    throw new Error(
-      "访问网页失败：必须提供一个有效的、以 http 或 https 开头的 URL。"
-    );
-  }
-
-  const gotoOptions = waitForNetworkIdle ? { waitUntil: "networkidle0" } : undefined;
-  const resolution = await discoverCanonicalDocsUrl(url);
-  const targetUrl = resolution.resolvedUrl;
-
-  await assertFetchableDocsUrl(targetUrl, fetch, url);
-
-  let data = await callToolApi<{
-    markdown: string;
-    success: boolean;
-    browserMsUsed?: number;
-    source: string;
-  }>(thunkApi, "/api/cf-markdown", { url: targetUrl, gotoOptions }, { withAuth: true });
-
-  let finalUrl = targetUrl;
-  const advertisedMarkdownUrl = extractAdvertisedMarkdownUrl(data.markdown ?? "", targetUrl);
-  if (advertisedMarkdownUrl && advertisedMarkdownUrl !== targetUrl) {
-    finalUrl = advertisedMarkdownUrl;
-    data = await callToolApi<{
-      markdown: string;
-      success: boolean;
-      browserMsUsed?: number;
-      source: string;
-    }>(thunkApi, "/api/cf-markdown", { url: finalUrl, gotoOptions }, { withAuth: true });
-  }
-
-  const markdown = data.markdown ?? "";
-  const extractionIssue = detectExtractionIssue(markdown, finalUrl);
-  if (extractionIssue) {
-    throw new Error(extractionIssue.message);
-  }
-
-  const seconds = data.browserMsUsed ? (data.browserMsUsed / 1000).toFixed(2) : "?";
-  const statusMsg =
-    `✅ 已成功获取网页内容 (URL: ${finalUrl})\n` +
-    `🌐 渲染引擎: Cloudflare Browser Rendering\n` +
-    `⏱ 浏览器耗时: ${seconds}s | 字符数: ${markdown.length}` +
-    (finalUrl !== url ? `\n🧭 文档地址已规范化: ${url} → ${finalUrl}` : "");
-
-  return {
-    rawData: finalUrl === url ? markdown : `[Resolved URL] ${finalUrl}\n\n${markdown}`,
-    displayData: `${statusMsg}\n\n${markdown}`,
-  };
-}