@trim21/personal-pi-extensions 0.0.335 → 0.0.336

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trim21/personal-pi-extensions",
-  "version": "0.0.335",
+  "version": "0.0.336",
   "type": "module",
   "description": "Custom pi coding-agent extensions: bwrap sandbox, workspace guard, opencode edit, and more",
   "keywords": [
@@ -37,6 +37,7 @@
     "@earendil-works/pi-coding-agent": "^0.84.1",
     "@eslint/js": "10.0.1",
     "@types/node": "^24.13.3",
+    "@types/turndown": "^5.0.6",
     "@typescript/native": "npm:typescript@^7.0.2",
     "@vitest/coverage-v8": "^4.1.10",
     "eslint": "^10.8.1",
@@ -65,7 +66,8 @@
       "src/gh-readonly.ts",
       "src/spawn-agent.ts",
       "src/system-prompt/index.ts",
-      "src/talk/index.ts"
+      "src/talk/index.ts",
+      "src/web/index.ts"
     ],
     "skills": [
       "src/talk/skills"
@@ -86,9 +88,12 @@
   },
   "dependencies": {
     "@cortexkit/aft-bridge": "0.51.2",
+    "@mozilla/readability": "^0.6.0",
     "@vscode/tree-sitter-wasm": "^0.3.1",
     "jsonc-parser": "^3.3.1",
+    "linkedom": "^0.16.0",
     "minimatch": "^10.2.6",
+    "turndown": "^7.2.0",
     "vscode-jsonrpc": "^9.0.1",
     "vscode-languageserver-types": "^3.18.0",
     "web-tree-sitter": "^0.26.12"

package/src/web/config.ts

@@ -0,0 +1,31 @@
+/**
+ * web 扩展的本地配置读取：Search1API key。
+ * 路径可注入，便于测试。
+ */
+import { readFile } from "node:fs/promises";
+import { homedir } from "node:os";
+import { join } from "node:path";
+
+import { Type } from "typebox";
+import { Value } from "typebox/value";
+
+const webSearchSchema = Type.Object({
+  search1apiApiKey: Type.Optional(Type.String()),
+});
+
+export function webSearchConfigPath(): string {
+  return join(homedir(), ".pi", "web-search.json");
+}
+
+/** ~/.pi/web-search.json 的 search1apiApiKey，或 SEARCH1API_KEY 环境变量。 */
+export async function loadSearch1ApiKey(path = webSearchConfigPath()): Promise<string | undefined> {
+  const envKey = process.env.SEARCH1API_KEY;
+  if (envKey) return envKey;
+  try {
+    const raw = await readFile(path, "utf8");
+    const parsed = Value.Parse(webSearchSchema, JSON.parse(raw));
+    return parsed.search1apiApiKey?.trim() || undefined;
+  } catch {
+    return undefined;
+  }
+}

package/src/web/fetch.ts

@@ -0,0 +1,183 @@
+/**
+ * web_fetch：抓取 URL 并提取正文为 markdown。
+ *
+ * SSRF 防护：DNS 预解析 + 拒绝私有/保留地址 + 每跳重定向重新校验，
+ * 防止把 agent 变成内网探测口。正文提取用 readability 主内容算法。
+ */
+import { lookup } from "node:dns/promises";
+import { isIP } from "node:net";
+
+import { Readability } from "@mozilla/readability";
+import { parseHTML } from "linkedom";
+import TurndownService from "turndown";
+
+const MAX_REDIRECTS = 5;
+const TIMEOUT_MS = 30_000;
+const MAX_BYTES = 5 * 1024 * 1024;
+const MIN_USEFUL_CONTENT = 200;
+
+function isPrivateIpv4(ip: string): boolean {
+  const parts = ip.split(".").map(Number);
+  if (parts.length !== 4 || parts.some((p) => Number.isNaN(p))) return false;
+  const [a, b, c] = parts;
+  if (a === 0) return true; // 0.0.0.0/8
+  if (a === 10) return true; // 10.0.0.0/8
+  if (a === 127) return true; // 127.0.0.0/8 loopback
+  if (a === 169 && b === 254) return true; // 169.254.0.0/16 link-local
+  if (a === 172 && b >= 16 && b <= 31) return true; // 172.16.0.0/12
+  if (a === 192 && b === 168) return true; // 192.168.0.0/16
+  if (a === 100 && b >= 64 && b <= 127) return true; // 100.64.0.0/10 CGNAT
+  if (a === 198 && (b === 18 || b === 19)) return true; // 198.18.0.0/15 benchmarking
+  if (a === 192 && b === 0 && c === 0) return true; // 192.0.0.0/24
+  return a >= 224; // 224.0.0.0/3 multicast + reserved
+}
+
+function isPrivateIpv6(ip: string): boolean {
+  const lower = ip.toLowerCase();
+  if (lower === "::" || lower === "::1") return true; // unspecified / loopback
+  if (lower.startsWith("fc") || lower.startsWith("fd")) return true; // fc00::/7 ULA
+  if (/^fe[89ab]/.test(lower)) return true; // fe80::/10 link-local
+  if (lower.startsWith("::ffff:")) return isPrivateIpv4(lower.slice(7));
+  return false;
+}
+
+export function isPrivateAddress(ip: string): boolean {
+  const version = isIP(ip);
+  if (version === 4) return isPrivateIpv4(ip);
+  if (version === 6) return isPrivateIpv6(ip);
+  return true; // 非 IP 一律拒绝
+}
+
+/** 解析 hostname 的全部地址，任一私有即拒绝，返回解析结果 */
+export async function assertPublicHostname(
+  hostname: string,
+  lookupFn: (hostname: string) => Promise<{ address: string }[]> = (h) => lookup(h, { all: true }),
+): Promise<void> {
+  let addresses: { address: string }[];
+  try {
+    addresses = await lookupFn(hostname);
+  } catch (error) {
+    throw new Error(
+      `域名解析失败 ${hostname}: ${error instanceof Error ? error.message : String(error)}`,
+      { cause: error },
+    );
+  }
+  const blocked = addresses.find(({ address }) => isPrivateAddress(address));
+  if (blocked) {
+    throw new Error(`拒绝访问内网地址 ${hostname} (${blocked.address})`);
+  }
+}
+
+function withTimeout(signal: AbortSignal | undefined, ms: number): AbortSignal {
+  const timeout = AbortSignal.timeout(ms);
+  return signal ? AbortSignal.any([signal, timeout]) : timeout;
+}
+
+interface FetchedPage {
+  url: string;
+  title: string;
+  markdown: string;
+}
+
+/** 手动跟随重定向，每跳重新做 SSRF 校验（防 DNS rebinding 简化处理） */
+async function fetchWithRedirects(
+  url: URL,
+  signal: AbortSignal | undefined,
+  fetchFn: typeof fetch = fetch,
+): Promise<Response> {
+  let current = url;
+  for (let redirects = 0; ; redirects++) {
+    await assertPublicHostname(current.hostname);
+    const response = await fetchFn(current, {
+      redirect: "manual",
+      signal: withTimeout(signal, TIMEOUT_MS),
+      headers: { "user-agent": "Mozilla/5.0 (compatible; pi-web-fetch/1.0)" },
+    });
+    const location = response.headers.get("location");
+    if (location && response.status >= 300 && response.status < 400) {
+      if (redirects >= MAX_REDIRECTS) {
+        throw new Error(`重定向次数超过上限 (${MAX_REDIRECTS})`);
+      }
+      current = new URL(location, current);
+      if (current.protocol !== "http:" && current.protocol !== "https:") {
+        throw new Error(`不支持的协议: ${current.protocol}`);
+      }
+      continue;
+    }
+    return response;
+  }
+}
+
+export async function fetchPage(url: string, signal?: AbortSignal): Promise<FetchedPage> {
+  let target: URL;
+  try {
+    target = new URL(url);
+  } catch {
+    throw new Error(`无效的 URL: ${url}`);
+  }
+  if (target.protocol !== "http:" && target.protocol !== "https:") {
+    throw new Error(`只支持 http/https，收到: ${target.protocol}`);
+  }
+
+  const response = await fetchWithRedirects(target, signal);
+  if (!response.ok) {
+    throw new Error(`HTTP ${response.status} ${response.statusText}`);
+  }
+  const contentType = response.headers.get("content-type") ?? "";
+  if (!contentType.includes("text/html") && !contentType.includes("text/plain")) {
+    throw new Error(`不支持的内容类型: ${contentType || "unknown"}`);
+  }
+
+  const declaredLength = Number(response.headers.get("content-length") ?? "0");
+  if (declaredLength > MAX_BYTES) {
+    throw new Error(`页面过大 (${declaredLength} bytes)，上限 ${MAX_BYTES}`);
+  }
+
+  let html = "";
+  if (response.body) {
+    const reader = response.body.getReader();
+    const decoder = new TextDecoder();
+    for (;;) {
+      const chunk = (await reader.read()) as { done: boolean; value: Uint8Array };
+      if (chunk.done) break;
+      html += decoder.decode(chunk.value, { stream: true });
+      if (Buffer.byteLength(html, "utf8") > MAX_BYTES) {
+        throw new Error(`页面过大，上限 ${MAX_BYTES} bytes`);
+      }
+    }
+    html += decoder.decode();
+  }
+
+  return extractMarkdown(html, response.url);
+}
+
+/** 提取用到的 document 最小接口（linkedom 类型是 any，显式标注避免 unsafe） */
+interface ParsedDocument {
+  title: string | null;
+  body: { textContent: string | null } | null;
+}
+
+/** 从 HTML 提取标题 + 正文 markdown（readability 主内容 → turndown） */
+export function extractMarkdown(html: string, sourceUrl: string): FetchedPage {
+  const parsed = parseHTML(html) as { document: ParsedDocument };
+  const document = parsed.document;
+  // tsconfig 无 DOM lib；Readability 构造参数声明为 DOM Document，运行时只用到
+  // linkedom document 的兼容方法，cast 桥接即可
+  const article = new Readability(parsed.document).parse();
+  let title = article?.title ?? document.title?.trim() ?? sourceUrl;
+  if (typeof title !== "string" || title.length === 0) title = sourceUrl;
+  let body = article?.content;
+  if (!body || body.length === 0) {
+    body = document.body?.textContent ?? "";
+  }
+  if (typeof body !== "string" || body.trim().length < MIN_USEFUL_CONTENT) {
+    throw new Error("页面没有可提取的正文内容");
+  }
+  const markdown = new TurndownService({
+    headingStyle: "atx",
+    codeBlockStyle: "fenced",
+  })
+    .turndown(body)
+    .trim();
+  return { url: sourceUrl, title, markdown };
+}

package/src/web/index.ts

@@ -0,0 +1,174 @@
+/**
+ * web 扩展：自研 web_search + web_fetch，替代 pi-web-access。
+ *
+ * - web_search：Search1API 搜索（key 读 ~/.pi/web-search.json 的
+ *   search1apiApiKey 或 SEARCH1API_KEY），直接返回整理后的结构化结果。
+ * - web_fetch：抓取 URL，SSRF 防护 + readability 提取正文为 markdown。
+ */
+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import { Type } from "typebox";
+
+import { loadSearch1ApiKey } from "./config.js";
+import { fetchPage } from "./fetch.js";
+import { type SearchHit, searchWeb } from "./search.js";
+
+const MAX_MARKDOWN_BYTES = 100 * 1024;
+
+function truncateMarkdown(text: string): { text: string; truncated: boolean } {
+  if (Buffer.byteLength(text, "utf8") <= MAX_MARKDOWN_BYTES) return { text, truncated: false };
+  const bytes = Buffer.from(text, "utf8");
+  const sliced = bytes.subarray(0, MAX_MARKDOWN_BYTES).toString("utf8");
+  const cut = sliced.lastIndexOf("\n", sliced.length - 1);
+  return { text: (cut > 0 ? sliced.slice(0, cut) : sliced) + "\n…(已截断)", truncated: true };
+}
+
+export default function webTools(pi: ExtensionAPI) {
+  pi.registerTool({
+    name: "web_search",
+    label: "Web Search",
+    description:
+      "Search the web via Search1API and return structured results " +
+      "(title/url/snippet, optionally inline content).",
+    promptSnippet: "Search the web",
+    parameters: Type.Object({
+      query: Type.Optional(
+        Type.String({ description: "Search query (mutually exclusive with queries)" }),
+      ),
+      queries: Type.Optional(
+        Type.Array(Type.String(), { description: "Multiple queries searched in sequence" }),
+      ),
+      numResults: Type.Optional(
+        Type.Number({ description: "Results per query (default: 5, max: 50)" }),
+      ),
+      recencyFilter: Type.Optional(
+        Type.String({ description: "Filter by recency: day, week, month, year" }),
+      ),
+      domainFilter: Type.Optional(
+        Type.Array(Type.String(), { description: "Limit to domains; prefix with - to exclude" }),
+      ),
+      searchService: Type.Optional(
+        Type.String({
+          description:
+            "Search engine: google, bing, duckduckgo, github, arxiv, reddit, youtube, etc.",
+        }),
+      ),
+      includeContent: Type.Optional(
+        Type.Boolean({ description: "Inline-fetch content for the top results (up to 5)" }),
+      ),
+    }),
+    async execute(_id, params, signal, onUpdate) {
+      try {
+        const queries = [...(params.query ? [params.query] : []), ...(params.queries ?? [])]
+          .map((q) => q.trim())
+          .filter(Boolean)
+          .slice(0, 4);
+        if (queries.length === 0) {
+          return {
+            isError: true,
+            content: [{ type: "text", text: "需要提供 query 或 queries（至少一个搜索词）。" }],
+            details: { error: "no query" },
+          };
+        }
+
+        const apiKey = await loadSearch1ApiKey();
+        if (!apiKey) {
+          return {
+            isError: true,
+            content: [
+              {
+                type: "text",
+                text: "未找到 Search1API key：请在 ~/.pi/web-search.json 配置 search1apiApiKey，或设置 SEARCH1API_KEY 环境变量。",
+              },
+            ],
+            details: { error: "search1api key not configured" },
+          };
+        }
+
+        onUpdate?.({
+          content: [{ type: "text", text: `正在搜索: ${queries.join(" / ")}` }],
+          details: {},
+        });
+
+        const common = {
+          numResults: params.numResults,
+          recencyFilter: params.recencyFilter,
+          domainFilter: params.domainFilter,
+          searchService: params.searchService,
+          includeContent: params.includeContent,
+          signal,
+        };
+        const results = await Promise.all(queries.map((query) => searchWeb(query, apiKey, common)));
+
+        // 按 URL 去重合并
+        const seen = new Set<string>();
+        const hits: SearchHit[] = [];
+        for (const result of results) {
+          for (const hit of result.hits) {
+            if (seen.has(hit.url)) continue;
+            seen.add(hit.url);
+            hits.push(hit);
+          }
+        }
+        if (hits.length === 0) {
+          return {
+            content: [{ type: "text", text: "没有找到结果。" }],
+            details: { query: queries, count: 0 },
+          };
+        }
+
+        return {
+          content: [{ type: "text", text: JSON.stringify(hits, null, 2) }],
+          details: {
+            query: queries,
+            provider: "search1api",
+            count: hits.length,
+            results: hits,
+          },
+        };
+      } catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        return {
+          isError: true,
+          content: [{ type: "text", text: `搜索失败: ${message}` }],
+          details: { error: message },
+        };
+      }
+    },
+  });
+
+  pi.registerTool({
+    name: "web_fetch",
+    label: "Web Fetch",
+    description:
+      "Fetch a URL and extract the main content as markdown. SSRF-protected: refuses " +
+      "private/internal addresses. Only http/https HTML pages are supported.",
+    promptSnippet: "Fetch a web page and extract its content",
+    parameters: Type.Object({
+      url: Type.String({ description: "The URL to fetch" }),
+    }),
+    async execute(_id, params, signal) {
+      try {
+        const page = await fetchPage(params.url, signal);
+        const { text, truncated } = truncateMarkdown(page.markdown);
+        const details: Record<string, unknown> = {
+          url: page.url,
+          title: page.title,
+          bytes: Buffer.byteLength(page.markdown, "utf8"),
+          truncated,
+        };
+        return {
+          content: [{ type: "text", text }],
+          details,
+        };
+      } catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        const details: Record<string, unknown> = { error: message, url: params.url };
+        return {
+          isError: true,
+          content: [{ type: "text", text: `抓取失败: ${message}` }],
+          details,
+        };
+      }
+    },
+  });
+}

package/src/web/search.ts

@@ -0,0 +1,145 @@
+/**
+ * Search1API 搜索：请求构造、响应解析与结果整理。
+ *
+ * 搜索响应不做 AI 预消化，直接返回整理后的结构化结果（title/url/snippet，
+ * crawl_results 开启时含内联正文）——模型自己读，零额外模型调用。
+ */
+import { Type } from "typebox";
+import { Value } from "typebox/value";
+
+const SEARCH_URL = "https://api.search1api.com/search";
+const TIMEOUT_MS = 60_000;
+
+const hitSchema = Type.Object({
+  title: Type.Union([Type.String(), Type.Null()]),
+  link: Type.Union([Type.String(), Type.Null()]),
+  snippet: Type.Union([Type.String(), Type.Null()]),
+  content: Type.Optional(Type.Union([Type.String(), Type.Null()])),
+});
+
+const searchResponseSchema = Type.Object(
+  { results: Type.Array(hitSchema) },
+  { additionalProperties: true },
+);
+
+export interface SearchHit {
+  title: string;
+  url: string;
+  snippet: string;
+  content?: string;
+}
+
+export interface SearchResult {
+  query: string;
+  hits: SearchHit[];
+}
+
+export interface SearchOptions {
+  numResults?: number;
+  recencyFilter?: string;
+  domainFilter?: string[];
+  searchService?: string;
+  /** 内联抓取前几个结果的正文（crawl_results），可省去后续 web_fetch */
+  includeContent?: boolean;
+  signal?: AbortSignal;
+}
+
+function clampNumResults(value: number | undefined): number {
+  if (typeof value !== "number" || !Number.isFinite(value)) return 5;
+  return Math.max(1, Math.min(Math.floor(value), 50));
+}
+
+function normalizeDomain(raw: string): string | undefined {
+  const input = raw.trim().toLowerCase().replace(/^-/, "").trim();
+  if (!input) return undefined;
+  try {
+    const host = input.includes("://")
+      ? new URL(input).hostname
+      : new URL(`https://${input}`).hostname;
+    return host.replaceAll(/^\.+|\.+$/g, "") || undefined;
+  } catch {
+    return input.split("/", 1)[0]?.split(":", 1)[0] || undefined;
+  }
+}
+
+function splitDomainFilter(domainFilter: string[] | undefined): {
+  includeSites: string[];
+  excludeSites: string[];
+} {
+  const includeSites: string[] = [];
+  const excludeSites: string[] = [];
+  for (const raw of domainFilter ?? []) {
+    const domain = normalizeDomain(raw);
+    if (!domain) continue;
+    (raw.trimStart().startsWith("-") ? excludeSites : includeSites).push(domain);
+  }
+  return { includeSites, excludeSites };
+}
+
+export function buildSearchBody(query: string, options: SearchOptions): Record<string, unknown> {
+  const numResults = clampNumResults(options.numResults);
+  const { includeSites, excludeSites } = splitDomainFilter(options.domainFilter);
+  const body: Record<string, unknown> = {
+    query,
+    max_results: numResults,
+    crawl_results: options.includeContent ? Math.min(numResults, 5) : 0,
+  };
+  if (options.recencyFilter) body.time_range = options.recencyFilter;
+  if (options.searchService) body.search_service = options.searchService;
+  if (includeSites.length > 0) body.include_sites = includeSites;
+  if (excludeSites.length > 0) body.exclude_sites = excludeSites;
+  return body;
+}
+
+function mapHits(
+  results: {
+    title: string | null;
+    link: string | null;
+    snippet: string | null;
+    content?: string | null;
+  }[],
+): SearchHit[] {
+  const hits: SearchHit[] = [];
+  for (const item of results) {
+    const url = item.link?.trim();
+    if (!url) continue;
+    const hit: SearchHit = {
+      title: item.title?.trim() || url,
+      url,
+      snippet: item.snippet?.replaceAll(/\s+/g, " ").trim() || "",
+    };
+    const content = item.content?.trim();
+    if (content) hit.content = content;
+    hits.push(hit);
+  }
+  return hits;
+}
+
+/** 合并调用方 signal 与超时；调用方未传时仍有超时兜底 */
+function withTimeout(signal: AbortSignal | undefined, ms: number): AbortSignal {
+  const timeout = AbortSignal.timeout(ms);
+  return signal ? AbortSignal.any([signal, timeout]) : timeout;
+}
+
+export async function searchWeb(
+  query: string,
+  apiKey: string,
+  options: SearchOptions = {},
+): Promise<SearchResult> {
+  const response = await fetch(SEARCH_URL, {
+    method: "POST",
+    headers: {
+      Authorization: `Bearer ${apiKey}`,
+      "Content-Type": "application/json",
+    },
+    body: JSON.stringify(buildSearchBody(query, options)),
+    signal: withTimeout(options.signal, TIMEOUT_MS),
+  });
+
+  const raw = await response.text();
+  if (!response.ok) {
+    throw new Error(`Search1API error ${response.status}: ${raw.slice(0, 300)}`);
+  }
+  const parsed = Value.Parse(searchResponseSchema, JSON.parse(raw));
+  return { query, hits: mapHits(parsed.results) };
+}