@thincoder/core 0.9.1

package/tools/web.mjs

@@ -0,0 +1,224 @@
+import { DESC, truncate, stripTags, htmlToText, isPrivateHost } from "./shared.mjs";
+import { URL } from "node:url";
+import { proxyFetch } from "../proxy.mjs";
+
+export const UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
+const FETCH_TIMEOUT = 15_000
+// §14 D-TF3：网络失败错误文本追加 proxy 提示——纯文本提示，不自动路由（2026-08-31 裁定：proxy 显式传不自动应用）
+const PROXY_HINT = "network failure — retry with proxy: 'http://host:port' if the target is blocked"
+
+// ── Web search (Bing; direct by default, through proxy when configured and web toggle on) ──
+
+function extractBing(html) {
+  const results = []
+  // RSS format: <item><title>..</title><link>..</link><description>..</description></item>
+  for (const m of html.matchAll(/<item>([\s\S]*?)<\/item>/gi)) {
+    const block = m[1]
+    const title = block.match(/<title>([\s\S]*?)<\/title>/)?.[1] ?? ""
+    const href = block.match(/<link>([\s\S]*?)<\/link>/)?.[1] ?? ""
+    const snippet = block.match(/<description>([\s\S]*?)<\/description>/)?.[1] ?? ""
+    if (!href) continue
+    results.push({ href, title: stripTags(title), snippet: stripTags(snippet) })
+  }
+  // Fallback: HTML b_algo blocks (older server-rendered pages)
+  if (results.length === 0) {
+    const blocks = html.split('<li class="b_algo"').slice(1)
+    for (const block of blocks) {
+      const link = block.match(/<h2[^>]*><a[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/)
+      if (!link) continue
+      const snippet = block.match(/<p[^>]*>([\s\S]*?)<\/p>/)
+      results.push({ href: link[1], title: stripTags(link[2]), snippet: snippet ? stripTags(snippet[1]) : "" })
+    }
+  }
+  return results
+}
+
+function bingUrl(query, page) {
+  let u = `https://www.bing.com/search?q=${encodeURIComponent(query)}&format=rss&setlang=en`
+  if (page > 1) u += `&first=${(page - 1) * 10 + 1}`
+  return u
+}
+
+const ENGINES = [{ name: "bing", label: "Bing", url: bingUrl, extract: extractBing, ua: UA }]
+const ENGINE_NAMES = ENGINES.map(e => e.name)
+
+/** Structured search via Tavily (optional — config.websearch.apiKey). Returns
+ *  { engine, results } or null to fall back to Bing HTML scraping.
+ *  proxyUri: explicit per-call proxy (args.proxy) — never the config.json one. */
+async function fetchTavily(query, limit, ctx, proxyUri) {
+  const apiKey = ctx?.agent?.config?.websearch?.apiKey
+  if (!apiKey) return null
+  const ctrl = new AbortController()
+  const timer = setTimeout(() => ctrl.abort(), FETCH_TIMEOUT)
+  try {
+    const response = await proxyFetch("https://api.tavily.com/search", {
+      method: "POST",
+      headers: { "Content-Type": "application/json", "Authorization": `Bearer ${apiKey}` },
+      body: JSON.stringify({ query, search_depth: "basic", max_results: limit, include_answer: false, include_raw_content: false }),
+      signal: ctrl.signal,
+    }, proxyUri)
+    if (!response.ok) return null
+    const data = await response.json()
+    const results = (Array.isArray(data.results) ? data.results : []).map((r) => ({
+      href: r.url, title: r.title ?? "", snippet: r.content ?? "", _engine: "tavily",
+    }))
+    return { engine: "tavily", results }
+  } catch { return null }
+  finally { clearTimeout(timer) }
+}
+
+async function fetchEngine(engine, query, page, proxyUri) {
+  const ctrl = new AbortController()
+  const timer = setTimeout(() => ctrl.abort(), FETCH_TIMEOUT)
+  try {
+    const response = await proxyFetch(engine.url(query, page), {
+      headers: { "User-Agent": engine.ua, "Accept-Language": "en-US,en;q=0.9,zh-CN;q=0.8" },
+      signal: ctrl.signal,
+    }, proxyUri)
+    if (!response.ok) return null
+    const html = await response.text()
+    const results = engine.extract(html)
+    return { engine: engine.name, results }
+  } catch { return null }
+  finally { clearTimeout(timer) }
+}
+
+export const websearchTool = {
+  name: "websearch",
+  description: DESC("websearch"),
+  parameters: {
+    type: "object",
+    properties: {
+      query: { type: "string", description: "Search query" },
+      limit: { type: "number", description: "Max results (default 8, max 20)" },
+      engine: { type: "string", enum: ENGINE_NAMES, description: "Specific engine — \"bing\" (Bing). Omit to search all engines concurrently." },
+      page: { type: "number", description: "Page number for pagination (1-based, default 1). Only used when engine is specified." },
+      proxy: { type: "string", description: "http://host:port explicit proxy (optional) — use ONLY when passed; no proxy = direct. config.json proxy is NOT auto-applied (2026-08-31 ruling: fixed config breaks domestic sites)" },
+    },
+    required: ["query"],
+  },
+  readonly: true,
+  async execute(args, ctx) {
+    const limit = Math.min(args.limit ?? 8, 20)
+    const page = Math.max(1, args.page ?? 1)
+    // 2026-08-31 ruling: proxy is a PER-CALL decision (args.proxy), never the config.json
+    // fixed configuration. Model picks by target: github/foreign → pass proxy; gitee/domestic → omit.
+    const proxyUri = args.proxy ?? null
+    // Structured search first when a Tavily key is configured — stable, dated,
+    // no HTML scraping. Falls back to Bing silently.
+    const tavily = await fetchTavily(args.query, limit, ctx, proxyUri)
+    if (tavily && tavily.results.length > 0) {
+      return truncate(tavily.results.slice(0, limit).map((r, i) => `${i + 1}. [tavily] ${r.title}\n   ${r.href}\n   ${r.snippet}`).join("\n\n"))
+    }
+    if (args.engine) {
+      const engine = ENGINES.find(e => e.name === args.engine)
+      if (!engine) return `Unknown engine '${args.engine}'. Available: ${ENGINE_NAMES.join(", ")}`
+      const fetched = await fetchEngine(engine, args.query, page, proxyUri)
+      if (!fetched || fetched.results.length === 0) return "(no results)"
+      return truncate(fetched.results.slice(0, limit).map((r, i) => `${i + 1}. [${engine.label}] ${r.title}\n   ${r.href}\n   ${r.snippet}`).join("\n\n"))
+    }
+    const promises = ENGINES.map(e => fetchEngine(e, args.query, 1, proxyUri))
+    const fetched = (await Promise.all(promises)).filter(Boolean)
+    if (fetched.length === 0) return "(no results)"
+    const merged = [], indexes = fetched.map(() => 0)
+    let done = false
+    while (!done && merged.length < limit) {
+      done = true
+      for (let i = 0; i < fetched.length; i++) {
+        if (indexes[i] < fetched[i].results.length) {
+          merged.push({ ...fetched[i].results[indexes[i]], _engine: fetched[i].engine })
+          indexes[i]++; done = false
+          if (merged.length >= limit) break
+        }
+      }
+    }
+    return truncate(merged.slice(0, limit).map((r, i) => `${i + 1}. [${r._engine}] ${r.title}\n   ${r.href}\n   ${r.snippet}`).join("\n\n"))
+  },
+}
+
+// ── Fetch tool (with proxy support) ──────
+
+function isPrivateUrl(urlStr) {
+  let u; try { u = new URL(urlStr) } catch { return true }
+  return isPrivateHost(u.hostname)
+}
+
+// proxyFetch returns a native Response (Headers object, needs .get()) without proxy,
+// but a Response-like with a plain lowercase-keyed Record through the CONNECT tunnel — handle both.
+function headerOf(res, name) {
+  const h = res.headers
+  if (!h) return null
+  if (typeof h.get === "function") return h.get(name)
+  return h[name.toLowerCase()] ?? null
+}
+
+/** Validate a redirect target: resolve relative → absolute, http/https only, and
+ *  SSRF-checked. Returns { target } on success or { error } (never follows into a
+ *  private host — the redirect SSRF bypass). */
+export function resolveRedirectTarget(loc, baseUrl) {
+  let target
+  try { target = new URL(loc, baseUrl).toString() } catch { return { error: "invalid redirect location" } }
+  if (!/^https?:\/\//.test(target)) return { error: "redirect target must be http/https" }
+  if (isPrivateUrl(target)) return { error: "redirect target is internal/private/metadata" }
+  return { target }
+}
+
+/** Heuristic: a fetch result that is a near-empty shell or a region-block page hides
+ *  the real content. Flags the pattern so the model switches strategy instead of
+ *  re-fetching blind (2026-08-31: 21-char MiniMax SPA shell + "App unavailable in
+ *  region" Claude page both wasted an hour of round trips). */
+export function detectSparseHtml(html, text) {
+  if (text.length >= 300) return ""
+  if (/unavailable in (your )?region|not available in (your )?region|app-unavailable|enable.?javascript|just a moment|attention required|cf-browser-verification/i.test(html)) {
+    return "\n\n[fetch hint: page looks region-blocked or JS-gated (body <300 chars). Try a search MCP tool (configured provider) or the site's .md/raw/mirror endpoint]"
+  }
+  if (/<div[^>]+id="(app|root)"[^>]*>\s*<\/div>|id="app"|id="root"/i.test(html)) {
+    return "\n\n[fetch hint: page is a JS-rendered SPA shell (body <300 chars) — content loads client-side. Try a search MCP tool or the site's .md/API endpoint]"
+  }
+  return ""
+}
+
+export const fetchTool = {
+  name: "fetch",
+  description: DESC("fetch"),
+  parameters: {
+    type: "object",
+    properties: {
+      url: { type: "string", description: "http/https URL" },
+      proxy: { type: "string", description: "http://host:port explicit proxy (optional) — use ONLY when passed; no proxy = direct. config.json proxy is NOT auto-applied (2026-08-31 ruling); pick per target (github/foreign sites need a proxy, gitee/domestic don't)" },
+    },
+    required: ["url"],
+  },
+  readonly: true,
+  async execute(args, _ctx) {
+    if (!/^https?:\/\//.test(args.url)) throw new Error("url must start with http:// or https://")
+    if (isPrivateUrl(args.url)) throw new Error("fetch blocked: internal/private/metadata addresses are not allowed")
+    try {
+      // 2026-08-31 ruling: per-call decision — args.proxy when passed, direct otherwise.
+      const proxyUri = args.proxy ?? null
+      const response = await proxyFetch(args.url, { headers: { "User-Agent": UA } }, proxyUri)
+      if (!response.ok) {
+        if ([301, 302, 307, 308].includes(response.status)) {
+          const loc = headerOf(response, "location")
+          if (loc) {
+            // SSRF-check the redirect target (resolve relative → absolute) before
+            // following — a 3xx must not bounce a public URL into a private host.
+            const r = resolveRedirectTarget(loc, args.url)
+            if (r.error) throw new Error(`fetch failed: ${r.error}`)
+            const r2 = await proxyFetch(r.target, { headers: { "User-Agent": UA } }, proxyUri)
+            if (!r2.ok) throw new Error(`fetch failed: HTTP ${r2.status}\n${PROXY_HINT}`)
+            const ct2 = headerOf(r2, "content-type") ?? ""
+            const b2 = await r2.text()
+            if (ct2.includes("text/html")) { const t = htmlToText(b2); return truncate(t + detectSparseHtml(b2, t)) }
+            return truncate(b2)
+          }
+        }
+        throw new Error(`fetch failed: HTTP ${response.status}\n${PROXY_HINT}`)
+      }
+      const ct = headerOf(response, "content-type") ?? ""
+      const body = await response.text()
+      if (ct.includes("text/html")) { const t = htmlToText(body); return truncate(t + detectSparseHtml(body, t)) }
+      return truncate(body)
+    } catch (e) { throw new Error(`fetch failed: ${e.cause?.code ?? e.message}\n${PROXY_HINT}`, { cause: e }) }
+  },
+}

package/tools/write-path.mjs

@@ -0,0 +1,191 @@
+/**
+ * tools/write-path.mjs — 核内**单一写路径点**（编辑器编辑径注入缝——CORE-UNIFICATION §2.13.5）。
+ *
+ * 缺口（本档存在的原因）：宿主侧对「编辑器里打开着的文件」有专门处理——先 `applyEdit`
+ * 再保存。不保存会让缓冲变脏而磁盘仍旧 ⇒ 下一次编辑撞上我们自己的 isDirty 护栏，外部
+ * 写入又与用户随后的保存互相竞写（split-brain 数据丢失）。核内原先写盘直调
+ * `node:fs/promises`、散在 5 个档的 8 个写点上 ⇒ **没有可注入的一跳** ⇒ 端侧接核后会
+ * 丢掉编辑器径。本模块就是那一跳：核内**工具面的 8 个编辑写点**全改调 `writeThroughPath`
+ * （模型面直写 `agent-tools/batch-segment.mjs` = 设计登记的 S2 前置门——不在本缝覆盖面；
+ * 口径与 `CORE-UNIFICATION.md` §2.13.5 落地收正同源）。
+ *
+ * 默认径 = CLI 语义（**零行为变**）：`writeFile` + 记账。端侧
+ * `configureWritePath({ openDoc, applyEdit, isDirty })` 覆盖 ⇒ 编辑器径；**缺省不覆盖**。
+ * 核内**零端名分支**（契约 5——本档只认三个函数名，不出现任何端名）。
+ *
+ * 记账面（dirty-file tracking）**同源住本档**：核内写盘只此一处 ⇒ 记账面只此一处（Map/Set +
+ * 函数）。默认径的记账由本档代落——`meta.record` ⇒ `recordWrite`、`op="delete"` ⇒ `markDirty`；
+ * 调用方不再自行 `recordWrite` / `markDirty`（**不重复记账**）。**唯一例外 = apply_patch 的批级
+ * 记账**（两段式全部成功后由调用方一次 `markDirty`——批级语义，与改前逐字同）。`file.mjs` 只做
+ * 再导出，保既有 import 面稳定（模块拆分写优先纪律）。
+ */
+import { writeFile, unlink, rename, cp, rm } from "node:fs/promises"
+
+/** apply_patch 两段式的暂存后缀（单点定义——阶段一落暂存档、阶段二提交、失败清理均用它）。 */
+export const TMP_SUFFIX = ".thincoder-tmp"
+
+// ────────────────────────────────────────
+// 记账面（read-before-insert 护栏的数据源）
+// ────────────────────────────────────────
+// insert_after 以行号为锚——最易漂移的定位方式。每个写点把文件标记为 dirty；
+// insert_after 拒绝在 dirty 文件上运行，直到模型重新 read（拿到新行号）。
+const dirtyPaths = new Set()
+export function markDirty(abs) { dirtyPaths.add(abs) }
+export function clearDirty(abs) { dirtyPaths.delete(abs) }
+export function isDirty(abs) { return dirtyPaths.has(abs) }
+
+// 写入工具记录受影响行范围——insert_after 精确判定：after_line 在未受影响区
+// （< lastWrite.startLine）→ 行号未漂移 → 允许；受影响区内 → 拒绝（护栏保留）；
+// write 全文重写 → 全文件受影响，任何 after_line 拒绝。
+const lastWrites = new Map() // abs → { type: 'write'|'edit'|'insert', startLine, shift }
+export function recordWrite(abs, write) {
+  lastWrites.set(abs, write)
+  dirtyPaths.delete(abs) // 本 session 写入——等效于刚 read 过（快照在 lastWrites）
+}
+export function lastWriteOf(abs) { return lastWrites.get(abs) }
+export function clearLastWrite(abs) { lastWrites.delete(abs) }
+
+// ────────────────────────────────────────
+// 注入面（端侧编辑器径）
+// ────────────────────────────────────────
+
+let injected = null
+
+/**
+ * 覆盖默认写路径（**端装配层**调用；核内不调用——缺省不覆盖）。
+ *
+ * @param {{openDoc?: (abs: string) => any, isDirty?: (doc: any) => boolean,
+ *          applyEdit?: (doc: any, content: string, meta: object) => Promise<void>}|null} impl
+ *   · `openDoc(abs)` → doc | null   该路径是否被宿主打开（**每个写点都会问一次**——门禁面）
+ *   · `isDirty(doc)` → boolean      打开且脏 ⇒ 拒写（split-brain 护栏，**所有写点一律适用**）；
+ *     **给了 `applyEdit` 就必须给 `isDirty`**（否则抛配置错——否则 = 静默退化为无护栏编辑器径）
+ *   · `applyEdit(doc, content, meta)` → 端侧应用并落盘（内容写点专用——`applyEdit` + save）；
+ *     返回 **null** = 未处理 ⇒ 回退默认径（§2.13.5 契约；返回其他值 = 已处理）
+ * 传 null / 非对象 ⇒ 撤销注入（回默认 fs 径）。
+ */
+export function configureWritePath(impl) {
+  injected = impl && typeof impl === "object" ? impl : null
+}
+
+/** 撤销注入（测试与端装配生命周期用——缺省态 = 默认 fs 径）。 */
+export function resetWritePath() { injected = null }
+
+/** 脏缓冲拒写文案（**端中立**——与宿主侧现措辞同句；S2 接线后用户可见文案由本条统一）。 */
+export const dirtyRefusalMessage = (abs) =>
+  `File has unsaved changes in the editor: ${abs}. Save or discard before allowing automated edits.`
+
+/**
+ * 门禁单点（src 面与 dest 面共用——每面各问一次 `openDoc`）：
+ * fail-closed（给了 `applyEdit` 却没给 `isDirty` ⇒ 脏态不可知 ⇒ 抛配置错，不静默退化为
+ * 无护栏编辑器径）；打开且脏 ⇒ 抛端中立拒写文案（split-brain 护栏）。
+ * @returns doc | null（未打开 ⇒ null——该面无门禁）
+ */
+function gateOpenDoc(impl, abs) {
+  const doc = impl.openDoc(abs)
+  if (!doc) return null
+  if (typeof impl.applyEdit === "function" && typeof impl.isDirty !== "function") {
+    throw new Error("writeThroughPath: injection provides applyEdit without isDirty — refusing to write to an open file whose dirty state is unknown")
+  }
+  if (typeof impl.isDirty === "function" && impl.isDirty(doc)) throw new Error(dirtyRefusalMessage(abs))
+  return doc
+}
+
+/**
+ * 试注入面：命中 ⇒ 编辑器径（`{ written: true, via: "editor" }`）；未处理 ⇒ null（回默认）。
+ * 未打开（`openDoc` 返回假值）/ 无注入面 / 非内容写点 ⇒ 未处理。
+ * **dest 面门禁**（move / rename / **copy** 的覆盖面——父侧裁定补丁，copy 扩面同批）：
+ * 路径操作会覆盖目标档 ⇒ 目标档与源档**同过门禁**（覆盖「编辑器打开且脏」的目标档 =
+ * 直接吞掉用户缓冲，与 src 面同源 split-brain 风险）⇒ 拒写。src 未打开**不豁免** dest 门
+ * （两面各问一次）。copy 不写 src、但覆 dest（`cp { force }`）⇒ 同过 dest 门。
+ */
+async function tryInjectedPath(abs, content, meta) {
+  const impl = injected
+  if (!impl?.openDoc) return null
+  const op = meta.op ?? "write"
+  const doc = gateOpenDoc(impl, abs) // 每个写点都问一次（门禁面——见 configureWritePath）
+  if ((op === "move" || op === "rename" || op === "copy") && typeof meta.dest === "string" && meta.dest.length > 0) {
+    gateOpenDoc(impl, meta.dest) // dest 面过门禁（覆盖目标档——与 src 同源拒写判据）
+  }
+  // 编辑器径只承载**内容写点**（op="write"）；delete / 路径操作（copy/move/rename）与
+  // commit（默认径两段式的收尾）无端侧载体 ⇒ 门禁过后回默认径。
+  if (!doc || op !== "write") return null
+  if (typeof impl.applyEdit !== "function") return null
+  const handled = await impl.applyEdit(doc, content, meta)
+  if (handled === null) return null // 契约：null = 未处理 ⇒ 回退默认径（防“报成功但无落盘”）
+  if (meta.record) recordWrite(abs, meta.record) // 编辑器径同样是「我们写的」——记账同源
+  return { written: true, via: "editor" }
+}
+
+/**
+ * 默认径（无注入面 / 注入面未处理）——**CLI 语义，零行为变**。
+ *
+ * meta:
+ *   · `op`     "write"（默认）| "delete" | "copy" | "move" | "rename" | "commit"
+ *   · `record` op="write"：写后记账快照（recordWrite 第二参——受影响区）
+ *   · `dest`   op ∈ {copy, move, rename}：目标绝对路径
+ *   · `stage`  op="write"：落到 `<abs>.thincoder-tmp`（**不提交、不记账**）——apply_patch
+ *              阶段一；提交由阶段二 `op="commit"` 收口（两段式原子保持）。**注入径例外**：
+ *              编辑器径无暂存档（即时提交）——此时 stage 写直接落目标档并返回
+ *              `via:"editor"`，调用方须读 `via` 决定是否跳过 `op="commit"`（apply_patch 已如此）
+ */
+async function defaultPath(abs, content, meta) {
+  const op = meta.op ?? "write"
+  switch (op) {
+    case "write": {
+      if (meta.stage) {
+        await writeFile(abs + TMP_SUFFIX, content, "utf8")
+        return { written: true, via: "staged" }
+      }
+      await writeFile(abs, content, "utf8")
+      if (meta.record) recordWrite(abs, meta.record)
+      return { written: true, via: "fs" }
+    }
+    case "commit": {
+      await rename(abs + TMP_SUFFIX, abs)
+      return { written: true, via: "committed" }
+    }
+    case "delete": {
+      await unlink(abs)
+      markDirty(abs)
+      return { written: true, via: "fs" }
+    }
+    case "copy": {
+      await cp(abs, meta.dest, { recursive: true, force: true })
+      return { written: true, via: "fs" }
+    }
+    case "move":
+    case "rename": {
+      // move 与 rename 共享 rename 系统调用；跨设备 move 回退 copy + rm（既有语义）。
+      try {
+        await rename(abs, meta.dest)
+      } catch (e) {
+        if (e?.code !== "EXDEV") throw e
+        await cp(abs, meta.dest, { recursive: true, force: true })
+        await rm(abs, { recursive: true, force: true })
+      }
+      return { written: true, via: "fs" }
+    }
+    default:
+      throw new Error(`writeThroughPath: unknown op "${op}"`)
+  }
+}
+
+/**
+ * 核内**单一写路径点**——全部工具写点经此落盘。
+ *
+ * @param {string} abs 目标绝对路径
+ * @param {string|null} content 写入内容（op="write" 必填；delete / copy / move / rename 传 null）
+ * @param {object} [meta] 见 defaultPath 的字段表
+ * @returns {Promise<{written: boolean, via: "editor"|"fs"|"staged"|"committed"}>}
+ *   `via` = 实际落盘径（"editor" = 注入面已提交；"staged" = 已落暂存待 `op="commit"`）；
+ *   `via:"editor"` 与 `meta.stage` 组合 = 编辑器径已直接提交目标档（无暂存档）
+ */
+export async function writeThroughPath(abs, content, meta = {}) {
+  // `op="commit"` = 默认径两段式的收尾（暂存档 → 目标）——编辑器径是即时提交、不产生暂存
+  // 档 ⇒ 无端侧载体，不再问注入面（避免无意义的门禁调用；目标档的门禁已在 staging 问过）。
+  if ((meta.op ?? "write") !== "commit") {
+    const handled = await tryInjectedPath(abs, content, meta)
+    if (handled) return handled
+  }
+  return await defaultPath(abs, content, meta)
+}