decorated-pi 0.7.3 → 0.8.0

package/README.md

@@ -14,7 +14,9 @@ pi install /path/to/decorated-pi
 
 ### 1. Token Efficiency
 
-Multiple layers of token savings that compound across every session. 
+Multiple layers of token savings that compound across every session.
+
+**Talk Normal Prompt** — injects a compact response-style prompt adapted from [talk-normal](https://github.com/hexiecs/talk-normal), trimming filler, summary stamps, conditional follow-up menus, and verbose framing. This reduces assistant output tokens and keeps visible reasoning / explanation blocks tighter.
 
 **RTK** — integrates [RTK](https://github.com/rtk-ai/rtk) to compress bash output into structured summaries, so the LLM never sees raw noise. **Just install the CLI, zero config**.
 

package/commands/usage.ts

@@ -401,7 +401,11 @@ export function formatCost(c: number): string {
 
 export function formatHitRate(hitRate: number, turns: number): string {
   if (turns === 0) return "—";
-  return `${hitRate.toFixed(1)}%`;
+  // Truncate (not round) beyond 1 decimal place: 99.97 → "99.9%", not
+  // "100.0%". A true 100.0 (every input-side token served from cache) is
+  // the only way to reach "100.0%".
+  const truncated = Math.floor(hitRate * 10) / 10;
+  return `${truncated.toFixed(1)}%`;
 }
 
 export type ColumnId = "input" | "output" | "cacheRead" | "cacheWrite" | "hitRate" | "cost";

package/hooks/compaction.ts

@@ -1,5 +1,5 @@
 /**
- * compaction — custom compaction model + auto-resume.
+ * compaction — custom compaction model.
  *
  * On `session_before_compact`, runs pi-coding-agent's `compact()` against
  * the model configured in settings (rather than the agent's current model)
@@ -7,82 +7,25 @@
  * default. If the configured model is missing, auth fails, or the call
  * throws, we fall through (return undefined) and pi runs its own compaction.
  *
- * On `session_compact`, if the compaction was auto-triggered, sends a
- * "continue" message to resume the agent loop.
+ * Auto-retry/resume is handled by pi natively; `reason` and `willRetry`
+ * on compaction events describe manual, threshold, and overflow flows.
  */
 
-import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
 import { compact } from "@earendil-works/pi-coding-agent";
 import type { Model } from "@earendil-works/pi-ai";
-import { isContextOverflow } from "@earendil-works/pi-ai";
-import * as fs from "node:fs";
-import * as os from "node:os";
-import { resolve } from "node:path";
 import { getCompactModelKey } from "../settings.js";
 import { parseModelKey } from "../settings.js";
 import type { Module, Skeleton } from "./skeleton.js";
 
-interface PiCompactionSettings {
-  enabled: boolean;
-}
-
-interface AutoCompactionCandidate {
-  messages: any[];
-  usage: { tokens: number | null; contextWindow: number } | undefined;
-}
-
-const DEFAULT_PI_COMPACTION_SETTINGS: PiCompactionSettings = {
-  enabled: true,
-};
-
-function readJsonObject(filePath: string): any | undefined {
-  try {
-    if (!fs.existsSync(filePath)) return undefined;
-    const parsed = JSON.parse(fs.readFileSync(filePath, "utf-8"));
-    return parsed && typeof parsed === "object" ? parsed : undefined;
-  } catch {
-    return undefined;
-  }
-}
+type CompactionReason = "manual" | "threshold" | "overflow";
 
-function loadPiCompactionSettings(cwd: string): PiCompactionSettings {
-  const globalSettings = readJsonObject(resolve(os.homedir(), ".pi", "agent", "settings.json"));
-  const projectSettings = readJsonObject(resolve(cwd, ".pi", "settings.json"));
-  const merged = {
-    ...DEFAULT_PI_COMPACTION_SETTINGS,
-    ...(globalSettings?.compaction ?? {}),
-    ...(projectSettings?.compaction ?? {}),
-  };
-  return {
-    enabled: merged.enabled !== false,
-  };
+interface CompactionEventMetadata {
+  reason: CompactionReason;
+  willRetry: boolean;
 }
 
-function getLastAssistantMessage(messages: any[]): any | undefined {
-  for (let i = messages.length - 1; i >= 0; i--) {
-    if (messages[i]?.role === "assistant") return messages[i];
-  }
-  return undefined;
-}
-
-/** Mirror pi's _runAutoCompaction(reason, willRetry) decision:
- *   - overflow → willRetry=true → auto-resume the agent
- *   - threshold → willRetry=false → no auto-resume (user continues manually)
- *   - manual /compact → no resume
- * Detected from the post-agent-end candidate: overflow shows up as a
- * context-overflow error on the last assistant message; threshold is a
- * pre-emptive compaction and we intentionally skip auto-resume for it. */
-function shouldAutoResumeCompaction(
-  candidate: AutoCompactionCandidate | null,
-  settings: PiCompactionSettings,
-  customInstructions?: string,
-): boolean {
-  if (customInstructions !== undefined) return false;
-  if (!candidate || !settings.enabled) return false;
-  const lastAssistant = getLastAssistantMessage(candidate.messages);
-  if (!lastAssistant) return false;
-  const contextWindow = candidate.usage?.contextWindow ?? 0;
-  return contextWindow > 0 && isContextOverflow(lastAssistant, contextWindow);
+function formatCompactionMode(event: CompactionEventMetadata): string {
+  return event.willRetry ? `${event.reason}, retrying` : event.reason;
 }
 
 function getConfiguredCompactModel(registry: any): Model<any> | null {
@@ -93,71 +36,11 @@ function getConfiguredCompactModel(registry: any): Model<any> | null {
   return registry.find(parsed.provider, parsed.modelId) ?? null;
 }
 
-/** Per-session state. Indexed by sessionId (from ctx.sessionManager) so
- *  two concurrent pi sessions don't trample each other's flags. */
-interface SessionState {
-  postAgentEndCandidate: AutoCompactionCandidate | null;
-  currentCompactionIsAuto: boolean;
-}
-
-const sessionStates = new Map<string, SessionState>();
-
-function getSessionState(sessionId: string): SessionState {
-  let s = sessionStates.get(sessionId);
-  if (!s) {
-    s = { postAgentEndCandidate: null, currentCompactionIsAuto: false };
-    sessionStates.set(sessionId, s);
-  }
-  return s;
-}
-
 export const compactionModule: Module = {
   name: "compaction",
   hooks: {
-    session_shutdown: [
-      (_event, ctx) => {
-        // Clean up state when the session ends so the Map doesn't grow.
-        sessionStates.delete(ctx.sessionManager.getSessionId());
-      },
-    ],
-    input: [
-      (_event, ctx) => {
-        getSessionState(ctx.sessionManager.getSessionId()).postAgentEndCandidate = null;
-      },
-    ],
-    before_agent_start: [
-      (_event, ctx) => {
-        getSessionState(ctx.sessionManager.getSessionId()).postAgentEndCandidate = null;
-      },
-    ],
-    agent_start: [
-      (_event, ctx) => {
-        getSessionState(ctx.sessionManager.getSessionId()).postAgentEndCandidate = null;
-      },
-    ],
-    agent_end: [
-      (event, ctx) => {
-        getSessionState(ctx.sessionManager.getSessionId()).postAgentEndCandidate = {
-          messages: event.messages,
-          usage: ctx.getContextUsage(),
-        };
-      },
-    ],
     session_before_compact: [
       async (event, ctx, pi) => {
-        const sessionState = getSessionState(ctx.sessionManager.getSessionId());
-        const compactionSettings = loadPiCompactionSettings(ctx.cwd);
-        // Pi only auto-compacts after agent_end (see _checkCompaction in
-        // agent-session.js), so we detect "auto" via the post-agent-end
-        // overflow heuristic. Manual /compact carries customInstructions
-        // and skips auto-resume.
-        const isAutoResume = shouldAutoResumeCompaction(
-          sessionState.postAgentEndCandidate,
-          compactionSettings,
-          event.customInstructions,
-        );
-        sessionState.postAgentEndCandidate = null;
-
         const model = getConfiguredCompactModel(ctx.modelRegistry);
         if (!model) return; // No custom compact model configured → let pi run its default compaction.
         const auth = await ctx.modelRegistry.getApiKeyAndHeaders(model);
@@ -168,7 +51,7 @@ export const compactionModule: Module = {
         const { preparation, customInstructions, signal } = event;
         if (ctx.hasUI) {
           ctx.ui.notify(
-            `🗜️ Compacting with ${model.id} (${preparation.tokensBefore.toLocaleString()} tokens)...`,
+            `📦 Compacting with ${model.id} (${formatCompactionMode(event)}, ${preparation.tokensBefore.toLocaleString()} tokens)...`,
             "info",
           );
         }
@@ -189,11 +72,6 @@ export const compactionModule: Module = {
             signal,
             thinkingLevel,
           );
-          // Only mark as auto-resumed if OUR hook did the compaction. If we
-          // had fallen through to pi's default path above, pi's own
-          // _runAutoCompaction would call agent.continue() on overflow;
-          // marking here would cause a duplicate resume message on top.
-          sessionState.currentCompactionIsAuto = isAutoResume;
           return { compaction: result };
         } catch (err) {
           if (signal.aborted) return;
@@ -209,19 +87,6 @@ export const compactionModule: Module = {
         }
       },
     ],
-    session_compact: [
-      (_event, ctx, pi) => {
-        const sessionState = getSessionState(ctx.sessionManager.getSessionId());
-        const shouldResume = sessionState.currentCompactionIsAuto;
-        sessionState.currentCompactionIsAuto = false;
-        if (!shouldResume) return;
-        pi.sendMessage({
-          customType: "auto_compact_resume",
-          content: "The context was just auto-compacted. Continue the current task based on the summary above. Do not repeat completed work. If unsure about progress, briefly summarize current state then continue.",
-          display: false,
-        }, { triggerTurn: true });
-      },
-    ],
   },
 };
 
@@ -230,5 +95,5 @@ export function setupCompaction(sk: Skeleton): void {
 }
 
 export const __modelIntegrationTest = {
-  shouldAutoResumeCompaction,
+  formatCompactionMode,
 };

package/hooks/skeleton.ts

@@ -25,7 +25,8 @@ export type HookEvent =
   | "agent_end"
   | "input"
   | "tool_call"
-  | "tool_result";
+  | "tool_result"
+  | "message_end";
 
 // ─── Handler modes ─────────────────────────────────────────────────────────
 
@@ -67,6 +68,7 @@ export interface Module {
     input?: ParallelHandler<"input">[];
     tool_call?: ComposeHandler<"tool_call">[];
     tool_result?: ComposeHandler<"tool_result">[];
+    message_end?: ComposeHandler<"message_end">[];
   };
 }
 
@@ -99,6 +101,7 @@ const COMPOSE_EVENTS = new Set<HookEvent>([
   "before_agent_start",
   "tool_call",
   "tool_result",
+  "message_end",
 ]);
 
 /** Events whose handler return value is propagated to the extension runner

package/hooks/smart-at.ts

@@ -6,6 +6,10 @@
  * frecency + git status, and returns ranked results. We create one
  * FileFinder per session and query it directly for every @ prefix.
  *
+ * Result handling trusts FFF's native score: re-sort by score
+ * descending (shorter path breaks ties), no substring or git-ignore
+ * post-filtering.
+ *
  * KNOWN FFF LIMITATION (v0.9.4):
  * FFF only returns directories that directly contain files. Intermediate
  * folders that only hold subdirectories (e.g. product/module/apmanage/ where
@@ -24,7 +28,6 @@ import type { Module, Skeleton } from "./skeleton.js";
 const AT_BOUNDARY = new Set<string>([" ", "\t", "(", "["]);
 
 const AUTOCOMPLETE_LIMIT = 20;
-const FFF_SUPERSET = 100;
 const WIDGET_FOOTER = "\x1b[2mpowered by fff\x1b[0m";
 
 interface AutocompleteItem {
@@ -45,10 +48,6 @@ function atPrefix(text: string): string | null {
   return null;
 }
 
-function isGitIgnored(item: MixedItem): boolean {
-  return item.type === "file" && item.item.gitStatus === "ignored";
-}
-
 function toAutocompleteItem(item: MixedItem): AutocompleteItem {
   const path = item.item.relativePath;
   const label = item.type === "file"
@@ -65,23 +64,20 @@ interface BuiltResult {
   items: AutocompleteItem[];
 }
 
-/** Take FFF's ranked results, drop git-ignored files, and for non-empty
- *  queries keep only paths that contain the query as a substring. This
- *  stops FFF's loose fuzzy matching from ranking every file in a directory
- *  that happens to share a few letters (e.g. "mm/" for query "mmc"). */
-function buildResult(items: MixedItem[], lowerQuery: string): BuiltResult | null {
-  const filtered = items
-    .filter((it) => !isGitIgnored(it))
-    .filter(
-      (it) =>
-        !lowerQuery || it.item.relativePath.toLowerCase().includes(lowerQuery),
-    )
+/** Take FFF's mixed-search results and re-sort by native score descending
+ *  (score ties broken by shorter path first), then map to autocomplete
+ *  items. Trust FFF's fuzzy+frecency score; no substring or git-ignore
+ *  post-filtering. */
+function buildResult(items: MixedItem[], scores: { total: number }[]): BuiltResult | null {
+  const ranked = items
+    .map((item, index) => ({ item, score: scores[index]?.total ?? 0 }))
+    .sort((a, b) => b.score - a.score || a.item.item.relativePath.length - b.item.item.relativePath.length)
     .slice(0, AUTOCOMPLETE_LIMIT)
-    .map(toAutocompleteItem);
-  return filtered.length ? { items: filtered } : null;
+    .map((entry) => toAutocompleteItem(entry.item));
+  return ranked.length ? { items: ranked } : null;
 }
 
-export const __smartAtTest = { atPrefix };
+export const __smartAtTest = { atPrefix, buildResult };
 
 /** Active FileFinder for the current session; freed in session_shutdown
  *  to avoid leaking FFF's native handle + LMDB mmap regions. */
@@ -148,9 +144,8 @@ export const smartAtModule: Module = {
             }
 
             const query = prefix.slice(1);
-            const lowerQuery = query.toLowerCase();
-            const r = finder.mixedSearch(lowerQuery, {
-              pageSize: lowerQuery ? FFF_SUPERSET : AUTOCOMPLETE_LIMIT,
+            const r = finder.mixedSearch(query.toLowerCase(), {
+              pageSize: AUTOCOMPLETE_LIMIT,
             });
             if (!r.ok) {
               ctx.ui.setWidget("smart-at", undefined);
@@ -178,7 +173,7 @@ export const smartAtModule: Module = {
               return null;
             }
 
-            const result = buildResult(r.value.items, lowerQuery);
+            const result = buildResult(r.value.items, r.value.scores);
             if (!result) {
               ctx.ui.setWidget("smart-at", undefined);
               return null;

package/hooks/thinking-label-strip.ts

@@ -0,0 +1,79 @@
+/**
+ * thinking-label-strip — remove stray <thinking></thinking> tags from
+ * assistant thinking blocks.
+ *
+ * Some models wrap their own reasoning text in literal <thinking></thinking>
+ * tags inside the `thinking` field of a ThinkingContent block. The tag
+ * itself is noise — pi already marks the block as type "thinking", so the
+ * tag is redundant. When it leaks into the context it confuses downstream
+ * consumers (compaction, other models that echo it back) and pollutes the
+ * session transcript.
+ *
+ * This hook intercepts `message_end` for assistant messages and strips
+ * every <thinking></thinking> pair from each ThinkingContent block's text,
+ * preserving the reasoning between the tags. The edit is persistent: the
+ * stored message is replaced, so the context sent to subsequent LLM calls
+ * no longer contains the literal tag.
+ */
+
+import type { Module } from "./skeleton.js";
+
+const THINKING_OPEN = "<thinking>";
+const THINKING_CLOSE = "</thinking>";
+
+/** Strip a leading <thinking> and/or trailing </thinking> from `text`.
+ *  Models often emit these tags as delimiters inside the `thinking` field;
+ *  pi already marks the block type, so the tags are noise that leaks into
+ *  the context and the TUI. We only trim at the head/tail — tags buried in
+ *  the middle are part of the reasoning text and left alone, which avoids
+ *  corrupting the model's argument if it literally discusses the tag. */
+export function stripThinkingLabels(text: string): string {
+  if (!text.includes(THINKING_OPEN) && !text.includes(THINKING_CLOSE)) {
+    return text;
+  }
+  let out = text;
+  // trimStart would loop-strip repeating opens; we want a single leading
+  // open tag (optionally with whitespace before it) removed.
+  const leadingOpen = out.match(/^\s*<thinking>/);
+  if (leadingOpen) {
+    out = out.slice(leadingOpen[0].length);
+  }
+  const trailingClose = out.match(/<\/thinking>\s*$/);
+  if (trailingClose) {
+    out = out.slice(0, out.length - trailingClose[0].length);
+  }
+  return out;
+}
+
+/** Return a new content array with thinking blocks cleaned, or undefined
+ *  if nothing changed. */
+function cleanContent(content: any[]): any[] | undefined {
+  let changed = false;
+  const next = content.map((block) => {
+    if (block && block.type === "thinking" && typeof block.thinking === "string") {
+      const cleaned = stripThinkingLabels(block.thinking);
+      if (cleaned !== block.thinking) {
+        changed = true;
+        return { ...block, thinking: cleaned };
+      }
+    }
+    return block;
+  });
+  return changed ? next : undefined;
+}
+
+export const thinkingLabelStripModule: Module = {
+  name: "thinking-label-strip",
+  hooks: {
+    message_end: [
+      (event: any) => {
+        const message = event?.message;
+        if (!message || message.role !== "assistant") return undefined;
+        if (!Array.isArray(message.content)) return undefined;
+        const cleaned = cleanContent(message.content);
+        if (!cleaned) return undefined;
+        return { message: { ...message, content: cleaned } };
+      },
+    ],
+  },
+};

package/index.ts

@@ -23,8 +23,12 @@ import { createSkeleton } from "./hooks/skeleton.js";
 import { setupRedact, REDACT_GUIDANCE } from "./hooks/redact.js";
 import { externalizeModule } from "./hooks/externalize.js";
 import { normalizeCodeblocksModule } from "./hooks/normalize-codeblocks.js";
+import { thinkingLabelStripModule } from "./hooks/thinking-label-strip.js";
 import { trackMtimeModule } from "./hooks/track-mtime.js";
-import { injectAgentsMdModule, INJECT_AGENTS_MD_GUIDANCE } from "./hooks/inject-agents-md.js";
+import {
+    injectAgentsMdModule,
+    INJECT_AGENTS_MD_GUIDANCE,
+} from "./hooks/inject-agents-md.js";
 import { imageVisionModule } from "./hooks/image-vision.js";
 import { smartAtModule } from "./hooks/smart-at.js";
 import { sessionTitleModule } from "./hooks/session-title.js";
@@ -39,7 +43,11 @@ import { registerLspTools } from "./tools/lsp/tools.js";
 import { LspServerManager } from "./tools/lsp/manager.js";
 import { collectLspDependencyStatuses } from "./tools/lsp/servers.js";
 import { registerAskTool } from "./tools/ask/index.js";
-import { resolveMcpConfigs, migrateLegacyGlobalMcpConfig, collectMcpDependencyStatuses } from "./tools/mcp/config.js";
+import {
+    resolveMcpConfigs,
+    migrateLegacyGlobalMcpConfig,
+    collectMcpDependencyStatuses,
+} from "./tools/mcp/config.js";
 import { ensureMcpServerReady } from "./hooks/mcp.js";
 
 import { registerDpModelCommand } from "./commands/dp-model.js";
@@ -56,80 +64,98 @@ import { captureModuleSnapshot, isModuleEnabled } from "./settings.js";
 // its constant and pushing it here. No priority / sort logic — just push.
 
 const BASE_GUIDANCE = [
-  "## Decorated Pi Guidance",
-  "",
-  "### Workflow, how to approach tasks",
-  "- Before acting on a prompt, do sufficient research on the existing state — read files, search, investigate — and only proceed once you have a clear picture.",
-  "- Exercise caution when performing any **write** operations, especially when you are in a research or exploration phase.",
-  "- Before modifying code, match the user's existing code style (naming, formatting, patterns). Do not re-modify lines the user has manually edited since your last change.",
-  "",
-  "### Filesystem Safety, where NOT to write",
-  "- CAUTION: Do not perform write operations in the following directories unless explicitly instructed: `node_modules`, `venv`, `env`, `__pycache__`, `.git` or any other hidden directories.",
+    "## Decorated Pi Guidance",
+    "",
+    "### Workflow, how to approach tasks",
+    "- Before acting on a prompt, do sufficient research on the existing state — read files, search, investigate — and only proceed once you have a clear picture.",
+    "- Exercise caution when performing any **write** operations, especially when you are in a research or exploration phase.",
+    "- Before modifying code, match the user's existing code style (naming, formatting, patterns). Do not re-modify lines the user has manually edited since your last change.",
+    "",
+    "### Filesystem Safety, where NOT to write",
+    "- CAUTION: Do not perform write operations in the following directories unless explicitly instructed: `node_modules`, `venv`, `env`, `__pycache__`, `.git` or any other hidden directories.",
+].join("\n");
+
+// Adapted from hexiecs/talk-normal (MIT), prompt-chatgpt.md v0.6.2.
+const TALK_NORMAL_GUIDANCE = [
+    "## Your talking style",
+    "",
+    "- Be direct and informative. No filler, no fluff, but give enough to be useful.",
+    "- Lead with the answer; add context only if it helps.",
+    "- Prefer direct positive claims. Avoid negation-based contrastive phrasing like `不是X，而是Y` / `it's not X, it's Y`; state the positive claim directly.",
+    "- Kill filler: `I'd be happy to`, `Great question`, `It's worth noting`, `Certainly`, `Of course`, `首先`, `值得注意的是`, `综上所述`.",
+    "- Never restate the question.",
+    "- Yes/no questions: answer first, then give one sentence of reasoning.",
+    "- Comparisons: give a recommendation with brief reasoning, not a balanced essay.",
+    "- Code: give the code plus a usage example when non-trivial. Skip preambles like `Certainly! Here is...`.",
+    "- Use bullets or numbered lists only when the content has real parallel or sequential structure.",
+    "- Do not end with conditional follow-up offers like `If you want, I can...` / `如果你愿意，我还可以...`; take the real next step or name it directly.",
+    "- Do not use summary-stamp closings like `In summary`, `Hope this helps`, `一句话总结`, `总结一下`, `简而言之`; state the final claim directly.",
 ].join("\n");
 
 /** Remove the injected Pi documentation block from the base system prompt.
  *  Matches a line containing "Pi documentation" and deletes it plus all
  *  following non-empty lines, stopping at the first blank line. */
 export function stripPiDocsBlock(prompt: string): string {
-  const lines = prompt.split("\n");
-  const out: string[] = [];
-  let i = 0;
-  while (i < lines.length) {
-    const line = lines[i];
-    if (line.includes("Pi documentation")) {
-      i++;
-      while (i < lines.length && lines[i].trim() !== "") i++;
-      // Drop the terminating blank line as well so we don't leave orphan whitespace.
-      if (i < lines.length && lines[i].trim() === "") i++;
-      continue;
+    const lines = prompt.split("\n");
+    const out: string[] = [];
+    let i = 0;
+    while (i < lines.length) {
+        const line = lines[i];
+        if (line.includes("Pi documentation")) {
+            i++;
+            while (i < lines.length && lines[i].trim() !== "") i++;
+            // Drop the terminating blank line as well so we don't leave orphan whitespace.
+            if (i < lines.length && lines[i].trim() === "") i++;
+            continue;
+        }
+        out.push(line);
+        i++;
     }
-    out.push(line);
-    i++;
-  }
-  return out.join("\n");
+    return out.join("\n");
 }
 
 /** Sort the <available_skills> block in the system prompt by skill name.
  *  Pi core appends extension-provided skills after user/project skills and does
  *  not sort the XML; this makes the final prompt stable and cache-friendly. */
 export function sortSkillsInSystemPrompt(prompt: string): string {
-  const startMarker = "\n<available_skills>";
-  const endMarker = "</available_skills>";
-  const startIdx = prompt.indexOf(startMarker);
-  if (startIdx === -1) return prompt;
-  const endIdx = prompt.indexOf(endMarker, startIdx);
-  if (endIdx === -1) return prompt;
-
-  const before = prompt.slice(0, startIdx + startMarker.length);
-  const after = prompt.slice(endIdx);
-  const inner = prompt.slice(startIdx + startMarker.length, endIdx);
-
-  const chunks: string[][] = [];
-  let current: string[] = [];
-  for (const line of inner.split("\n")) {
-    const trimmed = line.trim();
-    if (trimmed === "<skill>") {
-      current = [line];
-    } else if (trimmed === "</skill>") {
-      current.push(line);
-      chunks.push(current);
-      current = [];
-    } else if (current.length > 0) {
-      current.push(line);
+    const startMarker = "\n<available_skills>";
+    const endMarker = "</available_skills>";
+    const startIdx = prompt.indexOf(startMarker);
+    if (startIdx === -1) return prompt;
+    const endIdx = prompt.indexOf(endMarker, startIdx);
+    if (endIdx === -1) return prompt;
+
+    const before = prompt.slice(0, startIdx + startMarker.length);
+    const after = prompt.slice(endIdx);
+    const inner = prompt.slice(startIdx + startMarker.length, endIdx);
+
+    const chunks: string[][] = [];
+    let current: string[] = [];
+    for (const line of inner.split("\n")) {
+        const trimmed = line.trim();
+        if (trimmed === "<skill>") {
+            current = [line];
+        } else if (trimmed === "</skill>") {
+            current.push(line);
+            chunks.push(current);
+            current = [];
+        } else if (current.length > 0) {
+            current.push(line);
+        }
     }
-  }
 
-  const nameOf = (chunk: string[]) => {
-    const line = chunk.find((l) => l.trim().startsWith("<name>"));
-    if (!line) return "";
-    const t = line.trim();
-    return t.slice(6, t.indexOf("</name>"));
-  };
+    const nameOf = (chunk: string[]) => {
+        const line = chunk.find((l) => l.trim().startsWith("<name>"));
+        if (!line) return "";
+        const t = line.trim();
+        return t.slice(6, t.indexOf("</name>"));
+    };
 
-  chunks.sort((a, b) => nameOf(a).localeCompare(nameOf(b)));
+    chunks.sort((a, b) => nameOf(a).localeCompare(nameOf(b)));
 
-  const sortedInner = "\n" + chunks.map((chunk) => chunk.join("\n")).join("\n") + "\n";
-  return before + sortedInner + after;
+    const sortedInner =
+        "\n" + chunks.map((chunk) => chunk.join("\n")).join("\n") + "\n";
+    return before + sortedInner + after;
 }
 
 /** Build the list of guideline strings to inject, in prompt order.
@@ -139,146 +165,153 @@ export function sortSkillsInSystemPrompt(prompt: string): string {
  *  injected). The mcp module check lives in `resolveMcpConfigs`, so this
  *  code only needs to look at the server's own `enabled` flag. */
 function buildGuidelines(): string[] {
-  const out: string[] = [
-    BASE_GUIDANCE,
-    INJECT_AGENTS_MD_GUIDANCE,   // from hooks/inject-agents-md.ts — always on
-  ];
-  if (isModuleEnabled("secretRedaction")) out.push(REDACT_GUIDANCE);
-  return out;
+    const out: string[] = [
+        BASE_GUIDANCE,
+        TALK_NORMAL_GUIDANCE,
+        INJECT_AGENTS_MD_GUIDANCE, // from hooks/inject-agents-md.ts — always on
+    ];
+    if (isModuleEnabled("secretRedaction")) out.push(REDACT_GUIDANCE);
+    return out;
 }
 
-function canRegisterMcpServer(config: { name: string; command?: string }, deps: Array<{ module: string; state: string }>): boolean {
-  if (!config.command) return true;
-  const dep = deps.find((d) => d.module === `mcp:${config.name}`);
-  return dep ? dep.state === "ok" : true;
+function canRegisterMcpServer(
+    config: { name: string; command?: string },
+    deps: Array<{ module: string; state: string }>,
+): boolean {
+    if (!config.command) return true;
+    const dep = deps.find((d) => d.module === `mcp:${config.name}`);
+    return dep ? dep.state === "ok" : true;
 }
 
 /** Absolute path to the plugin's builtin skills directory.
  *  Used by `resources_discover` so the skill travels with the plugin
  *  regardless of which project pi is running in. */
 export function getBuiltinSkillPaths(): string[] {
-  return [fileURLToPath(new URL("./skills", import.meta.url))];
+    return [fileURLToPath(new URL("./skills", import.meta.url))];
 }
 
 /** Register the plugin's builtin skill paths with Pi core. */
 function installBuiltinSkills(pi: ExtensionAPI): void {
-  pi.on("resources_discover", async (_event: any) => ({
-    skillPaths: getBuiltinSkillPaths(),
-  }));
+    pi.on("resources_discover", async (_event: any) => ({
+        skillPaths: getBuiltinSkillPaths(),
+    }));
 }
 
 /** Install a single before_agent_start handler that appends every
  *  guideline in order, stripping the volatile "Current date: …" line
  *  for cache stability. Idempotent — re-injection is a no-op via marker. */
 function installGuidelines(pi: ExtensionAPI): void {
-  const blocks = buildGuidelines();
-  const joined = blocks.join("\n\n");
-  const marker = "## Decorated Pi Guidance";
-
-  pi.on("before_agent_start", async (event: any) => {
-    if (!event.systemPrompt) return undefined;
-    let prompt: string = stripPiDocsBlock(event.systemPrompt);
-    prompt = sortSkillsInSystemPrompt(prompt);
-    prompt = prompt.replace(/\nCurrent date: \d{4}-\d{2}-\d{2}/, "");
-    if (prompt.includes(marker)) return undefined; // already injected this turn
-    return { systemPrompt: `${prompt}\n\n${joined}` };
-  });
+    const blocks = buildGuidelines();
+    const joined = blocks.join("\n\n");
+    const marker = "## Decorated Pi Guidance";
+
+    pi.on("before_agent_start", async (event: any) => {
+        if (!event.systemPrompt) return undefined;
+        let prompt: string = stripPiDocsBlock(event.systemPrompt);
+        prompt = sortSkillsInSystemPrompt(prompt);
+        prompt = prompt.replace(/\nCurrent date: \d{4}-\d{2}-\d{2}/, "");
+        if (prompt.includes(marker)) return undefined; // already injected this turn
+        return { systemPrompt: `${prompt}\n\n${joined}` };
+    });
 }
 
 export default async function (pi: ExtensionAPI) {
-  // Snapshot the module settings that pi is about to load. /dp-settings
-  // compares against this to avoid prompting for reload when the user
-  // has only returned the settings to the currently-loaded state.
-  captureModuleSnapshot();
-
-  // ── Skeleton (hooks) ───────────────────────────────────────────────────
-  const sk = createSkeleton();
-
-  // Order matters for tool_result compose chain:
-  //   1. redact → normalize-codeblocks → externalize → track-mtime → inject-agents-md → image-vision → wakatime
-  // The first module registered for a given event runs first (compose chain).
-  if (isModuleEnabled("secretRedaction")) setupRedact(sk);
-  sk.register(normalizeCodeblocksModule);
-  sk.register(externalizeModule);
-  sk.register(trackMtimeModule);
-  sk.register(injectAgentsMdModule);
-  sk.register(imageVisionModule);
-
-  // session_start handlers (parallel)
-  // pi-tool-filter must register first so native tools are dropped before
-  // anything else inspects the tool list.
-  sk.register(piToolFilterModule);
-  sk.register(sessionTitleModule);
-  if (isModuleEnabled("atOverride")) sk.register(smartAtModule);
-  if (isModuleEnabled("wakatime")) sk.register(wakatimeModule);
-
-  // Compaction + RTK (these also install their own pi.on via setup<>()).
-  setupCompaction(sk);
-  if (isModuleEnabled("rtk")) setupRtk(sk);
-  if (isModuleEnabled("wakatime")) setupWakatime(sk, pi);
-
-  // ── Tools (conditional on module switches) ────────────────────────────
-  if (isModuleEnabled("patchOverrideEdit")) registerPatchTool(pi);
-  if (isModuleEnabled("lsp")) {
-    const lspDeps = collectLspDependencyStatuses(process.cwd());
-    if (lspDeps.some((d) => d.state === "ok")) {
-      registerLspTools(pi, new LspServerManager());
-    }
-    for (const dep of lspDeps) {
-      if (dep.state !== "ok") {
-        sk.declareMissing({
-          name: dep.label,
-          module: "lsp",
-          hint: dep.detail,
-        });
-      }
-    }
-  }
-  if (isModuleEnabled("ask")) registerAskTool(pi);
-
-  // MCP: hook, tools, and /mcp command are gated together. Disabling the
-  // module means no session_start handler runs, no tools register, no
-  // /mcp command is available, and no background connections are attempted.
-  if (isModuleEnabled("mcp")) {
-    // One-time migration: legacy global MCP configs in
-    // ~/.pi/agent/decorated-pi.json move to ~/.pi/agent/mcp.json. Run
-    // explicitly here so `loadGlobalMcpConfigs` stays pure.
-    migrateLegacyGlobalMcpConfig();
-    sk.register(mcpModule);
-    const mcpDeps = collectMcpDependencyStatuses(process.cwd());
-    for (const dep of mcpDeps) {
-      if (dep.state !== "ok") {
-        sk.declareMissing({
-          name: dep.label,  // binary name (e.g. "codegraph")
-          module: "mcp",
-          hint: dep.detail,
-        });
-      }
+    // Snapshot the module settings that pi is about to load. /dp-settings
+    // compares against this to avoid prompting for reload when the user
+    // has only returned the settings to the currently-loaded state.
+    captureModuleSnapshot();
+
+    // ── Skeleton (hooks) ───────────────────────────────────────────────────
+    const sk = createSkeleton();
+
+    // Order matters for tool_result compose chain:
+    //   1. redact → normalize-codeblocks → externalize → track-mtime → inject-agents-md → image-vision → wakatime
+    // The first module registered for a given event runs first (compose chain).
+    if (isModuleEnabled("secretRedaction")) setupRedact(sk);
+    sk.register(normalizeCodeblocksModule);
+    sk.register(thinkingLabelStripModule);
+    sk.register(externalizeModule);
+    sk.register(trackMtimeModule);
+    sk.register(injectAgentsMdModule);
+    sk.register(imageVisionModule);
+
+    // session_start handlers (parallel)
+    // pi-tool-filter must register first so native tools are dropped before
+    // anything else inspects the tool list.
+    sk.register(piToolFilterModule);
+    sk.register(sessionTitleModule);
+    if (isModuleEnabled("atOverride")) sk.register(smartAtModule);
+    if (isModuleEnabled("wakatime")) sk.register(wakatimeModule);
+
+    // Compaction + RTK (these also install their own pi.on via setup<>()).
+    setupCompaction(sk);
+    if (isModuleEnabled("rtk")) setupRtk(sk);
+    if (isModuleEnabled("wakatime")) setupWakatime(sk, pi);
+
+    // ── Tools (conditional on module switches) ────────────────────────────
+    if (isModuleEnabled("patchOverrideEdit")) registerPatchTool(pi);
+    if (isModuleEnabled("lsp")) {
+        const lspDeps = collectLspDependencyStatuses(process.cwd());
+        if (lspDeps.some((d) => d.state === "ok")) {
+            registerLspTools(pi, new LspServerManager());
+        }
+        for (const dep of lspDeps) {
+            if (dep.state !== "ok") {
+                sk.declareMissing({
+                    name: dep.label,
+                    module: "lsp",
+                    hint: dep.detail,
+                });
+            }
+        }
     }
-    const configs = resolveMcpConfigs(process.cwd()).filter(s => s.enabled);
-    // Per-server readiness: cache hit → register from cache (fast).
-    // Cache miss → connect synchronously, write cache, then register
-    // live tools. This blocks startup only for cache-miss servers.
-    // Skip servers whose binary is missing (dependency not met).
-    for (const config of configs) {
-      if (!canRegisterMcpServer(config, mcpDeps)) continue;
-      await ensureMcpServerReady(pi, config, process.cwd());
+    if (isModuleEnabled("ask")) registerAskTool(pi);
+
+    // MCP: hook, tools, and /mcp command are gated together. Disabling the
+    // module means no session_start handler runs, no tools register, no
+    // /mcp command is available, and no background connections are attempted.
+    if (isModuleEnabled("mcp")) {
+        // One-time migration: legacy global MCP configs in
+        // ~/.pi/agent/decorated-pi.json move to ~/.pi/agent/mcp.json. Run
+        // explicitly here so `loadGlobalMcpConfigs` stays pure.
+        migrateLegacyGlobalMcpConfig();
+        sk.register(mcpModule);
+        const mcpDeps = collectMcpDependencyStatuses(process.cwd());
+        for (const dep of mcpDeps) {
+            if (dep.state !== "ok") {
+                sk.declareMissing({
+                    name: dep.label, // binary name (e.g. "codegraph")
+                    module: "mcp",
+                    hint: dep.detail,
+                });
+            }
+        }
+        const configs = resolveMcpConfigs(process.cwd()).filter(
+            (s) => s.enabled,
+        );
+        // Per-server readiness: cache hit → register from cache (fast).
+        // Cache miss → connect synchronously, write cache, then register
+        // live tools. This blocks startup only for cache-miss servers.
+        // Skip servers whose binary is missing (dependency not met).
+        for (const config of configs) {
+            if (!canRegisterMcpServer(config, mcpDeps)) continue;
+            await ensureMcpServerReady(pi, config, process.cwd());
+        }
+        registerMcpStatusCommand(pi);
     }
-    registerMcpStatusCommand(pi);
-  }
 
-  // ── Builtin skills (travel with the plugin in every project) ─────────────
-  installBuiltinSkills(pi);
+    // ── Builtin skills (travel with the plugin in every project) ─────────────
+    installBuiltinSkills(pi);
 
-  // ── System-prompt guidelines (single handler, array order = prompt order) ──
-  installGuidelines(pi);
+    // ── System-prompt guidelines (single handler, array order = prompt order) ──
+    installGuidelines(pi);
 
-  // ── Commands ──────────────────────────────────────────────────────────
-  registerDpModelCommand(pi);
-  registerDpSettingsCommand(pi);
-  if (isModuleEnabled("retry")) registerRetryCommand(pi);
-  if (isModuleEnabled("usage")) registerUsageCommand(pi);
+    // ── Commands ──────────────────────────────────────────────────────────
+    registerDpModelCommand(pi);
+    registerDpSettingsCommand(pi);
+    if (isModuleEnabled("retry")) registerRetryCommand(pi);
+    if (isModuleEnabled("usage")) registerUsageCommand(pi);
 
-  // ── Install skeleton (last) ────────────────────────────────────────────
-  sk.install(pi);
+    // ── Install skeleton (last) ────────────────────────────────────────────
+    sk.install(pi);
 }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "decorated-pi",
-    "version": "0.7.3",
+    "version": "0.8.0",
     "description": "decorated-pi is a practical enhancement pack for pi coding agent — token-efficient workflow, cache-friendly design, and smarter tools.",
     "keywords": [
         "pi",
@@ -26,7 +26,9 @@
     "dependencies": {
         "@ff-labs/fff-node": "^0.9.4",
         "@modelcontextprotocol/sdk": "^1.29.0",
+        "chardet": "^2.2.0",
         "file-type": "^21.3.4",
+        "iconv-lite": "^0.7.2",
         "openai": "^6.37.0"
     },
     "peerDependencies": {

package/tools/ask/index.ts

@@ -17,7 +17,7 @@ const askQuestionSchema = Type.Object({
     { description: "text = free input, single = one option, multi = many options" },
   ),
   question: Type.String({ description: "Question text shown to the user." }),
-  options: Type.Optional(Type.Array(Type.String(), { description: "Options for single or multi choice." })),
+  options: Type.Optional(Type.Array(Type.String(), { description: "Options for single or multi choice. Each option MUST be a plain string (not an object). Example: [\"选项A\", \"选项B\", \"选项C\"]. The user picks by index; do NOT pass {id,text} objects." })),
   default: Type.Optional(Type.String({ description: "Default answer. For multi, comma-separated values." })),
 });
 

package/tools/patch/core.ts

@@ -11,7 +11,12 @@
 
 import * as fs from "node:fs";
 import * as path from "node:path";
-import * as fsPromises from "node:fs/promises";
+import {
+  detectFileEncoding,
+  readFileDecoded,
+  writeFileEncoded,
+  type FileEncoding,
+} from "./encoding.js";
 
 // ═══════════════════════════════════════════════════════════════════════════
 // Types
@@ -63,6 +68,12 @@ export interface ReplacementInfo {
   oldLines: string[];
   /** The new lines that replaced them */
   newLines: string[];
+  /** Verbatim normalized replacement text (for byte-faithful writeback). */
+  newStr?: string;
+  /** Offset in normalized content where the matched region starts (writeback). */
+  normStart?: number;
+  /** Offset one past the matched region in normalized content (writeback). */
+  normEnd?: number;
   /** Optional anchor text (first line only, for hunk display) */
   anchor?: string;
   /** Anchor was provided but not found, and patch fell back to global old_str search */
@@ -156,13 +167,20 @@ function applyOverwrite(
   content: string,
   result: PatchResult,
 ): void {
-  const oldContent = fs.existsSync(absPath) ? fs.readFileSync(absPath, "utf8") : "";
+  // Detect encoding from the existing file so we round-trip in the same
+  // bytes. New files default to UTF-8 (no BOM).
+  const enc: FileEncoding | null = fs.existsSync(absPath)
+    ? detectFileEncoding(absPath)
+    : null;
+  const oldContent = enc ? readFileDecoded(absPath, enc) : "";
 
   // Write to temp file in the same directory (same filesystem → mv is atomic)
   ensureParentDir(absPath);
   const dir = path.dirname(absPath);
   const tmpName = path.join(dir, `.pi-patch-${randomId()}.tmp`);
-  fs.writeFileSync(tmpName, content, "utf8");
+  // Overwrite semantics: write exactly what the caller passed, in the
+  // detected encoding (UTF-8 for new files).
+  writeFileEncoded(tmpName, content, enc ?? { encoding: "utf-8", hasBOM: false, isUtf8: true });
   fs.renameSync(tmpName, absPath);
 
   if (oldContent) {
@@ -287,8 +305,8 @@ async function applyEdits(
     throw new ApplyError(`Cannot patch directory: ${displayPath}`);
   }
 
-  const rawContent = fs.readFileSync(absPath, "utf8");
-  const lineEnding = detectLineEnding(rawContent);
+  const enc = detectFileEncoding(absPath);
+  const rawContent = readFileDecoded(absPath, enc);
   const originalContent = normalizeLineEndings(rawContent);
 
   // Precompute line offsets for the original file (used throughout)
@@ -349,6 +367,8 @@ async function applyEdits(
 
       const oldStartLine = lineAtOffset(lineOffsets, matchIdx - cumulativeOffset);
       const oldEndLine = lineAtOffset(lineOffsets, matchIdx - cumulativeOffset + oldNorm.length - 1);
+      const normStart = matchIdx - cumulativeOffset;
+      const normEnd = normStart + oldNorm.length;
 
       content =
         content.substring(0, matchIdx) +
@@ -364,6 +384,9 @@ async function applyEdits(
         newEndLine: 0,
         oldLines: oldNorm.split("\n").filter((l, i, arr) => !(i === arr.length - 1 && l === "")),
         newLines: newNorm.split("\n").filter((l, i, arr) => !(i === arr.length - 1 && l === "")),
+        newStr: newNorm,
+        normStart,
+        normEnd,
         anchor: displayAnchor ? displayAnchor.split("\n")[0] : undefined,
         anchorMissing,
       });
@@ -392,12 +415,18 @@ async function applyEdits(
       result.diff = fileDiff;
     }
 
-    const finalContent = restoreLineEndings(content, lineEnding);
-    if (lineEnding === "\r\n" && rawContent.includes("\r\n")) {
-      result.warnings.push(`${displayPath}: CRLF line endings were normalized to LF during editing.`);
-    }
+    const finalContent = spliceOntoRaw(
+      rawContent,
+      cleanReplacements
+        .map((r) => ({
+          normStart: r.normStart ?? 0,
+          normEnd: r.normEnd ?? 0,
+          newStr: r.newStr ?? r.newLines.join("\n"),
+        }))
+        .sort((a, b) => a.normStart - b.normStart),
+    );
 
-    fs.writeFileSync(absPath, finalContent, "utf8");
+    writeFileEncoded(absPath, finalContent, enc);
     result.modified.push(displayPath);
     result.replacements.set(displayPath, cleanReplacements);
     return;
@@ -495,14 +524,16 @@ async function applyEdits(
     result.diff = fileDiff;
   }
 
-  // Restore line endings
-  const finalContent = restoreLineEndings(content, lineEnding);
-
-  if (lineEnding === "\r\n" && rawContent.includes("\r\n")) {
-    result.warnings.push(`${displayPath}: CRLF line endings were normalized to LF during editing.`);
-  }
+  const finalContent = spliceOntoRaw(
+    rawContent,
+    sorted.map((p) => ({
+      normStart: p.matchIdx,
+      normEnd: p.matchIdx + p.oldNorm.length,
+      newStr: p.newNorm,
+    })),
+  );
 
-  fs.writeFileSync(absPath, finalContent, "utf8");
+  writeFileEncoded(absPath, finalContent, enc);
   result.modified.push(displayPath);
   result.replacements.set(displayPath, replacements);
 }
@@ -541,7 +572,8 @@ export async function computePatchPreview(
         return { error: "File not found" };
       }
 
-      const rawContent = await fsPromises.readFile(absPath, "utf8");
+      const enc = detectFileEncoding(absPath);
+      const rawContent = readFileDecoded(absPath, enc);
     const lineOffsets = buildLineOffsets(rawContent);
     const totalLines = lineOffsets.length - 1;
     let content = normalizeLineEndings(rawContent);
@@ -1053,16 +1085,68 @@ function ensureParentDir(absPath: string): void {
   if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
 }
 
-function detectLineEnding(content: string): string {
-  return content.includes("\r\n") ? "\r\n" : "\n";
-}
-
 function normalizeLineEndings(text: string): string {
   return text.replace(/\r\n/g, "\n").replace(/\r/g, "\n");
 }
 
-function restoreLineEndings(text: string, ending: string): string {
-  return ending === "\r\n" ? text.replace(/\n/g, "\r\n") : text;
+/** A replacement expressed in coordinates of the normalized (\n-only) content,
+ *  paired with the exact new_str to drop in. Used by spliceOntoRaw to rebuild
+ *  the file byte-for-byte on the original rawContent. */
+interface RawSplice {
+  /** Offset in the normalized content where the matched text starts. */
+  normStart: number;
+  /** Offset in the normalized content one past the matched text. */
+  normEnd: number;
+  /** Verbatim replacement text (already normalized to \n). */
+  newStr: string;
+}
+
+/** Map every index of the normalized content to its offset in rawContent.
+ *  The two strings differ only by `\r` bytes (CRLF→LF normalization removed
+ *  them), so we walk both in lockstep. O(n). */
+function buildNormToRawMap(raw: string, norm: string): Int32Array {
+  const map = new Int32Array(norm.length + 1);
+  let ri = 0;
+  for (let ni = 0; ni <= norm.length; ni++) {
+    // Skip any `\r` in raw that the normalization folded into `\n`.
+    // norm[ni] corresponds to raw[ri]; when norm advances past a `\n` that
+    // came from `\r\n`, raw must skip the `\r` first.
+    if (ni < norm.length) {
+      map[ni] = ri;
+      const ch = norm.charCodeAt(ni);
+      const rawCh = raw.charCodeAt(ri);
+      if (ch === 10 /* \n */ && rawCh === 13 /* \r */) {
+        // raw had \r\n; advance past \r then \n
+        ri += 2;
+      } else {
+        ri += 1;
+      }
+    } else {
+      map[ni] = raw.length;
+    }
+  }
+  return map;
+}
+
+/** Rebuild the file on top of the original rawContent: untouched regions keep
+ *  their original bytes (including CRLF / mixed endings), edited regions get
+ *  the verbatim newStr the caller supplied. Splices must be sorted by
+ *  normStart and non-overlapping. */
+function spliceOntoRaw(rawContent: string, splices: RawSplice[]): string {
+  if (splices.length === 0) return rawContent;
+  const norm = normalizeLineEndings(rawContent);
+  const map = buildNormToRawMap(rawContent, norm);
+  let out = "";
+  let rawCursor = 0;
+  for (const s of splices) {
+    const rawStart = map[s.normStart] ?? 0;
+    const rawEnd = map[s.normEnd] ?? rawContent.length;
+    if (rawStart > rawCursor) out += rawContent.substring(rawCursor, rawStart);
+    out += s.newStr;
+    rawCursor = rawEnd;
+  }
+  if (rawCursor < rawContent.length) out += rawContent.substring(rawCursor);
+  return out;
 }
 
 // ═══════════════════════════════════════════════════════════════════════════
@@ -1203,6 +1287,9 @@ function collapseSequentialReplacements(
         newEndLine: merged.oldStartLine + next.newLines.length - 1,
         oldLines: merged.oldLines,
         newLines: next.newLines,
+        newStr: next.newStr,
+        normStart: merged.normStart,
+        normEnd: merged.normEnd,
         anchor: undefined,
         anchorMissing: merged.anchorMissing || next.anchorMissing,
       };
@@ -1530,4 +1617,6 @@ export const __patchCoreTest = {
   truncate,
   collapseSequentialReplacements,
   generateReplacementDiff,
+  spliceOntoRaw,
+  buildNormToRawMap,
 };

package/tools/patch/encoding.ts

@@ -0,0 +1,172 @@
+/**
+ * Encoding detection and conversion for patch.
+ *
+ * - chardet analyses raw bytes (BOM + histogram heuristics) → encoding name.
+ * - iconv-lite decodes/encodes between Buffer and string.
+ *
+ * For UTF-8 (the overwhelmingly common case) we bypass iconv-lite and use
+ * Node's native string<->Buffer conversion — zero behaviour change vs the
+ * pre-encoding-aware tool, and no iconv runtime cost for ASCII/UTF-8 files.
+ */
+
+import * as fs from "node:fs";
+import chardet, { type Match } from "chardet";
+import * as iconv from "iconv-lite";
+
+export interface FileEncoding {
+  /** iconv-lite-compatible encoding name (e.g. "utf-8", "gb18030", "utf-16le"). */
+  encoding: string;
+  /** True when the file starts with a BOM that should be preserved on write. */
+  hasBOM: boolean;
+  /** True when the file is UTF-8 (with or without BOM). Native path is used. */
+  isUtf8: boolean;
+}
+
+// chardet emits "UTF-8" for plain UTF-8. iconv-lite accepts any case.
+const UTF8_ALIASES = new Set(["utf-8", "utf8", "ascii"]);
+const UTF8_BOM = Buffer.from([0xef, 0xbb, 0xbf]);
+const UTF16LE_BOM = Buffer.from([0xff, 0xfe]);
+const UTF16BE_BOM = Buffer.from([0xfe, 0xff]);
+
+/** Returns true if `buf` is valid UTF-8 (every byte parses, no U+FFFD
+ *  replacement). Pure-ASCII and any well-formed UTF-8 qualify. */
+function isValidUtf8(buf: Buffer): boolean {
+  // Writing then reading back would corrupt invalid sequences into U+FFFD;
+  // detect that by comparing round-trip byte lengths. The cheap, correct
+  // check is Node's built-in: toString('utf8') replaces bad sequences with
+  // U+FFFD, so we scan the decoded string for it.
+  if (buf.length === 0) return true;
+  const s = buf.toString("utf8");
+  return !s.includes("\uFFFD");
+}
+
+function looksLikeUtf8Bom(buf: Buffer): boolean {
+  return buf.length >= 3 && buf[0] === 0xef && buf[1] === 0xbb && buf[2] === 0xbf;
+}
+
+function looksLikeUtf16LeBom(buf: Buffer): boolean {
+  return buf.length >= 2 && buf[0] === 0xff && buf[1] === 0xfe;
+}
+
+function looksLikeUtf16BeBom(buf: Buffer): boolean {
+  return buf.length >= 2 && buf[0] === 0xfe && buf[1] === 0xff;
+}
+
+/**
+ * Detect a file's encoding by reading its bytes.
+ * Falls back to UTF-8 if detection fails or the detected encoding is not
+ * supported by iconv-lite.
+ */
+export function detectFileEncoding(filePath: string): FileEncoding {
+  const buf = fs.readFileSync(filePath);
+
+  // BOM takes precedence for the UTF-16/UTF-32 family (endianness matters).
+  if (looksLikeUtf8Bom(buf)) {
+    return { encoding: "utf-8", hasBOM: true, isUtf8: true };
+  }
+  if (looksLikeUtf16LeBom(buf)) {
+    return { encoding: "utf-16le", hasBOM: true, isUtf8: false };
+  }
+  if (looksLikeUtf16BeBom(buf)) {
+    return { encoding: "utf-16be", hasBOM: true, isUtf8: false };
+  }
+
+  // Any well-formed UTF-8 (including pure ASCII) is UTF-8. This short-circuits
+  // chardet's tendency to mis-classify short or ASCII-only buffers as exotic
+  // encodings (e.g. 2-byte "x\n" as utf-32le).
+  if (isValidUtf8(buf)) {
+    return { encoding: "utf-8", hasBOM: false, isUtf8: true };
+  }
+
+  // Not valid UTF-8 — trust chardet's heuristic for the legacy encoding.
+  // chardet mis-classifies short or mixed CJK samples (it often ties GBK,
+  // Big5, Shift_JIS, EUC-JP, EUC-KR at the same low confidence). Use the
+  // full ranked list and prefer Chinese encodings — GB18030 is a superset
+  // of GBK/GB2312 and the most common non-UTF-8 encoding for Chinese text,
+  // which is the primary use case for this tool.
+  const candidates = chardet.analyse(buf);
+  let encoding = pickLegacyEncoding(candidates);
+  let hasBOM = false;
+
+  // Verify iconv-lite actually ships a codec for this label; otherwise fall
+  // back to UTF-8 (the previous behaviour) instead of throwing mid-edit.
+  if (!iconv.encodingExists(encoding)) {
+    encoding = "utf-8";
+    hasBOM = false;
+  }
+
+  // Last-resort safety net: if the chosen encoding still produces U+FFFD
+  // replacement chars on decode, the file is not actually in that encoding
+  // and we would corrupt it on write-back. Fall back to ISO-8859-1 (Latin1),
+  // which is a lossless 1:1 byte<->code-point mapping for 0x00–0xFF and can
+  // never introduce U+FFFD — round-tripping is byte-identical.
+  if (!UTF8_ALIASES.has(encoding) && iconv.decode(buf, encoding).includes("\uFFFD")) {
+    encoding = "iso-8859-1";
+    hasBOM = false;
+  }
+
+  return {
+    encoding,
+    hasBOM,
+    isUtf8: UTF8_ALIASES.has(encoding),
+  };
+}
+
+/** Preference order for breaking chardet ties. Chinese first (GB18030 is a
+ *  superset of GBK/GB2312), then other CJK, then everything else. */
+const ENCODING_PRIORITY: string[] = [
+  "gb18030", "gbk", "gb2312",  // Chinese (mainland)
+  "big5",                       // Chinese (traditional)
+  "shift_jis", "euc-jp",        // Japanese
+  "euc-kr", "windows-949",      // Korean
+  "windows-1252", "iso-8859-1", // Western (rarely reached: isValidUtf8 short-circuits)
+];
+
+function pickLegacyEncoding(candidates: Match[]): string {
+  if (candidates.length === 0) return "iso-8859-1";
+  const top = candidates[0]!.confidence;
+  // Keep only candidates tied with the top confidence (±5 tolerance —
+  // chardet's scores are coarse).
+  const tied = candidates.filter((c) => Math.abs(c.confidence - top) <= 5);
+  for (const pref of ENCODING_PRIORITY) {
+    const hit = tied.find((c) => c.name.toLowerCase() === pref);
+    if (hit) return pref;
+  }
+  // We only reach here when isValidUtf8 already failed (there are high
+  // bytes), so "ascii" / "UTF-8" from chardet are wrong. ISO-8859-1 is the
+  // safe single-byte fallback — lossless for 0x00–0xFF, no U+FFFD.
+  return "iso-8859-1";
+}
+
+/** Read a file as a string, decoding via the detected encoding. */
+export function readFileDecoded(filePath: string, enc: FileEncoding): string {
+  const buf = fs.readFileSync(filePath);
+  if (enc.isUtf8) {
+    // Native path: identical to the old fs.readFileSync(p, "utf8").
+    // For UTF-8 with BOM, slice off the BOM so it does not leak into content
+    // matching (it would prefix the first line and break old_str lookups).
+    const slice = enc.hasBOM ? buf.subarray(3) : buf;
+    return slice.toString("utf8");
+  }
+  return iconv.decode(buf, enc.encoding, { stripBOM: true });
+}
+
+/** Write a string back to a file, encoding via the detected encoding. */
+export function writeFileEncoded(
+  filePath: string,
+  content: string,
+  enc: FileEncoding,
+): void {
+  if (enc.isUtf8) {
+    if (enc.hasBOM) {
+      const body = Buffer.from(content, "utf8");
+      const out = Buffer.concat([UTF8_BOM, body]);
+      fs.writeFileSync(filePath, out);
+      return;
+    }
+    fs.writeFileSync(filePath, content, "utf8");
+    return;
+  }
+  const buf = iconv.encode(content, enc.encoding, { addBOM: enc.hasBOM });
+  fs.writeFileSync(filePath, buf);
+}