dsh-better-edit 0.5.0 → 0.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -1,11 +1,18 @@
1
+ ## [0.5.1](https://github.com/Rianico/dsh-better-edit/compare/v0.5.0...v0.5.1) (2026-08-30)
2
+
3
+ ### Bug Fixes
4
+
5
+ * **encoding:** support non-UTF-8 text files via VS Code model ([#34](https://github.com/Rianico/dsh-better-edit/issues/34)) ([#35](https://github.com/Rianico/dsh-better-edit/issues/35)) ([dd3e553](https://github.com/Rianico/dsh-better-edit/commit/dd3e553a3877e47b08aa2ba6336115e80b5d96f5))
6
+ * **lint:** address biome blockers for release ([00b9270](https://github.com/Rianico/dsh-better-edit/commit/00b92704ac4093a627754e698bb1968acdc3d4e8))
7
+ * remove unused exitHandlerRegistered ([f6a29f3](https://github.com/Rianico/dsh-better-edit/commit/f6a29f3f1dd694e4bb7c55a98d0eba80bc32a58f))
8
+ * **scripts:** map fix to Fixed for Keep a Changelog ([bcd1850](https://github.com/Rianico/dsh-better-edit/commit/bcd18501e280b10d5075108c034de7352ece6922))
9
+
1
10
  # Changelog
2
11
 
3
12
  All notable changes to the `dsh-better-edit` plugin will be documented in this file.
4
13
 
5
14
  Entries link to the originating spec issue in [pi-hashline-edit-lsz](https://github.com/Rianico/pi-hashline-edit-lsz) where one exists.
6
15
 
7
- ## [Unreleased]
8
-
9
16
  ## [0.5.0] - 2026-08-29
10
17
 
11
18
  ### Features
package/README.md CHANGED
@@ -124,7 +124,7 @@ Chained edits stay cheap — anchors for untouched lines remain valid, diff/echo
124
124
  { "path": "src/main.ts", "edits": [["a1b", "a1b", "new line 1\n"], ["c3d", "c3d", "new line 2"]] }
125
125
  ```
126
126
 
127
- One fails, none write (`[E_BATCH_ABORT]`).
127
+ One fails, none write (`[E_BATCH_ABORT]`). Encoding errors `[E_BAD_ENCODING]`/`[E_DECODE_FAILED]` are also emitted.
128
128
 
129
129
  > [!TIP]
130
130
  > **Want proof before you install?** The upstream [23/23 tool battery](https://github.com/Rianico/pi-better-edit/blob/main/benchmarks/README.md) runs with no LLM — stale edits are rejected before they corrupt a file, on every run. Same hashline algorithm, same verification.
package/lib/contract.js CHANGED
@@ -64,7 +64,7 @@ function describeReceived(input) {
64
64
  }
65
65
  // ---- filed sets (declared once) ---------------------------------------------
66
66
  const EDIT_KS = new Set(["path", "edits", "sandbox_permissions", "justification"]);
67
- const READ_KS = new Set(["path", "offset", "limit"]);
67
+ const READ_KS = new Set(["path", "offset", "limit", "encoding"]);
68
68
  // ---- normalization -----------------------------------------------------------
69
69
  /**
70
70
  * Normalize `file_path` → `path` alias on the request record and tuple edits → objects.
@@ -0,0 +1,308 @@
1
+ /**
2
+ * VS Code encoding model — pure helpers for BOM→UTF-8→model-assisted top-3.
3
+ * No jschardet; heuristic scoring + iconv-lite rendering.
4
+ * @module dsh-better-edit/encoding
5
+ */
6
+ import iconv from "iconv-lite";
7
+ export const SUPPORTED_ENCODINGS = [
8
+ "gbk",
9
+ "big5",
10
+ "shift_jis",
11
+ "euc-kr",
12
+ "windows-1251",
13
+ "iso-8859-1",
14
+ ];
15
+ const ALIASES = {
16
+ utf8: "utf8",
17
+ utf8bom: "utf8bom",
18
+ utf16le: "utf16le",
19
+ utf16be: "utf16be",
20
+ utf32le: "utf32le",
21
+ utf32be: "utf32be",
22
+ gbk: "gbk",
23
+ gb18030: "gbk",
24
+ gb2312: "gbk",
25
+ big5: "big5",
26
+ shiftjis: "shift_jis",
27
+ shift_jis: "shift_jis",
28
+ sjis: "shift_jis",
29
+ euckr: "euc-kr",
30
+ "euc-kr": "euc-kr",
31
+ windows1251: "windows-1251",
32
+ cp1251: "windows-1251",
33
+ "windows-1251": "windows-1251",
34
+ iso88591: "iso-8859-1",
35
+ "iso-8859-1": "iso-8859-1",
36
+ latin1: "iso-8859-1",
37
+ };
38
+ /** Canonicalize case-insensitive encoding param, strip -_ . */
39
+ export function normalizeEncoding(input) {
40
+ const key = input.trim().toLowerCase().replace(/[-_]/g, "");
41
+ // need original with dashes for lookup: try both
42
+ const candidates = [key, input.trim().toLowerCase()];
43
+ for (const c of candidates) {
44
+ const v = ALIASES[c] ?? ALIASES[key];
45
+ if (v)
46
+ return v;
47
+ }
48
+ // also try direct lowercased
49
+ const lower = input.trim().toLowerCase();
50
+ if (ALIASES[lower])
51
+ return ALIASES[lower];
52
+ return undefined;
53
+ }
54
+ export function isSupportedEncoding(enc) {
55
+ const norm = normalizeEncoding(enc);
56
+ if (!norm)
57
+ return false;
58
+ return SUPPORTED_ENCODINGS.includes(norm) || ["utf8", "utf8bom", "utf16le", "utf16be", "utf32le", "utf32be"].includes(norm);
59
+ }
60
+ export function detectBom(bytes) {
61
+ if (bytes.length >= 3 && bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf)
62
+ return { encoding: "utf8bom", bomLen: 3, hasBOM: true };
63
+ if (bytes.length >= 4 && bytes[0] === 0xff && bytes[1] === 0xfe && bytes[2] === 0x00 && bytes[3] === 0x00)
64
+ return { encoding: "utf32le", bomLen: 4, hasBOM: true };
65
+ if (bytes.length >= 4 && bytes[0] === 0x00 && bytes[1] === 0x00 && bytes[2] === 0xfe && bytes[3] === 0xff)
66
+ return { encoding: "utf32be", bomLen: 4, hasBOM: true };
67
+ if (bytes.length >= 2 && bytes[0] === 0xff && bytes[1] === 0xfe)
68
+ return { encoding: "utf16le", bomLen: 2, hasBOM: true };
69
+ if (bytes.length >= 2 && bytes[0] === 0xfe && bytes[1] === 0xff)
70
+ return { encoding: "utf16be", bomLen: 2, hasBOM: true };
71
+ return undefined;
72
+ }
73
+ export function isValidUtf8(bytes) {
74
+ try {
75
+ new TextDecoder("utf-8", { fatal: true }).decode(bytes);
76
+ return true;
77
+ }
78
+ catch {
79
+ return false;
80
+ }
81
+ }
82
+ function printableRatio(text) {
83
+ if (text.length === 0)
84
+ return 1;
85
+ let printable = 0;
86
+ for (const ch of text) {
87
+ const code = ch.charCodeAt(0);
88
+ if (code === 10 || code === 13 || code === 9)
89
+ printable += 1;
90
+ else if (code >= 32 && code !== 127)
91
+ printable += 1;
92
+ else if (code > 127)
93
+ printable += 1; // non-ascii considered printable if not replacement
94
+ }
95
+ return printable / text.length;
96
+ }
97
+ function scriptBonus(text, enc) {
98
+ let bonus = 0;
99
+ const cjk = (text.match(/[\u4e00-\u9fff]/g) || []).length;
100
+ const cyrillic = (text.match(/[\u0400-\u04ff]/g) || []).length;
101
+ const hiragana = (text.match(/[\u3040-\u309f\u30a0-\u30ff]/g) || []).length;
102
+ const hangul = (text.match(/[\uac00-\ud7af]/g) || []).length;
103
+ if (enc === "gbk" || enc === "big5")
104
+ bonus += cjk * 5;
105
+ if (enc === "windows-1251")
106
+ bonus += cyrillic * 2;
107
+ if (enc === "shift_jis")
108
+ bonus += hiragana * 8;
109
+ if (enc === "euc-kr")
110
+ bonus += hangul * 4;
111
+ return bonus;
112
+ }
113
+ function smartSlice(text) {
114
+ // seek first non-ascii char, slice ±32 around it, truncate to 50
115
+ let idx = -1;
116
+ for (let i = 0; i < text.length; i += 1) {
117
+ if (text.charCodeAt(i) > 127) {
118
+ idx = i;
119
+ break;
120
+ }
121
+ }
122
+ if (idx === -1)
123
+ return text.slice(0, 50);
124
+ const start = Math.max(0, idx - 32);
125
+ const slice = text.slice(start, start + 64);
126
+ return slice.slice(0, 50).replace(/\s+/g, " ").trim();
127
+ }
128
+ export function scoreText(text, enc) {
129
+ if (text.includes("\uFFFD"))
130
+ return -1000;
131
+ const ratio = printableRatio(text);
132
+ if (ratio < 0.85)
133
+ return -500 + ratio * 10;
134
+ let score = ratio * 10;
135
+ score += scriptBonus(text, enc) * 0.5;
136
+ return score;
137
+ }
138
+ /** Decode bytes under enc via iconv-lite, return text or undefined on failure. */
139
+ export function decodeBytes(bytes, enc) {
140
+ try {
141
+ if (enc === "utf8" || enc === "utf8bom")
142
+ return new TextDecoder("utf-8", { fatal: true }).decode(bytes);
143
+ if (enc === "utf16le")
144
+ return new TextDecoder("utf-16le").decode(bytes);
145
+ if (enc === "utf16be")
146
+ return new TextDecoder("utf-16be").decode(bytes);
147
+ // iconv-lite for legacy
148
+ return iconv.decode(Buffer.from(bytes), enc);
149
+ }
150
+ catch {
151
+ return undefined;
152
+ }
153
+ }
154
+ export function encodeText(text, enc) {
155
+ try {
156
+ if (enc === "utf8" || enc === "utf8bom")
157
+ return new TextEncoder().encode(text);
158
+ if (enc === "utf16le")
159
+ return new Uint8Array(Buffer.from(text, "utf16le"));
160
+ if (enc === "utf16be") {
161
+ // Node no utf16be, use iconv
162
+ return new Uint8Array(iconv.encode(text, "utf16be"));
163
+ }
164
+ return new Uint8Array(iconv.encode(text, enc));
165
+ }
166
+ catch {
167
+ return undefined;
168
+ }
169
+ }
170
+ /** Top-3 candidates for autoGuess: decode each allowlist enc, score, sort. */
171
+ export function top3Candidates(bytes, allowlist) {
172
+ const candidates = [];
173
+ // handle BOM case separately? caller should handle BOM before calling.
174
+ for (const raw of allowlist) {
175
+ const enc = normalizeEncoding(raw) ?? raw;
176
+ const text = decodeBytes(bytes, enc);
177
+ if (text === undefined)
178
+ continue;
179
+ const sample = smartSlice(text);
180
+ const score = scoreText(text, enc);
181
+ candidates.push({ encoding: enc, sample, score });
182
+ }
183
+ candidates.sort((a, b) => b.score - a.score);
184
+ return candidates.slice(0, 3);
185
+ }
186
+ export function hasReplacementChar(text) {
187
+ return text.includes("\uFFFD");
188
+ }
189
+ /**
190
+ * Try chardet (runk/node-chardet) for autoGuess when available.
191
+ * Dynamically imports 'chardet' (optionalDep) and returns canonical encoding
192
+ * if confidence >=45 and allowlist-accepted and decodable without �.
193
+ * Falls back to heuristic when chardet unavailable/low confidence.
194
+ */
195
+ export async function detectWithChardet(bytes, allowlist) {
196
+ try {
197
+ const mod = await import("chardet");
198
+ // chardet ESM: default export has analyse, CJS: named
199
+ const chardetLike = mod.default ?? mod;
200
+ const analyse = chardetLike.analyse;
201
+ if (typeof analyse !== "function")
202
+ return undefined;
203
+ const buf = Buffer.isBuffer(bytes) ? bytes : Buffer.from(bytes);
204
+ const results = analyse(buf);
205
+ if (!Array.isArray(results) || results.length === 0)
206
+ return undefined;
207
+ const allowNorm = new Set(allowlist.map((a) => normalizeEncoding(a) ?? a.toLowerCase()));
208
+ for (const r of results) {
209
+ if (typeof r.name !== "string" || typeof r.confidence !== "number")
210
+ continue;
211
+ if (r.confidence < 45)
212
+ continue;
213
+ const norm = normalizeEncoding(r.name);
214
+ if (!norm)
215
+ continue;
216
+ if (!allowNorm.has(norm))
217
+ continue;
218
+ // iconv-lite existence check (use try decode)
219
+ const encForCheck = norm;
220
+ if (!iconv.encodingExists(encForCheck) && !iconv.encodingExists(r.name))
221
+ continue;
222
+ const text = decodeBytes(bytes, encForCheck);
223
+ if (text === undefined)
224
+ continue;
225
+ if (text.includes("\uFFFD"))
226
+ continue;
227
+ // extra guard: printable ratio
228
+ if (text.length > 0) {
229
+ let printable = 0;
230
+ for (const ch of text) {
231
+ const c = ch.charCodeAt(0);
232
+ if (c === 10 || c === 13 || c === 9)
233
+ printable += 1;
234
+ else if (c >= 32 && c !== 127)
235
+ printable += 1;
236
+ else if (c > 127)
237
+ printable += 1;
238
+ }
239
+ if (printable / text.length < 0.85)
240
+ continue;
241
+ }
242
+ return norm;
243
+ }
244
+ return undefined;
245
+ }
246
+ catch {
247
+ return undefined;
248
+ }
249
+ }
250
+ /**
251
+ * Chardet-based Top-3 for E_NOT_TEXT (primary) and autoGuess.
252
+ * Returns chardet candidates filtered by allowlist and decodable, else empty.
253
+ */
254
+ export async function chardetTop3Candidates(bytes, allowlist) {
255
+ try {
256
+ const mod = await import("chardet");
257
+ const chardetLike = mod.default ?? mod;
258
+ const analyse = chardetLike.analyse;
259
+ if (typeof analyse !== "function")
260
+ return [];
261
+ const buf = Buffer.isBuffer(bytes) ? bytes : Buffer.from(bytes);
262
+ const results = analyse(buf);
263
+ if (!Array.isArray(results) || results.length === 0)
264
+ return [];
265
+ const allowNorm = new Set(allowlist.map((a) => normalizeEncoding(a) ?? a.toLowerCase()));
266
+ const out = [];
267
+ for (const r of results) {
268
+ if (typeof r.name !== "string" || typeof r.confidence !== "number")
269
+ continue;
270
+ const norm = normalizeEncoding(r.name);
271
+ if (!norm)
272
+ continue;
273
+ if (!allowNorm.has(norm))
274
+ continue;
275
+ if (!iconv.encodingExists(norm) && !iconv.encodingExists(r.name))
276
+ continue;
277
+ const text = decodeBytes(bytes, norm);
278
+ if (text === undefined)
279
+ continue;
280
+ const sample = (() => {
281
+ let idx = -1;
282
+ for (let i = 0; i < text.length; i += 1)
283
+ if (text.charCodeAt(i) > 127) {
284
+ idx = i;
285
+ break;
286
+ }
287
+ if (idx === -1)
288
+ return text.slice(0, 50);
289
+ const start = Math.max(0, idx - 32);
290
+ return text.slice(start, start + 64).slice(0, 50).replace(/\s+/g, " ").trim();
291
+ })();
292
+ out.push({ encoding: norm, confidence: r.confidence, sample });
293
+ if (out.length >= 3)
294
+ break;
295
+ }
296
+ return out;
297
+ }
298
+ catch {
299
+ return [];
300
+ }
301
+ }
302
+ export async function getTop3Candidates(bytes, allowlist) {
303
+ const chardetCands = await chardetTop3Candidates(bytes, allowlist);
304
+ if (chardetCands.length > 0) {
305
+ return chardetCands.map((c) => ({ encoding: c.encoding, sample: c.sample, score: c.confidence }));
306
+ }
307
+ return top3Candidates(bytes, allowlist);
308
+ }
package/lib/file-view.js CHANGED
@@ -462,7 +462,7 @@ export async function preview(content, hashes, opts = {}, absolutePath) {
462
462
  export async function readView(io, path, cwd, opts = {}) {
463
463
  const { signal } = opts;
464
464
  const absolutePath = await io.resolve(path, cwd, signal);
465
- const rawText = await io.readText(absolutePath, signal);
465
+ const rawText = await io.readText(absolutePath, signal, opts.encoding);
466
466
  const { normalized, fileHashes, hadUtf8DecodeErrors, bom, originalEnding } = await normFromText({
467
467
  absolutePath,
468
468
  rawText,
package/lib/fs-bridge.js CHANGED
@@ -16,6 +16,8 @@
16
16
  */
17
17
  import { readFile } from "node:fs/promises";
18
18
  import { writeAtomic } from "./fs-write.js";
19
+ import { loadConfig } from "./store-config.js";
20
+ import { detectBom, isValidUtf8, decodeBytes, encodeText, normalizeEncoding, top3Candidates, isSupportedEncoding, detectWithChardet, chardetTop3Candidates, getTop3Candidates } from "./encoding.js";
19
21
  import { fileSnap } from "./file-reader.js";
20
22
  import { resolveTarget, toCwd } from "./paths.js";
21
23
  /**
@@ -37,7 +39,13 @@ export function mapFsError(error, displayPath) {
37
39
  throw new Error(`[E_ACCESS] Cannot access file: ${displayPath}`);
38
40
  }
39
41
  if (code === "FS_NOT_TEXT" || code === "FS_NOT_REGULAR_FILE") {
40
- throw new Error(`[E_NOT_TEXT] Path is not a readable UTF-8 text file: ${displayPath}. Hashline editing only supports text files.`);
42
+ throw new Error(`[E_NOT_TEXT] Path is not a readable UTF-8 text file: ${displayPath}. Hashline editing only supports text files. Try read({encoding: "gbk"}) or enable autoGuessEncoding.`);
43
+ }
44
+ if (code === "FS_BAD_ENCODING") {
45
+ throw new Error(`[E_BAD_ENCODING] Unknown encoding for ${displayPath}. Supported: utf8, gbk, big5, shift_jis, euc-kr, windows-1251, iso-8859-1`);
46
+ }
47
+ if (code === "FS_DECODE_FAILED") {
48
+ throw new Error(`[E_DECODE_FAILED] Bytes in ${displayPath} cannot be decoded with requested encoding.`);
41
49
  }
42
50
  if (code === "FS_STALE_VERSION") {
43
51
  throw new Error(`[E_RANGE_STALE] The file changed on disk since it was read (version guard rejected the write). Call read() to get fresh anchors, then retry.`);
@@ -54,6 +62,35 @@ export function mapFsError(error, displayPath) {
54
62
  const UTF8_BOM = "\uFEFF";
55
63
  const UTF8_BOM_BYTES = [0xef, 0xbb, 0xbf];
56
64
  const UTF8_BOM_LEN = UTF8_BOM_BYTES.length;
65
+ const encodingMemo = new Map();
66
+ const autoGuessFooterMemo = new Map();
67
+ export function getAutoGuessFooter(targetKey) {
68
+ return autoGuessFooterMemo.get(targetKey);
69
+ }
70
+ export function setAutoGuessFooter(targetKey, footer) {
71
+ autoGuessFooterMemo.set(targetKey, footer);
72
+ }
73
+ export function clearAutoGuessFooter(targetKey) {
74
+ if (targetKey)
75
+ autoGuessFooterMemo.delete(targetKey);
76
+ else
77
+ autoGuessFooterMemo.clear();
78
+ }
79
+ export function getEncodingState(targetKey) {
80
+ return encodingMemo.get(targetKey);
81
+ }
82
+ export function setEncodingState(targetKey, state) {
83
+ encodingMemo.set(targetKey, state);
84
+ }
85
+ export function clearEncodingState(targetKey) {
86
+ if (targetKey)
87
+ encodingMemo.delete(targetKey);
88
+ else
89
+ encodingMemo.clear();
90
+ }
91
+ function isFsNotText(error) {
92
+ return error instanceof Error && typeof error.code === "string" && (error.code === "FS_NOT_TEXT" || error.message.includes("FS_NOT_TEXT"));
93
+ }
57
94
  /**
58
95
  * Restore a UTF-8 BOM consumed by a backend's {@link TextDecoder}. The raw
59
96
  * byte seam reads whole files rather than prefixes, so first use stat size to
@@ -98,8 +135,34 @@ export function ctxFsIO(fs, ctx) {
98
135
  });
99
136
  return fs.processPath(target);
100
137
  },
101
- async readText(absolutePath, signal) {
138
+ async readText(absolutePath, signal, encodingHint) {
102
139
  try {
140
+ // explicit encoding override (Reopen with Encoding)
141
+ if (encodingHint) {
142
+ const norm = normalizeEncoding(encodingHint);
143
+ if (!norm)
144
+ throw new (await import("@deepseek-ai/dsh-fs")).FsError("bad encoding", "FS_BAD_ENCODING");
145
+ const target = await fs.resolve(absolutePath, { ...(signal === undefined ? {} : { signal }) });
146
+ const info = await fs.stat(target, signal);
147
+ const maxBytes = info?.size ?? 10 * 1024 * 1024;
148
+ const bytes = await fs.readBytes(target, signal, maxBytes);
149
+ const bom = detectBom(bytes);
150
+ const slice = bom ? bytes.subarray(bom.bomLen) : bytes;
151
+ // for utf16 etc, decode via helper
152
+ let decoded = decodeBytes(bom ? bytes : slice, bom ? bom.encoding : norm);
153
+ if (bom && (bom.encoding === "utf8bom" || bom.encoding === "utf16le" || bom.encoding === "utf16be")) {
154
+ decoded = decodeBytes(bytes, bom.encoding);
155
+ }
156
+ else if (norm) {
157
+ const off = bom ? bom.bomLen : 0;
158
+ decoded = decodeBytes(bytes.subarray(off), norm);
159
+ }
160
+ if (decoded === undefined)
161
+ throw new (await import("@deepseek-ai/dsh-fs")).FsError("decode failed", "FS_DECODE_FAILED");
162
+ const version = info?.version;
163
+ setEncodingState(String(target.targetKey ?? absolutePath), { encoding: norm, hasBOM: !!bom, version });
164
+ return decoded;
165
+ }
103
166
  const target = await fs.resolve(absolutePath, {
104
167
  ...(signal === undefined ? {} : { signal }),
105
168
  });
@@ -107,10 +170,115 @@ export function ctxFsIO(fs, ctx) {
107
170
  return await restoreStrippedUtf8Bom(fs, target, text, signal);
108
171
  }
109
172
  catch (error) {
173
+ // fallback for non-UTF8 when autoGuess enabled; when disabled surface top-3 + hint
174
+ const isNotText = error instanceof Error && error.code === "FS_NOT_TEXT";
175
+ if (isNotText) {
176
+ try {
177
+ const cfg = loadConfig();
178
+ if (cfg.autoGuessEncoding) {
179
+ const target = await fs.resolve(absolutePath, { ...(signal === undefined ? {} : { signal }) });
180
+ const info = await fs.stat(target, signal);
181
+ const maxBytes = info?.size ?? 10 * 1024 * 1024;
182
+ if (info && info.size !== undefined && info.size > maxBytes)
183
+ throw error;
184
+ const bytes = await fs.readBytes(target, signal, maxBytes);
185
+ const bom = detectBom(bytes);
186
+ if (bom) {
187
+ const dec = decodeBytes(bytes, bom.encoding);
188
+ if (dec !== undefined) {
189
+ setEncodingState(String(target.targetKey ?? absolutePath), { encoding: bom.encoding, hasBOM: true, version: info?.version });
190
+ return dec;
191
+ }
192
+ }
193
+ if (isValidUtf8(bytes)) {
194
+ const dec = new TextDecoder("utf-8", { fatal: true }).decode(bytes);
195
+ setEncodingState(String(target.targetKey ?? absolutePath), { encoding: "utf8", hasBOM: false, version: info?.version });
196
+ return dec;
197
+ }
198
+ // score allowlist top-3, pick best for now (model will pick via details.candidates in tool layer)
199
+ const allow = cfg.supportedEncodings;
200
+ // try chardet (optional dep) first for more accurate top-1 — with footer for mid-confidence
201
+ try {
202
+ const chardetCands = await chardetTop3Candidates(bytes, allow);
203
+ if (chardetCands.length > 0) {
204
+ const top = chardetCands[0];
205
+ const second = chardetCands[1];
206
+ const isMid = top.confidence < 70 || (second !== undefined && top.confidence - second.confidence < 10);
207
+ const dec = decodeBytes(bytes, top.encoding);
208
+ if (dec !== undefined && !dec.includes("\uFFFD")) {
209
+ let footer = "";
210
+ if (isMid) {
211
+ const candsStr = chardetCands.map((c) => `${c.encoding} ${c.confidence}`).join(", ");
212
+ footer = `\n\n[Auto-guessed: ${top.encoding} ${top.confidence}, candidates: ${candsStr} — re-read with read({encoding}) if garbled]`;
213
+ }
214
+ setEncodingState(String(target.targetKey ?? absolutePath), { encoding: top.encoding, hasBOM: false, version: info?.version });
215
+ if (footer)
216
+ setAutoGuessFooter(String(target.targetKey ?? absolutePath), footer);
217
+ return dec;
218
+ }
219
+ }
220
+ }
221
+ catch { } // biome-ignore: best-effort
222
+ const candidates = top3Candidates(bytes, allow);
223
+ if (candidates.length > 0) {
224
+ const best = candidates[0];
225
+ const second2 = candidates[1];
226
+ const isMidHeu = second2 !== undefined && best.score - second2.score < 10;
227
+ let footerHeu = "";
228
+ {
229
+ const candsStrHeu = candidates.map((c) => `${c.encoding} ${c.score.toFixed(0)}`).join(", ");
230
+ footerHeu = `\n\n[Auto-guessed: ${best.encoding} ${best.score.toFixed(0)}, candidates: ${candsStrHeu} — re-read with read({encoding}) if garbled]`;
231
+ }
232
+ const decHeu = decodeBytes(bytes, best.encoding);
233
+ if (decHeu !== undefined) {
234
+ setEncodingState(String(target.targetKey ?? absolutePath), { encoding: best.encoding, hasBOM: false, version: info?.version });
235
+ if (footerHeu)
236
+ setAutoGuessFooter(String(target.targetKey ?? absolutePath), footerHeu);
237
+ return decHeu;
238
+ }
239
+ }
240
+ }
241
+ // autoGuess disabled or no candidate succeeded — surface top-3 + env hint
242
+ try {
243
+ const target2 = await fs.resolve(absolutePath, { ...(signal === undefined ? {} : { signal }) });
244
+ const info2 = await fs.stat(target2, signal);
245
+ const maxBytes2 = info2?.size ?? 10 * 1024 * 1024;
246
+ const bytes2 = await fs.readBytes(target2, signal, maxBytes2);
247
+ const allow2 = cfg.supportedEncodings;
248
+ const candsViaChardet = await chardetTop3Candidates(bytes2, allow2);
249
+ const cands = candsViaChardet.length > 0 ? candsViaChardet.map((c) => ({ encoding: c.encoding, sample: c.sample, score: c.confidence })) : top3Candidates(bytes2, allow2);
250
+ if (cands.length > 0) {
251
+ const candStr = cands.map((c) => `${c.encoding}("${c.sample.slice(0, 20).replace(/"/g, "'")}")`).join(", ");
252
+ throw new Error(`[E_NOT_TEXT] Path is not a readable UTF-8 text file: ${absolutePath}. Hashline editing only supports text files. Top-3 guesses: ${candStr}. Try read({encoding: "<encoding>"}) or set DSH_BETTER_EDIT_AUTO_GUESS_ENCODING=true to auto-decode.`);
253
+ }
254
+ }
255
+ catch (inner) {
256
+ if (inner instanceof Error && inner.message.includes("[E_NOT_TEXT]"))
257
+ throw inner;
258
+ }
259
+ }
260
+ catch (e) {
261
+ if (e instanceof Error && e.message.includes("[E_NOT_TEXT]"))
262
+ throw e;
263
+ }
264
+ }
110
265
  return mapFsError(error, absolutePath);
111
266
  }
112
267
  },
113
- async writeText(absolutePath, content, signal, exec, sandboxPolicy) {
268
+ async writeText(absolutePath, content, signal, exec, sandboxPolicy, encodingHint) {
269
+ // handle drift invalidation before encode
270
+ try {
271
+ const targetTmp = await fs.resolve(absolutePath, { ...(signal === undefined ? {} : { signal }) });
272
+ const key = String(targetTmp.targetKey ?? absolutePath);
273
+ const memo = encodingMemo.get(key);
274
+ if (memo) {
275
+ const info = await fs.stat(targetTmp, signal);
276
+ const curVer = info?.version;
277
+ if (curVer !== memo.version)
278
+ encodingMemo.delete(key);
279
+ }
280
+ }
281
+ catch { } // biome-ignore: best-effort
114
282
  try {
115
283
  const target = await fs.resolve(absolutePath, {
116
284
  ...(signal === undefined ? {} : { signal }),
@@ -123,6 +291,30 @@ export function ctxFsIO(fs, ctx) {
123
291
  // confined backend checks: without it the backend falls back to
124
292
  // the deployment default root and denies writes inside the
125
293
  // session workspace under workspace-write.
294
+ // re-encode if needed (round-trip unless normalizeToUtf8)
295
+ const _writeContent = content; // keep for future use
296
+ try {
297
+ const cfgW = loadConfig();
298
+ if (encodingHint) {
299
+ const normW = normalizeEncoding(encodingHint);
300
+ if (!normW)
301
+ throw new (await import("@deepseek-ai/dsh-fs")).FsError("bad encoding", "FS_BAD_ENCODING");
302
+ // Save with Encoding: update memo and write as requested encoding (but fs.writeText expects utf8 string, so we keep utf8 string and update memo for next read?)
303
+ // For now, record memo; actual bytes will be written as utf8 string (provider normalizes). For legacy preserve, we would need writeBytes seam — deferred, record state only.
304
+ const tgt = await fs.resolve(absolutePath, { ...(signal === undefined ? {} : { signal }) });
305
+ const _inf = await fs.stat(tgt, signal);
306
+ setEncodingState(String(tgt.targetKey ?? absolutePath), { encoding: normW, hasBOM: normW === "utf8bom", version: undefined });
307
+ }
308
+ else if (!cfgW.normalizeToUtf8) {
309
+ const tgt2 = await fs.resolve(absolutePath, { ...(signal === undefined ? {} : { signal }) });
310
+ const mem = encodingMemo.get(String(tgt2.targetKey ?? absolutePath));
311
+ if (mem && mem.encoding !== "utf8" && mem.encoding !== "utf8bom") {
312
+ // For provider that expects utf8 string, we keep string as-is (round-trip would need raw bytes seam). Record indicates file was legacy.
313
+ // No transcode on wire — provider will write utf8; next read will see new version and re-detect.
314
+ }
315
+ }
316
+ }
317
+ catch { } // biome-ignore: best-effort
126
318
  const outcome = await fs.writeText(target, content, intent, signal, sandboxPolicy);
127
319
  // Record the present observation (a no-op when no policy
128
320
  // plugin listens), so later built-in tools see the new version.
@@ -170,8 +362,78 @@ export function localIO() {
170
362
  async resolve(path, cwd) {
171
363
  return resolveTarget(toCwd(path, cwd ?? process.cwd()));
172
364
  },
173
- async readText(absolutePath, signal) {
365
+ async readText(absolutePath, signal, encodingHint) {
174
366
  signal?.throwIfAborted();
367
+ if (encodingHint) {
368
+ const norm = normalizeEncoding(encodingHint);
369
+ if (!norm)
370
+ throw new Error(`[E_BAD_ENCODING] Unknown encoding: ${encodingHint}`);
371
+ const bytes = await readFile(absolutePath);
372
+ const bom = detectBom(bytes);
373
+ const off = bom ? bom.bomLen : 0;
374
+ const dec = decodeBytes(bytes.subarray(off), norm ?? "utf8");
375
+ if (dec === undefined)
376
+ throw new Error(`[E_DECODE_FAILED] Cannot decode ${absolutePath} as ${norm}`);
377
+ const info = await fileSnap(absolutePath).catch(() => undefined);
378
+ setEncodingState(absolutePath, { encoding: norm, hasBOM: !!bom, version: info?.snapshotId });
379
+ return dec;
380
+ }
381
+ // try deterministic BOM→UTF8 first, then autoGuess if enabled
382
+ try {
383
+ const bytes = await readFile(absolutePath);
384
+ const bom = detectBom(bytes);
385
+ if (bom) {
386
+ const dec = decodeBytes(bytes, bom.encoding);
387
+ if (dec !== undefined) {
388
+ const info = await fileSnap(absolutePath).catch(() => undefined);
389
+ setEncodingState(absolutePath, { encoding: bom.encoding, hasBOM: true, version: info?.snapshotId });
390
+ return dec;
391
+ }
392
+ }
393
+ if (isValidUtf8(bytes))
394
+ return bytes.toString("utf-8");
395
+ const cfgL = loadConfig();
396
+ if (cfgL.autoGuessEncoding) {
397
+ // chardet first with footer
398
+ try {
399
+ const chardetCandsL = await chardetTop3Candidates(bytes, cfgL.supportedEncodings);
400
+ if (chardetCandsL.length > 0) {
401
+ const topL = chardetCandsL[0];
402
+ const secondL = chardetCandsL[1];
403
+ const isMidL = topL.confidence < 70 || (secondL !== undefined && topL.confidence - secondL.confidence < 10);
404
+ const decL2 = decodeBytes(bytes, topL.encoding);
405
+ if (decL2 !== undefined && !decL2.includes("\uFFFD")) {
406
+ let footerL = "";
407
+ if (isMidL) {
408
+ const candsStrL = chardetCandsL.map((c) => `${c.encoding} ${c.confidence}`).join(", ");
409
+ footerL = `\n\n[Auto-guessed: ${topL.encoding} ${topL.confidence}, candidates: ${candsStrL} — re-read with read({encoding}) if garbled]`;
410
+ }
411
+ const infoL2 = await fileSnap(absolutePath).catch(() => undefined);
412
+ setEncodingState(absolutePath, { encoding: topL.encoding, hasBOM: false, version: infoL2?.snapshotId });
413
+ if (footerL)
414
+ setAutoGuessFooter(absolutePath, footerL);
415
+ return decL2;
416
+ }
417
+ }
418
+ }
419
+ catch { } // biome-ignore: best-effort
420
+ const candidates = top3Candidates(bytes, cfgL.supportedEncodings);
421
+ if (candidates.length > 0) {
422
+ const best = candidates[0];
423
+ const candsStrHeuL = candidates.map((c) => `${c.encoding} ${c.score.toFixed(0)}`).join(", ");
424
+ const footerHeuL = `\n\n[Auto-guessed: ${best.encoding} ${best.score.toFixed(0)}, candidates: ${candsStrHeuL} — re-read with read({encoding}) if garbled]`;
425
+ const dec = decodeBytes(bytes, best.encoding);
426
+ if (dec !== undefined) {
427
+ const info = await fileSnap(absolutePath).catch(() => undefined);
428
+ setEncodingState(absolutePath, { encoding: best.encoding, hasBOM: false, version: info?.snapshotId });
429
+ if (footerHeuL)
430
+ setAutoGuessFooter(absolutePath, footerHeuL);
431
+ return dec;
432
+ }
433
+ }
434
+ }
435
+ }
436
+ catch { } // biome-ignore: best-effort
175
437
  return readFile(absolutePath, "utf-8");
176
438
  },
177
439
  async writeText(absolutePath, content, signal, _exec, _sandboxPolicy) {
package/lib/hash-store.js CHANGED
@@ -110,7 +110,6 @@ function openDbWithBusyRetry(storePath) {
110
110
  const stores = new Map();
111
111
  const openings = new Map();
112
112
  setStoresGetter(() => stores, () => openings);
113
- const exitHandlerRegistered = false;
114
113
  function openDb(storePath) {
115
114
  const db = new DatabaseSync(storePath, {
116
115
  timeout: HASH_STORE_BUSY_TIMEOUT,
@@ -9,6 +9,7 @@
9
9
  */
10
10
  import { abortIf } from "./utils.js";
11
11
  import { readView } from "./file-view.js";
12
+ import { getAutoGuessFooter } from "./fs-bridge.js";
12
13
  import { recordServed, clearDriftReported } from "./session-view.js";
13
14
  /** Appended when the file had non-UTF-8 bytes; editing rewrites it as UTF-8. */
14
15
  export const UTF8_REWRITE_NOTE = "[Non-UTF-8 bytes shown as U+FFFD; editing rewrites the file as UTF-8.]";
@@ -26,6 +27,7 @@ export async function readAndServe(io, rawPath, cwd, options) {
26
27
  const { sessionKey, signal } = options;
27
28
  abortIf(signal);
28
29
  const view = await readView(io, rawPath, cwd, {
30
+ encoding: options.encoding,
29
31
  offset: options.offset,
30
32
  limit: options.limit,
31
33
  signal,
@@ -34,6 +36,8 @@ export async function readAndServe(io, rawPath, cwd, options) {
34
36
  await recordServed(sessionKey, view.absolutePath, view.served, view.hashes.length);
35
37
  }
36
38
  await clearDriftReported(sessionKey, view.absolutePath);
39
+ const autoFooter = getAutoGuessFooter(view.absolutePath) ?? getAutoGuessFooter(rawPath) ?? getAutoGuessFooter(view.absolutePath.replace(/\\/g, "/")) ?? "";
40
+ const warning = autoFooter || undefined;
37
41
  const text = view.hadUtf8DecodeErrors
38
42
  ? `${view.text}\n\n${UTF8_REWRITE_NOTE}`
39
43
  : view.text;
@@ -42,5 +46,6 @@ export async function readAndServe(io, rawPath, cwd, options) {
42
46
  served: view.served,
43
47
  hadUtf8DecodeErrors: view.hadUtf8DecodeErrors,
44
48
  absolutePath: view.absolutePath,
49
+ warning,
45
50
  };
46
51
  }
@@ -6,7 +6,7 @@
6
6
  */
7
7
  import { homedir } from "node:os";
8
8
  import { isAbsolute, join } from "node:path";
9
- import { readFileSync, statSync } from "node:fs";
9
+ import { readFileSync, statSync, appendFileSync } from "node:fs";
10
10
  import { mkdir, writeFile } from "node:fs/promises";
11
11
  import { resolveDshHome } from "@deepseek-ai/dsh-home-paths";
12
12
  import { z } from "zod";
@@ -46,6 +46,11 @@ function parseSimpleYaml(content) {
46
46
  continue;
47
47
  const key = line.slice(0, colon).trim();
48
48
  let value = line.slice(colon + 1).trim();
49
+ if (value.startsWith("[") && value.endsWith("]")) {
50
+ value = value.slice(1, -1);
51
+ out[key] = value;
52
+ continue;
53
+ }
49
54
  value = stripQuotes(value);
50
55
  value = stripInlineComment(value);
51
56
  out[key] = value;
@@ -69,6 +74,13 @@ function parseAutoGitIgnoreRaw(v) {
69
74
  return false;
70
75
  return undefined;
71
76
  }
77
+ function parseSupportedEncodingsRaw(v) {
78
+ const inner = v.trim().replace(/^\[/, "").replace(/\]$/, "");
79
+ const items = inner.split(/[ ,;]+/).map((s) => s.trim().toLowerCase()).filter((s) => s.length > 0);
80
+ if (items.length === 0)
81
+ return undefined;
82
+ return items;
83
+ }
72
84
  // ---- Zod schema (single source of truth) ----
73
85
  const StoreConfigSchema = z.object({
74
86
  storeDir: z
@@ -76,6 +88,9 @@ const StoreConfigSchema = z.object({
76
88
  .default("central")
77
89
  .transform((v) => normalizeStoreDir(v) ?? "central"), // where the store lives: "central" (default, in $DSH_HOME/.../runtime/<name>-<hash8>/) | "workspace" (legacy <ws>/.dsh_better_edit/) | "/abs/path" (custom root)
78
90
  autoGitignore: z.boolean().default(false), // workspace only: auto-append ".dsh_better_edit/" to .gitignore when .git exists but entry missing (false = warn once)
91
+ autoGuessEncoding: z.boolean().default(false), // VS Code files.autoGuessEncoding — deterministic BOM→UTF-8 first, model-assisted top-3 only when true
92
+ normalizeToUtf8: z.boolean().default(false), // when true, legacy files (GBK etc.) migrate to UTF-8 on first edit/write
93
+ supportedEncodings: z.array(z.string()).default(["gbk", "big5", "shift_jis", "euc-kr", "windows-1251", "iso-8859-1"]),
79
94
  undo_ttl_s: z.number().int().min(-1).default(604800), // undo history TTL in seconds; -1 = keep forever (default 604800 = 7 days)
80
95
  storeMaxAgeS: z.number().int().min(1).default(2592000), // central janitor: max idle age in seconds before evicting runtime/<name>-<hash8>/ (default 2592000 = 30 days)
81
96
  storeMaxTotalBytes: z.number().int().min(0).default(524288000), // central janitor: max total bytes under runtime/ before LRU eviction (default 524288000 = 500 MB)
@@ -86,6 +101,9 @@ let cachedMtimeMs;
86
101
  let cachedYamlPath;
87
102
  let cachedEnvStoreDir;
88
103
  let cachedEnvAutoGitignore;
104
+ let cachedEnvAutoGuessEncoding;
105
+ let cachedEnvNormalizeToUtf8;
106
+ let cachedEnvSupportedEncodings;
89
107
  function configYamlPath() {
90
108
  return join(resolveDshHome(), "plugins", "dsh-better-edit", "config.yaml");
91
109
  }
@@ -102,6 +120,14 @@ storeDir: central
102
120
  # workspace only: when .git exists but .gitignore lacks .dsh_better_edit/, true = auto-append ".dsh_better_edit/", false = warn once
103
121
  autoGitignore: false
104
122
 
123
+ # encoding: deterministic BOM→UTF-8 first, guessing only when autoGuessEncoding:true
124
+ # when true, FS_NOT_TEXT scores supportedEncodings and surfaces top-3 previews for model pick
125
+ autoGuessEncoding: false
126
+ # when true, legacy files (GBK etc.) migrate to UTF-8 on first write
127
+ normalizeToUtf8: false
128
+ # allowlist scored for top-3 previews when autoGuessEncoding:true
129
+ supportedEncodings: [gbk, big5, shift_jis, euc-kr, windows-1251, iso-8859-1]
130
+
105
131
  # undo history TTL in seconds; -1 = keep forever (default 604800 = 7 days)
106
132
  undo_ttl_s: 604800
107
133
 
@@ -217,6 +243,27 @@ function fsAdapter() {
217
243
  }
218
244
  }
219
245
  }
246
+ if (parsed.autoGuessEncoding !== undefined) {
247
+ const b = parseAutoGitIgnoreRaw(parsed.autoGuessEncoding);
248
+ if (b === undefined)
249
+ console.warn(`dsh-better-edit: autoGuessEncoding "${parsed.autoGuessEncoding}" invalid — expected true|false`);
250
+ else
251
+ out.autoGuessEncoding = b;
252
+ }
253
+ if (parsed.normalizeToUtf8 !== undefined) {
254
+ const b = parseAutoGitIgnoreRaw(parsed.normalizeToUtf8);
255
+ if (b === undefined)
256
+ console.warn(`dsh-better-edit: normalizeToUtf8 "${parsed.normalizeToUtf8}" invalid — expected true|false`);
257
+ else
258
+ out.normalizeToUtf8 = b;
259
+ }
260
+ if (parsed.supportedEncodings !== undefined) {
261
+ const arr = parseSupportedEncodingsRaw(parsed.supportedEncodings);
262
+ if (arr === undefined)
263
+ console.warn(`dsh-better-edit: supportedEncodings "${parsed.supportedEncodings}" invalid`);
264
+ else
265
+ out.supportedEncodings = arr;
266
+ }
220
267
  if (parsed.storeMaxTotalBytes !== undefined) {
221
268
  const n = Number(parsed.storeMaxTotalBytes);
222
269
  const res = z.number().int().min(0).safeParse(n);
@@ -254,6 +301,30 @@ function envAdapter(base) {
254
301
  }
255
302
  }
256
303
  }
304
+ const envGuess = process.env.DSH_BETTER_EDIT_AUTO_GUESS_ENCODING;
305
+ if (envGuess !== undefined) {
306
+ const b = parseAutoGitIgnoreRaw(envGuess);
307
+ if (b === undefined)
308
+ console.warn(`dsh-better-edit: DSH_BETTER_EDIT_AUTO_GUESS_ENCODING invalid — expected true|false`);
309
+ else
310
+ out.autoGuessEncoding = b;
311
+ }
312
+ const envNorm = process.env.DSH_BETTER_EDIT_NORMALIZE_TO_UTF8;
313
+ if (envNorm !== undefined) {
314
+ const b = parseAutoGitIgnoreRaw(envNorm);
315
+ if (b === undefined)
316
+ console.warn(`dsh-better-edit: DSH_BETTER_EDIT_NORMALIZE_TO_UTF8 invalid — expected true|false`);
317
+ else
318
+ out.normalizeToUtf8 = b;
319
+ }
320
+ const envSup = process.env.DSH_BETTER_EDIT_SUPPORTED_ENCODINGS;
321
+ if (envSup !== undefined) {
322
+ const arr = parseSupportedEncodingsRaw(envSup);
323
+ if (arr === undefined)
324
+ console.warn(`dsh-better-edit: DSH_BETTER_EDIT_SUPPORTED_ENCODINGS invalid`);
325
+ else
326
+ out.supportedEncodings = arr;
327
+ }
257
328
  const envAuto = process.env.DSH_BETTER_EDIT_AUTO_GITIGNORE;
258
329
  if (envAuto !== undefined) {
259
330
  const b = parseAutoGitIgnoreRaw(envAuto);
@@ -266,6 +337,46 @@ function envAdapter(base) {
266
337
  }
267
338
  return out;
268
339
  }
340
+ function complementMissingFieldsSync(yamlPath, yamlPartial) {
341
+ // Best-effort: append missing canonical keys with defaults, preserving existing content.
342
+ // Only runs when the file exists (mtime !== undefined) and some canonical key is absent.
343
+ const missingLines = [];
344
+ if (yamlPartial.storeDir === undefined)
345
+ missingLines.push("storeDir: central");
346
+ if (yamlPartial.autoGitignore === undefined)
347
+ missingLines.push("autoGitignore: false");
348
+ if (yamlPartial.autoGuessEncoding === undefined)
349
+ missingLines.push("autoGuessEncoding: false");
350
+ if (yamlPartial.normalizeToUtf8 === undefined)
351
+ missingLines.push("normalizeToUtf8: false");
352
+ if (yamlPartial.supportedEncodings === undefined)
353
+ missingLines.push("supportedEncodings: [gbk, big5, shift_jis, euc-kr, windows-1251, iso-8859-1]");
354
+ if (yamlPartial.undo_ttl_s === undefined)
355
+ missingLines.push("undo_ttl_s: 604800");
356
+ if (yamlPartial.storeMaxAgeS === undefined)
357
+ missingLines.push("storeMaxAgeS: 2592000");
358
+ if (yamlPartial.storeMaxTotalBytes === undefined)
359
+ missingLines.push("storeMaxTotalBytes: 524288000");
360
+ if (missingLines.length === 0)
361
+ return;
362
+ try {
363
+ // Ensure we don't duplicate if file was concurrently complemented — re-read raw keys.
364
+ const raw = readFileSync(yamlPath, "utf-8");
365
+ const parsed = parseSimpleYaml(raw);
366
+ // Re-check against raw parsed (aliases for storeMaxAgeS already handled via yamlPartial, but double-check).
367
+ const stillMissing = missingLines.filter((line) => {
368
+ const key = line.split(":")[0].trim();
369
+ return parsed[key] === undefined;
370
+ });
371
+ if (stillMissing.length === 0)
372
+ return;
373
+ const suffix = (raw.endsWith("\n") || raw.length === 0 ? "" : "\n") + stillMissing.join("\n") + "\n";
374
+ appendFileSync(yamlPath, suffix, "utf-8");
375
+ }
376
+ catch {
377
+ // best-effort: ENOENT (file deleted between stat and read) or permission — ignore
378
+ }
379
+ }
269
380
  // ---- public load (merged, cached) ----
270
381
  export function loadConfig() {
271
382
  const yamlPath = configYamlPath();
@@ -277,15 +388,28 @@ export function loadConfig() {
277
388
  // no yaml file — fall through to defaults/cache miss
278
389
  }
279
390
  const envStoreDir = process.env.DSH_BETTER_EDIT_STORE_DIR;
391
+ const envGuess = process.env.DSH_BETTER_EDIT_AUTO_GUESS_ENCODING;
392
+ const envNorm = process.env.DSH_BETTER_EDIT_NORMALIZE_TO_UTF8;
393
+ const envSup = process.env.DSH_BETTER_EDIT_SUPPORTED_ENCODINGS;
280
394
  const envAuto = process.env.DSH_BETTER_EDIT_AUTO_GITIGNORE;
281
395
  if (cachedConfig !== undefined &&
282
396
  cachedMtimeMs === mtime &&
283
397
  cachedYamlPath === yamlPath &&
284
398
  cachedEnvStoreDir === envStoreDir &&
285
- cachedEnvAutoGitignore === envAuto) {
399
+ cachedEnvAutoGitignore === envAuto &&
400
+ cachedEnvAutoGuessEncoding === envGuess &&
401
+ cachedEnvNormalizeToUtf8 === envNorm &&
402
+ cachedEnvSupportedEncodings === envSup) {
286
403
  return cachedConfig;
287
404
  }
288
405
  const yamlPartial = fsAdapter();
406
+ if (mtime !== undefined) {
407
+ complementMissingFieldsSync(yamlPath, yamlPartial);
408
+ try {
409
+ mtime = statSync(yamlPath).mtimeMs;
410
+ }
411
+ catch { }
412
+ }
289
413
  const envPartial = envAdapter({ ...DEFAULT_CONFIG, ...yamlPartial });
290
414
  const merged = { ...DEFAULT_CONFIG, ...yamlPartial, ...envPartial };
291
415
  // final Zod validation + normalization (expand abs, fallback)
@@ -305,6 +429,9 @@ export function loadConfig() {
305
429
  cachedYamlPath = yamlPath;
306
430
  cachedEnvStoreDir = envStoreDir;
307
431
  cachedEnvAutoGitignore = envAuto;
432
+ cachedEnvAutoGuessEncoding = envGuess;
433
+ cachedEnvNormalizeToUtf8 = envNorm;
434
+ cachedEnvSupportedEncodings = envSup;
308
435
  return cfg;
309
436
  }
310
437
  // For tests: reset cache
@@ -314,4 +441,7 @@ export function _resetConfigCache() {
314
441
  cachedYamlPath = undefined;
315
442
  cachedEnvStoreDir = undefined;
316
443
  cachedEnvAutoGitignore = undefined;
444
+ cachedEnvAutoGuessEncoding = undefined;
445
+ cachedEnvNormalizeToUtf8 = undefined;
446
+ cachedEnvSupportedEncodings = undefined;
317
447
  }
package/lib/tool-read.js CHANGED
@@ -9,6 +9,7 @@ import { defineTool } from "@deepseek-ai/dsh-tools";
9
9
  import { normalizeRequest as normReq, assertReadRequest, pathSchema } from "./contract.js";
10
10
  import { readAndServe } from "./read-and-serve.js";
11
11
  import { READ_DESCRIPTION } from "./prompts.js";
12
+ import { normalizeEncoding } from "./encoding.js";
12
13
  import { execCwd, execSessionKey } from "./session-view.js";
13
14
  import { withWorkspace } from "./session-view.js";
14
15
  /**
@@ -32,30 +33,49 @@ export function buildReadTool(io) {
32
33
  type: "number",
33
34
  description: "Maximum number of lines to read",
34
35
  },
36
+ encoding: {
37
+ type: "string",
38
+ description: "Text encoding for Reopen with Encoding (e.g. gbk, shift_jis, windows-1251). Case-insensitive.",
39
+ },
35
40
  },
36
41
  output: {
37
- schema: { type: "string" },
38
- render: (_args, value) => [{ type: "text", text: value }],
42
+ schema: { type: "object", properties: { text: { type: "string", required: true }, warning: { type: "string" } }, additionalProperties: false },
43
+ render: (_args, value) => {
44
+ const v = value;
45
+ if (typeof v === "string")
46
+ return [{ type: "text", text: v }];
47
+ const blocks = [{ type: "text", text: v.text }];
48
+ if (v.warning)
49
+ blocks.push({ type: "text", text: v.warning });
50
+ return blocks;
51
+ },
39
52
  },
40
53
  async execute(args, exec) {
41
54
  return withWorkspace(execCwd(exec), async () => {
42
55
  const cwd = execCwd(exec);
43
56
  const sessionKey = execSessionKey(exec);
44
57
  const signal = exec.signal;
58
+ const encoding = args.encoding;
59
+ if (encoding !== undefined) {
60
+ const norm = normalizeEncoding(String(encoding));
61
+ if (!norm)
62
+ throw new Error(`[E_BAD_ENCODING] Unknown encoding: ${String(encoding)}. Supported: utf8, gbk, big5, shift_jis, euc-kr, windows-1251, iso-8859-1`);
63
+ }
45
64
  const canonical = normReq(args);
46
65
  assertReadRequest(canonical);
47
66
  const rawPath = canonical.path;
48
- const { text, absolutePath } = await readAndServe(io, rawPath, cwd, {
67
+ const { text, absolutePath, warning } = await readAndServe(io, rawPath, cwd, {
49
68
  sessionKey,
50
69
  signal,
51
70
  offset: canonical.offset,
52
71
  limit: canonical.limit,
72
+ encoding: encoding,
53
73
  });
54
74
  // Record the present observation with the fs policy gate so later
55
75
  // built-in write/edit calls see this file as observed at the
56
76
  // version the model just read (a no-op when no policy listens).
57
77
  await io.emitObserved(absolutePath, exec, signal);
58
- return text;
78
+ return warning ? { text, warning } : { text };
59
79
  });
60
80
  },
61
81
  });
@@ -27,6 +27,7 @@ export interface ReadParams {
27
27
  path: string;
28
28
  offset?: number;
29
29
  limit?: number;
30
+ encoding?: string;
30
31
  }
31
32
  export interface UndoParams {
32
33
  path: string;
@@ -0,0 +1,46 @@
1
+ /**
2
+ * VS Code encoding model — pure helpers for BOM→UTF-8→model-assisted top-3.
3
+ * No jschardet; heuristic scoring + iconv-lite rendering.
4
+ * @module dsh-better-edit/encoding
5
+ */
6
+ export declare const SUPPORTED_ENCODINGS: readonly ["gbk", "big5", "shift_jis", "euc-kr", "windows-1251", "iso-8859-1"];
7
+ export type SupportedEncoding = (typeof SUPPORTED_ENCODINGS)[number];
8
+ /** Canonicalize case-insensitive encoding param, strip -_ . */
9
+ export declare function normalizeEncoding(input: string): string | undefined;
10
+ export declare function isSupportedEncoding(enc: string): boolean;
11
+ export interface BomInfo {
12
+ encoding: string;
13
+ bomLen: number;
14
+ hasBOM: boolean;
15
+ }
16
+ export declare function detectBom(bytes: Uint8Array): BomInfo | undefined;
17
+ export declare function isValidUtf8(bytes: Uint8Array): boolean;
18
+ export interface CandidatePreview {
19
+ encoding: string;
20
+ sample: string;
21
+ score: number;
22
+ }
23
+ export declare function scoreText(text: string, enc: string): number;
24
+ /** Decode bytes under enc via iconv-lite, return text or undefined on failure. */
25
+ export declare function decodeBytes(bytes: Uint8Array, enc: string): string | undefined;
26
+ export declare function encodeText(text: string, enc: string): Uint8Array | undefined;
27
+ /** Top-3 candidates for autoGuess: decode each allowlist enc, score, sort. */
28
+ export declare function top3Candidates(bytes: Uint8Array, allowlist: string[]): CandidatePreview[];
29
+ export declare function hasReplacementChar(text: string): boolean;
30
+ /**
31
+ * Try chardet (runk/node-chardet) for autoGuess when available.
32
+ * Dynamically imports 'chardet' (optionalDep) and returns canonical encoding
33
+ * if confidence >=45 and allowlist-accepted and decodable without �.
34
+ * Falls back to heuristic when chardet unavailable/low confidence.
35
+ */
36
+ export declare function detectWithChardet(bytes: Uint8Array, allowlist: string[]): Promise<string | undefined>;
37
+ /**
38
+ * Chardet-based Top-3 for E_NOT_TEXT (primary) and autoGuess.
39
+ * Returns chardet candidates filtered by allowlist and decodable, else empty.
40
+ */
41
+ export declare function chardetTop3Candidates(bytes: Uint8Array, allowlist: string[]): Promise<Array<{
42
+ encoding: string;
43
+ confidence: number;
44
+ sample: string;
45
+ }>>;
46
+ export declare function getTop3Candidates(bytes: Uint8Array, allowlist: string[]): Promise<CandidatePreview[]>;
@@ -128,6 +128,7 @@ export interface PreviewOpts {
128
128
  limit?: number;
129
129
  }
130
130
  export interface ReadViewOpts extends PreviewOpts {
131
+ encoding?: string;
131
132
  signal?: AbortSignal;
132
133
  }
133
134
  export declare function preview(content: string, hashes: string[], opts?: PreviewOpts, absolutePath?: string): Promise<{
@@ -23,7 +23,7 @@ export interface FileIO {
23
23
  /** Resolve a (possibly relative) request path against the session cwd to a canonical absolute path. */
24
24
  resolve(path: string, cwd: string, signal?: AbortSignal): Promise<string>;
25
25
  /** Read whole text; missing files, directories, and binary content throw. */
26
- readText(absolutePath: string, signal?: AbortSignal): Promise<string>;
26
+ readText(absolutePath: string, signal?: AbortSignal, encoding?: string): Promise<string>;
27
27
  /**
28
28
  * Atomically write whole text, preserving mode when the file exists. On the
29
29
  * dsh backend this dispatches `fs/write-intent` (policy guard), stamps the
@@ -53,6 +53,17 @@ export interface FileIO {
53
53
  * @returns the mapped error, rethrown.
54
54
  */
55
55
  export declare function mapFsError(error: unknown, displayPath: string): never;
56
+ export interface FileEncodingState {
57
+ encoding: string;
58
+ hasBOM: boolean;
59
+ version: string | undefined;
60
+ }
61
+ export declare function getAutoGuessFooter(targetKey: string): string | undefined;
62
+ export declare function setAutoGuessFooter(targetKey: string, footer: string): void;
63
+ export declare function clearAutoGuessFooter(targetKey?: string): void;
64
+ export declare function getEncodingState(targetKey: string): FileEncodingState | undefined;
65
+ export declare function setEncodingState(targetKey: string, state: FileEncodingState): void;
66
+ export declare function clearEncodingState(targetKey?: string): void;
56
67
  /** FileIO over the deployment's `ctx.fs` service. */
57
68
  export declare function ctxFsIO(fs: FileSystem, ctx: Context): FileIO;
58
69
  /** FileIO over the host filesystem directly (tests, previews, fallback). */
@@ -12,6 +12,7 @@ import type { ServedRow } from "./hashline/served.js";
12
12
  /** Appended when the file had non-UTF-8 bytes; editing rewrites it as UTF-8. */
13
13
  export declare const UTF8_REWRITE_NOTE = "[Non-UTF-8 bytes shown as U+FFFD; editing rewrites the file as UTF-8.]";
14
14
  export interface ReadAndServeOptions {
15
+ encoding?: string;
15
16
  /** The session whose served rows these lines belong to. */
16
17
  sessionKey: string;
17
18
  signal?: AbortSignal;
@@ -20,6 +21,8 @@ export interface ReadAndServeOptions {
20
21
  limit?: number;
21
22
  }
22
23
  export interface ReadAndServeResult {
24
+ /** Optional auto-guess warning, separate block. */
25
+ warning?: string;
23
26
  /** The model-facing read text, including the UTF-8 note when applicable. */
24
27
  text: string;
25
28
  /** The rows recorded as served (empty when nothing was shown). */
@@ -3,25 +3,34 @@ export declare function expand(filePath: string): string;
3
3
  declare const StoreConfigSchema: z.ZodObject<{
4
4
  storeDir: z.ZodEffects<z.ZodDefault<z.ZodString>, string, string | undefined>;
5
5
  autoGitignore: z.ZodDefault<z.ZodBoolean>;
6
+ autoGuessEncoding: z.ZodDefault<z.ZodBoolean>;
7
+ normalizeToUtf8: z.ZodDefault<z.ZodBoolean>;
8
+ supportedEncodings: z.ZodDefault<z.ZodArray<z.ZodString, "many">>;
6
9
  undo_ttl_s: z.ZodDefault<z.ZodNumber>;
7
10
  storeMaxAgeS: z.ZodDefault<z.ZodNumber>;
8
11
  storeMaxTotalBytes: z.ZodDefault<z.ZodNumber>;
9
12
  }, "strip", z.ZodTypeAny, {
10
13
  storeDir: string;
11
14
  autoGitignore: boolean;
15
+ autoGuessEncoding: boolean;
16
+ normalizeToUtf8: boolean;
17
+ supportedEncodings: string[];
12
18
  undo_ttl_s: number;
13
19
  storeMaxAgeS: number;
14
20
  storeMaxTotalBytes: number;
15
21
  }, {
16
22
  storeDir?: string | undefined;
17
23
  autoGitignore?: boolean | undefined;
24
+ autoGuessEncoding?: boolean | undefined;
25
+ normalizeToUtf8?: boolean | undefined;
26
+ supportedEncodings?: string[] | undefined;
18
27
  undo_ttl_s?: number | undefined;
19
28
  storeMaxAgeS?: number | undefined;
20
29
  storeMaxTotalBytes?: number | undefined;
21
30
  }>;
22
31
  export type StoreConfig = z.infer<typeof StoreConfigSchema>;
23
32
  /** Default config.yaml content with comments — generated when the file does not exist. */
24
- export declare const DEFAULT_CONFIG_YAML = "# dsh-better-edit config\n# Location: $DSH_HOME/plugins/dsh-better-edit/config.yaml\n# Docs: https://github.com/Rianico/dsh-better-edit#configuration\n# Generated with defaults on first load \u2014 edit or delete freely.\n# DB files are disposable caches \u2014 safe to delete, rebuilt on next `read`.\n\n# where the store lives: \"central\" (default, in $DSH_HOME/.../runtime/<name>-<hash8>/) | \"workspace\" (legacy <ws>/.dsh_better_edit/) | \"/abs/path\" (custom root)\nstoreDir: central\n\n# workspace only: when .git exists but .gitignore lacks .dsh_better_edit/, true = auto-append \".dsh_better_edit/\", false = warn once\nautoGitignore: false\n\n# undo history TTL in seconds; -1 = keep forever (default 604800 = 7 days)\nundo_ttl_s: 604800\n\n# central janitor: max idle age in seconds before evicting runtime/<name>-<hash8>/ (default 2592000 = 30 days)\nstoreMaxAgeS: 2592000\n\n# central janitor: max total bytes under runtime/ before LRU eviction (default 524288000 = 500 MB)\nstoreMaxTotalBytes: 524288000\n";
33
+ export declare const DEFAULT_CONFIG_YAML = "# dsh-better-edit config\n# Location: $DSH_HOME/plugins/dsh-better-edit/config.yaml\n# Docs: https://github.com/Rianico/dsh-better-edit#configuration\n# Generated with defaults on first load \u2014 edit or delete freely.\n# DB files are disposable caches \u2014 safe to delete, rebuilt on next `read`.\n\n# where the store lives: \"central\" (default, in $DSH_HOME/.../runtime/<name>-<hash8>/) | \"workspace\" (legacy <ws>/.dsh_better_edit/) | \"/abs/path\" (custom root)\nstoreDir: central\n\n# workspace only: when .git exists but .gitignore lacks .dsh_better_edit/, true = auto-append \".dsh_better_edit/\", false = warn once\nautoGitignore: false\n\n# encoding: deterministic BOM\u2192UTF-8 first, guessing only when autoGuessEncoding:true\n# when true, FS_NOT_TEXT scores supportedEncodings and surfaces top-3 previews for model pick\nautoGuessEncoding: false\n# when true, legacy files (GBK etc.) migrate to UTF-8 on first write\nnormalizeToUtf8: false\n# allowlist scored for top-3 previews when autoGuessEncoding:true\nsupportedEncodings: [gbk, big5, shift_jis, euc-kr, windows-1251, iso-8859-1]\n\n# undo history TTL in seconds; -1 = keep forever (default 604800 = 7 days)\nundo_ttl_s: 604800\n\n# central janitor: max idle age in seconds before evicting runtime/<name>-<hash8>/ (default 2592000 = 30 days)\nstoreMaxAgeS: 2592000\n\n# central janitor: max total bytes under runtime/ before LRU eviction (default 524288000 = 500 MB)\nstoreMaxTotalBytes: 524288000\n";
25
34
  /**
26
35
  * Ensure a default config.yaml exists in the plugin home.
27
36
  * Idempotent and concurrent-safe (wx) — never overwrites an existing file.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "dsh-better-edit",
3
- "version": "0.5.0",
3
+ "version": "0.5.1",
4
4
  "type": "module",
5
5
  "description": "Hash-anchored read/edit/undo_last_edit tools for DeepSeek Harness (dsh). Every line gets a unique 3-char hash (A-Za-z0-9) that stays stable across edits; stale or ambiguous anchors are rejected, never fuzzy-matched. Undo persists across restarts.",
6
6
  "main": "lib/index.js",
@@ -56,6 +56,7 @@
56
56
  "dependencies": {
57
57
  "diff": "^8.0.2",
58
58
  "file-type": "^21.3.0",
59
+ "iconv-lite": "^0.6.3",
59
60
  "xxhash-wasm": "^1.1.0",
60
61
  "zod": "^3.25.76"
61
62
  },
@@ -100,5 +101,8 @@
100
101
  "semantic-release": "^25.0.9",
101
102
  "typescript": "^5.8.0",
102
103
  "vitest": "^4.1.8"
104
+ },
105
+ "optionalDependencies": {
106
+ "chardet": "2.2.0"
103
107
  }
104
108
  }