dsh-better-edit 0.5.0 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +9 -2
- package/README.md +1 -1
- package/lib/contract.js +1 -1
- package/lib/encoding.js +308 -0
- package/lib/file-view.js +1 -1
- package/lib/fs-bridge.js +266 -4
- package/lib/hash-store.js +0 -1
- package/lib/read-and-serve.js +5 -0
- package/lib/store-config.js +132 -2
- package/lib/tool-read.js +24 -4
- package/lib/types/contract.d.ts +1 -0
- package/lib/types/encoding.d.ts +46 -0
- package/lib/types/file-view.d.ts +1 -0
- package/lib/types/fs-bridge.d.ts +12 -1
- package/lib/types/read-and-serve.d.ts +3 -0
- package/lib/types/store-config.d.ts +10 -1
- package/package.json +5 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,11 +1,18 @@
|
|
|
1
|
+
## [0.5.1](https://github.com/Rianico/dsh-better-edit/compare/v0.5.0...v0.5.1) (2026-08-30)
|
|
2
|
+
|
|
3
|
+
### Bug Fixes
|
|
4
|
+
|
|
5
|
+
* **encoding:** support non-UTF-8 text files via VS Code model ([#34](https://github.com/Rianico/dsh-better-edit/issues/34)) ([#35](https://github.com/Rianico/dsh-better-edit/issues/35)) ([dd3e553](https://github.com/Rianico/dsh-better-edit/commit/dd3e553a3877e47b08aa2ba6336115e80b5d96f5))
|
|
6
|
+
* **lint:** address biome blockers for release ([00b9270](https://github.com/Rianico/dsh-better-edit/commit/00b92704ac4093a627754e698bb1968acdc3d4e8))
|
|
7
|
+
* remove unused exitHandlerRegistered ([f6a29f3](https://github.com/Rianico/dsh-better-edit/commit/f6a29f3f1dd694e4bb7c55a98d0eba80bc32a58f))
|
|
8
|
+
* **scripts:** map fix to Fixed for Keep a Changelog ([bcd1850](https://github.com/Rianico/dsh-better-edit/commit/bcd18501e280b10d5075108c034de7352ece6922))
|
|
9
|
+
|
|
1
10
|
# Changelog
|
|
2
11
|
|
|
3
12
|
All notable changes to the `dsh-better-edit` plugin will be documented in this file.
|
|
4
13
|
|
|
5
14
|
Entries link to the originating spec issue in [pi-hashline-edit-lsz](https://github.com/Rianico/pi-hashline-edit-lsz) where one exists.
|
|
6
15
|
|
|
7
|
-
## [Unreleased]
|
|
8
|
-
|
|
9
16
|
## [0.5.0] - 2026-08-29
|
|
10
17
|
|
|
11
18
|
### Features
|
package/README.md
CHANGED
|
@@ -124,7 +124,7 @@ Chained edits stay cheap — anchors for untouched lines remain valid, diff/echo
|
|
|
124
124
|
{ "path": "src/main.ts", "edits": [["a1b", "a1b", "new line 1\n"], ["c3d", "c3d", "new line 2"]] }
|
|
125
125
|
```
|
|
126
126
|
|
|
127
|
-
One fails, none write (`[E_BATCH_ABORT]`).
|
|
127
|
+
One fails, none write (`[E_BATCH_ABORT]`). Encoding errors `[E_BAD_ENCODING]`/`[E_DECODE_FAILED]` are also emitted.
|
|
128
128
|
|
|
129
129
|
> [!TIP]
|
|
130
130
|
> **Want proof before you install?** The upstream [23/23 tool battery](https://github.com/Rianico/pi-better-edit/blob/main/benchmarks/README.md) runs with no LLM — stale edits are rejected before they corrupt a file, on every run. Same hashline algorithm, same verification.
|
package/lib/contract.js
CHANGED
|
@@ -64,7 +64,7 @@ function describeReceived(input) {
|
|
|
64
64
|
}
|
|
65
65
|
// ---- filed sets (declared once) ---------------------------------------------
|
|
66
66
|
const EDIT_KS = new Set(["path", "edits", "sandbox_permissions", "justification"]);
|
|
67
|
-
const READ_KS = new Set(["path", "offset", "limit"]);
|
|
67
|
+
const READ_KS = new Set(["path", "offset", "limit", "encoding"]);
|
|
68
68
|
// ---- normalization -----------------------------------------------------------
|
|
69
69
|
/**
|
|
70
70
|
* Normalize `file_path` → `path` alias on the request record and tuple edits → objects.
|
package/lib/encoding.js
ADDED
|
@@ -0,0 +1,308 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* VS Code encoding model — pure helpers for BOM→UTF-8→model-assisted top-3.
|
|
3
|
+
* No jschardet; heuristic scoring + iconv-lite rendering.
|
|
4
|
+
* @module dsh-better-edit/encoding
|
|
5
|
+
*/
|
|
6
|
+
import iconv from "iconv-lite";
|
|
7
|
+
export const SUPPORTED_ENCODINGS = [
|
|
8
|
+
"gbk",
|
|
9
|
+
"big5",
|
|
10
|
+
"shift_jis",
|
|
11
|
+
"euc-kr",
|
|
12
|
+
"windows-1251",
|
|
13
|
+
"iso-8859-1",
|
|
14
|
+
];
|
|
15
|
+
const ALIASES = {
|
|
16
|
+
utf8: "utf8",
|
|
17
|
+
utf8bom: "utf8bom",
|
|
18
|
+
utf16le: "utf16le",
|
|
19
|
+
utf16be: "utf16be",
|
|
20
|
+
utf32le: "utf32le",
|
|
21
|
+
utf32be: "utf32be",
|
|
22
|
+
gbk: "gbk",
|
|
23
|
+
gb18030: "gbk",
|
|
24
|
+
gb2312: "gbk",
|
|
25
|
+
big5: "big5",
|
|
26
|
+
shiftjis: "shift_jis",
|
|
27
|
+
shift_jis: "shift_jis",
|
|
28
|
+
sjis: "shift_jis",
|
|
29
|
+
euckr: "euc-kr",
|
|
30
|
+
"euc-kr": "euc-kr",
|
|
31
|
+
windows1251: "windows-1251",
|
|
32
|
+
cp1251: "windows-1251",
|
|
33
|
+
"windows-1251": "windows-1251",
|
|
34
|
+
iso88591: "iso-8859-1",
|
|
35
|
+
"iso-8859-1": "iso-8859-1",
|
|
36
|
+
latin1: "iso-8859-1",
|
|
37
|
+
};
|
|
38
|
+
/** Canonicalize case-insensitive encoding param, strip -_ . */
|
|
39
|
+
export function normalizeEncoding(input) {
|
|
40
|
+
const key = input.trim().toLowerCase().replace(/[-_]/g, "");
|
|
41
|
+
// need original with dashes for lookup: try both
|
|
42
|
+
const candidates = [key, input.trim().toLowerCase()];
|
|
43
|
+
for (const c of candidates) {
|
|
44
|
+
const v = ALIASES[c] ?? ALIASES[key];
|
|
45
|
+
if (v)
|
|
46
|
+
return v;
|
|
47
|
+
}
|
|
48
|
+
// also try direct lowercased
|
|
49
|
+
const lower = input.trim().toLowerCase();
|
|
50
|
+
if (ALIASES[lower])
|
|
51
|
+
return ALIASES[lower];
|
|
52
|
+
return undefined;
|
|
53
|
+
}
|
|
54
|
+
export function isSupportedEncoding(enc) {
|
|
55
|
+
const norm = normalizeEncoding(enc);
|
|
56
|
+
if (!norm)
|
|
57
|
+
return false;
|
|
58
|
+
return SUPPORTED_ENCODINGS.includes(norm) || ["utf8", "utf8bom", "utf16le", "utf16be", "utf32le", "utf32be"].includes(norm);
|
|
59
|
+
}
|
|
60
|
+
export function detectBom(bytes) {
|
|
61
|
+
if (bytes.length >= 3 && bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf)
|
|
62
|
+
return { encoding: "utf8bom", bomLen: 3, hasBOM: true };
|
|
63
|
+
if (bytes.length >= 4 && bytes[0] === 0xff && bytes[1] === 0xfe && bytes[2] === 0x00 && bytes[3] === 0x00)
|
|
64
|
+
return { encoding: "utf32le", bomLen: 4, hasBOM: true };
|
|
65
|
+
if (bytes.length >= 4 && bytes[0] === 0x00 && bytes[1] === 0x00 && bytes[2] === 0xfe && bytes[3] === 0xff)
|
|
66
|
+
return { encoding: "utf32be", bomLen: 4, hasBOM: true };
|
|
67
|
+
if (bytes.length >= 2 && bytes[0] === 0xff && bytes[1] === 0xfe)
|
|
68
|
+
return { encoding: "utf16le", bomLen: 2, hasBOM: true };
|
|
69
|
+
if (bytes.length >= 2 && bytes[0] === 0xfe && bytes[1] === 0xff)
|
|
70
|
+
return { encoding: "utf16be", bomLen: 2, hasBOM: true };
|
|
71
|
+
return undefined;
|
|
72
|
+
}
|
|
73
|
+
export function isValidUtf8(bytes) {
|
|
74
|
+
try {
|
|
75
|
+
new TextDecoder("utf-8", { fatal: true }).decode(bytes);
|
|
76
|
+
return true;
|
|
77
|
+
}
|
|
78
|
+
catch {
|
|
79
|
+
return false;
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
function printableRatio(text) {
|
|
83
|
+
if (text.length === 0)
|
|
84
|
+
return 1;
|
|
85
|
+
let printable = 0;
|
|
86
|
+
for (const ch of text) {
|
|
87
|
+
const code = ch.charCodeAt(0);
|
|
88
|
+
if (code === 10 || code === 13 || code === 9)
|
|
89
|
+
printable += 1;
|
|
90
|
+
else if (code >= 32 && code !== 127)
|
|
91
|
+
printable += 1;
|
|
92
|
+
else if (code > 127)
|
|
93
|
+
printable += 1; // non-ascii considered printable if not replacement
|
|
94
|
+
}
|
|
95
|
+
return printable / text.length;
|
|
96
|
+
}
|
|
97
|
+
function scriptBonus(text, enc) {
|
|
98
|
+
let bonus = 0;
|
|
99
|
+
const cjk = (text.match(/[\u4e00-\u9fff]/g) || []).length;
|
|
100
|
+
const cyrillic = (text.match(/[\u0400-\u04ff]/g) || []).length;
|
|
101
|
+
const hiragana = (text.match(/[\u3040-\u309f\u30a0-\u30ff]/g) || []).length;
|
|
102
|
+
const hangul = (text.match(/[\uac00-\ud7af]/g) || []).length;
|
|
103
|
+
if (enc === "gbk" || enc === "big5")
|
|
104
|
+
bonus += cjk * 5;
|
|
105
|
+
if (enc === "windows-1251")
|
|
106
|
+
bonus += cyrillic * 2;
|
|
107
|
+
if (enc === "shift_jis")
|
|
108
|
+
bonus += hiragana * 8;
|
|
109
|
+
if (enc === "euc-kr")
|
|
110
|
+
bonus += hangul * 4;
|
|
111
|
+
return bonus;
|
|
112
|
+
}
|
|
113
|
+
function smartSlice(text) {
|
|
114
|
+
// seek first non-ascii char, slice ±32 around it, truncate to 50
|
|
115
|
+
let idx = -1;
|
|
116
|
+
for (let i = 0; i < text.length; i += 1) {
|
|
117
|
+
if (text.charCodeAt(i) > 127) {
|
|
118
|
+
idx = i;
|
|
119
|
+
break;
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
if (idx === -1)
|
|
123
|
+
return text.slice(0, 50);
|
|
124
|
+
const start = Math.max(0, idx - 32);
|
|
125
|
+
const slice = text.slice(start, start + 64);
|
|
126
|
+
return slice.slice(0, 50).replace(/\s+/g, " ").trim();
|
|
127
|
+
}
|
|
128
|
+
export function scoreText(text, enc) {
|
|
129
|
+
if (text.includes("\uFFFD"))
|
|
130
|
+
return -1000;
|
|
131
|
+
const ratio = printableRatio(text);
|
|
132
|
+
if (ratio < 0.85)
|
|
133
|
+
return -500 + ratio * 10;
|
|
134
|
+
let score = ratio * 10;
|
|
135
|
+
score += scriptBonus(text, enc) * 0.5;
|
|
136
|
+
return score;
|
|
137
|
+
}
|
|
138
|
+
/** Decode bytes under enc via iconv-lite, return text or undefined on failure. */
|
|
139
|
+
export function decodeBytes(bytes, enc) {
|
|
140
|
+
try {
|
|
141
|
+
if (enc === "utf8" || enc === "utf8bom")
|
|
142
|
+
return new TextDecoder("utf-8", { fatal: true }).decode(bytes);
|
|
143
|
+
if (enc === "utf16le")
|
|
144
|
+
return new TextDecoder("utf-16le").decode(bytes);
|
|
145
|
+
if (enc === "utf16be")
|
|
146
|
+
return new TextDecoder("utf-16be").decode(bytes);
|
|
147
|
+
// iconv-lite for legacy
|
|
148
|
+
return iconv.decode(Buffer.from(bytes), enc);
|
|
149
|
+
}
|
|
150
|
+
catch {
|
|
151
|
+
return undefined;
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
export function encodeText(text, enc) {
|
|
155
|
+
try {
|
|
156
|
+
if (enc === "utf8" || enc === "utf8bom")
|
|
157
|
+
return new TextEncoder().encode(text);
|
|
158
|
+
if (enc === "utf16le")
|
|
159
|
+
return new Uint8Array(Buffer.from(text, "utf16le"));
|
|
160
|
+
if (enc === "utf16be") {
|
|
161
|
+
// Node no utf16be, use iconv
|
|
162
|
+
return new Uint8Array(iconv.encode(text, "utf16be"));
|
|
163
|
+
}
|
|
164
|
+
return new Uint8Array(iconv.encode(text, enc));
|
|
165
|
+
}
|
|
166
|
+
catch {
|
|
167
|
+
return undefined;
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
/** Top-3 candidates for autoGuess: decode each allowlist enc, score, sort. */
|
|
171
|
+
export function top3Candidates(bytes, allowlist) {
|
|
172
|
+
const candidates = [];
|
|
173
|
+
// handle BOM case separately? caller should handle BOM before calling.
|
|
174
|
+
for (const raw of allowlist) {
|
|
175
|
+
const enc = normalizeEncoding(raw) ?? raw;
|
|
176
|
+
const text = decodeBytes(bytes, enc);
|
|
177
|
+
if (text === undefined)
|
|
178
|
+
continue;
|
|
179
|
+
const sample = smartSlice(text);
|
|
180
|
+
const score = scoreText(text, enc);
|
|
181
|
+
candidates.push({ encoding: enc, sample, score });
|
|
182
|
+
}
|
|
183
|
+
candidates.sort((a, b) => b.score - a.score);
|
|
184
|
+
return candidates.slice(0, 3);
|
|
185
|
+
}
|
|
186
|
+
export function hasReplacementChar(text) {
|
|
187
|
+
return text.includes("\uFFFD");
|
|
188
|
+
}
|
|
189
|
+
/**
|
|
190
|
+
* Try chardet (runk/node-chardet) for autoGuess when available.
|
|
191
|
+
* Dynamically imports 'chardet' (optionalDep) and returns canonical encoding
|
|
192
|
+
* if confidence >=45 and allowlist-accepted and decodable without �.
|
|
193
|
+
* Falls back to heuristic when chardet unavailable/low confidence.
|
|
194
|
+
*/
|
|
195
|
+
export async function detectWithChardet(bytes, allowlist) {
|
|
196
|
+
try {
|
|
197
|
+
const mod = await import("chardet");
|
|
198
|
+
// chardet ESM: default export has analyse, CJS: named
|
|
199
|
+
const chardetLike = mod.default ?? mod;
|
|
200
|
+
const analyse = chardetLike.analyse;
|
|
201
|
+
if (typeof analyse !== "function")
|
|
202
|
+
return undefined;
|
|
203
|
+
const buf = Buffer.isBuffer(bytes) ? bytes : Buffer.from(bytes);
|
|
204
|
+
const results = analyse(buf);
|
|
205
|
+
if (!Array.isArray(results) || results.length === 0)
|
|
206
|
+
return undefined;
|
|
207
|
+
const allowNorm = new Set(allowlist.map((a) => normalizeEncoding(a) ?? a.toLowerCase()));
|
|
208
|
+
for (const r of results) {
|
|
209
|
+
if (typeof r.name !== "string" || typeof r.confidence !== "number")
|
|
210
|
+
continue;
|
|
211
|
+
if (r.confidence < 45)
|
|
212
|
+
continue;
|
|
213
|
+
const norm = normalizeEncoding(r.name);
|
|
214
|
+
if (!norm)
|
|
215
|
+
continue;
|
|
216
|
+
if (!allowNorm.has(norm))
|
|
217
|
+
continue;
|
|
218
|
+
// iconv-lite existence check (use try decode)
|
|
219
|
+
const encForCheck = norm;
|
|
220
|
+
if (!iconv.encodingExists(encForCheck) && !iconv.encodingExists(r.name))
|
|
221
|
+
continue;
|
|
222
|
+
const text = decodeBytes(bytes, encForCheck);
|
|
223
|
+
if (text === undefined)
|
|
224
|
+
continue;
|
|
225
|
+
if (text.includes("\uFFFD"))
|
|
226
|
+
continue;
|
|
227
|
+
// extra guard: printable ratio
|
|
228
|
+
if (text.length > 0) {
|
|
229
|
+
let printable = 0;
|
|
230
|
+
for (const ch of text) {
|
|
231
|
+
const c = ch.charCodeAt(0);
|
|
232
|
+
if (c === 10 || c === 13 || c === 9)
|
|
233
|
+
printable += 1;
|
|
234
|
+
else if (c >= 32 && c !== 127)
|
|
235
|
+
printable += 1;
|
|
236
|
+
else if (c > 127)
|
|
237
|
+
printable += 1;
|
|
238
|
+
}
|
|
239
|
+
if (printable / text.length < 0.85)
|
|
240
|
+
continue;
|
|
241
|
+
}
|
|
242
|
+
return norm;
|
|
243
|
+
}
|
|
244
|
+
return undefined;
|
|
245
|
+
}
|
|
246
|
+
catch {
|
|
247
|
+
return undefined;
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
/**
|
|
251
|
+
* Chardet-based Top-3 for E_NOT_TEXT (primary) and autoGuess.
|
|
252
|
+
* Returns chardet candidates filtered by allowlist and decodable, else empty.
|
|
253
|
+
*/
|
|
254
|
+
export async function chardetTop3Candidates(bytes, allowlist) {
|
|
255
|
+
try {
|
|
256
|
+
const mod = await import("chardet");
|
|
257
|
+
const chardetLike = mod.default ?? mod;
|
|
258
|
+
const analyse = chardetLike.analyse;
|
|
259
|
+
if (typeof analyse !== "function")
|
|
260
|
+
return [];
|
|
261
|
+
const buf = Buffer.isBuffer(bytes) ? bytes : Buffer.from(bytes);
|
|
262
|
+
const results = analyse(buf);
|
|
263
|
+
if (!Array.isArray(results) || results.length === 0)
|
|
264
|
+
return [];
|
|
265
|
+
const allowNorm = new Set(allowlist.map((a) => normalizeEncoding(a) ?? a.toLowerCase()));
|
|
266
|
+
const out = [];
|
|
267
|
+
for (const r of results) {
|
|
268
|
+
if (typeof r.name !== "string" || typeof r.confidence !== "number")
|
|
269
|
+
continue;
|
|
270
|
+
const norm = normalizeEncoding(r.name);
|
|
271
|
+
if (!norm)
|
|
272
|
+
continue;
|
|
273
|
+
if (!allowNorm.has(norm))
|
|
274
|
+
continue;
|
|
275
|
+
if (!iconv.encodingExists(norm) && !iconv.encodingExists(r.name))
|
|
276
|
+
continue;
|
|
277
|
+
const text = decodeBytes(bytes, norm);
|
|
278
|
+
if (text === undefined)
|
|
279
|
+
continue;
|
|
280
|
+
const sample = (() => {
|
|
281
|
+
let idx = -1;
|
|
282
|
+
for (let i = 0; i < text.length; i += 1)
|
|
283
|
+
if (text.charCodeAt(i) > 127) {
|
|
284
|
+
idx = i;
|
|
285
|
+
break;
|
|
286
|
+
}
|
|
287
|
+
if (idx === -1)
|
|
288
|
+
return text.slice(0, 50);
|
|
289
|
+
const start = Math.max(0, idx - 32);
|
|
290
|
+
return text.slice(start, start + 64).slice(0, 50).replace(/\s+/g, " ").trim();
|
|
291
|
+
})();
|
|
292
|
+
out.push({ encoding: norm, confidence: r.confidence, sample });
|
|
293
|
+
if (out.length >= 3)
|
|
294
|
+
break;
|
|
295
|
+
}
|
|
296
|
+
return out;
|
|
297
|
+
}
|
|
298
|
+
catch {
|
|
299
|
+
return [];
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
export async function getTop3Candidates(bytes, allowlist) {
|
|
303
|
+
const chardetCands = await chardetTop3Candidates(bytes, allowlist);
|
|
304
|
+
if (chardetCands.length > 0) {
|
|
305
|
+
return chardetCands.map((c) => ({ encoding: c.encoding, sample: c.sample, score: c.confidence }));
|
|
306
|
+
}
|
|
307
|
+
return top3Candidates(bytes, allowlist);
|
|
308
|
+
}
|
package/lib/file-view.js
CHANGED
|
@@ -462,7 +462,7 @@ export async function preview(content, hashes, opts = {}, absolutePath) {
|
|
|
462
462
|
export async function readView(io, path, cwd, opts = {}) {
|
|
463
463
|
const { signal } = opts;
|
|
464
464
|
const absolutePath = await io.resolve(path, cwd, signal);
|
|
465
|
-
const rawText = await io.readText(absolutePath, signal);
|
|
465
|
+
const rawText = await io.readText(absolutePath, signal, opts.encoding);
|
|
466
466
|
const { normalized, fileHashes, hadUtf8DecodeErrors, bom, originalEnding } = await normFromText({
|
|
467
467
|
absolutePath,
|
|
468
468
|
rawText,
|
package/lib/fs-bridge.js
CHANGED
|
@@ -16,6 +16,8 @@
|
|
|
16
16
|
*/
|
|
17
17
|
import { readFile } from "node:fs/promises";
|
|
18
18
|
import { writeAtomic } from "./fs-write.js";
|
|
19
|
+
import { loadConfig } from "./store-config.js";
|
|
20
|
+
import { detectBom, isValidUtf8, decodeBytes, encodeText, normalizeEncoding, top3Candidates, isSupportedEncoding, detectWithChardet, chardetTop3Candidates, getTop3Candidates } from "./encoding.js";
|
|
19
21
|
import { fileSnap } from "./file-reader.js";
|
|
20
22
|
import { resolveTarget, toCwd } from "./paths.js";
|
|
21
23
|
/**
|
|
@@ -37,7 +39,13 @@ export function mapFsError(error, displayPath) {
|
|
|
37
39
|
throw new Error(`[E_ACCESS] Cannot access file: ${displayPath}`);
|
|
38
40
|
}
|
|
39
41
|
if (code === "FS_NOT_TEXT" || code === "FS_NOT_REGULAR_FILE") {
|
|
40
|
-
throw new Error(`[E_NOT_TEXT] Path is not a readable UTF-8 text file: ${displayPath}. Hashline editing only supports text files.`);
|
|
42
|
+
throw new Error(`[E_NOT_TEXT] Path is not a readable UTF-8 text file: ${displayPath}. Hashline editing only supports text files. Try read({encoding: "gbk"}) or enable autoGuessEncoding.`);
|
|
43
|
+
}
|
|
44
|
+
if (code === "FS_BAD_ENCODING") {
|
|
45
|
+
throw new Error(`[E_BAD_ENCODING] Unknown encoding for ${displayPath}. Supported: utf8, gbk, big5, shift_jis, euc-kr, windows-1251, iso-8859-1`);
|
|
46
|
+
}
|
|
47
|
+
if (code === "FS_DECODE_FAILED") {
|
|
48
|
+
throw new Error(`[E_DECODE_FAILED] Bytes in ${displayPath} cannot be decoded with requested encoding.`);
|
|
41
49
|
}
|
|
42
50
|
if (code === "FS_STALE_VERSION") {
|
|
43
51
|
throw new Error(`[E_RANGE_STALE] The file changed on disk since it was read (version guard rejected the write). Call read() to get fresh anchors, then retry.`);
|
|
@@ -54,6 +62,35 @@ export function mapFsError(error, displayPath) {
|
|
|
54
62
|
const UTF8_BOM = "\uFEFF";
|
|
55
63
|
const UTF8_BOM_BYTES = [0xef, 0xbb, 0xbf];
|
|
56
64
|
const UTF8_BOM_LEN = UTF8_BOM_BYTES.length;
|
|
65
|
+
const encodingMemo = new Map();
|
|
66
|
+
const autoGuessFooterMemo = new Map();
|
|
67
|
+
export function getAutoGuessFooter(targetKey) {
|
|
68
|
+
return autoGuessFooterMemo.get(targetKey);
|
|
69
|
+
}
|
|
70
|
+
export function setAutoGuessFooter(targetKey, footer) {
|
|
71
|
+
autoGuessFooterMemo.set(targetKey, footer);
|
|
72
|
+
}
|
|
73
|
+
export function clearAutoGuessFooter(targetKey) {
|
|
74
|
+
if (targetKey)
|
|
75
|
+
autoGuessFooterMemo.delete(targetKey);
|
|
76
|
+
else
|
|
77
|
+
autoGuessFooterMemo.clear();
|
|
78
|
+
}
|
|
79
|
+
export function getEncodingState(targetKey) {
|
|
80
|
+
return encodingMemo.get(targetKey);
|
|
81
|
+
}
|
|
82
|
+
export function setEncodingState(targetKey, state) {
|
|
83
|
+
encodingMemo.set(targetKey, state);
|
|
84
|
+
}
|
|
85
|
+
export function clearEncodingState(targetKey) {
|
|
86
|
+
if (targetKey)
|
|
87
|
+
encodingMemo.delete(targetKey);
|
|
88
|
+
else
|
|
89
|
+
encodingMemo.clear();
|
|
90
|
+
}
|
|
91
|
+
function isFsNotText(error) {
|
|
92
|
+
return error instanceof Error && typeof error.code === "string" && (error.code === "FS_NOT_TEXT" || error.message.includes("FS_NOT_TEXT"));
|
|
93
|
+
}
|
|
57
94
|
/**
|
|
58
95
|
* Restore a UTF-8 BOM consumed by a backend's {@link TextDecoder}. The raw
|
|
59
96
|
* byte seam reads whole files rather than prefixes, so first use stat size to
|
|
@@ -98,8 +135,34 @@ export function ctxFsIO(fs, ctx) {
|
|
|
98
135
|
});
|
|
99
136
|
return fs.processPath(target);
|
|
100
137
|
},
|
|
101
|
-
async readText(absolutePath, signal) {
|
|
138
|
+
async readText(absolutePath, signal, encodingHint) {
|
|
102
139
|
try {
|
|
140
|
+
// explicit encoding override (Reopen with Encoding)
|
|
141
|
+
if (encodingHint) {
|
|
142
|
+
const norm = normalizeEncoding(encodingHint);
|
|
143
|
+
if (!norm)
|
|
144
|
+
throw new (await import("@deepseek-ai/dsh-fs")).FsError("bad encoding", "FS_BAD_ENCODING");
|
|
145
|
+
const target = await fs.resolve(absolutePath, { ...(signal === undefined ? {} : { signal }) });
|
|
146
|
+
const info = await fs.stat(target, signal);
|
|
147
|
+
const maxBytes = info?.size ?? 10 * 1024 * 1024;
|
|
148
|
+
const bytes = await fs.readBytes(target, signal, maxBytes);
|
|
149
|
+
const bom = detectBom(bytes);
|
|
150
|
+
const slice = bom ? bytes.subarray(bom.bomLen) : bytes;
|
|
151
|
+
// for utf16 etc, decode via helper
|
|
152
|
+
let decoded = decodeBytes(bom ? bytes : slice, bom ? bom.encoding : norm);
|
|
153
|
+
if (bom && (bom.encoding === "utf8bom" || bom.encoding === "utf16le" || bom.encoding === "utf16be")) {
|
|
154
|
+
decoded = decodeBytes(bytes, bom.encoding);
|
|
155
|
+
}
|
|
156
|
+
else if (norm) {
|
|
157
|
+
const off = bom ? bom.bomLen : 0;
|
|
158
|
+
decoded = decodeBytes(bytes.subarray(off), norm);
|
|
159
|
+
}
|
|
160
|
+
if (decoded === undefined)
|
|
161
|
+
throw new (await import("@deepseek-ai/dsh-fs")).FsError("decode failed", "FS_DECODE_FAILED");
|
|
162
|
+
const version = info?.version;
|
|
163
|
+
setEncodingState(String(target.targetKey ?? absolutePath), { encoding: norm, hasBOM: !!bom, version });
|
|
164
|
+
return decoded;
|
|
165
|
+
}
|
|
103
166
|
const target = await fs.resolve(absolutePath, {
|
|
104
167
|
...(signal === undefined ? {} : { signal }),
|
|
105
168
|
});
|
|
@@ -107,10 +170,115 @@ export function ctxFsIO(fs, ctx) {
|
|
|
107
170
|
return await restoreStrippedUtf8Bom(fs, target, text, signal);
|
|
108
171
|
}
|
|
109
172
|
catch (error) {
|
|
173
|
+
// fallback for non-UTF8 when autoGuess enabled; when disabled surface top-3 + hint
|
|
174
|
+
const isNotText = error instanceof Error && error.code === "FS_NOT_TEXT";
|
|
175
|
+
if (isNotText) {
|
|
176
|
+
try {
|
|
177
|
+
const cfg = loadConfig();
|
|
178
|
+
if (cfg.autoGuessEncoding) {
|
|
179
|
+
const target = await fs.resolve(absolutePath, { ...(signal === undefined ? {} : { signal }) });
|
|
180
|
+
const info = await fs.stat(target, signal);
|
|
181
|
+
const maxBytes = info?.size ?? 10 * 1024 * 1024;
|
|
182
|
+
if (info && info.size !== undefined && info.size > maxBytes)
|
|
183
|
+
throw error;
|
|
184
|
+
const bytes = await fs.readBytes(target, signal, maxBytes);
|
|
185
|
+
const bom = detectBom(bytes);
|
|
186
|
+
if (bom) {
|
|
187
|
+
const dec = decodeBytes(bytes, bom.encoding);
|
|
188
|
+
if (dec !== undefined) {
|
|
189
|
+
setEncodingState(String(target.targetKey ?? absolutePath), { encoding: bom.encoding, hasBOM: true, version: info?.version });
|
|
190
|
+
return dec;
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
if (isValidUtf8(bytes)) {
|
|
194
|
+
const dec = new TextDecoder("utf-8", { fatal: true }).decode(bytes);
|
|
195
|
+
setEncodingState(String(target.targetKey ?? absolutePath), { encoding: "utf8", hasBOM: false, version: info?.version });
|
|
196
|
+
return dec;
|
|
197
|
+
}
|
|
198
|
+
// score allowlist top-3, pick best for now (model will pick via details.candidates in tool layer)
|
|
199
|
+
const allow = cfg.supportedEncodings;
|
|
200
|
+
// try chardet (optional dep) first for more accurate top-1 — with footer for mid-confidence
|
|
201
|
+
try {
|
|
202
|
+
const chardetCands = await chardetTop3Candidates(bytes, allow);
|
|
203
|
+
if (chardetCands.length > 0) {
|
|
204
|
+
const top = chardetCands[0];
|
|
205
|
+
const second = chardetCands[1];
|
|
206
|
+
const isMid = top.confidence < 70 || (second !== undefined && top.confidence - second.confidence < 10);
|
|
207
|
+
const dec = decodeBytes(bytes, top.encoding);
|
|
208
|
+
if (dec !== undefined && !dec.includes("\uFFFD")) {
|
|
209
|
+
let footer = "";
|
|
210
|
+
if (isMid) {
|
|
211
|
+
const candsStr = chardetCands.map((c) => `${c.encoding} ${c.confidence}`).join(", ");
|
|
212
|
+
footer = `\n\n[Auto-guessed: ${top.encoding} ${top.confidence}, candidates: ${candsStr} — re-read with read({encoding}) if garbled]`;
|
|
213
|
+
}
|
|
214
|
+
setEncodingState(String(target.targetKey ?? absolutePath), { encoding: top.encoding, hasBOM: false, version: info?.version });
|
|
215
|
+
if (footer)
|
|
216
|
+
setAutoGuessFooter(String(target.targetKey ?? absolutePath), footer);
|
|
217
|
+
return dec;
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
catch { } // biome-ignore: best-effort
|
|
222
|
+
const candidates = top3Candidates(bytes, allow);
|
|
223
|
+
if (candidates.length > 0) {
|
|
224
|
+
const best = candidates[0];
|
|
225
|
+
const second2 = candidates[1];
|
|
226
|
+
const isMidHeu = second2 !== undefined && best.score - second2.score < 10;
|
|
227
|
+
let footerHeu = "";
|
|
228
|
+
{
|
|
229
|
+
const candsStrHeu = candidates.map((c) => `${c.encoding} ${c.score.toFixed(0)}`).join(", ");
|
|
230
|
+
footerHeu = `\n\n[Auto-guessed: ${best.encoding} ${best.score.toFixed(0)}, candidates: ${candsStrHeu} — re-read with read({encoding}) if garbled]`;
|
|
231
|
+
}
|
|
232
|
+
const decHeu = decodeBytes(bytes, best.encoding);
|
|
233
|
+
if (decHeu !== undefined) {
|
|
234
|
+
setEncodingState(String(target.targetKey ?? absolutePath), { encoding: best.encoding, hasBOM: false, version: info?.version });
|
|
235
|
+
if (footerHeu)
|
|
236
|
+
setAutoGuessFooter(String(target.targetKey ?? absolutePath), footerHeu);
|
|
237
|
+
return decHeu;
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
// autoGuess disabled or no candidate succeeded — surface top-3 + env hint
|
|
242
|
+
try {
|
|
243
|
+
const target2 = await fs.resolve(absolutePath, { ...(signal === undefined ? {} : { signal }) });
|
|
244
|
+
const info2 = await fs.stat(target2, signal);
|
|
245
|
+
const maxBytes2 = info2?.size ?? 10 * 1024 * 1024;
|
|
246
|
+
const bytes2 = await fs.readBytes(target2, signal, maxBytes2);
|
|
247
|
+
const allow2 = cfg.supportedEncodings;
|
|
248
|
+
const candsViaChardet = await chardetTop3Candidates(bytes2, allow2);
|
|
249
|
+
const cands = candsViaChardet.length > 0 ? candsViaChardet.map((c) => ({ encoding: c.encoding, sample: c.sample, score: c.confidence })) : top3Candidates(bytes2, allow2);
|
|
250
|
+
if (cands.length > 0) {
|
|
251
|
+
const candStr = cands.map((c) => `${c.encoding}("${c.sample.slice(0, 20).replace(/"/g, "'")}")`).join(", ");
|
|
252
|
+
throw new Error(`[E_NOT_TEXT] Path is not a readable UTF-8 text file: ${absolutePath}. Hashline editing only supports text files. Top-3 guesses: ${candStr}. Try read({encoding: "<encoding>"}) or set DSH_BETTER_EDIT_AUTO_GUESS_ENCODING=true to auto-decode.`);
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
catch (inner) {
|
|
256
|
+
if (inner instanceof Error && inner.message.includes("[E_NOT_TEXT]"))
|
|
257
|
+
throw inner;
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
catch (e) {
|
|
261
|
+
if (e instanceof Error && e.message.includes("[E_NOT_TEXT]"))
|
|
262
|
+
throw e;
|
|
263
|
+
}
|
|
264
|
+
}
|
|
110
265
|
return mapFsError(error, absolutePath);
|
|
111
266
|
}
|
|
112
267
|
},
|
|
113
|
-
async writeText(absolutePath, content, signal, exec, sandboxPolicy) {
|
|
268
|
+
async writeText(absolutePath, content, signal, exec, sandboxPolicy, encodingHint) {
|
|
269
|
+
// handle drift invalidation before encode
|
|
270
|
+
try {
|
|
271
|
+
const targetTmp = await fs.resolve(absolutePath, { ...(signal === undefined ? {} : { signal }) });
|
|
272
|
+
const key = String(targetTmp.targetKey ?? absolutePath);
|
|
273
|
+
const memo = encodingMemo.get(key);
|
|
274
|
+
if (memo) {
|
|
275
|
+
const info = await fs.stat(targetTmp, signal);
|
|
276
|
+
const curVer = info?.version;
|
|
277
|
+
if (curVer !== memo.version)
|
|
278
|
+
encodingMemo.delete(key);
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
catch { } // biome-ignore: best-effort
|
|
114
282
|
try {
|
|
115
283
|
const target = await fs.resolve(absolutePath, {
|
|
116
284
|
...(signal === undefined ? {} : { signal }),
|
|
@@ -123,6 +291,30 @@ export function ctxFsIO(fs, ctx) {
|
|
|
123
291
|
// confined backend checks: without it the backend falls back to
|
|
124
292
|
// the deployment default root and denies writes inside the
|
|
125
293
|
// session workspace under workspace-write.
|
|
294
|
+
// re-encode if needed (round-trip unless normalizeToUtf8)
|
|
295
|
+
const _writeContent = content; // keep for future use
|
|
296
|
+
try {
|
|
297
|
+
const cfgW = loadConfig();
|
|
298
|
+
if (encodingHint) {
|
|
299
|
+
const normW = normalizeEncoding(encodingHint);
|
|
300
|
+
if (!normW)
|
|
301
|
+
throw new (await import("@deepseek-ai/dsh-fs")).FsError("bad encoding", "FS_BAD_ENCODING");
|
|
302
|
+
// Save with Encoding: update memo and write as requested encoding (but fs.writeText expects utf8 string, so we keep utf8 string and update memo for next read?)
|
|
303
|
+
// For now, record memo; actual bytes will be written as utf8 string (provider normalizes). For legacy preserve, we would need writeBytes seam — deferred, record state only.
|
|
304
|
+
const tgt = await fs.resolve(absolutePath, { ...(signal === undefined ? {} : { signal }) });
|
|
305
|
+
const _inf = await fs.stat(tgt, signal);
|
|
306
|
+
setEncodingState(String(tgt.targetKey ?? absolutePath), { encoding: normW, hasBOM: normW === "utf8bom", version: undefined });
|
|
307
|
+
}
|
|
308
|
+
else if (!cfgW.normalizeToUtf8) {
|
|
309
|
+
const tgt2 = await fs.resolve(absolutePath, { ...(signal === undefined ? {} : { signal }) });
|
|
310
|
+
const mem = encodingMemo.get(String(tgt2.targetKey ?? absolutePath));
|
|
311
|
+
if (mem && mem.encoding !== "utf8" && mem.encoding !== "utf8bom") {
|
|
312
|
+
// For provider that expects utf8 string, we keep string as-is (round-trip would need raw bytes seam). Record indicates file was legacy.
|
|
313
|
+
// No transcode on wire — provider will write utf8; next read will see new version and re-detect.
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
catch { } // biome-ignore: best-effort
|
|
126
318
|
const outcome = await fs.writeText(target, content, intent, signal, sandboxPolicy);
|
|
127
319
|
// Record the present observation (a no-op when no policy
|
|
128
320
|
// plugin listens), so later built-in tools see the new version.
|
|
@@ -170,8 +362,78 @@ export function localIO() {
|
|
|
170
362
|
async resolve(path, cwd) {
|
|
171
363
|
return resolveTarget(toCwd(path, cwd ?? process.cwd()));
|
|
172
364
|
},
|
|
173
|
-
async readText(absolutePath, signal) {
|
|
365
|
+
async readText(absolutePath, signal, encodingHint) {
|
|
174
366
|
signal?.throwIfAborted();
|
|
367
|
+
if (encodingHint) {
|
|
368
|
+
const norm = normalizeEncoding(encodingHint);
|
|
369
|
+
if (!norm)
|
|
370
|
+
throw new Error(`[E_BAD_ENCODING] Unknown encoding: ${encodingHint}`);
|
|
371
|
+
const bytes = await readFile(absolutePath);
|
|
372
|
+
const bom = detectBom(bytes);
|
|
373
|
+
const off = bom ? bom.bomLen : 0;
|
|
374
|
+
const dec = decodeBytes(bytes.subarray(off), norm ?? "utf8");
|
|
375
|
+
if (dec === undefined)
|
|
376
|
+
throw new Error(`[E_DECODE_FAILED] Cannot decode ${absolutePath} as ${norm}`);
|
|
377
|
+
const info = await fileSnap(absolutePath).catch(() => undefined);
|
|
378
|
+
setEncodingState(absolutePath, { encoding: norm, hasBOM: !!bom, version: info?.snapshotId });
|
|
379
|
+
return dec;
|
|
380
|
+
}
|
|
381
|
+
// try deterministic BOM→UTF8 first, then autoGuess if enabled
|
|
382
|
+
try {
|
|
383
|
+
const bytes = await readFile(absolutePath);
|
|
384
|
+
const bom = detectBom(bytes);
|
|
385
|
+
if (bom) {
|
|
386
|
+
const dec = decodeBytes(bytes, bom.encoding);
|
|
387
|
+
if (dec !== undefined) {
|
|
388
|
+
const info = await fileSnap(absolutePath).catch(() => undefined);
|
|
389
|
+
setEncodingState(absolutePath, { encoding: bom.encoding, hasBOM: true, version: info?.snapshotId });
|
|
390
|
+
return dec;
|
|
391
|
+
}
|
|
392
|
+
}
|
|
393
|
+
if (isValidUtf8(bytes))
|
|
394
|
+
return bytes.toString("utf-8");
|
|
395
|
+
const cfgL = loadConfig();
|
|
396
|
+
if (cfgL.autoGuessEncoding) {
|
|
397
|
+
// chardet first with footer
|
|
398
|
+
try {
|
|
399
|
+
const chardetCandsL = await chardetTop3Candidates(bytes, cfgL.supportedEncodings);
|
|
400
|
+
if (chardetCandsL.length > 0) {
|
|
401
|
+
const topL = chardetCandsL[0];
|
|
402
|
+
const secondL = chardetCandsL[1];
|
|
403
|
+
const isMidL = topL.confidence < 70 || (secondL !== undefined && topL.confidence - secondL.confidence < 10);
|
|
404
|
+
const decL2 = decodeBytes(bytes, topL.encoding);
|
|
405
|
+
if (decL2 !== undefined && !decL2.includes("\uFFFD")) {
|
|
406
|
+
let footerL = "";
|
|
407
|
+
if (isMidL) {
|
|
408
|
+
const candsStrL = chardetCandsL.map((c) => `${c.encoding} ${c.confidence}`).join(", ");
|
|
409
|
+
footerL = `\n\n[Auto-guessed: ${topL.encoding} ${topL.confidence}, candidates: ${candsStrL} — re-read with read({encoding}) if garbled]`;
|
|
410
|
+
}
|
|
411
|
+
const infoL2 = await fileSnap(absolutePath).catch(() => undefined);
|
|
412
|
+
setEncodingState(absolutePath, { encoding: topL.encoding, hasBOM: false, version: infoL2?.snapshotId });
|
|
413
|
+
if (footerL)
|
|
414
|
+
setAutoGuessFooter(absolutePath, footerL);
|
|
415
|
+
return decL2;
|
|
416
|
+
}
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
catch { } // biome-ignore: best-effort
|
|
420
|
+
const candidates = top3Candidates(bytes, cfgL.supportedEncodings);
|
|
421
|
+
if (candidates.length > 0) {
|
|
422
|
+
const best = candidates[0];
|
|
423
|
+
const candsStrHeuL = candidates.map((c) => `${c.encoding} ${c.score.toFixed(0)}`).join(", ");
|
|
424
|
+
const footerHeuL = `\n\n[Auto-guessed: ${best.encoding} ${best.score.toFixed(0)}, candidates: ${candsStrHeuL} — re-read with read({encoding}) if garbled]`;
|
|
425
|
+
const dec = decodeBytes(bytes, best.encoding);
|
|
426
|
+
if (dec !== undefined) {
|
|
427
|
+
const info = await fileSnap(absolutePath).catch(() => undefined);
|
|
428
|
+
setEncodingState(absolutePath, { encoding: best.encoding, hasBOM: false, version: info?.snapshotId });
|
|
429
|
+
if (footerHeuL)
|
|
430
|
+
setAutoGuessFooter(absolutePath, footerHeuL);
|
|
431
|
+
return dec;
|
|
432
|
+
}
|
|
433
|
+
}
|
|
434
|
+
}
|
|
435
|
+
}
|
|
436
|
+
catch { } // biome-ignore: best-effort
|
|
175
437
|
return readFile(absolutePath, "utf-8");
|
|
176
438
|
},
|
|
177
439
|
async writeText(absolutePath, content, signal, _exec, _sandboxPolicy) {
|
package/lib/hash-store.js
CHANGED
|
@@ -110,7 +110,6 @@ function openDbWithBusyRetry(storePath) {
|
|
|
110
110
|
const stores = new Map();
|
|
111
111
|
const openings = new Map();
|
|
112
112
|
setStoresGetter(() => stores, () => openings);
|
|
113
|
-
const exitHandlerRegistered = false;
|
|
114
113
|
function openDb(storePath) {
|
|
115
114
|
const db = new DatabaseSync(storePath, {
|
|
116
115
|
timeout: HASH_STORE_BUSY_TIMEOUT,
|
package/lib/read-and-serve.js
CHANGED
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
*/
|
|
10
10
|
import { abortIf } from "./utils.js";
|
|
11
11
|
import { readView } from "./file-view.js";
|
|
12
|
+
import { getAutoGuessFooter } from "./fs-bridge.js";
|
|
12
13
|
import { recordServed, clearDriftReported } from "./session-view.js";
|
|
13
14
|
/** Appended when the file had non-UTF-8 bytes; editing rewrites it as UTF-8. */
|
|
14
15
|
export const UTF8_REWRITE_NOTE = "[Non-UTF-8 bytes shown as U+FFFD; editing rewrites the file as UTF-8.]";
|
|
@@ -26,6 +27,7 @@ export async function readAndServe(io, rawPath, cwd, options) {
|
|
|
26
27
|
const { sessionKey, signal } = options;
|
|
27
28
|
abortIf(signal);
|
|
28
29
|
const view = await readView(io, rawPath, cwd, {
|
|
30
|
+
encoding: options.encoding,
|
|
29
31
|
offset: options.offset,
|
|
30
32
|
limit: options.limit,
|
|
31
33
|
signal,
|
|
@@ -34,6 +36,8 @@ export async function readAndServe(io, rawPath, cwd, options) {
|
|
|
34
36
|
await recordServed(sessionKey, view.absolutePath, view.served, view.hashes.length);
|
|
35
37
|
}
|
|
36
38
|
await clearDriftReported(sessionKey, view.absolutePath);
|
|
39
|
+
const autoFooter = getAutoGuessFooter(view.absolutePath) ?? getAutoGuessFooter(rawPath) ?? getAutoGuessFooter(view.absolutePath.replace(/\\/g, "/")) ?? "";
|
|
40
|
+
const warning = autoFooter || undefined;
|
|
37
41
|
const text = view.hadUtf8DecodeErrors
|
|
38
42
|
? `${view.text}\n\n${UTF8_REWRITE_NOTE}`
|
|
39
43
|
: view.text;
|
|
@@ -42,5 +46,6 @@ export async function readAndServe(io, rawPath, cwd, options) {
|
|
|
42
46
|
served: view.served,
|
|
43
47
|
hadUtf8DecodeErrors: view.hadUtf8DecodeErrors,
|
|
44
48
|
absolutePath: view.absolutePath,
|
|
49
|
+
warning,
|
|
45
50
|
};
|
|
46
51
|
}
|
package/lib/store-config.js
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
*/
|
|
7
7
|
import { homedir } from "node:os";
|
|
8
8
|
import { isAbsolute, join } from "node:path";
|
|
9
|
-
import { readFileSync, statSync } from "node:fs";
|
|
9
|
+
import { readFileSync, statSync, appendFileSync } from "node:fs";
|
|
10
10
|
import { mkdir, writeFile } from "node:fs/promises";
|
|
11
11
|
import { resolveDshHome } from "@deepseek-ai/dsh-home-paths";
|
|
12
12
|
import { z } from "zod";
|
|
@@ -46,6 +46,11 @@ function parseSimpleYaml(content) {
|
|
|
46
46
|
continue;
|
|
47
47
|
const key = line.slice(0, colon).trim();
|
|
48
48
|
let value = line.slice(colon + 1).trim();
|
|
49
|
+
if (value.startsWith("[") && value.endsWith("]")) {
|
|
50
|
+
value = value.slice(1, -1);
|
|
51
|
+
out[key] = value;
|
|
52
|
+
continue;
|
|
53
|
+
}
|
|
49
54
|
value = stripQuotes(value);
|
|
50
55
|
value = stripInlineComment(value);
|
|
51
56
|
out[key] = value;
|
|
@@ -69,6 +74,13 @@ function parseAutoGitIgnoreRaw(v) {
|
|
|
69
74
|
return false;
|
|
70
75
|
return undefined;
|
|
71
76
|
}
|
|
77
|
+
function parseSupportedEncodingsRaw(v) {
|
|
78
|
+
const inner = v.trim().replace(/^\[/, "").replace(/\]$/, "");
|
|
79
|
+
const items = inner.split(/[ ,;]+/).map((s) => s.trim().toLowerCase()).filter((s) => s.length > 0);
|
|
80
|
+
if (items.length === 0)
|
|
81
|
+
return undefined;
|
|
82
|
+
return items;
|
|
83
|
+
}
|
|
72
84
|
// ---- Zod schema (single source of truth) ----
|
|
73
85
|
const StoreConfigSchema = z.object({
|
|
74
86
|
storeDir: z
|
|
@@ -76,6 +88,9 @@ const StoreConfigSchema = z.object({
|
|
|
76
88
|
.default("central")
|
|
77
89
|
.transform((v) => normalizeStoreDir(v) ?? "central"), // where the store lives: "central" (default, in $DSH_HOME/.../runtime/<name>-<hash8>/) | "workspace" (legacy <ws>/.dsh_better_edit/) | "/abs/path" (custom root)
|
|
78
90
|
autoGitignore: z.boolean().default(false), // workspace only: auto-append ".dsh_better_edit/" to .gitignore when .git exists but entry missing (false = warn once)
|
|
91
|
+
autoGuessEncoding: z.boolean().default(false), // VS Code files.autoGuessEncoding — deterministic BOM→UTF-8 first, model-assisted top-3 only when true
|
|
92
|
+
normalizeToUtf8: z.boolean().default(false), // when true, legacy files (GBK etc.) migrate to UTF-8 on first edit/write
|
|
93
|
+
supportedEncodings: z.array(z.string()).default(["gbk", "big5", "shift_jis", "euc-kr", "windows-1251", "iso-8859-1"]),
|
|
79
94
|
undo_ttl_s: z.number().int().min(-1).default(604800), // undo history TTL in seconds; -1 = keep forever (default 604800 = 7 days)
|
|
80
95
|
storeMaxAgeS: z.number().int().min(1).default(2592000), // central janitor: max idle age in seconds before evicting runtime/<name>-<hash8>/ (default 2592000 = 30 days)
|
|
81
96
|
storeMaxTotalBytes: z.number().int().min(0).default(524288000), // central janitor: max total bytes under runtime/ before LRU eviction (default 524288000 = 500 MB)
|
|
@@ -86,6 +101,9 @@ let cachedMtimeMs;
|
|
|
86
101
|
let cachedYamlPath;
|
|
87
102
|
let cachedEnvStoreDir;
|
|
88
103
|
let cachedEnvAutoGitignore;
|
|
104
|
+
let cachedEnvAutoGuessEncoding;
|
|
105
|
+
let cachedEnvNormalizeToUtf8;
|
|
106
|
+
let cachedEnvSupportedEncodings;
|
|
89
107
|
function configYamlPath() {
|
|
90
108
|
return join(resolveDshHome(), "plugins", "dsh-better-edit", "config.yaml");
|
|
91
109
|
}
|
|
@@ -102,6 +120,14 @@ storeDir: central
|
|
|
102
120
|
# workspace only: when .git exists but .gitignore lacks .dsh_better_edit/, true = auto-append ".dsh_better_edit/", false = warn once
|
|
103
121
|
autoGitignore: false
|
|
104
122
|
|
|
123
|
+
# encoding: deterministic BOM→UTF-8 first, guessing only when autoGuessEncoding:true
|
|
124
|
+
# when true, FS_NOT_TEXT scores supportedEncodings and surfaces top-3 previews for model pick
|
|
125
|
+
autoGuessEncoding: false
|
|
126
|
+
# when true, legacy files (GBK etc.) migrate to UTF-8 on first write
|
|
127
|
+
normalizeToUtf8: false
|
|
128
|
+
# allowlist scored for top-3 previews when autoGuessEncoding:true
|
|
129
|
+
supportedEncodings: [gbk, big5, shift_jis, euc-kr, windows-1251, iso-8859-1]
|
|
130
|
+
|
|
105
131
|
# undo history TTL in seconds; -1 = keep forever (default 604800 = 7 days)
|
|
106
132
|
undo_ttl_s: 604800
|
|
107
133
|
|
|
@@ -217,6 +243,27 @@ function fsAdapter() {
|
|
|
217
243
|
}
|
|
218
244
|
}
|
|
219
245
|
}
|
|
246
|
+
if (parsed.autoGuessEncoding !== undefined) {
|
|
247
|
+
const b = parseAutoGitIgnoreRaw(parsed.autoGuessEncoding);
|
|
248
|
+
if (b === undefined)
|
|
249
|
+
console.warn(`dsh-better-edit: autoGuessEncoding "${parsed.autoGuessEncoding}" invalid — expected true|false`);
|
|
250
|
+
else
|
|
251
|
+
out.autoGuessEncoding = b;
|
|
252
|
+
}
|
|
253
|
+
if (parsed.normalizeToUtf8 !== undefined) {
|
|
254
|
+
const b = parseAutoGitIgnoreRaw(parsed.normalizeToUtf8);
|
|
255
|
+
if (b === undefined)
|
|
256
|
+
console.warn(`dsh-better-edit: normalizeToUtf8 "${parsed.normalizeToUtf8}" invalid — expected true|false`);
|
|
257
|
+
else
|
|
258
|
+
out.normalizeToUtf8 = b;
|
|
259
|
+
}
|
|
260
|
+
if (parsed.supportedEncodings !== undefined) {
|
|
261
|
+
const arr = parseSupportedEncodingsRaw(parsed.supportedEncodings);
|
|
262
|
+
if (arr === undefined)
|
|
263
|
+
console.warn(`dsh-better-edit: supportedEncodings "${parsed.supportedEncodings}" invalid`);
|
|
264
|
+
else
|
|
265
|
+
out.supportedEncodings = arr;
|
|
266
|
+
}
|
|
220
267
|
if (parsed.storeMaxTotalBytes !== undefined) {
|
|
221
268
|
const n = Number(parsed.storeMaxTotalBytes);
|
|
222
269
|
const res = z.number().int().min(0).safeParse(n);
|
|
@@ -254,6 +301,30 @@ function envAdapter(base) {
|
|
|
254
301
|
}
|
|
255
302
|
}
|
|
256
303
|
}
|
|
304
|
+
const envGuess = process.env.DSH_BETTER_EDIT_AUTO_GUESS_ENCODING;
|
|
305
|
+
if (envGuess !== undefined) {
|
|
306
|
+
const b = parseAutoGitIgnoreRaw(envGuess);
|
|
307
|
+
if (b === undefined)
|
|
308
|
+
console.warn(`dsh-better-edit: DSH_BETTER_EDIT_AUTO_GUESS_ENCODING invalid — expected true|false`);
|
|
309
|
+
else
|
|
310
|
+
out.autoGuessEncoding = b;
|
|
311
|
+
}
|
|
312
|
+
const envNorm = process.env.DSH_BETTER_EDIT_NORMALIZE_TO_UTF8;
|
|
313
|
+
if (envNorm !== undefined) {
|
|
314
|
+
const b = parseAutoGitIgnoreRaw(envNorm);
|
|
315
|
+
if (b === undefined)
|
|
316
|
+
console.warn(`dsh-better-edit: DSH_BETTER_EDIT_NORMALIZE_TO_UTF8 invalid — expected true|false`);
|
|
317
|
+
else
|
|
318
|
+
out.normalizeToUtf8 = b;
|
|
319
|
+
}
|
|
320
|
+
const envSup = process.env.DSH_BETTER_EDIT_SUPPORTED_ENCODINGS;
|
|
321
|
+
if (envSup !== undefined) {
|
|
322
|
+
const arr = parseSupportedEncodingsRaw(envSup);
|
|
323
|
+
if (arr === undefined)
|
|
324
|
+
console.warn(`dsh-better-edit: DSH_BETTER_EDIT_SUPPORTED_ENCODINGS invalid`);
|
|
325
|
+
else
|
|
326
|
+
out.supportedEncodings = arr;
|
|
327
|
+
}
|
|
257
328
|
const envAuto = process.env.DSH_BETTER_EDIT_AUTO_GITIGNORE;
|
|
258
329
|
if (envAuto !== undefined) {
|
|
259
330
|
const b = parseAutoGitIgnoreRaw(envAuto);
|
|
@@ -266,6 +337,46 @@ function envAdapter(base) {
|
|
|
266
337
|
}
|
|
267
338
|
return out;
|
|
268
339
|
}
|
|
340
|
+
function complementMissingFieldsSync(yamlPath, yamlPartial) {
|
|
341
|
+
// Best-effort: append missing canonical keys with defaults, preserving existing content.
|
|
342
|
+
// Only runs when the file exists (mtime !== undefined) and some canonical key is absent.
|
|
343
|
+
const missingLines = [];
|
|
344
|
+
if (yamlPartial.storeDir === undefined)
|
|
345
|
+
missingLines.push("storeDir: central");
|
|
346
|
+
if (yamlPartial.autoGitignore === undefined)
|
|
347
|
+
missingLines.push("autoGitignore: false");
|
|
348
|
+
if (yamlPartial.autoGuessEncoding === undefined)
|
|
349
|
+
missingLines.push("autoGuessEncoding: false");
|
|
350
|
+
if (yamlPartial.normalizeToUtf8 === undefined)
|
|
351
|
+
missingLines.push("normalizeToUtf8: false");
|
|
352
|
+
if (yamlPartial.supportedEncodings === undefined)
|
|
353
|
+
missingLines.push("supportedEncodings: [gbk, big5, shift_jis, euc-kr, windows-1251, iso-8859-1]");
|
|
354
|
+
if (yamlPartial.undo_ttl_s === undefined)
|
|
355
|
+
missingLines.push("undo_ttl_s: 604800");
|
|
356
|
+
if (yamlPartial.storeMaxAgeS === undefined)
|
|
357
|
+
missingLines.push("storeMaxAgeS: 2592000");
|
|
358
|
+
if (yamlPartial.storeMaxTotalBytes === undefined)
|
|
359
|
+
missingLines.push("storeMaxTotalBytes: 524288000");
|
|
360
|
+
if (missingLines.length === 0)
|
|
361
|
+
return;
|
|
362
|
+
try {
|
|
363
|
+
// Ensure we don't duplicate if file was concurrently complemented — re-read raw keys.
|
|
364
|
+
const raw = readFileSync(yamlPath, "utf-8");
|
|
365
|
+
const parsed = parseSimpleYaml(raw);
|
|
366
|
+
// Re-check against raw parsed (aliases for storeMaxAgeS already handled via yamlPartial, but double-check).
|
|
367
|
+
const stillMissing = missingLines.filter((line) => {
|
|
368
|
+
const key = line.split(":")[0].trim();
|
|
369
|
+
return parsed[key] === undefined;
|
|
370
|
+
});
|
|
371
|
+
if (stillMissing.length === 0)
|
|
372
|
+
return;
|
|
373
|
+
const suffix = (raw.endsWith("\n") || raw.length === 0 ? "" : "\n") + stillMissing.join("\n") + "\n";
|
|
374
|
+
appendFileSync(yamlPath, suffix, "utf-8");
|
|
375
|
+
}
|
|
376
|
+
catch {
|
|
377
|
+
// best-effort: ENOENT (file deleted between stat and read) or permission — ignore
|
|
378
|
+
}
|
|
379
|
+
}
|
|
269
380
|
// ---- public load (merged, cached) ----
|
|
270
381
|
export function loadConfig() {
|
|
271
382
|
const yamlPath = configYamlPath();
|
|
@@ -277,15 +388,28 @@ export function loadConfig() {
|
|
|
277
388
|
// no yaml file — fall through to defaults/cache miss
|
|
278
389
|
}
|
|
279
390
|
const envStoreDir = process.env.DSH_BETTER_EDIT_STORE_DIR;
|
|
391
|
+
const envGuess = process.env.DSH_BETTER_EDIT_AUTO_GUESS_ENCODING;
|
|
392
|
+
const envNorm = process.env.DSH_BETTER_EDIT_NORMALIZE_TO_UTF8;
|
|
393
|
+
const envSup = process.env.DSH_BETTER_EDIT_SUPPORTED_ENCODINGS;
|
|
280
394
|
const envAuto = process.env.DSH_BETTER_EDIT_AUTO_GITIGNORE;
|
|
281
395
|
if (cachedConfig !== undefined &&
|
|
282
396
|
cachedMtimeMs === mtime &&
|
|
283
397
|
cachedYamlPath === yamlPath &&
|
|
284
398
|
cachedEnvStoreDir === envStoreDir &&
|
|
285
|
-
cachedEnvAutoGitignore === envAuto
|
|
399
|
+
cachedEnvAutoGitignore === envAuto &&
|
|
400
|
+
cachedEnvAutoGuessEncoding === envGuess &&
|
|
401
|
+
cachedEnvNormalizeToUtf8 === envNorm &&
|
|
402
|
+
cachedEnvSupportedEncodings === envSup) {
|
|
286
403
|
return cachedConfig;
|
|
287
404
|
}
|
|
288
405
|
const yamlPartial = fsAdapter();
|
|
406
|
+
if (mtime !== undefined) {
|
|
407
|
+
complementMissingFieldsSync(yamlPath, yamlPartial);
|
|
408
|
+
try {
|
|
409
|
+
mtime = statSync(yamlPath).mtimeMs;
|
|
410
|
+
}
|
|
411
|
+
catch { }
|
|
412
|
+
}
|
|
289
413
|
const envPartial = envAdapter({ ...DEFAULT_CONFIG, ...yamlPartial });
|
|
290
414
|
const merged = { ...DEFAULT_CONFIG, ...yamlPartial, ...envPartial };
|
|
291
415
|
// final Zod validation + normalization (expand abs, fallback)
|
|
@@ -305,6 +429,9 @@ export function loadConfig() {
|
|
|
305
429
|
cachedYamlPath = yamlPath;
|
|
306
430
|
cachedEnvStoreDir = envStoreDir;
|
|
307
431
|
cachedEnvAutoGitignore = envAuto;
|
|
432
|
+
cachedEnvAutoGuessEncoding = envGuess;
|
|
433
|
+
cachedEnvNormalizeToUtf8 = envNorm;
|
|
434
|
+
cachedEnvSupportedEncodings = envSup;
|
|
308
435
|
return cfg;
|
|
309
436
|
}
|
|
310
437
|
// For tests: reset cache
|
|
@@ -314,4 +441,7 @@ export function _resetConfigCache() {
|
|
|
314
441
|
cachedYamlPath = undefined;
|
|
315
442
|
cachedEnvStoreDir = undefined;
|
|
316
443
|
cachedEnvAutoGitignore = undefined;
|
|
444
|
+
cachedEnvAutoGuessEncoding = undefined;
|
|
445
|
+
cachedEnvNormalizeToUtf8 = undefined;
|
|
446
|
+
cachedEnvSupportedEncodings = undefined;
|
|
317
447
|
}
|
package/lib/tool-read.js
CHANGED
|
@@ -9,6 +9,7 @@ import { defineTool } from "@deepseek-ai/dsh-tools";
|
|
|
9
9
|
import { normalizeRequest as normReq, assertReadRequest, pathSchema } from "./contract.js";
|
|
10
10
|
import { readAndServe } from "./read-and-serve.js";
|
|
11
11
|
import { READ_DESCRIPTION } from "./prompts.js";
|
|
12
|
+
import { normalizeEncoding } from "./encoding.js";
|
|
12
13
|
import { execCwd, execSessionKey } from "./session-view.js";
|
|
13
14
|
import { withWorkspace } from "./session-view.js";
|
|
14
15
|
/**
|
|
@@ -32,30 +33,49 @@ export function buildReadTool(io) {
|
|
|
32
33
|
type: "number",
|
|
33
34
|
description: "Maximum number of lines to read",
|
|
34
35
|
},
|
|
36
|
+
encoding: {
|
|
37
|
+
type: "string",
|
|
38
|
+
description: "Text encoding for Reopen with Encoding (e.g. gbk, shift_jis, windows-1251). Case-insensitive.",
|
|
39
|
+
},
|
|
35
40
|
},
|
|
36
41
|
output: {
|
|
37
|
-
schema: { type: "string" },
|
|
38
|
-
render: (_args, value) =>
|
|
42
|
+
schema: { type: "object", properties: { text: { type: "string", required: true }, warning: { type: "string" } }, additionalProperties: false },
|
|
43
|
+
render: (_args, value) => {
|
|
44
|
+
const v = value;
|
|
45
|
+
if (typeof v === "string")
|
|
46
|
+
return [{ type: "text", text: v }];
|
|
47
|
+
const blocks = [{ type: "text", text: v.text }];
|
|
48
|
+
if (v.warning)
|
|
49
|
+
blocks.push({ type: "text", text: v.warning });
|
|
50
|
+
return blocks;
|
|
51
|
+
},
|
|
39
52
|
},
|
|
40
53
|
async execute(args, exec) {
|
|
41
54
|
return withWorkspace(execCwd(exec), async () => {
|
|
42
55
|
const cwd = execCwd(exec);
|
|
43
56
|
const sessionKey = execSessionKey(exec);
|
|
44
57
|
const signal = exec.signal;
|
|
58
|
+
const encoding = args.encoding;
|
|
59
|
+
if (encoding !== undefined) {
|
|
60
|
+
const norm = normalizeEncoding(String(encoding));
|
|
61
|
+
if (!norm)
|
|
62
|
+
throw new Error(`[E_BAD_ENCODING] Unknown encoding: ${String(encoding)}. Supported: utf8, gbk, big5, shift_jis, euc-kr, windows-1251, iso-8859-1`);
|
|
63
|
+
}
|
|
45
64
|
const canonical = normReq(args);
|
|
46
65
|
assertReadRequest(canonical);
|
|
47
66
|
const rawPath = canonical.path;
|
|
48
|
-
const { text, absolutePath } = await readAndServe(io, rawPath, cwd, {
|
|
67
|
+
const { text, absolutePath, warning } = await readAndServe(io, rawPath, cwd, {
|
|
49
68
|
sessionKey,
|
|
50
69
|
signal,
|
|
51
70
|
offset: canonical.offset,
|
|
52
71
|
limit: canonical.limit,
|
|
72
|
+
encoding: encoding,
|
|
53
73
|
});
|
|
54
74
|
// Record the present observation with the fs policy gate so later
|
|
55
75
|
// built-in write/edit calls see this file as observed at the
|
|
56
76
|
// version the model just read (a no-op when no policy listens).
|
|
57
77
|
await io.emitObserved(absolutePath, exec, signal);
|
|
58
|
-
return text;
|
|
78
|
+
return warning ? { text, warning } : { text };
|
|
59
79
|
});
|
|
60
80
|
},
|
|
61
81
|
});
|
package/lib/types/contract.d.ts
CHANGED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* VS Code encoding model — pure helpers for BOM→UTF-8→model-assisted top-3.
|
|
3
|
+
* No jschardet; heuristic scoring + iconv-lite rendering.
|
|
4
|
+
* @module dsh-better-edit/encoding
|
|
5
|
+
*/
|
|
6
|
+
export declare const SUPPORTED_ENCODINGS: readonly ["gbk", "big5", "shift_jis", "euc-kr", "windows-1251", "iso-8859-1"];
|
|
7
|
+
export type SupportedEncoding = (typeof SUPPORTED_ENCODINGS)[number];
|
|
8
|
+
/** Canonicalize case-insensitive encoding param, strip -_ . */
|
|
9
|
+
export declare function normalizeEncoding(input: string): string | undefined;
|
|
10
|
+
export declare function isSupportedEncoding(enc: string): boolean;
|
|
11
|
+
export interface BomInfo {
|
|
12
|
+
encoding: string;
|
|
13
|
+
bomLen: number;
|
|
14
|
+
hasBOM: boolean;
|
|
15
|
+
}
|
|
16
|
+
export declare function detectBom(bytes: Uint8Array): BomInfo | undefined;
|
|
17
|
+
export declare function isValidUtf8(bytes: Uint8Array): boolean;
|
|
18
|
+
export interface CandidatePreview {
|
|
19
|
+
encoding: string;
|
|
20
|
+
sample: string;
|
|
21
|
+
score: number;
|
|
22
|
+
}
|
|
23
|
+
export declare function scoreText(text: string, enc: string): number;
|
|
24
|
+
/** Decode bytes under enc via iconv-lite, return text or undefined on failure. */
|
|
25
|
+
export declare function decodeBytes(bytes: Uint8Array, enc: string): string | undefined;
|
|
26
|
+
export declare function encodeText(text: string, enc: string): Uint8Array | undefined;
|
|
27
|
+
/** Top-3 candidates for autoGuess: decode each allowlist enc, score, sort. */
|
|
28
|
+
export declare function top3Candidates(bytes: Uint8Array, allowlist: string[]): CandidatePreview[];
|
|
29
|
+
export declare function hasReplacementChar(text: string): boolean;
|
|
30
|
+
/**
|
|
31
|
+
* Try chardet (runk/node-chardet) for autoGuess when available.
|
|
32
|
+
* Dynamically imports 'chardet' (optionalDep) and returns canonical encoding
|
|
33
|
+
* if confidence >=45 and allowlist-accepted and decodable without �.
|
|
34
|
+
* Falls back to heuristic when chardet unavailable/low confidence.
|
|
35
|
+
*/
|
|
36
|
+
export declare function detectWithChardet(bytes: Uint8Array, allowlist: string[]): Promise<string | undefined>;
|
|
37
|
+
/**
|
|
38
|
+
* Chardet-based Top-3 for E_NOT_TEXT (primary) and autoGuess.
|
|
39
|
+
* Returns chardet candidates filtered by allowlist and decodable, else empty.
|
|
40
|
+
*/
|
|
41
|
+
export declare function chardetTop3Candidates(bytes: Uint8Array, allowlist: string[]): Promise<Array<{
|
|
42
|
+
encoding: string;
|
|
43
|
+
confidence: number;
|
|
44
|
+
sample: string;
|
|
45
|
+
}>>;
|
|
46
|
+
export declare function getTop3Candidates(bytes: Uint8Array, allowlist: string[]): Promise<CandidatePreview[]>;
|
package/lib/types/file-view.d.ts
CHANGED
|
@@ -128,6 +128,7 @@ export interface PreviewOpts {
|
|
|
128
128
|
limit?: number;
|
|
129
129
|
}
|
|
130
130
|
export interface ReadViewOpts extends PreviewOpts {
|
|
131
|
+
encoding?: string;
|
|
131
132
|
signal?: AbortSignal;
|
|
132
133
|
}
|
|
133
134
|
export declare function preview(content: string, hashes: string[], opts?: PreviewOpts, absolutePath?: string): Promise<{
|
package/lib/types/fs-bridge.d.ts
CHANGED
|
@@ -23,7 +23,7 @@ export interface FileIO {
|
|
|
23
23
|
/** Resolve a (possibly relative) request path against the session cwd to a canonical absolute path. */
|
|
24
24
|
resolve(path: string, cwd: string, signal?: AbortSignal): Promise<string>;
|
|
25
25
|
/** Read whole text; missing files, directories, and binary content throw. */
|
|
26
|
-
readText(absolutePath: string, signal?: AbortSignal): Promise<string>;
|
|
26
|
+
readText(absolutePath: string, signal?: AbortSignal, encoding?: string): Promise<string>;
|
|
27
27
|
/**
|
|
28
28
|
* Atomically write whole text, preserving mode when the file exists. On the
|
|
29
29
|
* dsh backend this dispatches `fs/write-intent` (policy guard), stamps the
|
|
@@ -53,6 +53,17 @@ export interface FileIO {
|
|
|
53
53
|
* @returns the mapped error, rethrown.
|
|
54
54
|
*/
|
|
55
55
|
export declare function mapFsError(error: unknown, displayPath: string): never;
|
|
56
|
+
export interface FileEncodingState {
|
|
57
|
+
encoding: string;
|
|
58
|
+
hasBOM: boolean;
|
|
59
|
+
version: string | undefined;
|
|
60
|
+
}
|
|
61
|
+
export declare function getAutoGuessFooter(targetKey: string): string | undefined;
|
|
62
|
+
export declare function setAutoGuessFooter(targetKey: string, footer: string): void;
|
|
63
|
+
export declare function clearAutoGuessFooter(targetKey?: string): void;
|
|
64
|
+
export declare function getEncodingState(targetKey: string): FileEncodingState | undefined;
|
|
65
|
+
export declare function setEncodingState(targetKey: string, state: FileEncodingState): void;
|
|
66
|
+
export declare function clearEncodingState(targetKey?: string): void;
|
|
56
67
|
/** FileIO over the deployment's `ctx.fs` service. */
|
|
57
68
|
export declare function ctxFsIO(fs: FileSystem, ctx: Context): FileIO;
|
|
58
69
|
/** FileIO over the host filesystem directly (tests, previews, fallback). */
|
|
@@ -12,6 +12,7 @@ import type { ServedRow } from "./hashline/served.js";
|
|
|
12
12
|
/** Appended when the file had non-UTF-8 bytes; editing rewrites it as UTF-8. */
|
|
13
13
|
export declare const UTF8_REWRITE_NOTE = "[Non-UTF-8 bytes shown as U+FFFD; editing rewrites the file as UTF-8.]";
|
|
14
14
|
export interface ReadAndServeOptions {
|
|
15
|
+
encoding?: string;
|
|
15
16
|
/** The session whose served rows these lines belong to. */
|
|
16
17
|
sessionKey: string;
|
|
17
18
|
signal?: AbortSignal;
|
|
@@ -20,6 +21,8 @@ export interface ReadAndServeOptions {
|
|
|
20
21
|
limit?: number;
|
|
21
22
|
}
|
|
22
23
|
export interface ReadAndServeResult {
|
|
24
|
+
/** Optional auto-guess warning, separate block. */
|
|
25
|
+
warning?: string;
|
|
23
26
|
/** The model-facing read text, including the UTF-8 note when applicable. */
|
|
24
27
|
text: string;
|
|
25
28
|
/** The rows recorded as served (empty when nothing was shown). */
|
|
@@ -3,25 +3,34 @@ export declare function expand(filePath: string): string;
|
|
|
3
3
|
declare const StoreConfigSchema: z.ZodObject<{
|
|
4
4
|
storeDir: z.ZodEffects<z.ZodDefault<z.ZodString>, string, string | undefined>;
|
|
5
5
|
autoGitignore: z.ZodDefault<z.ZodBoolean>;
|
|
6
|
+
autoGuessEncoding: z.ZodDefault<z.ZodBoolean>;
|
|
7
|
+
normalizeToUtf8: z.ZodDefault<z.ZodBoolean>;
|
|
8
|
+
supportedEncodings: z.ZodDefault<z.ZodArray<z.ZodString, "many">>;
|
|
6
9
|
undo_ttl_s: z.ZodDefault<z.ZodNumber>;
|
|
7
10
|
storeMaxAgeS: z.ZodDefault<z.ZodNumber>;
|
|
8
11
|
storeMaxTotalBytes: z.ZodDefault<z.ZodNumber>;
|
|
9
12
|
}, "strip", z.ZodTypeAny, {
|
|
10
13
|
storeDir: string;
|
|
11
14
|
autoGitignore: boolean;
|
|
15
|
+
autoGuessEncoding: boolean;
|
|
16
|
+
normalizeToUtf8: boolean;
|
|
17
|
+
supportedEncodings: string[];
|
|
12
18
|
undo_ttl_s: number;
|
|
13
19
|
storeMaxAgeS: number;
|
|
14
20
|
storeMaxTotalBytes: number;
|
|
15
21
|
}, {
|
|
16
22
|
storeDir?: string | undefined;
|
|
17
23
|
autoGitignore?: boolean | undefined;
|
|
24
|
+
autoGuessEncoding?: boolean | undefined;
|
|
25
|
+
normalizeToUtf8?: boolean | undefined;
|
|
26
|
+
supportedEncodings?: string[] | undefined;
|
|
18
27
|
undo_ttl_s?: number | undefined;
|
|
19
28
|
storeMaxAgeS?: number | undefined;
|
|
20
29
|
storeMaxTotalBytes?: number | undefined;
|
|
21
30
|
}>;
|
|
22
31
|
export type StoreConfig = z.infer<typeof StoreConfigSchema>;
|
|
23
32
|
/** Default config.yaml content with comments — generated when the file does not exist. */
|
|
24
|
-
export declare const DEFAULT_CONFIG_YAML = "# dsh-better-edit config\n# Location: $DSH_HOME/plugins/dsh-better-edit/config.yaml\n# Docs: https://github.com/Rianico/dsh-better-edit#configuration\n# Generated with defaults on first load \u2014 edit or delete freely.\n# DB files are disposable caches \u2014 safe to delete, rebuilt on next `read`.\n\n# where the store lives: \"central\" (default, in $DSH_HOME/.../runtime/<name>-<hash8>/) | \"workspace\" (legacy <ws>/.dsh_better_edit/) | \"/abs/path\" (custom root)\nstoreDir: central\n\n# workspace only: when .git exists but .gitignore lacks .dsh_better_edit/, true = auto-append \".dsh_better_edit/\", false = warn once\nautoGitignore: false\n\n# undo history TTL in seconds; -1 = keep forever (default 604800 = 7 days)\nundo_ttl_s: 604800\n\n# central janitor: max idle age in seconds before evicting runtime/<name>-<hash8>/ (default 2592000 = 30 days)\nstoreMaxAgeS: 2592000\n\n# central janitor: max total bytes under runtime/ before LRU eviction (default 524288000 = 500 MB)\nstoreMaxTotalBytes: 524288000\n";
|
|
33
|
+
export declare const DEFAULT_CONFIG_YAML = "# dsh-better-edit config\n# Location: $DSH_HOME/plugins/dsh-better-edit/config.yaml\n# Docs: https://github.com/Rianico/dsh-better-edit#configuration\n# Generated with defaults on first load \u2014 edit or delete freely.\n# DB files are disposable caches \u2014 safe to delete, rebuilt on next `read`.\n\n# where the store lives: \"central\" (default, in $DSH_HOME/.../runtime/<name>-<hash8>/) | \"workspace\" (legacy <ws>/.dsh_better_edit/) | \"/abs/path\" (custom root)\nstoreDir: central\n\n# workspace only: when .git exists but .gitignore lacks .dsh_better_edit/, true = auto-append \".dsh_better_edit/\", false = warn once\nautoGitignore: false\n\n# encoding: deterministic BOM\u2192UTF-8 first, guessing only when autoGuessEncoding:true\n# when true, FS_NOT_TEXT scores supportedEncodings and surfaces top-3 previews for model pick\nautoGuessEncoding: false\n# when true, legacy files (GBK etc.) migrate to UTF-8 on first write\nnormalizeToUtf8: false\n# allowlist scored for top-3 previews when autoGuessEncoding:true\nsupportedEncodings: [gbk, big5, shift_jis, euc-kr, windows-1251, iso-8859-1]\n\n# undo history TTL in seconds; -1 = keep forever (default 604800 = 7 days)\nundo_ttl_s: 604800\n\n# central janitor: max idle age in seconds before evicting runtime/<name>-<hash8>/ (default 2592000 = 30 days)\nstoreMaxAgeS: 2592000\n\n# central janitor: max total bytes under runtime/ before LRU eviction (default 524288000 = 500 MB)\nstoreMaxTotalBytes: 524288000\n";
|
|
25
34
|
/**
|
|
26
35
|
* Ensure a default config.yaml exists in the plugin home.
|
|
27
36
|
* Idempotent and concurrent-safe (wx) — never overwrites an existing file.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "dsh-better-edit",
|
|
3
|
-
"version": "0.5.
|
|
3
|
+
"version": "0.5.1",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Hash-anchored read/edit/undo_last_edit tools for DeepSeek Harness (dsh). Every line gets a unique 3-char hash (A-Za-z0-9) that stays stable across edits; stale or ambiguous anchors are rejected, never fuzzy-matched. Undo persists across restarts.",
|
|
6
6
|
"main": "lib/index.js",
|
|
@@ -56,6 +56,7 @@
|
|
|
56
56
|
"dependencies": {
|
|
57
57
|
"diff": "^8.0.2",
|
|
58
58
|
"file-type": "^21.3.0",
|
|
59
|
+
"iconv-lite": "^0.6.3",
|
|
59
60
|
"xxhash-wasm": "^1.1.0",
|
|
60
61
|
"zod": "^3.25.76"
|
|
61
62
|
},
|
|
@@ -100,5 +101,8 @@
|
|
|
100
101
|
"semantic-release": "^25.0.9",
|
|
101
102
|
"typescript": "^5.8.0",
|
|
102
103
|
"vitest": "^4.1.8"
|
|
104
|
+
},
|
|
105
|
+
"optionalDependencies": {
|
|
106
|
+
"chardet": "2.2.0"
|
|
103
107
|
}
|
|
104
108
|
}
|