dsh-better-edit 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +20 -2
- package/README.md +1 -1
- package/lib/contract.js +1 -1
- package/lib/edit-engine.js +4 -454
- package/lib/edit-response.js +24 -2
- package/lib/encoding.js +308 -0
- package/lib/file-encoding-state.js +263 -0
- package/lib/file-view.js +1 -1
- package/lib/fs-bridge.js +124 -19
- package/lib/hash-store.js +1 -2
- package/lib/hashline/anchor-pipeline.js +19 -165
- package/lib/index.js +2 -2
- package/lib/mutation/engine.js +388 -0
- package/lib/mutation.js +84 -11
- package/lib/paths.js +1 -1
- package/lib/read-and-serve.js +5 -0
- package/lib/session-view.js +16 -49
- package/lib/store/index.js +28 -0
- package/lib/store-config.js +132 -2
- package/lib/store-lifecycle.js +1 -1
- package/lib/tool-edit.js +2 -2
- package/lib/tool-read.js +26 -6
- package/lib/tool-undo.js +2 -2
- package/lib/types/contract.d.ts +1 -0
- package/lib/types/edit-diff.d.ts +1 -1
- package/lib/types/edit-engine.d.ts +4 -176
- package/lib/types/edit-response.d.ts +1 -1
- package/lib/types/encoding.d.ts +46 -0
- package/lib/types/file-encoding-state.d.ts +77 -0
- package/lib/types/file-view.d.ts +2 -1
- package/lib/types/fs-bridge.d.ts +5 -1
- package/lib/types/hashline/anchor-pipeline.d.ts +6 -20
- package/lib/types/hashline/index.d.ts +1 -1
- package/lib/types/mutation/engine.d.ts +143 -0
- package/lib/types/mutation.d.ts +30 -5
- package/lib/types/paths.d.ts +1 -1
- package/lib/types/read-and-serve.d.ts +4 -1
- package/lib/types/session-view.d.ts +16 -30
- package/lib/types/store/index.d.ts +25 -0
- package/lib/types/store-config.d.ts +10 -1
- package/lib/types/workspace-context.d.ts +23 -0
- package/lib/workspace-context.js +44 -0
- package/lib/write-hook.js +2 -1
- package/package.json +5 -1
package/lib/encoding.js
ADDED
|
@@ -0,0 +1,308 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* VS Code encoding model — pure helpers for BOM→UTF-8→model-assisted top-3.
|
|
3
|
+
* No jschardet; heuristic scoring + iconv-lite rendering.
|
|
4
|
+
* @module dsh-better-edit/encoding
|
|
5
|
+
*/
|
|
6
|
+
import iconv from "iconv-lite";
|
|
7
|
+
export const SUPPORTED_ENCODINGS = [
|
|
8
|
+
"gbk",
|
|
9
|
+
"big5",
|
|
10
|
+
"shift_jis",
|
|
11
|
+
"euc-kr",
|
|
12
|
+
"windows-1251",
|
|
13
|
+
"iso-8859-1",
|
|
14
|
+
];
|
|
15
|
+
const ALIASES = {
|
|
16
|
+
utf8: "utf8",
|
|
17
|
+
utf8bom: "utf8bom",
|
|
18
|
+
utf16le: "utf16le",
|
|
19
|
+
utf16be: "utf16be",
|
|
20
|
+
utf32le: "utf32le",
|
|
21
|
+
utf32be: "utf32be",
|
|
22
|
+
gbk: "gbk",
|
|
23
|
+
gb18030: "gbk",
|
|
24
|
+
gb2312: "gbk",
|
|
25
|
+
big5: "big5",
|
|
26
|
+
shiftjis: "shift_jis",
|
|
27
|
+
shift_jis: "shift_jis",
|
|
28
|
+
sjis: "shift_jis",
|
|
29
|
+
euckr: "euc-kr",
|
|
30
|
+
"euc-kr": "euc-kr",
|
|
31
|
+
windows1251: "windows-1251",
|
|
32
|
+
cp1251: "windows-1251",
|
|
33
|
+
"windows-1251": "windows-1251",
|
|
34
|
+
iso88591: "iso-8859-1",
|
|
35
|
+
"iso-8859-1": "iso-8859-1",
|
|
36
|
+
latin1: "iso-8859-1",
|
|
37
|
+
};
|
|
38
|
+
/** Canonicalize case-insensitive encoding param, strip -_ . */
|
|
39
|
+
export function normalizeEncoding(input) {
|
|
40
|
+
const key = input.trim().toLowerCase().replace(/[-_]/g, "");
|
|
41
|
+
// need original with dashes for lookup: try both
|
|
42
|
+
const candidates = [key, input.trim().toLowerCase()];
|
|
43
|
+
for (const c of candidates) {
|
|
44
|
+
const v = ALIASES[c] ?? ALIASES[key];
|
|
45
|
+
if (v)
|
|
46
|
+
return v;
|
|
47
|
+
}
|
|
48
|
+
// also try direct lowercased
|
|
49
|
+
const lower = input.trim().toLowerCase();
|
|
50
|
+
if (ALIASES[lower])
|
|
51
|
+
return ALIASES[lower];
|
|
52
|
+
return undefined;
|
|
53
|
+
}
|
|
54
|
+
export function isSupportedEncoding(enc) {
|
|
55
|
+
const norm = normalizeEncoding(enc);
|
|
56
|
+
if (!norm)
|
|
57
|
+
return false;
|
|
58
|
+
return SUPPORTED_ENCODINGS.includes(norm) || ["utf8", "utf8bom", "utf16le", "utf16be", "utf32le", "utf32be"].includes(norm);
|
|
59
|
+
}
|
|
60
|
+
export function detectBom(bytes) {
|
|
61
|
+
if (bytes.length >= 3 && bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf)
|
|
62
|
+
return { encoding: "utf8bom", bomLen: 3, hasBOM: true };
|
|
63
|
+
if (bytes.length >= 4 && bytes[0] === 0xff && bytes[1] === 0xfe && bytes[2] === 0x00 && bytes[3] === 0x00)
|
|
64
|
+
return { encoding: "utf32le", bomLen: 4, hasBOM: true };
|
|
65
|
+
if (bytes.length >= 4 && bytes[0] === 0x00 && bytes[1] === 0x00 && bytes[2] === 0xfe && bytes[3] === 0xff)
|
|
66
|
+
return { encoding: "utf32be", bomLen: 4, hasBOM: true };
|
|
67
|
+
if (bytes.length >= 2 && bytes[0] === 0xff && bytes[1] === 0xfe)
|
|
68
|
+
return { encoding: "utf16le", bomLen: 2, hasBOM: true };
|
|
69
|
+
if (bytes.length >= 2 && bytes[0] === 0xfe && bytes[1] === 0xff)
|
|
70
|
+
return { encoding: "utf16be", bomLen: 2, hasBOM: true };
|
|
71
|
+
return undefined;
|
|
72
|
+
}
|
|
73
|
+
export function isValidUtf8(bytes) {
|
|
74
|
+
try {
|
|
75
|
+
new TextDecoder("utf-8", { fatal: true }).decode(bytes);
|
|
76
|
+
return true;
|
|
77
|
+
}
|
|
78
|
+
catch {
|
|
79
|
+
return false;
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
function printableRatio(text) {
|
|
83
|
+
if (text.length === 0)
|
|
84
|
+
return 1;
|
|
85
|
+
let printable = 0;
|
|
86
|
+
for (const ch of text) {
|
|
87
|
+
const code = ch.charCodeAt(0);
|
|
88
|
+
if (code === 10 || code === 13 || code === 9)
|
|
89
|
+
printable += 1;
|
|
90
|
+
else if (code >= 32 && code !== 127)
|
|
91
|
+
printable += 1;
|
|
92
|
+
else if (code > 127)
|
|
93
|
+
printable += 1; // non-ascii considered printable if not replacement
|
|
94
|
+
}
|
|
95
|
+
return printable / text.length;
|
|
96
|
+
}
|
|
97
|
+
function scriptBonus(text, enc) {
|
|
98
|
+
let bonus = 0;
|
|
99
|
+
const cjk = (text.match(/[\u4e00-\u9fff]/g) || []).length;
|
|
100
|
+
const cyrillic = (text.match(/[\u0400-\u04ff]/g) || []).length;
|
|
101
|
+
const hiragana = (text.match(/[\u3040-\u309f\u30a0-\u30ff]/g) || []).length;
|
|
102
|
+
const hangul = (text.match(/[\uac00-\ud7af]/g) || []).length;
|
|
103
|
+
if (enc === "gbk" || enc === "big5")
|
|
104
|
+
bonus += cjk * 5;
|
|
105
|
+
if (enc === "windows-1251")
|
|
106
|
+
bonus += cyrillic * 2;
|
|
107
|
+
if (enc === "shift_jis")
|
|
108
|
+
bonus += hiragana * 8;
|
|
109
|
+
if (enc === "euc-kr")
|
|
110
|
+
bonus += hangul * 4;
|
|
111
|
+
return bonus;
|
|
112
|
+
}
|
|
113
|
+
function smartSlice(text) {
|
|
114
|
+
// seek first non-ascii char, slice ±32 around it, truncate to 50
|
|
115
|
+
let idx = -1;
|
|
116
|
+
for (let i = 0; i < text.length; i += 1) {
|
|
117
|
+
if (text.charCodeAt(i) > 127) {
|
|
118
|
+
idx = i;
|
|
119
|
+
break;
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
if (idx === -1)
|
|
123
|
+
return text.slice(0, 50);
|
|
124
|
+
const start = Math.max(0, idx - 32);
|
|
125
|
+
const slice = text.slice(start, start + 64);
|
|
126
|
+
return slice.slice(0, 50).replace(/\s+/g, " ").trim();
|
|
127
|
+
}
|
|
128
|
+
export function scoreText(text, enc) {
|
|
129
|
+
if (text.includes("\uFFFD"))
|
|
130
|
+
return -1000;
|
|
131
|
+
const ratio = printableRatio(text);
|
|
132
|
+
if (ratio < 0.85)
|
|
133
|
+
return -500 + ratio * 10;
|
|
134
|
+
let score = ratio * 10;
|
|
135
|
+
score += scriptBonus(text, enc) * 0.5;
|
|
136
|
+
return score;
|
|
137
|
+
}
|
|
138
|
+
/** Decode bytes under enc via iconv-lite, return text or undefined on failure. */
|
|
139
|
+
export function decodeBytes(bytes, enc) {
|
|
140
|
+
try {
|
|
141
|
+
if (enc === "utf8" || enc === "utf8bom")
|
|
142
|
+
return new TextDecoder("utf-8", { fatal: true }).decode(bytes);
|
|
143
|
+
if (enc === "utf16le")
|
|
144
|
+
return new TextDecoder("utf-16le").decode(bytes);
|
|
145
|
+
if (enc === "utf16be")
|
|
146
|
+
return new TextDecoder("utf-16be").decode(bytes);
|
|
147
|
+
// iconv-lite for legacy
|
|
148
|
+
return iconv.decode(Buffer.from(bytes), enc);
|
|
149
|
+
}
|
|
150
|
+
catch {
|
|
151
|
+
return undefined;
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
export function encodeText(text, enc) {
|
|
155
|
+
try {
|
|
156
|
+
if (enc === "utf8" || enc === "utf8bom")
|
|
157
|
+
return new TextEncoder().encode(text);
|
|
158
|
+
if (enc === "utf16le")
|
|
159
|
+
return new Uint8Array(Buffer.from(text, "utf16le"));
|
|
160
|
+
if (enc === "utf16be") {
|
|
161
|
+
// Node no utf16be, use iconv
|
|
162
|
+
return new Uint8Array(iconv.encode(text, "utf16be"));
|
|
163
|
+
}
|
|
164
|
+
return new Uint8Array(iconv.encode(text, enc));
|
|
165
|
+
}
|
|
166
|
+
catch {
|
|
167
|
+
return undefined;
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
/** Top-3 candidates for autoGuess: decode each allowlist enc, score, sort. */
|
|
171
|
+
export function top3Candidates(bytes, allowlist) {
|
|
172
|
+
const candidates = [];
|
|
173
|
+
// handle BOM case separately? caller should handle BOM before calling.
|
|
174
|
+
for (const raw of allowlist) {
|
|
175
|
+
const enc = normalizeEncoding(raw) ?? raw;
|
|
176
|
+
const text = decodeBytes(bytes, enc);
|
|
177
|
+
if (text === undefined)
|
|
178
|
+
continue;
|
|
179
|
+
const sample = smartSlice(text);
|
|
180
|
+
const score = scoreText(text, enc);
|
|
181
|
+
candidates.push({ encoding: enc, sample, score });
|
|
182
|
+
}
|
|
183
|
+
candidates.sort((a, b) => b.score - a.score);
|
|
184
|
+
return candidates.slice(0, 3);
|
|
185
|
+
}
|
|
186
|
+
export function hasReplacementChar(text) {
|
|
187
|
+
return text.includes("\uFFFD");
|
|
188
|
+
}
|
|
189
|
+
/**
|
|
190
|
+
* Try chardet (runk/node-chardet) for autoGuess when available.
|
|
191
|
+
* Dynamically imports 'chardet' (optionalDep) and returns canonical encoding
|
|
192
|
+
* if confidence >=45 and allowlist-accepted and decodable without �.
|
|
193
|
+
* Falls back to heuristic when chardet unavailable/low confidence.
|
|
194
|
+
*/
|
|
195
|
+
export async function detectWithChardet(bytes, allowlist) {
|
|
196
|
+
try {
|
|
197
|
+
const mod = await import("chardet");
|
|
198
|
+
// chardet ESM: default export has analyse, CJS: named
|
|
199
|
+
const chardetLike = mod.default ?? mod;
|
|
200
|
+
const analyse = chardetLike.analyse;
|
|
201
|
+
if (typeof analyse !== "function")
|
|
202
|
+
return undefined;
|
|
203
|
+
const buf = Buffer.isBuffer(bytes) ? bytes : Buffer.from(bytes);
|
|
204
|
+
const results = analyse(buf);
|
|
205
|
+
if (!Array.isArray(results) || results.length === 0)
|
|
206
|
+
return undefined;
|
|
207
|
+
const allowNorm = new Set(allowlist.map((a) => normalizeEncoding(a) ?? a.toLowerCase()));
|
|
208
|
+
for (const r of results) {
|
|
209
|
+
if (typeof r.name !== "string" || typeof r.confidence !== "number")
|
|
210
|
+
continue;
|
|
211
|
+
if (r.confidence < 45)
|
|
212
|
+
continue;
|
|
213
|
+
const norm = normalizeEncoding(r.name);
|
|
214
|
+
if (!norm)
|
|
215
|
+
continue;
|
|
216
|
+
if (!allowNorm.has(norm))
|
|
217
|
+
continue;
|
|
218
|
+
// iconv-lite existence check (use try decode)
|
|
219
|
+
const encForCheck = norm;
|
|
220
|
+
if (!iconv.encodingExists(encForCheck) && !iconv.encodingExists(r.name))
|
|
221
|
+
continue;
|
|
222
|
+
const text = decodeBytes(bytes, encForCheck);
|
|
223
|
+
if (text === undefined)
|
|
224
|
+
continue;
|
|
225
|
+
if (text.includes("\uFFFD"))
|
|
226
|
+
continue;
|
|
227
|
+
// extra guard: printable ratio
|
|
228
|
+
if (text.length > 0) {
|
|
229
|
+
let printable = 0;
|
|
230
|
+
for (const ch of text) {
|
|
231
|
+
const c = ch.charCodeAt(0);
|
|
232
|
+
if (c === 10 || c === 13 || c === 9)
|
|
233
|
+
printable += 1;
|
|
234
|
+
else if (c >= 32 && c !== 127)
|
|
235
|
+
printable += 1;
|
|
236
|
+
else if (c > 127)
|
|
237
|
+
printable += 1;
|
|
238
|
+
}
|
|
239
|
+
if (printable / text.length < 0.85)
|
|
240
|
+
continue;
|
|
241
|
+
}
|
|
242
|
+
return norm;
|
|
243
|
+
}
|
|
244
|
+
return undefined;
|
|
245
|
+
}
|
|
246
|
+
catch {
|
|
247
|
+
return undefined;
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
/**
|
|
251
|
+
* Chardet-based Top-3 for E_NOT_TEXT (primary) and autoGuess.
|
|
252
|
+
* Returns chardet candidates filtered by allowlist and decodable, else empty.
|
|
253
|
+
*/
|
|
254
|
+
export async function chardetTop3Candidates(bytes, allowlist) {
|
|
255
|
+
try {
|
|
256
|
+
const mod = await import("chardet");
|
|
257
|
+
const chardetLike = mod.default ?? mod;
|
|
258
|
+
const analyse = chardetLike.analyse;
|
|
259
|
+
if (typeof analyse !== "function")
|
|
260
|
+
return [];
|
|
261
|
+
const buf = Buffer.isBuffer(bytes) ? bytes : Buffer.from(bytes);
|
|
262
|
+
const results = analyse(buf);
|
|
263
|
+
if (!Array.isArray(results) || results.length === 0)
|
|
264
|
+
return [];
|
|
265
|
+
const allowNorm = new Set(allowlist.map((a) => normalizeEncoding(a) ?? a.toLowerCase()));
|
|
266
|
+
const out = [];
|
|
267
|
+
for (const r of results) {
|
|
268
|
+
if (typeof r.name !== "string" || typeof r.confidence !== "number")
|
|
269
|
+
continue;
|
|
270
|
+
const norm = normalizeEncoding(r.name);
|
|
271
|
+
if (!norm)
|
|
272
|
+
continue;
|
|
273
|
+
if (!allowNorm.has(norm))
|
|
274
|
+
continue;
|
|
275
|
+
if (!iconv.encodingExists(norm) && !iconv.encodingExists(r.name))
|
|
276
|
+
continue;
|
|
277
|
+
const text = decodeBytes(bytes, norm);
|
|
278
|
+
if (text === undefined)
|
|
279
|
+
continue;
|
|
280
|
+
const sample = (() => {
|
|
281
|
+
let idx = -1;
|
|
282
|
+
for (let i = 0; i < text.length; i += 1)
|
|
283
|
+
if (text.charCodeAt(i) > 127) {
|
|
284
|
+
idx = i;
|
|
285
|
+
break;
|
|
286
|
+
}
|
|
287
|
+
if (idx === -1)
|
|
288
|
+
return text.slice(0, 50);
|
|
289
|
+
const start = Math.max(0, idx - 32);
|
|
290
|
+
return text.slice(start, start + 64).slice(0, 50).replace(/\s+/g, " ").trim();
|
|
291
|
+
})();
|
|
292
|
+
out.push({ encoding: norm, confidence: r.confidence, sample });
|
|
293
|
+
if (out.length >= 3)
|
|
294
|
+
break;
|
|
295
|
+
}
|
|
296
|
+
return out;
|
|
297
|
+
}
|
|
298
|
+
catch {
|
|
299
|
+
return [];
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
export async function getTop3Candidates(bytes, allowlist) {
|
|
303
|
+
const chardetCands = await chardetTop3Candidates(bytes, allowlist);
|
|
304
|
+
if (chardetCands.length > 0) {
|
|
305
|
+
return chardetCands.map((c) => ({ encoding: c.encoding, sample: c.sample, score: c.confidence }));
|
|
306
|
+
}
|
|
307
|
+
return top3Candidates(bytes, allowlist);
|
|
308
|
+
}
|
|
@@ -0,0 +1,263 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* FileEncodingState — deep module owning the file encoding state round-trip.
|
|
3
|
+
*
|
|
4
|
+
* Single seam for deterministic admission (BOM → strict UTF-8 → autoGuess gate
|
|
5
|
+
* → Top-3 always pushed), canonical encoding enum, version-invalidated memo,
|
|
6
|
+
* and BOM+lineEnding round-trip. Previously scattered across fs-bridge
|
|
7
|
+
* (branching + fallback Map + Top-3), encoding.ts (pure scoring), and
|
|
8
|
+
* file-view/fs-write (stripBOM/detectEnding/encode back). Now one file owns
|
|
9
|
+
* the invariant; deleting it would scatter round-trip fidelity (deep).
|
|
10
|
+
*
|
|
11
|
+
* Interface is the test surface: pure decode is tested without filesystem,
|
|
12
|
+
* IO adapters (fs-bridge) are thin. `store-config.autoGuessEncoding` stays the
|
|
13
|
+
* only config knob — this module reads it but does not own it.
|
|
14
|
+
*
|
|
15
|
+
* @module dsh-better-edit/file-encoding-state
|
|
16
|
+
*/
|
|
17
|
+
import { detectBom, isValidUtf8, decodeBytes, normalizeEncoding, top3Candidates, chardetTop3Candidates } from "./encoding.js";
|
|
18
|
+
import { detectEnding } from "./edit-diff.js";
|
|
19
|
+
const encodingMemo = new Map();
|
|
20
|
+
const autoGuessFooterMemo = new Map();
|
|
21
|
+
export function getEncodingState(targetKey) {
|
|
22
|
+
return encodingMemo.get(targetKey);
|
|
23
|
+
}
|
|
24
|
+
export function setEncodingState(targetKey, state) {
|
|
25
|
+
encodingMemo.set(targetKey, state);
|
|
26
|
+
}
|
|
27
|
+
export function clearEncodingState(targetKey) {
|
|
28
|
+
if (targetKey)
|
|
29
|
+
encodingMemo.delete(targetKey);
|
|
30
|
+
else
|
|
31
|
+
encodingMemo.clear();
|
|
32
|
+
}
|
|
33
|
+
export function getAutoGuessFooter(targetKey) {
|
|
34
|
+
return autoGuessFooterMemo.get(targetKey);
|
|
35
|
+
}
|
|
36
|
+
export function setAutoGuessFooter(targetKey, footer) {
|
|
37
|
+
autoGuessFooterMemo.set(targetKey, footer);
|
|
38
|
+
}
|
|
39
|
+
export function clearAutoGuessFooter(targetKey) {
|
|
40
|
+
if (targetKey)
|
|
41
|
+
autoGuessFooterMemo.delete(targetKey);
|
|
42
|
+
else
|
|
43
|
+
autoGuessFooterMemo.clear();
|
|
44
|
+
}
|
|
45
|
+
/** Invalidate memo when FsVersion drifted (ADR-0012). */
|
|
46
|
+
export function invalidateIfStale(targetKey, currentVersion) {
|
|
47
|
+
const memo = encodingMemo.get(targetKey);
|
|
48
|
+
if (memo && memo.version !== currentVersion)
|
|
49
|
+
encodingMemo.delete(targetKey);
|
|
50
|
+
}
|
|
51
|
+
// ---------------------------------------------------------------------------
|
|
52
|
+
// Pure helpers — no IO
|
|
53
|
+
// ---------------------------------------------------------------------------
|
|
54
|
+
export function buildTop3ErrorMessage(displayPath, candidates) {
|
|
55
|
+
const candStr = candidates.map((c) => `${c.encoding}("${c.sample.slice(0, 20).replace(/"/g, "'")}")`).join(", ");
|
|
56
|
+
return `[E_NOT_TEXT] Path is not a readable UTF-8 text file: ${displayPath}. Hashline editing only supports text files. Top-3 guesses: ${candStr}. Try read({encoding: "<encoding>"}) or set DSH_BETTER_EDIT_AUTO_GUESS_ENCODING=true to auto-decode.`;
|
|
57
|
+
}
|
|
58
|
+
function buildAutoGuessFooterFromCandidates(candidates) {
|
|
59
|
+
// chardet path vs heuristic path share same footer shape, just score vs confidence
|
|
60
|
+
const candsStr = candidates
|
|
61
|
+
.map((c) => {
|
|
62
|
+
const score = c.confidence !== undefined ? String(c.confidence) : c.score !== undefined ? c.score.toFixed(0) : "0";
|
|
63
|
+
return `${c.encoding} ${score}`;
|
|
64
|
+
})
|
|
65
|
+
.join(", ");
|
|
66
|
+
const top = candidates[0];
|
|
67
|
+
const topScore = top.confidence !== undefined ? String(top.confidence) : top.score !== undefined ? top.score.toFixed(0) : "0";
|
|
68
|
+
return `\n\n[Auto-guessed: ${top.encoding} ${topScore}, candidates: ${candsStr} — re-read with read({encoding}) if garbled]`;
|
|
69
|
+
}
|
|
70
|
+
function isMidConfidenceChardet(top, second) {
|
|
71
|
+
return top.confidence < 70 || (second !== undefined && top.confidence - second.confidence < 10);
|
|
72
|
+
}
|
|
73
|
+
function isMidConfidenceHeuristic(best, second) {
|
|
74
|
+
return second !== undefined && best.score - second.score < 10;
|
|
75
|
+
}
|
|
76
|
+
/**
|
|
77
|
+
* Deterministic admission for a raw byte buffer. No IO, no side effects
|
|
78
|
+
* except Top-3 scoring. Caller owns recording the returned state via
|
|
79
|
+
* `recordOpenState` and handling the footer vs E_NOT_TEXT split.
|
|
80
|
+
*
|
|
81
|
+
* Order is fixed: BOM sniff always precedes strict UTF-8, which always
|
|
82
|
+
* precedes guessing. Probabilistic encodings are last resort, gated by
|
|
83
|
+
* `autoGuessEncoding`, but Top-3 is always computed for the error/ footer.
|
|
84
|
+
*/
|
|
85
|
+
export async function decodeForOpen(bytes, config, opts = {}) {
|
|
86
|
+
const hint = opts.encodingHint ? normalizeEncoding(opts.encodingHint) : undefined;
|
|
87
|
+
if (opts.encodingHint && !hint) {
|
|
88
|
+
throw new Error(`[E_BAD_ENCODING] Unknown encoding: ${opts.encodingHint}`);
|
|
89
|
+
}
|
|
90
|
+
// 1) Explicit hint (Reopen with Encoding) — caller already chose, no autoGuess
|
|
91
|
+
if (hint) {
|
|
92
|
+
const bom = detectBom(bytes);
|
|
93
|
+
// If BOM present, it wins for hasBOM but hint wins for decode (VS Code)
|
|
94
|
+
const off = bom ? bom.bomLen : 0;
|
|
95
|
+
const slice = bom ? bytes.subarray(off) : bytes;
|
|
96
|
+
// For BOM encodings, decode the full bytes (including BOM) via helper; otherwise hint
|
|
97
|
+
let decoded;
|
|
98
|
+
if (bom && (bom.encoding === "utf8bom" || bom.encoding === "utf16le" || bom.encoding === "utf16be" || bom.encoding === "utf32le" || bom.encoding === "utf32be")) {
|
|
99
|
+
// When hint differs from BOM, hint takes precedence for decoding the slice
|
|
100
|
+
// but we preserve hasBOM from BOM sniff. For utf variants, BOM encoding
|
|
101
|
+
// already handled; for explicit hint we decode slice with hint.
|
|
102
|
+
decoded = decodeBytes(slice, hint);
|
|
103
|
+
if (decoded === undefined)
|
|
104
|
+
throw new Error(`[E_DECODE_FAILED] Cannot decode bytes as ${hint}`);
|
|
105
|
+
return {
|
|
106
|
+
text: decoded,
|
|
107
|
+
encoding: hint,
|
|
108
|
+
hasBOM: !!bom,
|
|
109
|
+
lineEnding: detectEnding(decoded),
|
|
110
|
+
candidates: [],
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
decoded = decodeBytes(slice, hint);
|
|
114
|
+
if (decoded === undefined)
|
|
115
|
+
throw new Error(`[E_DECODE_FAILED] Cannot decode bytes as ${hint}`);
|
|
116
|
+
return {
|
|
117
|
+
text: decoded,
|
|
118
|
+
encoding: hint,
|
|
119
|
+
hasBOM: !!bom,
|
|
120
|
+
lineEnding: detectEnding(decoded),
|
|
121
|
+
candidates: [],
|
|
122
|
+
};
|
|
123
|
+
}
|
|
124
|
+
// 2) BOM sniff — deterministic, before UTF-8
|
|
125
|
+
const bom = detectBom(bytes);
|
|
126
|
+
if (bom) {
|
|
127
|
+
const decoded = decodeBytes(bytes, bom.encoding);
|
|
128
|
+
if (decoded !== undefined) {
|
|
129
|
+
return {
|
|
130
|
+
text: decoded,
|
|
131
|
+
encoding: bom.encoding,
|
|
132
|
+
hasBOM: true,
|
|
133
|
+
lineEnding: detectEnding(decoded),
|
|
134
|
+
candidates: [],
|
|
135
|
+
};
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
// 3) Strict UTF-8 fatal:true — deterministic
|
|
139
|
+
if (isValidUtf8(bytes)) {
|
|
140
|
+
const text = new TextDecoder("utf-8", { fatal: true }).decode(bytes);
|
|
141
|
+
return {
|
|
142
|
+
text,
|
|
143
|
+
encoding: "utf8",
|
|
144
|
+
hasBOM: false,
|
|
145
|
+
lineEnding: detectEnding(text),
|
|
146
|
+
candidates: [],
|
|
147
|
+
};
|
|
148
|
+
}
|
|
149
|
+
// 4) AutoGuess gate — probabilistic, last resort
|
|
150
|
+
if (config.autoGuessEncoding) {
|
|
151
|
+
// Try chardet first (Top-1 with footer for mid-confidence)
|
|
152
|
+
try {
|
|
153
|
+
const chardetCands = await chardetTop3Candidates(bytes, config.supportedEncodings);
|
|
154
|
+
if (chardetCands.length > 0) {
|
|
155
|
+
const top = chardetCands[0];
|
|
156
|
+
const second = chardetCands[1];
|
|
157
|
+
const isMid = isMidConfidenceChardet(top, second);
|
|
158
|
+
const dec = decodeBytes(bytes, top.encoding);
|
|
159
|
+
if (dec !== undefined && !dec.includes("\uFFFD")) {
|
|
160
|
+
let footer;
|
|
161
|
+
if (isMid)
|
|
162
|
+
footer = buildAutoGuessFooterFromCandidates(chardetCands);
|
|
163
|
+
return {
|
|
164
|
+
text: dec,
|
|
165
|
+
encoding: top.encoding,
|
|
166
|
+
hasBOM: false,
|
|
167
|
+
lineEnding: detectEnding(dec),
|
|
168
|
+
footer,
|
|
169
|
+
candidates: chardetCands.map((c) => ({ encoding: c.encoding, sample: c.sample, score: c.confidence })),
|
|
170
|
+
};
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
catch {
|
|
175
|
+
// best-effort
|
|
176
|
+
}
|
|
177
|
+
const candidates = top3Candidates(bytes, config.supportedEncodings);
|
|
178
|
+
if (candidates.length > 0) {
|
|
179
|
+
const best = candidates[0];
|
|
180
|
+
const second = candidates[1];
|
|
181
|
+
const isMid = isMidConfidenceHeuristic(best, second);
|
|
182
|
+
const dec = decodeBytes(bytes, best.encoding);
|
|
183
|
+
if (dec !== undefined) {
|
|
184
|
+
// Always produce footer for heuristic path (mid check decides but we always have candidates)
|
|
185
|
+
// Keep existing behavior: always a footer for heuristic, but mid decides inclusion — for now always include
|
|
186
|
+
// to match prior fs-bridge which always built footerHeu.
|
|
187
|
+
void isMid;
|
|
188
|
+
const footer = buildAutoGuessFooterFromCandidates(candidates);
|
|
189
|
+
return {
|
|
190
|
+
text: dec,
|
|
191
|
+
encoding: best.encoding,
|
|
192
|
+
hasBOM: false,
|
|
193
|
+
lineEnding: detectEnding(dec),
|
|
194
|
+
footer,
|
|
195
|
+
candidates,
|
|
196
|
+
};
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
// No candidate succeeded — fall through to E_NOT_TEXT with Top-3
|
|
200
|
+
}
|
|
201
|
+
// 5) E_NOT_TEXT + Top-3 always pushed (even when autoGuess off)
|
|
202
|
+
// Use chardet Top-3 when available, else heuristic Top-3 — same as fs-bridge fallback
|
|
203
|
+
let candidates = [];
|
|
204
|
+
try {
|
|
205
|
+
const chardetCands = await chardetTop3Candidates(bytes, config.supportedEncodings);
|
|
206
|
+
if (chardetCands.length > 0) {
|
|
207
|
+
candidates = chardetCands.map((c) => ({ encoding: c.encoding, sample: c.sample, score: c.confidence }));
|
|
208
|
+
}
|
|
209
|
+
else {
|
|
210
|
+
candidates = top3Candidates(bytes, config.supportedEncodings);
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
catch {
|
|
214
|
+
candidates = top3Candidates(bytes, config.supportedEncodings);
|
|
215
|
+
}
|
|
216
|
+
const display = opts.displayPath ?? "(unknown path)";
|
|
217
|
+
throw new Error(buildTop3ErrorMessage(display, candidates));
|
|
218
|
+
}
|
|
219
|
+
// ---------------------------------------------------------------------------
|
|
220
|
+
// Recording helpers — caller supplies targetKey + version after decode
|
|
221
|
+
// ---------------------------------------------------------------------------
|
|
222
|
+
export function recordOpenState(targetKey, text, encoding, hasBOM, version) {
|
|
223
|
+
const state = {
|
|
224
|
+
encoding,
|
|
225
|
+
hasBOM,
|
|
226
|
+
lineEnding: detectEnding(text),
|
|
227
|
+
version,
|
|
228
|
+
};
|
|
229
|
+
setEncodingState(targetKey, state);
|
|
230
|
+
return state;
|
|
231
|
+
}
|
|
232
|
+
export function recordFooter(targetKey, footer) {
|
|
233
|
+
if (footer)
|
|
234
|
+
setAutoGuessFooter(targetKey, footer);
|
|
235
|
+
}
|
|
236
|
+
/**
|
|
237
|
+
* Decide what bytes (as text, since provider expects string) to write and
|
|
238
|
+
* what new state to record. BOM and lineEnding are restored here when a
|
|
239
|
+
* previous state exists; otherwise defaults are UTF-8 without BOM and LF.
|
|
240
|
+
* For legacy encodings with normalizeToUtf8:false, we preserve the memo
|
|
241
|
+
* (provider will write UTF-8 string, next read re-detects via version bump).
|
|
242
|
+
*/
|
|
243
|
+
export function prepareForSave(content, existingState, opts = {}) {
|
|
244
|
+
if (opts.encodingHint) {
|
|
245
|
+
const norm = normalizeEncoding(opts.encodingHint);
|
|
246
|
+
if (!norm)
|
|
247
|
+
throw new Error(`[E_BAD_ENCODING] Unknown encoding: ${opts.encodingHint}`);
|
|
248
|
+
const hasBOM = norm === "utf8bom";
|
|
249
|
+
const lineEnding = existingState?.lineEnding ?? detectEnding(content);
|
|
250
|
+
// Caller will write `content` as UTF-8 string; record new state for next read.
|
|
251
|
+
return {
|
|
252
|
+
textToWrite: content,
|
|
253
|
+
newState: { encoding: norm, hasBOM, lineEnding, version: undefined },
|
|
254
|
+
};
|
|
255
|
+
}
|
|
256
|
+
if (existingState && !opts.normalizeToUtf8 && existingState.encoding !== "utf8" && existingState.encoding !== "utf8bom") {
|
|
257
|
+
// Legacy file, normalize false — preserve memo on wire (no transcode),
|
|
258
|
+
// provider writes UTF-8 string, version bump will invalidate on next read.
|
|
259
|
+
return { textToWrite: content };
|
|
260
|
+
}
|
|
261
|
+
// Default: UTF-8 without BOM, preserve lineEnding if known
|
|
262
|
+
return { textToWrite: content };
|
|
263
|
+
}
|
package/lib/file-view.js
CHANGED
|
@@ -462,7 +462,7 @@ export async function preview(content, hashes, opts = {}, absolutePath) {
|
|
|
462
462
|
export async function readView(io, path, cwd, opts = {}) {
|
|
463
463
|
const { signal } = opts;
|
|
464
464
|
const absolutePath = await io.resolve(path, cwd, signal);
|
|
465
|
-
const rawText = await io.readText(absolutePath, signal);
|
|
465
|
+
const rawText = await io.readText(absolutePath, signal, opts.encoding);
|
|
466
466
|
const { normalized, fileHashes, hadUtf8DecodeErrors, bom, originalEnding } = await normFromText({
|
|
467
467
|
absolutePath,
|
|
468
468
|
rawText,
|