dsh-better-edit 0.5.1 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,263 @@
1
+ /**
2
+ * FileEncodingState — deep module owning the file encoding state round-trip.
3
+ *
4
+ * Single seam for deterministic admission (BOM → strict UTF-8 → autoGuess gate
5
+ * → Top-3 always pushed), canonical encoding enum, version-invalidated memo,
6
+ * and BOM+lineEnding round-trip. Previously scattered across fs-bridge
7
+ * (branching + fallback Map + Top-3), encoding.ts (pure scoring), and
8
+ * file-view/fs-write (stripBOM/detectEnding/encode back). Now one file owns
9
+ * the invariant; deleting it would scatter round-trip fidelity (deep).
10
+ *
11
+ * Interface is the test surface: pure decode is tested without filesystem,
12
+ * IO adapters (fs-bridge) are thin. `store-config.autoGuessEncoding` stays the
13
+ * only config knob — this module reads it but does not own it.
14
+ *
15
+ * @module dsh-better-edit/file-encoding-state
16
+ */
17
+ import { detectBom, isValidUtf8, decodeBytes, normalizeEncoding, top3Candidates, chardetTop3Candidates } from "./encoding.js";
18
+ import { detectEnding } from "./edit-diff.js";
19
+ const encodingMemo = new Map();
20
+ const autoGuessFooterMemo = new Map();
21
+ export function getEncodingState(targetKey) {
22
+ return encodingMemo.get(targetKey);
23
+ }
24
+ export function setEncodingState(targetKey, state) {
25
+ encodingMemo.set(targetKey, state);
26
+ }
27
+ export function clearEncodingState(targetKey) {
28
+ if (targetKey)
29
+ encodingMemo.delete(targetKey);
30
+ else
31
+ encodingMemo.clear();
32
+ }
33
+ export function getAutoGuessFooter(targetKey) {
34
+ return autoGuessFooterMemo.get(targetKey);
35
+ }
36
+ export function setAutoGuessFooter(targetKey, footer) {
37
+ autoGuessFooterMemo.set(targetKey, footer);
38
+ }
39
+ export function clearAutoGuessFooter(targetKey) {
40
+ if (targetKey)
41
+ autoGuessFooterMemo.delete(targetKey);
42
+ else
43
+ autoGuessFooterMemo.clear();
44
+ }
45
+ /** Invalidate memo when FsVersion drifted (ADR-0012). */
46
+ export function invalidateIfStale(targetKey, currentVersion) {
47
+ const memo = encodingMemo.get(targetKey);
48
+ if (memo && memo.version !== currentVersion)
49
+ encodingMemo.delete(targetKey);
50
+ }
51
+ // ---------------------------------------------------------------------------
52
+ // Pure helpers — no IO
53
+ // ---------------------------------------------------------------------------
54
+ export function buildTop3ErrorMessage(displayPath, candidates) {
55
+ const candStr = candidates.map((c) => `${c.encoding}("${c.sample.slice(0, 20).replace(/"/g, "'")}")`).join(", ");
56
+ return `[E_NOT_TEXT] Path is not a readable UTF-8 text file: ${displayPath}. Hashline editing only supports text files. Top-3 guesses: ${candStr}. Try read({encoding: "<encoding>"}) or set DSH_BETTER_EDIT_AUTO_GUESS_ENCODING=true to auto-decode.`;
57
+ }
58
+ function buildAutoGuessFooterFromCandidates(candidates) {
59
+ // chardet path vs heuristic path share same footer shape, just score vs confidence
60
+ const candsStr = candidates
61
+ .map((c) => {
62
+ const score = c.confidence !== undefined ? String(c.confidence) : c.score !== undefined ? c.score.toFixed(0) : "0";
63
+ return `${c.encoding} ${score}`;
64
+ })
65
+ .join(", ");
66
+ const top = candidates[0];
67
+ const topScore = top.confidence !== undefined ? String(top.confidence) : top.score !== undefined ? top.score.toFixed(0) : "0";
68
+ return `\n\n[Auto-guessed: ${top.encoding} ${topScore}, candidates: ${candsStr} — re-read with read({encoding}) if garbled]`;
69
+ }
70
+ function isMidConfidenceChardet(top, second) {
71
+ return top.confidence < 70 || (second !== undefined && top.confidence - second.confidence < 10);
72
+ }
73
+ function isMidConfidenceHeuristic(best, second) {
74
+ return second !== undefined && best.score - second.score < 10;
75
+ }
76
+ /**
77
+ * Deterministic admission for a raw byte buffer. No IO, no side effects
78
+ * except Top-3 scoring. Caller owns recording the returned state via
79
+ * `recordOpenState` and handling the footer vs E_NOT_TEXT split.
80
+ *
81
+ * Order is fixed: BOM sniff always precedes strict UTF-8, which always
82
+ * precedes guessing. Probabilistic encodings are last resort, gated by
83
+ * `autoGuessEncoding`, but Top-3 is always computed for the error/ footer.
84
+ */
85
+ export async function decodeForOpen(bytes, config, opts = {}) {
86
+ const hint = opts.encodingHint ? normalizeEncoding(opts.encodingHint) : undefined;
87
+ if (opts.encodingHint && !hint) {
88
+ throw new Error(`[E_BAD_ENCODING] Unknown encoding: ${opts.encodingHint}`);
89
+ }
90
+ // 1) Explicit hint (Reopen with Encoding) — caller already chose, no autoGuess
91
+ if (hint) {
92
+ const bom = detectBom(bytes);
93
+ // If BOM present, it wins for hasBOM but hint wins for decode (VS Code)
94
+ const off = bom ? bom.bomLen : 0;
95
+ const slice = bom ? bytes.subarray(off) : bytes;
96
+ // For BOM encodings, decode the full bytes (including BOM) via helper; otherwise hint
97
+ let decoded;
98
+ if (bom && (bom.encoding === "utf8bom" || bom.encoding === "utf16le" || bom.encoding === "utf16be" || bom.encoding === "utf32le" || bom.encoding === "utf32be")) {
99
+ // When hint differs from BOM, hint takes precedence for decoding the slice
100
+ // but we preserve hasBOM from BOM sniff. For utf variants, BOM encoding
101
+ // already handled; for explicit hint we decode slice with hint.
102
+ decoded = decodeBytes(slice, hint);
103
+ if (decoded === undefined)
104
+ throw new Error(`[E_DECODE_FAILED] Cannot decode bytes as ${hint}`);
105
+ return {
106
+ text: decoded,
107
+ encoding: hint,
108
+ hasBOM: !!bom,
109
+ lineEnding: detectEnding(decoded),
110
+ candidates: [],
111
+ };
112
+ }
113
+ decoded = decodeBytes(slice, hint);
114
+ if (decoded === undefined)
115
+ throw new Error(`[E_DECODE_FAILED] Cannot decode bytes as ${hint}`);
116
+ return {
117
+ text: decoded,
118
+ encoding: hint,
119
+ hasBOM: !!bom,
120
+ lineEnding: detectEnding(decoded),
121
+ candidates: [],
122
+ };
123
+ }
124
+ // 2) BOM sniff — deterministic, before UTF-8
125
+ const bom = detectBom(bytes);
126
+ if (bom) {
127
+ const decoded = decodeBytes(bytes, bom.encoding);
128
+ if (decoded !== undefined) {
129
+ return {
130
+ text: decoded,
131
+ encoding: bom.encoding,
132
+ hasBOM: true,
133
+ lineEnding: detectEnding(decoded),
134
+ candidates: [],
135
+ };
136
+ }
137
+ }
138
+ // 3) Strict UTF-8 fatal:true — deterministic
139
+ if (isValidUtf8(bytes)) {
140
+ const text = new TextDecoder("utf-8", { fatal: true }).decode(bytes);
141
+ return {
142
+ text,
143
+ encoding: "utf8",
144
+ hasBOM: false,
145
+ lineEnding: detectEnding(text),
146
+ candidates: [],
147
+ };
148
+ }
149
+ // 4) AutoGuess gate — probabilistic, last resort
150
+ if (config.autoGuessEncoding) {
151
+ // Try chardet first (Top-1 with footer for mid-confidence)
152
+ try {
153
+ const chardetCands = await chardetTop3Candidates(bytes, config.supportedEncodings);
154
+ if (chardetCands.length > 0) {
155
+ const top = chardetCands[0];
156
+ const second = chardetCands[1];
157
+ const isMid = isMidConfidenceChardet(top, second);
158
+ const dec = decodeBytes(bytes, top.encoding);
159
+ if (dec !== undefined && !dec.includes("\uFFFD")) {
160
+ let footer;
161
+ if (isMid)
162
+ footer = buildAutoGuessFooterFromCandidates(chardetCands);
163
+ return {
164
+ text: dec,
165
+ encoding: top.encoding,
166
+ hasBOM: false,
167
+ lineEnding: detectEnding(dec),
168
+ footer,
169
+ candidates: chardetCands.map((c) => ({ encoding: c.encoding, sample: c.sample, score: c.confidence })),
170
+ };
171
+ }
172
+ }
173
+ }
174
+ catch {
175
+ // best-effort
176
+ }
177
+ const candidates = top3Candidates(bytes, config.supportedEncodings);
178
+ if (candidates.length > 0) {
179
+ const best = candidates[0];
180
+ const second = candidates[1];
181
+ const isMid = isMidConfidenceHeuristic(best, second);
182
+ const dec = decodeBytes(bytes, best.encoding);
183
+ if (dec !== undefined) {
184
+ // Always produce footer for heuristic path (mid check decides but we always have candidates)
185
+ // Keep existing behavior: always a footer for heuristic, but mid decides inclusion — for now always include
186
+ // to match prior fs-bridge which always built footerHeu.
187
+ void isMid;
188
+ const footer = buildAutoGuessFooterFromCandidates(candidates);
189
+ return {
190
+ text: dec,
191
+ encoding: best.encoding,
192
+ hasBOM: false,
193
+ lineEnding: detectEnding(dec),
194
+ footer,
195
+ candidates,
196
+ };
197
+ }
198
+ }
199
+ // No candidate succeeded — fall through to E_NOT_TEXT with Top-3
200
+ }
201
+ // 5) E_NOT_TEXT + Top-3 always pushed (even when autoGuess off)
202
+ // Use chardet Top-3 when available, else heuristic Top-3 — same as fs-bridge fallback
203
+ let candidates = [];
204
+ try {
205
+ const chardetCands = await chardetTop3Candidates(bytes, config.supportedEncodings);
206
+ if (chardetCands.length > 0) {
207
+ candidates = chardetCands.map((c) => ({ encoding: c.encoding, sample: c.sample, score: c.confidence }));
208
+ }
209
+ else {
210
+ candidates = top3Candidates(bytes, config.supportedEncodings);
211
+ }
212
+ }
213
+ catch {
214
+ candidates = top3Candidates(bytes, config.supportedEncodings);
215
+ }
216
+ const display = opts.displayPath ?? "(unknown path)";
217
+ throw new Error(buildTop3ErrorMessage(display, candidates));
218
+ }
219
+ // ---------------------------------------------------------------------------
220
+ // Recording helpers — caller supplies targetKey + version after decode
221
+ // ---------------------------------------------------------------------------
222
+ export function recordOpenState(targetKey, text, encoding, hasBOM, version) {
223
+ const state = {
224
+ encoding,
225
+ hasBOM,
226
+ lineEnding: detectEnding(text),
227
+ version,
228
+ };
229
+ setEncodingState(targetKey, state);
230
+ return state;
231
+ }
232
+ export function recordFooter(targetKey, footer) {
233
+ if (footer)
234
+ setAutoGuessFooter(targetKey, footer);
235
+ }
236
+ /**
237
+ * Decide what bytes (as text, since provider expects string) to write and
238
+ * what new state to record. BOM and lineEnding are restored here when a
239
+ * previous state exists; otherwise defaults are UTF-8 without BOM and LF.
240
+ * For legacy encodings with normalizeToUtf8:false, we preserve the memo
241
+ * (provider will write UTF-8 string, next read re-detects via version bump).
242
+ */
243
+ export function prepareForSave(content, existingState, opts = {}) {
244
+ if (opts.encodingHint) {
245
+ const norm = normalizeEncoding(opts.encodingHint);
246
+ if (!norm)
247
+ throw new Error(`[E_BAD_ENCODING] Unknown encoding: ${opts.encodingHint}`);
248
+ const hasBOM = norm === "utf8bom";
249
+ const lineEnding = existingState?.lineEnding ?? detectEnding(content);
250
+ // Caller will write `content` as UTF-8 string; record new state for next read.
251
+ return {
252
+ textToWrite: content,
253
+ newState: { encoding: norm, hasBOM, lineEnding, version: undefined },
254
+ };
255
+ }
256
+ if (existingState && !opts.normalizeToUtf8 && existingState.encoding !== "utf8" && existingState.encoding !== "utf8bom") {
257
+ // Legacy file, normalize false — preserve memo on wire (no transcode),
258
+ // provider writes UTF-8 string, version bump will invalidate on next read.
259
+ return { textToWrite: content };
260
+ }
261
+ // Default: UTF-8 without BOM, preserve lineEnding if known
262
+ return { textToWrite: content };
263
+ }