dsh-better-edit 0.5.1 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +30 -0
- package/lib/edit-engine.js +4 -454
- package/lib/edit-response.js +24 -2
- package/lib/file-encoding-state.js +263 -0
- package/lib/file-view.js +5 -1
- package/lib/fs-bridge.js +94 -251
- package/lib/hash-store.js +180 -2
- package/lib/hashline/anchor-pipeline.js +94 -166
- package/lib/hashline/hash-assign.js +15 -27
- package/lib/hashline/hash.js +5 -5
- package/lib/index.js +2 -2
- package/lib/mutation/engine.js +415 -0
- package/lib/mutation.js +105 -11
- package/lib/paths.js +1 -1
- package/lib/read-and-serve.js +25 -3
- package/lib/session-view.js +107 -58
- package/lib/store/index.js +28 -0
- package/lib/store-lifecycle.js +1 -1
- package/lib/tool-edit.js +2 -2
- package/lib/tool-read.js +2 -2
- package/lib/tool-undo.js +21 -8
- package/lib/types/edit-diff.d.ts +1 -1
- package/lib/types/edit-engine.d.ts +4 -176
- package/lib/types/edit-response.d.ts +1 -1
- package/lib/types/file-encoding-state.d.ts +77 -0
- package/lib/types/file-view.d.ts +7 -1
- package/lib/types/fs-bridge.d.ts +4 -11
- package/lib/types/hash-store.d.ts +15 -0
- package/lib/types/hashline/anchor-pipeline.d.ts +12 -21
- package/lib/types/hashline/hash-assign.d.ts +2 -2
- package/lib/types/hashline/hash.d.ts +1 -1
- package/lib/types/hashline/index.d.ts +1 -1
- package/lib/types/mutation/engine.d.ts +149 -0
- package/lib/types/mutation.d.ts +30 -5
- package/lib/types/paths.d.ts +1 -1
- package/lib/types/read-and-serve.d.ts +1 -1
- package/lib/types/session-view.d.ts +25 -31
- package/lib/types/store/index.d.ts +25 -0
- package/lib/types/workspace-context.d.ts +23 -0
- package/lib/workspace-context.js +44 -0
- package/lib/write-hook.js +2 -1
- package/package.json +1 -1
|
@@ -0,0 +1,263 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* FileEncodingState — deep module owning the file encoding state round-trip.
|
|
3
|
+
*
|
|
4
|
+
* Single seam for deterministic admission (BOM → strict UTF-8 → autoGuess gate
|
|
5
|
+
* → Top-3 always pushed), canonical encoding enum, version-invalidated memo,
|
|
6
|
+
* and BOM+lineEnding round-trip. Previously scattered across fs-bridge
|
|
7
|
+
* (branching + fallback Map + Top-3), encoding.ts (pure scoring), and
|
|
8
|
+
* file-view/fs-write (stripBOM/detectEnding/encode back). Now one file owns
|
|
9
|
+
* the invariant; deleting it would scatter round-trip fidelity (deep).
|
|
10
|
+
*
|
|
11
|
+
* Interface is the test surface: pure decode is tested without filesystem,
|
|
12
|
+
* IO adapters (fs-bridge) are thin. `store-config.autoGuessEncoding` stays the
|
|
13
|
+
* only config knob — this module reads it but does not own it.
|
|
14
|
+
*
|
|
15
|
+
* @module dsh-better-edit/file-encoding-state
|
|
16
|
+
*/
|
|
17
|
+
import { detectBom, isValidUtf8, decodeBytes, normalizeEncoding, top3Candidates, chardetTop3Candidates } from "./encoding.js";
|
|
18
|
+
import { detectEnding } from "./edit-diff.js";
|
|
19
|
+
const encodingMemo = new Map();
|
|
20
|
+
const autoGuessFooterMemo = new Map();
|
|
21
|
+
export function getEncodingState(targetKey) {
|
|
22
|
+
return encodingMemo.get(targetKey);
|
|
23
|
+
}
|
|
24
|
+
export function setEncodingState(targetKey, state) {
|
|
25
|
+
encodingMemo.set(targetKey, state);
|
|
26
|
+
}
|
|
27
|
+
export function clearEncodingState(targetKey) {
|
|
28
|
+
if (targetKey)
|
|
29
|
+
encodingMemo.delete(targetKey);
|
|
30
|
+
else
|
|
31
|
+
encodingMemo.clear();
|
|
32
|
+
}
|
|
33
|
+
export function getAutoGuessFooter(targetKey) {
|
|
34
|
+
return autoGuessFooterMemo.get(targetKey);
|
|
35
|
+
}
|
|
36
|
+
export function setAutoGuessFooter(targetKey, footer) {
|
|
37
|
+
autoGuessFooterMemo.set(targetKey, footer);
|
|
38
|
+
}
|
|
39
|
+
export function clearAutoGuessFooter(targetKey) {
|
|
40
|
+
if (targetKey)
|
|
41
|
+
autoGuessFooterMemo.delete(targetKey);
|
|
42
|
+
else
|
|
43
|
+
autoGuessFooterMemo.clear();
|
|
44
|
+
}
|
|
45
|
+
/** Invalidate memo when FsVersion drifted (ADR-0012). */
|
|
46
|
+
export function invalidateIfStale(targetKey, currentVersion) {
|
|
47
|
+
const memo = encodingMemo.get(targetKey);
|
|
48
|
+
if (memo && memo.version !== currentVersion)
|
|
49
|
+
encodingMemo.delete(targetKey);
|
|
50
|
+
}
|
|
51
|
+
// ---------------------------------------------------------------------------
|
|
52
|
+
// Pure helpers — no IO
|
|
53
|
+
// ---------------------------------------------------------------------------
|
|
54
|
+
export function buildTop3ErrorMessage(displayPath, candidates) {
|
|
55
|
+
const candStr = candidates.map((c) => `${c.encoding}("${c.sample.slice(0, 20).replace(/"/g, "'")}")`).join(", ");
|
|
56
|
+
return `[E_NOT_TEXT] Path is not a readable UTF-8 text file: ${displayPath}. Hashline editing only supports text files. Top-3 guesses: ${candStr}. Try read({encoding: "<encoding>"}) or set DSH_BETTER_EDIT_AUTO_GUESS_ENCODING=true to auto-decode.`;
|
|
57
|
+
}
|
|
58
|
+
function buildAutoGuessFooterFromCandidates(candidates) {
|
|
59
|
+
// chardet path vs heuristic path share same footer shape, just score vs confidence
|
|
60
|
+
const candsStr = candidates
|
|
61
|
+
.map((c) => {
|
|
62
|
+
const score = c.confidence !== undefined ? String(c.confidence) : c.score !== undefined ? c.score.toFixed(0) : "0";
|
|
63
|
+
return `${c.encoding} ${score}`;
|
|
64
|
+
})
|
|
65
|
+
.join(", ");
|
|
66
|
+
const top = candidates[0];
|
|
67
|
+
const topScore = top.confidence !== undefined ? String(top.confidence) : top.score !== undefined ? top.score.toFixed(0) : "0";
|
|
68
|
+
return `\n\n[Auto-guessed: ${top.encoding} ${topScore}, candidates: ${candsStr} — re-read with read({encoding}) if garbled]`;
|
|
69
|
+
}
|
|
70
|
+
function isMidConfidenceChardet(top, second) {
|
|
71
|
+
return top.confidence < 70 || (second !== undefined && top.confidence - second.confidence < 10);
|
|
72
|
+
}
|
|
73
|
+
function isMidConfidenceHeuristic(best, second) {
|
|
74
|
+
return second !== undefined && best.score - second.score < 10;
|
|
75
|
+
}
|
|
76
|
+
/**
|
|
77
|
+
* Deterministic admission for a raw byte buffer. No IO, no side effects
|
|
78
|
+
* except Top-3 scoring. Caller owns recording the returned state via
|
|
79
|
+
* `recordOpenState` and handling the footer vs E_NOT_TEXT split.
|
|
80
|
+
*
|
|
81
|
+
* Order is fixed: BOM sniff always precedes strict UTF-8, which always
|
|
82
|
+
* precedes guessing. Probabilistic encodings are last resort, gated by
|
|
83
|
+
* `autoGuessEncoding`, but Top-3 is always computed for the error/ footer.
|
|
84
|
+
*/
|
|
85
|
+
export async function decodeForOpen(bytes, config, opts = {}) {
|
|
86
|
+
const hint = opts.encodingHint ? normalizeEncoding(opts.encodingHint) : undefined;
|
|
87
|
+
if (opts.encodingHint && !hint) {
|
|
88
|
+
throw new Error(`[E_BAD_ENCODING] Unknown encoding: ${opts.encodingHint}`);
|
|
89
|
+
}
|
|
90
|
+
// 1) Explicit hint (Reopen with Encoding) — caller already chose, no autoGuess
|
|
91
|
+
if (hint) {
|
|
92
|
+
const bom = detectBom(bytes);
|
|
93
|
+
// If BOM present, it wins for hasBOM but hint wins for decode (VS Code)
|
|
94
|
+
const off = bom ? bom.bomLen : 0;
|
|
95
|
+
const slice = bom ? bytes.subarray(off) : bytes;
|
|
96
|
+
// For BOM encodings, decode the full bytes (including BOM) via helper; otherwise hint
|
|
97
|
+
let decoded;
|
|
98
|
+
if (bom && (bom.encoding === "utf8bom" || bom.encoding === "utf16le" || bom.encoding === "utf16be" || bom.encoding === "utf32le" || bom.encoding === "utf32be")) {
|
|
99
|
+
// When hint differs from BOM, hint takes precedence for decoding the slice
|
|
100
|
+
// but we preserve hasBOM from BOM sniff. For utf variants, BOM encoding
|
|
101
|
+
// already handled; for explicit hint we decode slice with hint.
|
|
102
|
+
decoded = decodeBytes(slice, hint);
|
|
103
|
+
if (decoded === undefined)
|
|
104
|
+
throw new Error(`[E_DECODE_FAILED] Cannot decode bytes as ${hint}`);
|
|
105
|
+
return {
|
|
106
|
+
text: decoded,
|
|
107
|
+
encoding: hint,
|
|
108
|
+
hasBOM: !!bom,
|
|
109
|
+
lineEnding: detectEnding(decoded),
|
|
110
|
+
candidates: [],
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
decoded = decodeBytes(slice, hint);
|
|
114
|
+
if (decoded === undefined)
|
|
115
|
+
throw new Error(`[E_DECODE_FAILED] Cannot decode bytes as ${hint}`);
|
|
116
|
+
return {
|
|
117
|
+
text: decoded,
|
|
118
|
+
encoding: hint,
|
|
119
|
+
hasBOM: !!bom,
|
|
120
|
+
lineEnding: detectEnding(decoded),
|
|
121
|
+
candidates: [],
|
|
122
|
+
};
|
|
123
|
+
}
|
|
124
|
+
// 2) BOM sniff — deterministic, before UTF-8
|
|
125
|
+
const bom = detectBom(bytes);
|
|
126
|
+
if (bom) {
|
|
127
|
+
const decoded = decodeBytes(bytes, bom.encoding);
|
|
128
|
+
if (decoded !== undefined) {
|
|
129
|
+
return {
|
|
130
|
+
text: decoded,
|
|
131
|
+
encoding: bom.encoding,
|
|
132
|
+
hasBOM: true,
|
|
133
|
+
lineEnding: detectEnding(decoded),
|
|
134
|
+
candidates: [],
|
|
135
|
+
};
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
// 3) Strict UTF-8 fatal:true — deterministic
|
|
139
|
+
if (isValidUtf8(bytes)) {
|
|
140
|
+
const text = new TextDecoder("utf-8", { fatal: true }).decode(bytes);
|
|
141
|
+
return {
|
|
142
|
+
text,
|
|
143
|
+
encoding: "utf8",
|
|
144
|
+
hasBOM: false,
|
|
145
|
+
lineEnding: detectEnding(text),
|
|
146
|
+
candidates: [],
|
|
147
|
+
};
|
|
148
|
+
}
|
|
149
|
+
// 4) AutoGuess gate — probabilistic, last resort
|
|
150
|
+
if (config.autoGuessEncoding) {
|
|
151
|
+
// Try chardet first (Top-1 with footer for mid-confidence)
|
|
152
|
+
try {
|
|
153
|
+
const chardetCands = await chardetTop3Candidates(bytes, config.supportedEncodings);
|
|
154
|
+
if (chardetCands.length > 0) {
|
|
155
|
+
const top = chardetCands[0];
|
|
156
|
+
const second = chardetCands[1];
|
|
157
|
+
const isMid = isMidConfidenceChardet(top, second);
|
|
158
|
+
const dec = decodeBytes(bytes, top.encoding);
|
|
159
|
+
if (dec !== undefined && !dec.includes("\uFFFD")) {
|
|
160
|
+
let footer;
|
|
161
|
+
if (isMid)
|
|
162
|
+
footer = buildAutoGuessFooterFromCandidates(chardetCands);
|
|
163
|
+
return {
|
|
164
|
+
text: dec,
|
|
165
|
+
encoding: top.encoding,
|
|
166
|
+
hasBOM: false,
|
|
167
|
+
lineEnding: detectEnding(dec),
|
|
168
|
+
footer,
|
|
169
|
+
candidates: chardetCands.map((c) => ({ encoding: c.encoding, sample: c.sample, score: c.confidence })),
|
|
170
|
+
};
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
catch {
|
|
175
|
+
// best-effort
|
|
176
|
+
}
|
|
177
|
+
const candidates = top3Candidates(bytes, config.supportedEncodings);
|
|
178
|
+
if (candidates.length > 0) {
|
|
179
|
+
const best = candidates[0];
|
|
180
|
+
const second = candidates[1];
|
|
181
|
+
const isMid = isMidConfidenceHeuristic(best, second);
|
|
182
|
+
const dec = decodeBytes(bytes, best.encoding);
|
|
183
|
+
if (dec !== undefined) {
|
|
184
|
+
// Always produce footer for heuristic path (mid check decides but we always have candidates)
|
|
185
|
+
// Keep existing behavior: always a footer for heuristic, but mid decides inclusion — for now always include
|
|
186
|
+
// to match prior fs-bridge which always built footerHeu.
|
|
187
|
+
void isMid;
|
|
188
|
+
const footer = buildAutoGuessFooterFromCandidates(candidates);
|
|
189
|
+
return {
|
|
190
|
+
text: dec,
|
|
191
|
+
encoding: best.encoding,
|
|
192
|
+
hasBOM: false,
|
|
193
|
+
lineEnding: detectEnding(dec),
|
|
194
|
+
footer,
|
|
195
|
+
candidates,
|
|
196
|
+
};
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
// No candidate succeeded — fall through to E_NOT_TEXT with Top-3
|
|
200
|
+
}
|
|
201
|
+
// 5) E_NOT_TEXT + Top-3 always pushed (even when autoGuess off)
|
|
202
|
+
// Use chardet Top-3 when available, else heuristic Top-3 — same as fs-bridge fallback
|
|
203
|
+
let candidates = [];
|
|
204
|
+
try {
|
|
205
|
+
const chardetCands = await chardetTop3Candidates(bytes, config.supportedEncodings);
|
|
206
|
+
if (chardetCands.length > 0) {
|
|
207
|
+
candidates = chardetCands.map((c) => ({ encoding: c.encoding, sample: c.sample, score: c.confidence }));
|
|
208
|
+
}
|
|
209
|
+
else {
|
|
210
|
+
candidates = top3Candidates(bytes, config.supportedEncodings);
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
catch {
|
|
214
|
+
candidates = top3Candidates(bytes, config.supportedEncodings);
|
|
215
|
+
}
|
|
216
|
+
const display = opts.displayPath ?? "(unknown path)";
|
|
217
|
+
throw new Error(buildTop3ErrorMessage(display, candidates));
|
|
218
|
+
}
|
|
219
|
+
// ---------------------------------------------------------------------------
|
|
220
|
+
// Recording helpers — caller supplies targetKey + version after decode
|
|
221
|
+
// ---------------------------------------------------------------------------
|
|
222
|
+
export function recordOpenState(targetKey, text, encoding, hasBOM, version) {
|
|
223
|
+
const state = {
|
|
224
|
+
encoding,
|
|
225
|
+
hasBOM,
|
|
226
|
+
lineEnding: detectEnding(text),
|
|
227
|
+
version,
|
|
228
|
+
};
|
|
229
|
+
setEncodingState(targetKey, state);
|
|
230
|
+
return state;
|
|
231
|
+
}
|
|
232
|
+
export function recordFooter(targetKey, footer) {
|
|
233
|
+
if (footer)
|
|
234
|
+
setAutoGuessFooter(targetKey, footer);
|
|
235
|
+
}
|
|
236
|
+
/**
|
|
237
|
+
* Decide what bytes (as text, since provider expects string) to write and
|
|
238
|
+
* what new state to record. BOM and lineEnding are restored here when a
|
|
239
|
+
* previous state exists; otherwise defaults are UTF-8 without BOM and LF.
|
|
240
|
+
* For legacy encodings with normalizeToUtf8:false, we preserve the memo
|
|
241
|
+
* (provider will write UTF-8 string, next read re-detects via version bump).
|
|
242
|
+
*/
|
|
243
|
+
export function prepareForSave(content, existingState, opts = {}) {
|
|
244
|
+
if (opts.encodingHint) {
|
|
245
|
+
const norm = normalizeEncoding(opts.encodingHint);
|
|
246
|
+
if (!norm)
|
|
247
|
+
throw new Error(`[E_BAD_ENCODING] Unknown encoding: ${opts.encodingHint}`);
|
|
248
|
+
const hasBOM = norm === "utf8bom";
|
|
249
|
+
const lineEnding = existingState?.lineEnding ?? detectEnding(content);
|
|
250
|
+
// Caller will write `content` as UTF-8 string; record new state for next read.
|
|
251
|
+
return {
|
|
252
|
+
textToWrite: content,
|
|
253
|
+
newState: { encoding: norm, hasBOM, lineEnding, version: undefined },
|
|
254
|
+
};
|
|
255
|
+
}
|
|
256
|
+
if (existingState && !opts.normalizeToUtf8 && existingState.encoding !== "utf8" && existingState.encoding !== "utf8bom") {
|
|
257
|
+
// Legacy file, normalize false — preserve memo on wire (no transcode),
|
|
258
|
+
// provider writes UTF-8 string, version bump will invalidate on next read.
|
|
259
|
+
return { textToWrite: content };
|
|
260
|
+
}
|
|
261
|
+
// Default: UTF-8 without BOM, preserve lineEnding if known
|
|
262
|
+
return { textToWrite: content };
|
|
263
|
+
}
|
package/lib/file-view.js
CHANGED
|
@@ -290,7 +290,7 @@ export async function normFromText(input) {
|
|
|
290
290
|
throw new Error(`[E_FILE_TOO_LARGE] ${displayPath} has ${lineCount} lines, exceeding the ${input.maxLines}-line edit limit. Hashline editing targets source-sized files; for very large files use write or a non-line-based approach.`);
|
|
291
291
|
}
|
|
292
292
|
}
|
|
293
|
-
const fileHashes = await lineHashes(normalized, absolutePath, undefined, input.store, input.noPersist !== true);
|
|
293
|
+
const fileHashes = await lineHashes(normalized, absolutePath, undefined, input.store, input.noPersist !== true, input.reservedHashes, input.retiredHashes);
|
|
294
294
|
return {
|
|
295
295
|
absolutePath,
|
|
296
296
|
normalized,
|
|
@@ -322,6 +322,8 @@ export async function readNormFile(path, cwd, options) {
|
|
|
322
322
|
maxLines: options?.maxLines,
|
|
323
323
|
store: options?.store,
|
|
324
324
|
noPersist: options?.noPersist,
|
|
325
|
+
reservedHashes: options?.reservedHashes,
|
|
326
|
+
retiredHashes: options?.retiredHashes,
|
|
325
327
|
hadUtf8DecodeErrors: file.hadUtf8DecodeErrors,
|
|
326
328
|
});
|
|
327
329
|
}
|
|
@@ -469,6 +471,8 @@ export async function readView(io, path, cwd, opts = {}) {
|
|
|
469
471
|
displayPath: path,
|
|
470
472
|
signal,
|
|
471
473
|
maxLines: MAX_HASH_LINES,
|
|
474
|
+
reservedHashes: opts.reservedHashes,
|
|
475
|
+
retiredHashes: opts.retiredHashes,
|
|
472
476
|
});
|
|
473
477
|
const r = await fmtReadPreview(normalized, { offset: opts.offset, limit: opts.limit }, fileHashes, absolutePath);
|
|
474
478
|
return {
|