dsh-better-edit 0.4.1 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +26 -1
- package/README.md +2 -1
- package/lib/contract.js +1 -1
- package/lib/edit-response.js +14 -11
- package/lib/encoding.js +308 -0
- package/lib/file-view.js +1 -1
- package/lib/fs-bridge.js +266 -4
- package/lib/fs-write.js +15 -6
- package/lib/hash-store.js +1 -1
- package/lib/hashline/anchor-pipeline.js +278 -43
- package/lib/hashline/hash-assign.js +18 -3
- package/lib/hashline/index.js +1 -1
- package/lib/prompts.js +14 -11
- package/lib/read-and-serve.js +5 -0
- package/lib/store-config.js +132 -2
- package/lib/tool-read.js +24 -4
- package/lib/types/contract.d.ts +1 -0
- package/lib/types/encoding.d.ts +46 -0
- package/lib/types/file-view.d.ts +1 -0
- package/lib/types/fs-bridge.d.ts +12 -1
- package/lib/types/hashline/anchor-pipeline.d.ts +7 -0
- package/lib/types/hashline/hash-assign.d.ts +2 -0
- package/lib/types/hashline/index.d.ts +1 -1
- package/lib/types/read-and-serve.d.ts +3 -0
- package/lib/types/store-config.d.ts +10 -1
- package/package.json +21 -3
package/CHANGELOG.md
CHANGED
|
@@ -1,10 +1,35 @@
|
|
|
1
|
+
## [0.5.1](https://github.com/Rianico/dsh-better-edit/compare/v0.5.0...v0.5.1) (2026-08-30)
|
|
2
|
+
|
|
3
|
+
### Bug Fixes
|
|
4
|
+
|
|
5
|
+
* **encoding:** support non-UTF-8 text files via VS Code model ([#34](https://github.com/Rianico/dsh-better-edit/issues/34)) ([#35](https://github.com/Rianico/dsh-better-edit/issues/35)) ([dd3e553](https://github.com/Rianico/dsh-better-edit/commit/dd3e553a3877e47b08aa2ba6336115e80b5d96f5))
|
|
6
|
+
* **lint:** address biome blockers for release ([00b9270](https://github.com/Rianico/dsh-better-edit/commit/00b92704ac4093a627754e698bb1968acdc3d4e8))
|
|
7
|
+
* remove unused exitHandlerRegistered ([f6a29f3](https://github.com/Rianico/dsh-better-edit/commit/f6a29f3f1dd694e4bb7c55a98d0eba80bc32a58f))
|
|
8
|
+
* **scripts:** map fix to Fixed for Keep a Changelog ([bcd1850](https://github.com/Rianico/dsh-better-edit/commit/bcd18501e280b10d5075108c034de7352ece6922))
|
|
9
|
+
|
|
1
10
|
# Changelog
|
|
2
11
|
|
|
3
12
|
All notable changes to the `dsh-better-edit` plugin will be documented in this file.
|
|
4
13
|
|
|
5
14
|
Entries link to the originating spec issue in [pi-hashline-edit-lsz](https://github.com/Rianico/pi-hashline-edit-lsz) where one exists.
|
|
6
15
|
|
|
7
|
-
## [
|
|
16
|
+
## [0.5.0] - 2026-08-29
|
|
17
|
+
|
|
18
|
+
### Features
|
|
19
|
+
|
|
20
|
+
- **edit:** route drift signals to user-facing details (pi-better-edit@6ec52f2)
|
|
21
|
+
- scaffold verify→release with biome+coverage 80%
|
|
22
|
+
|
|
23
|
+
### Bug Fixes
|
|
24
|
+
|
|
25
|
+
- **ci:** lower coverage threshold to 75 and add SAFETY for hash-store cast
|
|
26
|
+
- **tests:** align drift expectations with user-facing routing (6ec52f2)
|
|
27
|
+
- **hashline:** add E_EDIT_HASH_ECHO guard for served echo (pi-better-edit@65b8eb1)
|
|
28
|
+
- clarify HASH vs content without served
|
|
29
|
+
- **edit:** clarify HASH vs HASH│content and downgrade bare prefix warnings (pi-better-edit@a837e9d)
|
|
30
|
+
- **fs-write:** log stale-temp and sync failures (pi-better-edit@671c66d)
|
|
31
|
+
- **hashline:** heal multi-line staleness and stable anchoring (f94fb88)
|
|
32
|
+
- **git:** wire pre-push hook via husky path
|
|
8
33
|
|
|
9
34
|
## [0.4.1] - 2026-08-27
|
|
10
35
|
|
package/README.md
CHANGED
|
@@ -124,7 +124,7 @@ Chained edits stay cheap — anchors for untouched lines remain valid, diff/echo
|
|
|
124
124
|
{ "path": "src/main.ts", "edits": [["a1b", "a1b", "new line 1\n"], ["c3d", "c3d", "new line 2"]] }
|
|
125
125
|
```
|
|
126
126
|
|
|
127
|
-
One fails, none write (`[E_BATCH_ABORT]`).
|
|
127
|
+
One fails, none write (`[E_BATCH_ABORT]`). Encoding errors `[E_BAD_ENCODING]`/`[E_DECODE_FAILED]` are also emitted.
|
|
128
128
|
|
|
129
129
|
> [!TIP]
|
|
130
130
|
> **Want proof before you install?** The upstream [23/23 tool battery](https://github.com/Rianico/pi-better-edit/blob/main/benchmarks/README.md) runs with no LLM — stale edits are rejected before they corrupt a file, on every run. Same hashline algorithm, same verification.
|
|
@@ -272,6 +272,7 @@ atomically to that one file — one item per call is the norm, several same-file
|
|
|
272
272
|
| `[E_AMBIGUOUS_ANCHOR]` | An anchor matches multiple lines; call `read` for fresh anchors. |
|
|
273
273
|
| `[E_INVALID_PATCH]` | A `replacement_text` line is a diff-preview row (`+HASH│`, `-HASH│`, `- │`). The marker is stripped automatically with a warning. |
|
|
274
274
|
| `[E_BARE_HASH_PREFIX]` | A `replacement_text` line starts with a hash-like `HASH│` prefix. The prefix is stripped automatically with a warning. |
|
|
275
|
+
| `[E_EDIT_HASH_ECHO]` | A `replacement_text` line begins with the exact `HASH│` anchor served for the same session, path, and range-relative line (`E1`). The edit is refused; remove the copied anchors and retry. Nothing was written. |
|
|
275
276
|
| `[E_BAD_OP]` | Range start line is after range end line. The pair is swapped automatically with a warning. |
|
|
276
277
|
| `[E_WOULD_EMPTY]` | An edit would empty a non-empty file; use `write` instead. |
|
|
277
278
|
| `[E_NOT_FOUND]` | The path does not exist. |
|
package/lib/contract.js
CHANGED
|
@@ -64,7 +64,7 @@ function describeReceived(input) {
|
|
|
64
64
|
}
|
|
65
65
|
// ---- filed sets (declared once) ---------------------------------------------
|
|
66
66
|
const EDIT_KS = new Set(["path", "edits", "sandbox_permissions", "justification"]);
|
|
67
|
-
const READ_KS = new Set(["path", "offset", "limit"]);
|
|
67
|
+
const READ_KS = new Set(["path", "offset", "limit", "encoding"]);
|
|
68
68
|
// ---- normalization -----------------------------------------------------------
|
|
69
69
|
/**
|
|
70
70
|
* Normalize `file_path` → `path` alias on the request record and tuple edits → objects.
|
package/lib/edit-response.js
CHANGED
|
@@ -22,8 +22,9 @@ export function buildMetrics(args) {
|
|
|
22
22
|
return metrics;
|
|
23
23
|
}
|
|
24
24
|
export function finalizeResult(input) {
|
|
25
|
-
const
|
|
26
|
-
|
|
25
|
+
const modelWarnings = input.warnings?.filter((w) => !w.startsWith("Batch drift note:"));
|
|
26
|
+
const base = input.diff + warnBlock(modelWarnings);
|
|
27
|
+
return base;
|
|
27
28
|
}
|
|
28
29
|
export function finalizeToolResult(details) {
|
|
29
30
|
const text = finalizeResult({
|
|
@@ -44,8 +45,8 @@ export function buildNoop(input) {
|
|
|
44
45
|
const noopDetailsText = noopEdit
|
|
45
46
|
? `Edit for ${noopEdit.loc} is identical to current content:\n ${noopEdit.loc}: ${clipLine(noopEdit.currentContent)}`
|
|
46
47
|
: "The edit produced identical content.";
|
|
47
|
-
const
|
|
48
|
-
const text = `No changes made to ${path}\nClassification: noop\n${noopDetailsText}${warnBlock(
|
|
48
|
+
const modelWarnings = warnings?.filter((w) => !w.startsWith("Batch drift note:"));
|
|
49
|
+
const text = `No changes made to ${path}\nClassification: noop\n${noopDetailsText}${warnBlock(modelWarnings)}`;
|
|
49
50
|
const metrics = buildMetrics({
|
|
50
51
|
classification: "noop",
|
|
51
52
|
editsAttempted: editMeta.editsAttempted,
|
|
@@ -71,17 +72,17 @@ export function buildChanged(input) {
|
|
|
71
72
|
const diffResult = genDiff(originalNormalized, result, 1, resultHashes, originalHashes);
|
|
72
73
|
const addedLines = editMeta.addedLines;
|
|
73
74
|
const removedLines = editMeta.removedLines;
|
|
74
|
-
const
|
|
75
|
+
const modelWarnings = warnings?.filter((w) => !w.startsWith("Batch drift note:"));
|
|
76
|
+
const warningsBlock = warnBlock(modelWarnings);
|
|
75
77
|
const successPrefix = `Successfully edited in ${path}.`;
|
|
76
78
|
const lineSummary = addedLines > 0 || removedLines > 0
|
|
77
79
|
? ` Added ${addedLines} line(s), removed ${removedLines} line(s).`
|
|
78
80
|
: "";
|
|
79
|
-
const noticeBlock = driftBlock(driftNotice);
|
|
80
81
|
const text = resultLines.length === 0
|
|
81
|
-
? "File is empty. Use edit to insert content."
|
|
82
|
+
? "File is empty. Use edit to insert content."
|
|
82
83
|
: warningsBlock
|
|
83
|
-
? `${successPrefix}${lineSummary}${warningsBlock}
|
|
84
|
-
: `${successPrefix}${lineSummary}
|
|
84
|
+
? `${successPrefix}${lineSummary}${warningsBlock}`
|
|
85
|
+
: `${successPrefix}${lineSummary}`;
|
|
85
86
|
const metrics = buildMetrics({
|
|
86
87
|
classification: "applied",
|
|
87
88
|
editsAttempted: editMeta.editsAttempted,
|
|
@@ -123,7 +124,8 @@ export function buildBatchResult(sections) {
|
|
|
123
124
|
.filter((d) => d !== undefined)
|
|
124
125
|
.join("\n\n");
|
|
125
126
|
if (allNoop) {
|
|
126
|
-
const
|
|
127
|
+
const modelWarnings = warnings.filter((w) => !w.startsWith("Batch drift note:"));
|
|
128
|
+
const text = `No changes made. All ${totalEdits} edit(s) in the batch produced identical content.\nClassification: noop${warnBlock(modelWarnings)}`;
|
|
127
129
|
return {
|
|
128
130
|
content: [{ type: "text", text }],
|
|
129
131
|
details: {
|
|
@@ -158,7 +160,8 @@ export function buildBatchResult(sections) {
|
|
|
158
160
|
? ` Added ${addedLines} line(s), removed ${removedLines} line(s).`
|
|
159
161
|
: "";
|
|
160
162
|
const summary = `Successfully edited ${appliedFiles.length} file(s) — ${appliedTotal} of ${totalEdits} edit(s) applied${noopTotal > 0 ? ` (${noopTotal} noop)` : ""}.${lineSummary}`;
|
|
161
|
-
const
|
|
163
|
+
const modelWarnings = warnings.filter((w) => !w.startsWith("Batch drift note:"));
|
|
164
|
+
const text = `${summary}${warnBlock(modelWarnings)}`;
|
|
162
165
|
return {
|
|
163
166
|
content: [{ type: "text", text }],
|
|
164
167
|
details: {
|
package/lib/encoding.js
ADDED
|
@@ -0,0 +1,308 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* VS Code encoding model — pure helpers for BOM→UTF-8→model-assisted top-3.
|
|
3
|
+
* No jschardet; heuristic scoring + iconv-lite rendering.
|
|
4
|
+
* @module dsh-better-edit/encoding
|
|
5
|
+
*/
|
|
6
|
+
import iconv from "iconv-lite";
|
|
7
|
+
export const SUPPORTED_ENCODINGS = [
|
|
8
|
+
"gbk",
|
|
9
|
+
"big5",
|
|
10
|
+
"shift_jis",
|
|
11
|
+
"euc-kr",
|
|
12
|
+
"windows-1251",
|
|
13
|
+
"iso-8859-1",
|
|
14
|
+
];
|
|
15
|
+
const ALIASES = {
|
|
16
|
+
utf8: "utf8",
|
|
17
|
+
utf8bom: "utf8bom",
|
|
18
|
+
utf16le: "utf16le",
|
|
19
|
+
utf16be: "utf16be",
|
|
20
|
+
utf32le: "utf32le",
|
|
21
|
+
utf32be: "utf32be",
|
|
22
|
+
gbk: "gbk",
|
|
23
|
+
gb18030: "gbk",
|
|
24
|
+
gb2312: "gbk",
|
|
25
|
+
big5: "big5",
|
|
26
|
+
shiftjis: "shift_jis",
|
|
27
|
+
shift_jis: "shift_jis",
|
|
28
|
+
sjis: "shift_jis",
|
|
29
|
+
euckr: "euc-kr",
|
|
30
|
+
"euc-kr": "euc-kr",
|
|
31
|
+
windows1251: "windows-1251",
|
|
32
|
+
cp1251: "windows-1251",
|
|
33
|
+
"windows-1251": "windows-1251",
|
|
34
|
+
iso88591: "iso-8859-1",
|
|
35
|
+
"iso-8859-1": "iso-8859-1",
|
|
36
|
+
latin1: "iso-8859-1",
|
|
37
|
+
};
|
|
38
|
+
/** Canonicalize case-insensitive encoding param, strip -_ . */
|
|
39
|
+
export function normalizeEncoding(input) {
|
|
40
|
+
const key = input.trim().toLowerCase().replace(/[-_]/g, "");
|
|
41
|
+
// need original with dashes for lookup: try both
|
|
42
|
+
const candidates = [key, input.trim().toLowerCase()];
|
|
43
|
+
for (const c of candidates) {
|
|
44
|
+
const v = ALIASES[c] ?? ALIASES[key];
|
|
45
|
+
if (v)
|
|
46
|
+
return v;
|
|
47
|
+
}
|
|
48
|
+
// also try direct lowercased
|
|
49
|
+
const lower = input.trim().toLowerCase();
|
|
50
|
+
if (ALIASES[lower])
|
|
51
|
+
return ALIASES[lower];
|
|
52
|
+
return undefined;
|
|
53
|
+
}
|
|
54
|
+
export function isSupportedEncoding(enc) {
|
|
55
|
+
const norm = normalizeEncoding(enc);
|
|
56
|
+
if (!norm)
|
|
57
|
+
return false;
|
|
58
|
+
return SUPPORTED_ENCODINGS.includes(norm) || ["utf8", "utf8bom", "utf16le", "utf16be", "utf32le", "utf32be"].includes(norm);
|
|
59
|
+
}
|
|
60
|
+
export function detectBom(bytes) {
|
|
61
|
+
if (bytes.length >= 3 && bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf)
|
|
62
|
+
return { encoding: "utf8bom", bomLen: 3, hasBOM: true };
|
|
63
|
+
if (bytes.length >= 4 && bytes[0] === 0xff && bytes[1] === 0xfe && bytes[2] === 0x00 && bytes[3] === 0x00)
|
|
64
|
+
return { encoding: "utf32le", bomLen: 4, hasBOM: true };
|
|
65
|
+
if (bytes.length >= 4 && bytes[0] === 0x00 && bytes[1] === 0x00 && bytes[2] === 0xfe && bytes[3] === 0xff)
|
|
66
|
+
return { encoding: "utf32be", bomLen: 4, hasBOM: true };
|
|
67
|
+
if (bytes.length >= 2 && bytes[0] === 0xff && bytes[1] === 0xfe)
|
|
68
|
+
return { encoding: "utf16le", bomLen: 2, hasBOM: true };
|
|
69
|
+
if (bytes.length >= 2 && bytes[0] === 0xfe && bytes[1] === 0xff)
|
|
70
|
+
return { encoding: "utf16be", bomLen: 2, hasBOM: true };
|
|
71
|
+
return undefined;
|
|
72
|
+
}
|
|
73
|
+
export function isValidUtf8(bytes) {
|
|
74
|
+
try {
|
|
75
|
+
new TextDecoder("utf-8", { fatal: true }).decode(bytes);
|
|
76
|
+
return true;
|
|
77
|
+
}
|
|
78
|
+
catch {
|
|
79
|
+
return false;
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
function printableRatio(text) {
|
|
83
|
+
if (text.length === 0)
|
|
84
|
+
return 1;
|
|
85
|
+
let printable = 0;
|
|
86
|
+
for (const ch of text) {
|
|
87
|
+
const code = ch.charCodeAt(0);
|
|
88
|
+
if (code === 10 || code === 13 || code === 9)
|
|
89
|
+
printable += 1;
|
|
90
|
+
else if (code >= 32 && code !== 127)
|
|
91
|
+
printable += 1;
|
|
92
|
+
else if (code > 127)
|
|
93
|
+
printable += 1; // non-ascii considered printable if not replacement
|
|
94
|
+
}
|
|
95
|
+
return printable / text.length;
|
|
96
|
+
}
|
|
97
|
+
function scriptBonus(text, enc) {
|
|
98
|
+
let bonus = 0;
|
|
99
|
+
const cjk = (text.match(/[\u4e00-\u9fff]/g) || []).length;
|
|
100
|
+
const cyrillic = (text.match(/[\u0400-\u04ff]/g) || []).length;
|
|
101
|
+
const hiragana = (text.match(/[\u3040-\u309f\u30a0-\u30ff]/g) || []).length;
|
|
102
|
+
const hangul = (text.match(/[\uac00-\ud7af]/g) || []).length;
|
|
103
|
+
if (enc === "gbk" || enc === "big5")
|
|
104
|
+
bonus += cjk * 5;
|
|
105
|
+
if (enc === "windows-1251")
|
|
106
|
+
bonus += cyrillic * 2;
|
|
107
|
+
if (enc === "shift_jis")
|
|
108
|
+
bonus += hiragana * 8;
|
|
109
|
+
if (enc === "euc-kr")
|
|
110
|
+
bonus += hangul * 4;
|
|
111
|
+
return bonus;
|
|
112
|
+
}
|
|
113
|
+
function smartSlice(text) {
|
|
114
|
+
// seek first non-ascii char, slice ±32 around it, truncate to 50
|
|
115
|
+
let idx = -1;
|
|
116
|
+
for (let i = 0; i < text.length; i += 1) {
|
|
117
|
+
if (text.charCodeAt(i) > 127) {
|
|
118
|
+
idx = i;
|
|
119
|
+
break;
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
if (idx === -1)
|
|
123
|
+
return text.slice(0, 50);
|
|
124
|
+
const start = Math.max(0, idx - 32);
|
|
125
|
+
const slice = text.slice(start, start + 64);
|
|
126
|
+
return slice.slice(0, 50).replace(/\s+/g, " ").trim();
|
|
127
|
+
}
|
|
128
|
+
export function scoreText(text, enc) {
|
|
129
|
+
if (text.includes("\uFFFD"))
|
|
130
|
+
return -1000;
|
|
131
|
+
const ratio = printableRatio(text);
|
|
132
|
+
if (ratio < 0.85)
|
|
133
|
+
return -500 + ratio * 10;
|
|
134
|
+
let score = ratio * 10;
|
|
135
|
+
score += scriptBonus(text, enc) * 0.5;
|
|
136
|
+
return score;
|
|
137
|
+
}
|
|
138
|
+
/** Decode bytes under enc via iconv-lite, return text or undefined on failure. */
|
|
139
|
+
export function decodeBytes(bytes, enc) {
|
|
140
|
+
try {
|
|
141
|
+
if (enc === "utf8" || enc === "utf8bom")
|
|
142
|
+
return new TextDecoder("utf-8", { fatal: true }).decode(bytes);
|
|
143
|
+
if (enc === "utf16le")
|
|
144
|
+
return new TextDecoder("utf-16le").decode(bytes);
|
|
145
|
+
if (enc === "utf16be")
|
|
146
|
+
return new TextDecoder("utf-16be").decode(bytes);
|
|
147
|
+
// iconv-lite for legacy
|
|
148
|
+
return iconv.decode(Buffer.from(bytes), enc);
|
|
149
|
+
}
|
|
150
|
+
catch {
|
|
151
|
+
return undefined;
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
export function encodeText(text, enc) {
|
|
155
|
+
try {
|
|
156
|
+
if (enc === "utf8" || enc === "utf8bom")
|
|
157
|
+
return new TextEncoder().encode(text);
|
|
158
|
+
if (enc === "utf16le")
|
|
159
|
+
return new Uint8Array(Buffer.from(text, "utf16le"));
|
|
160
|
+
if (enc === "utf16be") {
|
|
161
|
+
// Node no utf16be, use iconv
|
|
162
|
+
return new Uint8Array(iconv.encode(text, "utf16be"));
|
|
163
|
+
}
|
|
164
|
+
return new Uint8Array(iconv.encode(text, enc));
|
|
165
|
+
}
|
|
166
|
+
catch {
|
|
167
|
+
return undefined;
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
/** Top-3 candidates for autoGuess: decode each allowlist enc, score, sort. */
|
|
171
|
+
export function top3Candidates(bytes, allowlist) {
|
|
172
|
+
const candidates = [];
|
|
173
|
+
// handle BOM case separately? caller should handle BOM before calling.
|
|
174
|
+
for (const raw of allowlist) {
|
|
175
|
+
const enc = normalizeEncoding(raw) ?? raw;
|
|
176
|
+
const text = decodeBytes(bytes, enc);
|
|
177
|
+
if (text === undefined)
|
|
178
|
+
continue;
|
|
179
|
+
const sample = smartSlice(text);
|
|
180
|
+
const score = scoreText(text, enc);
|
|
181
|
+
candidates.push({ encoding: enc, sample, score });
|
|
182
|
+
}
|
|
183
|
+
candidates.sort((a, b) => b.score - a.score);
|
|
184
|
+
return candidates.slice(0, 3);
|
|
185
|
+
}
|
|
186
|
+
export function hasReplacementChar(text) {
|
|
187
|
+
return text.includes("\uFFFD");
|
|
188
|
+
}
|
|
189
|
+
/**
|
|
190
|
+
* Try chardet (runk/node-chardet) for autoGuess when available.
|
|
191
|
+
* Dynamically imports 'chardet' (optionalDep) and returns canonical encoding
|
|
192
|
+
* if confidence >=45 and allowlist-accepted and decodable without �.
|
|
193
|
+
* Falls back to heuristic when chardet unavailable/low confidence.
|
|
194
|
+
*/
|
|
195
|
+
export async function detectWithChardet(bytes, allowlist) {
|
|
196
|
+
try {
|
|
197
|
+
const mod = await import("chardet");
|
|
198
|
+
// chardet ESM: default export has analyse, CJS: named
|
|
199
|
+
const chardetLike = mod.default ?? mod;
|
|
200
|
+
const analyse = chardetLike.analyse;
|
|
201
|
+
if (typeof analyse !== "function")
|
|
202
|
+
return undefined;
|
|
203
|
+
const buf = Buffer.isBuffer(bytes) ? bytes : Buffer.from(bytes);
|
|
204
|
+
const results = analyse(buf);
|
|
205
|
+
if (!Array.isArray(results) || results.length === 0)
|
|
206
|
+
return undefined;
|
|
207
|
+
const allowNorm = new Set(allowlist.map((a) => normalizeEncoding(a) ?? a.toLowerCase()));
|
|
208
|
+
for (const r of results) {
|
|
209
|
+
if (typeof r.name !== "string" || typeof r.confidence !== "number")
|
|
210
|
+
continue;
|
|
211
|
+
if (r.confidence < 45)
|
|
212
|
+
continue;
|
|
213
|
+
const norm = normalizeEncoding(r.name);
|
|
214
|
+
if (!norm)
|
|
215
|
+
continue;
|
|
216
|
+
if (!allowNorm.has(norm))
|
|
217
|
+
continue;
|
|
218
|
+
// iconv-lite existence check (use try decode)
|
|
219
|
+
const encForCheck = norm;
|
|
220
|
+
if (!iconv.encodingExists(encForCheck) && !iconv.encodingExists(r.name))
|
|
221
|
+
continue;
|
|
222
|
+
const text = decodeBytes(bytes, encForCheck);
|
|
223
|
+
if (text === undefined)
|
|
224
|
+
continue;
|
|
225
|
+
if (text.includes("\uFFFD"))
|
|
226
|
+
continue;
|
|
227
|
+
// extra guard: printable ratio
|
|
228
|
+
if (text.length > 0) {
|
|
229
|
+
let printable = 0;
|
|
230
|
+
for (const ch of text) {
|
|
231
|
+
const c = ch.charCodeAt(0);
|
|
232
|
+
if (c === 10 || c === 13 || c === 9)
|
|
233
|
+
printable += 1;
|
|
234
|
+
else if (c >= 32 && c !== 127)
|
|
235
|
+
printable += 1;
|
|
236
|
+
else if (c > 127)
|
|
237
|
+
printable += 1;
|
|
238
|
+
}
|
|
239
|
+
if (printable / text.length < 0.85)
|
|
240
|
+
continue;
|
|
241
|
+
}
|
|
242
|
+
return norm;
|
|
243
|
+
}
|
|
244
|
+
return undefined;
|
|
245
|
+
}
|
|
246
|
+
catch {
|
|
247
|
+
return undefined;
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
/**
|
|
251
|
+
* Chardet-based Top-3 for E_NOT_TEXT (primary) and autoGuess.
|
|
252
|
+
* Returns chardet candidates filtered by allowlist and decodable, else empty.
|
|
253
|
+
*/
|
|
254
|
+
export async function chardetTop3Candidates(bytes, allowlist) {
|
|
255
|
+
try {
|
|
256
|
+
const mod = await import("chardet");
|
|
257
|
+
const chardetLike = mod.default ?? mod;
|
|
258
|
+
const analyse = chardetLike.analyse;
|
|
259
|
+
if (typeof analyse !== "function")
|
|
260
|
+
return [];
|
|
261
|
+
const buf = Buffer.isBuffer(bytes) ? bytes : Buffer.from(bytes);
|
|
262
|
+
const results = analyse(buf);
|
|
263
|
+
if (!Array.isArray(results) || results.length === 0)
|
|
264
|
+
return [];
|
|
265
|
+
const allowNorm = new Set(allowlist.map((a) => normalizeEncoding(a) ?? a.toLowerCase()));
|
|
266
|
+
const out = [];
|
|
267
|
+
for (const r of results) {
|
|
268
|
+
if (typeof r.name !== "string" || typeof r.confidence !== "number")
|
|
269
|
+
continue;
|
|
270
|
+
const norm = normalizeEncoding(r.name);
|
|
271
|
+
if (!norm)
|
|
272
|
+
continue;
|
|
273
|
+
if (!allowNorm.has(norm))
|
|
274
|
+
continue;
|
|
275
|
+
if (!iconv.encodingExists(norm) && !iconv.encodingExists(r.name))
|
|
276
|
+
continue;
|
|
277
|
+
const text = decodeBytes(bytes, norm);
|
|
278
|
+
if (text === undefined)
|
|
279
|
+
continue;
|
|
280
|
+
const sample = (() => {
|
|
281
|
+
let idx = -1;
|
|
282
|
+
for (let i = 0; i < text.length; i += 1)
|
|
283
|
+
if (text.charCodeAt(i) > 127) {
|
|
284
|
+
idx = i;
|
|
285
|
+
break;
|
|
286
|
+
}
|
|
287
|
+
if (idx === -1)
|
|
288
|
+
return text.slice(0, 50);
|
|
289
|
+
const start = Math.max(0, idx - 32);
|
|
290
|
+
return text.slice(start, start + 64).slice(0, 50).replace(/\s+/g, " ").trim();
|
|
291
|
+
})();
|
|
292
|
+
out.push({ encoding: norm, confidence: r.confidence, sample });
|
|
293
|
+
if (out.length >= 3)
|
|
294
|
+
break;
|
|
295
|
+
}
|
|
296
|
+
return out;
|
|
297
|
+
}
|
|
298
|
+
catch {
|
|
299
|
+
return [];
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
export async function getTop3Candidates(bytes, allowlist) {
|
|
303
|
+
const chardetCands = await chardetTop3Candidates(bytes, allowlist);
|
|
304
|
+
if (chardetCands.length > 0) {
|
|
305
|
+
return chardetCands.map((c) => ({ encoding: c.encoding, sample: c.sample, score: c.confidence }));
|
|
306
|
+
}
|
|
307
|
+
return top3Candidates(bytes, allowlist);
|
|
308
|
+
}
|
package/lib/file-view.js
CHANGED
|
@@ -462,7 +462,7 @@ export async function preview(content, hashes, opts = {}, absolutePath) {
|
|
|
462
462
|
export async function readView(io, path, cwd, opts = {}) {
|
|
463
463
|
const { signal } = opts;
|
|
464
464
|
const absolutePath = await io.resolve(path, cwd, signal);
|
|
465
|
-
const rawText = await io.readText(absolutePath, signal);
|
|
465
|
+
const rawText = await io.readText(absolutePath, signal, opts.encoding);
|
|
466
466
|
const { normalized, fileHashes, hadUtf8DecodeErrors, bom, originalEnding } = await normFromText({
|
|
467
467
|
absolutePath,
|
|
468
468
|
rawText,
|