@hncode/hncode 0.3.4 → 0.3.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +131 -674
- package/README.md +5 -4
- package/README_CN.md +5 -4
- package/package.json +2 -2
- package/src/config.js +57 -19
- package/src/llm.js +7 -3
- package/src/plugin.js +10 -8
- package/src/term.js +84 -46
- package/src/tui.js +381 -155
package/README.md
CHANGED
|
@@ -118,12 +118,13 @@ Special thanks to the communities behind these projects for pushing the boundari
|
|
|
118
118
|
|
|
119
119
|
## 📄 License
|
|
120
120
|
|
|
121
|
-
This project is licensed under the [
|
|
121
|
+
This project is licensed under the [PolyForm Noncommercial License 1.0.0](https://github.com/NiceHello666/hncode/blob/main/LICENSE).
|
|
122
122
|
|
|
123
123
|
**In short:**
|
|
124
|
-
1. ✅
|
|
125
|
-
2. ✅
|
|
126
|
-
3. ❌
|
|
124
|
+
1. ✅ Free for noncommercial use — personal projects, learning, research, hobby work, and noncommercial organizations
|
|
125
|
+
2. ✅ You may modify and share it, provided you keep the license text and the noncommercial notice with any copy
|
|
126
|
+
3. ❌ No commercial use of any kind (selling it, shipping it inside a paid product, or using it to run a commercial service)
|
|
127
|
+
4. ℹ️ This is *not* a copyleft license: your own additions do not have to be published. Commercial use requires a separate license from the author.
|
|
127
128
|
|
|
128
129
|
---
|
|
129
130
|
|
package/README_CN.md
CHANGED
|
@@ -75,12 +75,13 @@ hncode 从以下几个优秀项目中汲取了灵感:
|
|
|
75
75
|
|
|
76
76
|
## 📄 许可证
|
|
77
77
|
|
|
78
|
-
本项目的许可受 [
|
|
78
|
+
本项目的许可受 [PolyForm Noncommercial License 1.0.0](https://github.com/NiceHello666/hncode/blob/main/LICENSE) 保护。
|
|
79
79
|
|
|
80
80
|
**简单来说:**
|
|
81
|
-
1. ✅
|
|
82
|
-
2. ✅
|
|
83
|
-
3. ❌
|
|
81
|
+
1. ✅ 非商业用途免费——个人项目、学习、研究、业余爱好,以及非商业组织
|
|
82
|
+
2. ✅ 可以修改和分享,但任何副本都必须附带许可证全文和「不得商用」的声明
|
|
83
|
+
3. ❌ 禁止任何形式的商业使用(出售、作为付费产品的一部分、或用于运营商业服务)
|
|
84
|
+
4. ℹ️ 这**不是** copyleft 许可:你自己新增的部分无需开源。商业使用需向作者单独获取授权。
|
|
84
85
|
|
|
85
86
|
---
|
|
86
87
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@hncode/hncode",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.6",
|
|
4
4
|
"description": "hncode — A cyan-blue TUI AI coding agent with zero runtime dependencies.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "src/index.js",
|
|
@@ -22,7 +22,7 @@
|
|
|
22
22
|
"engines": {
|
|
23
23
|
"node": ">=20"
|
|
24
24
|
},
|
|
25
|
-
"license": "
|
|
25
|
+
"license": "PolyForm-Noncommercial-1.0.0",
|
|
26
26
|
"author": "NiceHello666",
|
|
27
27
|
"keywords": [
|
|
28
28
|
"cli",
|
package/src/config.js
CHANGED
|
@@ -383,7 +383,7 @@ export function setConfigString(key, value) {
|
|
|
383
383
|
const opensMultiline = /^"""|^'''/.test(rest) && !/^"""[\s\S]*"""\s*$/.test(rest);
|
|
384
384
|
if (opensMultiline) skipping = true;
|
|
385
385
|
if (!replaced && value !== '' && value != null) {
|
|
386
|
-
out.push(...renderTomlString(key, String(value)));
|
|
386
|
+
out.push(...renderTomlString(key, String(value)).split('\n'));
|
|
387
387
|
replaced = true;
|
|
388
388
|
}
|
|
389
389
|
continue;
|
|
@@ -392,9 +392,9 @@ export function setConfigString(key, value) {
|
|
|
392
392
|
}
|
|
393
393
|
while (out.length && out[out.length - 1].trim() === '') out.pop();
|
|
394
394
|
if (!replaced && value !== '' && value != null) {
|
|
395
|
-
// Insert new root-level keys BEFORE the first [table] section so they
|
|
396
|
-
//
|
|
397
|
-
const rendered = renderTomlString(key, String(value));
|
|
395
|
+
// Insert new root-level keys BEFORE the first [table] section so they are
|
|
396
|
+
// parsed as root keys, not assigned to a table.
|
|
397
|
+
const rendered = renderTomlString(key, String(value)).split('\n');
|
|
398
398
|
if (firstTableIdx >= 0) {
|
|
399
399
|
out.splice(firstTableIdx, 0, '', ...rendered);
|
|
400
400
|
} else {
|
|
@@ -409,13 +409,26 @@ export function setConfigString(key, value) {
|
|
|
409
409
|
|
|
410
410
|
// Render `key = <value>` as TOML. Uses a multi-line basic string when the value
|
|
411
411
|
// contains newlines (readable, and TOML-valid), a plain one otherwise.
|
|
412
|
+
//
|
|
413
|
+
// Layout matters: TOML DISCARDS the newline immediately after `"""`, so the
|
|
414
|
+
// value begins on the following line and the closing `"""` must butt up against
|
|
415
|
+
// the last character. The previous form (""" + \n + body + \n + """) parsed back
|
|
416
|
+
// as `value + '\n'`, and because setConfigString writes back whatever it read,
|
|
417
|
+
// every read→write cycle appended another newline (a system prompt gained a
|
|
418
|
+
// blank line per edit).
|
|
419
|
+
//
|
|
420
|
+
// key = """
|
|
421
|
+
// <value, verbatim>
|
|
422
|
+
// """
|
|
412
423
|
function renderTomlString(key, value) {
|
|
413
424
|
if (value.includes('\n')) {
|
|
414
425
|
// Inside """…""" a backslash must be escaped; quotes are fine unless tripled.
|
|
415
426
|
const body = value.replace(/\\/g, '\\\\').replace(/"""/g, '\\"\\"\\"');
|
|
416
|
-
|
|
427
|
+
// `"""` then a newline (discarded by the parser), then the value, then the
|
|
428
|
+
// closing delimiter on the same line as the value's last line.
|
|
429
|
+
return [`${key} = """`, ...body.split('\n')].join('\n') + '"""';
|
|
417
430
|
}
|
|
418
|
-
return
|
|
431
|
+
return `${key} = "${value.replace(/\\/g, '\\\\').replace(/"/g, '\\"')}"`;
|
|
419
432
|
}
|
|
420
433
|
|
|
421
434
|
// Append (or replace) a [providers.<name>] table in config.toml without
|
|
@@ -425,21 +438,30 @@ export function addProvider(name, { base_url, api_key, protocol } = {}) {
|
|
|
425
438
|
let text = '';
|
|
426
439
|
try { text = fs.readFileSync(file, 'utf8'); } catch { /* new file */ }
|
|
427
440
|
const lines = text.split(/\r?\n/);
|
|
428
|
-
//
|
|
441
|
+
// Drop any existing [providers.<name>] block (and its key lines) so it is
|
|
442
|
+
// replaced rather than duplicated. Accepts the BARE and QUOTED header forms
|
|
443
|
+
// (this function writes the quoted one) and unescapes before comparing.
|
|
444
|
+
const unq = (s) => String(s).replace(/\\(.)/g, '$1');
|
|
445
|
+
const provRe = /^\s*\[providers\.\s*(?:"((?:[^"\\]|\\.)*)"|([^\]]+))\s*\]\s*$/;
|
|
429
446
|
const out = [];
|
|
430
447
|
let skipping = false;
|
|
431
448
|
for (const ln of lines) {
|
|
432
|
-
const m =
|
|
433
|
-
if (m) { skipping = m[1] === name; if (skipping) continue; }
|
|
449
|
+
const m = provRe.exec(ln);
|
|
450
|
+
if (m) { skipping = unq(m[1] !== undefined ? m[1] : m[2]) === name; if (skipping) continue; }
|
|
434
451
|
else if (/^\s*\[/.test(ln)) skipping = false;
|
|
435
452
|
if (!skipping) out.push(ln);
|
|
436
453
|
}
|
|
437
454
|
while (out.length && out[out.length - 1].trim() === '') out.pop();
|
|
438
455
|
out.push('');
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
456
|
+
// Escape backslashes BEFORE quotes (the other order double-escapes), and quote
|
|
457
|
+
// the table key too. Raw interpolation meant a provider name or URL holding a
|
|
458
|
+
// backslash or a quote wrote invalid TOML and broke the ENTIRE config file —
|
|
459
|
+
// every later launch failed until it was repaired by hand.
|
|
460
|
+
const q = (v) => `"${String(v).replace(/\\/g, '\\\\').replace(/"/g, '\\"')}"`;
|
|
461
|
+
out.push(`[providers.${q(name)}]`);
|
|
462
|
+
if (base_url) out.push(`base_url = ${q(base_url)}`);
|
|
463
|
+
if (api_key) out.push(`api_key = ${q(api_key)}`);
|
|
464
|
+
if (protocol) out.push(`protocol = ${q(protocol)}`);
|
|
443
465
|
fs.mkdirSync(path.dirname(file), { recursive: true });
|
|
444
466
|
fs.writeFileSync(file, out.join('\n') + '\n', 'utf8');
|
|
445
467
|
return file;
|
|
@@ -455,15 +477,25 @@ export function removeProvider(name) {
|
|
|
455
477
|
const lines = text.split(/\r?\n/);
|
|
456
478
|
const out = [];
|
|
457
479
|
let skipping = false;
|
|
480
|
+
// Table headers may be BARE (`[providers.p]`) or QUOTED (`[providers."p"]`) —
|
|
481
|
+
// addProvider now quotes them so a name containing a quote or backslash stays
|
|
482
|
+
// valid TOML. Match both and unescape before comparing: with the quoted form
|
|
483
|
+
// the old bare-only regex captured `"p"` (quotes included), `=== name` never
|
|
484
|
+
// held, and removeProvider silently deleted nothing.
|
|
485
|
+
const unq = (s) => String(s).replace(/\\(.)/g, '$1');
|
|
486
|
+
const provRe = /^\s*\[providers\.\s*(?:"((?:[^"\\]|\\.)*)"|([^\]]+))\s*\]\s*$/;
|
|
458
487
|
// A models key is `[models."provider/modelId"]`; match the provider prefix.
|
|
459
|
-
const modelRe = /^\s*\[models
|
|
488
|
+
const modelRe = /^\s*\[models\.\s*"((?:[^"\\]|\\.)*)"\s*\]\s*$/;
|
|
460
489
|
for (const ln of lines) {
|
|
461
|
-
const pm =
|
|
462
|
-
if (pm) {
|
|
490
|
+
const pm = provRe.exec(ln);
|
|
491
|
+
if (pm) {
|
|
492
|
+
skipping = unq(pm[1] !== undefined ? pm[1] : pm[2]) === name;
|
|
493
|
+
if (skipping) continue;
|
|
494
|
+
}
|
|
463
495
|
const mm = modelRe.exec(ln);
|
|
464
496
|
if (mm) {
|
|
465
497
|
// The model key starts with `<provider>/`; skip it if it is this provider's.
|
|
466
|
-
const keyProvider = mm[1].split('/')[0];
|
|
498
|
+
const keyProvider = unq(mm[1]).split('/')[0];
|
|
467
499
|
skipping = keyProvider === name;
|
|
468
500
|
if (skipping) continue;
|
|
469
501
|
} else if (/^\s*\[/.test(ln)) {
|
|
@@ -490,8 +522,14 @@ export function addModel(provider, modelId, opts = {}) {
|
|
|
490
522
|
try { text = fs.readFileSync(file, 'utf8'); } catch {}
|
|
491
523
|
const key = modelKey(provider, modelId);
|
|
492
524
|
if (new RegExp(`^\\s*\\[models\\."${escapeRe(key)}"\\]`, 'm').test(text)) return file; // already there
|
|
493
|
-
|
|
494
|
-
|
|
525
|
+
// EVERY interpolated string is escaped. These were written raw, so a model id
|
|
526
|
+
// (or provider) containing a quote or backslash produced malformed TOML and
|
|
527
|
+
// the ENTIRE config file stopped parsing — every later launch then failed
|
|
528
|
+
// until the file was repaired by hand. Provider ids and model ids come from
|
|
529
|
+
// remote APIs, so they are not trusted input.
|
|
530
|
+
const q = (v) => `"${String(v).replace(/\\/g, '\\\\').replace(/"/g, '\\"')}"`;
|
|
531
|
+
const lines = [`[models.${q(key)}]`, `provider = ${q(provider)}`, `model = ${q(modelId)}`];
|
|
532
|
+
if (opts.display_name) lines.push(`display_name = ${q(opts.display_name)}`);
|
|
495
533
|
// Persist the context window discovered from /v1/models so resolveConfig can
|
|
496
534
|
// size the context bar per model (not a global 128k).
|
|
497
535
|
if (opts.contextLength) lines.push(`context_length = ${Number(opts.contextLength)}`);
|
package/src/llm.js
CHANGED
|
@@ -12,7 +12,9 @@ import { effortWire } from './config.js';
|
|
|
12
12
|
import { applyPromptCache } from './cache.js';
|
|
13
13
|
|
|
14
14
|
export function openAiToolDefs(tools) {
|
|
15
|
-
|
|
15
|
+
// Guard: a caller that passes null/undefined (e.g. setToolsList(null)) used to
|
|
16
|
+
// throw on `.map`, failing the whole request. An empty tool list is valid.
|
|
17
|
+
return (Array.isArray(tools) ? tools : []).map((t) => ({
|
|
16
18
|
type: 'function',
|
|
17
19
|
function: {
|
|
18
20
|
name: t.name,
|
|
@@ -23,7 +25,7 @@ export function openAiToolDefs(tools) {
|
|
|
23
25
|
}
|
|
24
26
|
|
|
25
27
|
export function anthropicToolDefs(tools) {
|
|
26
|
-
return tools.map((t) => ({
|
|
28
|
+
return (Array.isArray(tools) ? tools : []).map((t) => ({
|
|
27
29
|
name: t.name,
|
|
28
30
|
description: t.description,
|
|
29
31
|
input_schema: t.parameters,
|
|
@@ -120,7 +122,9 @@ function buildBody(cfg, messages, streaming = true) {
|
|
|
120
122
|
|
|
121
123
|
// tool list used for defs (injected via setTools)
|
|
122
124
|
let llmToolsList = [];
|
|
123
|
-
|
|
125
|
+
// Only ever store an array: `null`/undefined here would make every later request
|
|
126
|
+
// throw when the defs are built.
|
|
127
|
+
export function setToolsList(list) { llmToolsList = Array.isArray(list) ? list : []; }
|
|
124
128
|
|
|
125
129
|
export class LLM {
|
|
126
130
|
constructor(cfg) {
|
package/src/plugin.js
CHANGED
|
@@ -152,17 +152,19 @@ export async function loadPlugins(dir) {
|
|
|
152
152
|
continue;
|
|
153
153
|
}
|
|
154
154
|
|
|
155
|
-
// Metadata
|
|
156
|
-
//
|
|
157
|
-
//
|
|
158
|
-
//
|
|
159
|
-
//
|
|
155
|
+
// Metadata can arrive three ways, and all three are in the wild:
|
|
156
|
+
// export default { name: 'x', version: '1' } (object)
|
|
157
|
+
// export default 'x' (plain string)
|
|
158
|
+
// export const meta = { name: 'x' } (named export)
|
|
159
|
+
// Only the default export used to be read, so a plugin using the named form
|
|
160
|
+
// showed up as its FILENAME with version 0.0.0 in /plugins. Check them all.
|
|
160
161
|
const def = (mod && mod.default) || {};
|
|
161
|
-
const
|
|
162
|
+
const defMeta = typeof def === 'string' ? { name: def } : def;
|
|
163
|
+
const namedMeta = (mod && mod.meta) || {};
|
|
162
164
|
const plugin = {
|
|
163
165
|
id: f,
|
|
164
|
-
name:
|
|
165
|
-
version:
|
|
166
|
+
name: defMeta.name || defMeta.meta?.name || namedMeta.name || namedMeta.meta?.name || f,
|
|
167
|
+
version: defMeta.version || defMeta.meta?.version || namedMeta.version || namedMeta.meta?.version || '0.0.0',
|
|
166
168
|
};
|
|
167
169
|
loadedPlugins.push(plugin);
|
|
168
170
|
installFn(API);
|
package/src/term.js
CHANGED
|
@@ -50,13 +50,47 @@ export function strWidth(s) {
|
|
|
50
50
|
// renders East Asian / wide / some symbol glyphs as 2 columns. Must be used for any
|
|
51
51
|
// cursor positioning and line wrapping so an I-shaped (insert) caret lines up.
|
|
52
52
|
// Source of truth: Unicode wide ranges + a small set of CJK-fused symbol glyphs.
|
|
53
|
+
//
|
|
54
|
+
// PERFORMANCE (this is the hottest function in the whole TUI — a profile of a
|
|
55
|
+
// 500-message transcript streaming markdown put ~48% of samples in this plus
|
|
56
|
+
// clusterWidth/clusterGlyphWidth/isWideByDefault):
|
|
57
|
+
// * a printable-ASCII fast path returns the length immediately. Agent output
|
|
58
|
+
// is overwhelmingly ASCII, and measuring it cost ~30 range comparisons per
|
|
59
|
+
// character for nothing. 1MB of ASCII went from 219ms to ~0.
|
|
60
|
+
// * everything else is memoised (see WIDTH_CACHE_MAX). Rows are re-measured
|
|
61
|
+
// every frame while a reply streams, and the same substrings come back
|
|
62
|
+
// again and again.
|
|
53
63
|
export function visualWidth(s) {
|
|
54
64
|
if (!s) return 0;
|
|
55
|
-
|
|
56
|
-
//
|
|
57
|
-
|
|
65
|
+
const str = typeof s === 'string' ? s : String(s);
|
|
66
|
+
// Fast path: printable ASCII measures exactly its length.
|
|
67
|
+
if (isPrintableAscii(str)) return str.length;
|
|
68
|
+
const hit = widthCache.get(str);
|
|
69
|
+
if (hit !== undefined) return hit;
|
|
70
|
+
const w = clusterWidth(str);
|
|
71
|
+
if (widthCache.size >= WIDTH_CACHE_MAX) {
|
|
72
|
+
// Cheap eviction: drop the oldest insertion (Map preserves order).
|
|
73
|
+
const oldest = widthCache.keys().next().value;
|
|
74
|
+
if (oldest !== undefined) widthCache.delete(oldest);
|
|
75
|
+
}
|
|
76
|
+
widthCache.set(str, w);
|
|
77
|
+
return w;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
// All code units printable ASCII? `\t`/newline/escape all fail, which is what we
|
|
81
|
+
// want: those have their own width rules (tab = 4, ESC starts an ANSI sequence).
|
|
82
|
+
function isPrintableAscii(str) {
|
|
83
|
+
for (let i = 0; i < str.length; i++) {
|
|
84
|
+
const c = str.charCodeAt(i);
|
|
85
|
+
if (c < 0x20 || c > 0x7e) return false;
|
|
86
|
+
}
|
|
87
|
+
return true;
|
|
58
88
|
}
|
|
59
89
|
|
|
90
|
+
const WIDTH_CACHE_MAX = 4096;
|
|
91
|
+
const widthCache = new Map();
|
|
92
|
+
|
|
93
|
+
|
|
60
94
|
// ---- East Asian Width tables (Unicode 15) --------------------------------
|
|
61
95
|
// Wide/Fullwidth ranges -> 2 columns. Everything else is 1 unless it is a
|
|
62
96
|
// zero-width character (see ZERO_WIDTH) or emoji presentation (see below).
|
|
@@ -151,69 +185,69 @@ export function charWidth(ch) {
|
|
|
151
185
|
// Tab renders as 4 spaces in most terminals.
|
|
152
186
|
if (c === 0x09) return 4;
|
|
153
187
|
if (c < 0x20 || c === 0x7F) return 0;
|
|
188
|
+
// FAST PATH: plain printable ASCII is exactly one cell wide. Without this each
|
|
189
|
+
// ASCII character paid for isZeroWidth + isWideByDefault (≈30 range tests) + a
|
|
190
|
+
// binary search, which is most of what made width measurement the hottest
|
|
191
|
+
// function in the TUI.
|
|
192
|
+
if (c >= 0x20 && c <= 0x7e) return 1;
|
|
154
193
|
if (isZeroWidth(c)) return 0;
|
|
155
194
|
if (isWideByDefault(c)) return 2;
|
|
156
195
|
if (inRanges(c, WIDE)) return 2;
|
|
157
196
|
return 1;
|
|
158
197
|
}
|
|
159
198
|
|
|
160
|
-
//
|
|
161
|
-
//
|
|
162
|
-
//
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
const len = (cp) => cp > 0xFFFF ? 2 : 1;
|
|
169
|
-
|
|
170
|
-
const firstCp = at(i);
|
|
171
|
-
let end = i + len(firstCp);
|
|
172
|
-
const startedRegional = isRegional(firstCp);
|
|
173
|
-
|
|
174
|
-
for (;;) {
|
|
175
|
-
if (end >= s.length) break;
|
|
176
|
-
const cp = at(end);
|
|
177
|
-
if (cp === 0x200D) { // ZWJ: absorb the next base too
|
|
178
|
-
end += 1;
|
|
179
|
-
if (end >= s.length) break;
|
|
180
|
-
end += len(at(end));
|
|
181
|
-
continue;
|
|
182
|
-
}
|
|
183
|
-
if (isZeroWidth(cp) || cp === 0xFE0F) { end += len(cp); continue; }
|
|
184
|
-
// A regional-indicator PAIR forms one flag glyph.
|
|
185
|
-
if (startedRegional && isRegional(cp)) { end += len(cp); break; }
|
|
186
|
-
break;
|
|
187
|
-
}
|
|
188
|
-
return { text: s.slice(i, end), next: end };
|
|
189
|
-
}
|
|
199
|
+
// Shared native segmenter. `Intl.Segmenter` implements the full Unicode
|
|
200
|
+
// grapheme-break algorithm in native code, so it is both more correct than the
|
|
201
|
+
// hand-rolled scanner it replaces (ZWJ families, flags, combining marks) and
|
|
202
|
+
// much cheaper — that scanner allocated a `{text, next}` object per character,
|
|
203
|
+
// which showed up as GC pressure in profiles.
|
|
204
|
+
const graphemeSegmenter = typeof Intl !== 'undefined' && typeof Intl.Segmenter === 'function'
|
|
205
|
+
? new Intl.Segmenter(undefined, { granularity: 'grapheme' })
|
|
206
|
+
: null;
|
|
190
207
|
|
|
191
208
|
// Width of a whole string measured by GRAPHEME CLUSTER: a ZWJ sequence or a
|
|
192
209
|
// flag counts 2 columns total, not the sum of its codepoints (which used to
|
|
193
210
|
// make 👨👩👧 measure 8 columns and break every row it appeared on).
|
|
194
211
|
export function clusterWidth(s) {
|
|
195
212
|
const str = String(s || '');
|
|
213
|
+
if (!str) return 0;
|
|
196
214
|
let w = 0;
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
215
|
+
if (graphemeSegmenter) {
|
|
216
|
+
// NOTE: an "ASCII runs are free" split was tried here and made things WORSE
|
|
217
|
+
// (a 500KB markdown string went 97ms -> 544ms): slicing the string into a
|
|
218
|
+
// run per CJK character costs far more than handing the whole thing to the
|
|
219
|
+
// native segmenter. The real win is the printable-ASCII fast path in
|
|
220
|
+
// visualWidth() plus the memo cache, not micro-splitting here.
|
|
221
|
+
for (const { segment } of graphemeSegmenter.segment(str)) w += clusterGlyphWidth(segment);
|
|
222
|
+
return w;
|
|
202
223
|
}
|
|
224
|
+
// Fallback for a runtime without Intl.Segmenter: per code point.
|
|
225
|
+
for (const ch of str) w += clusterGlyphWidth(ch);
|
|
203
226
|
return w;
|
|
204
227
|
}
|
|
228
|
+
// Width of one grapheme cluster. Allocation-free on the common path: a
|
|
205
229
|
|
|
230
|
+
// Width of one grapheme cluster. Allocation-free on the common path: a
|
|
231
|
+
// single-code-point cluster (every ASCII character, most CJK) returns
|
|
232
|
+
// charWidth directly instead of first building an array of code points.
|
|
206
233
|
function clusterGlyphWidth(cluster) {
|
|
207
|
-
const
|
|
234
|
+
const first = cluster.codePointAt(0);
|
|
235
|
+
if (first === undefined) return 0;
|
|
236
|
+
if (cluster.length <= (first > 0xFFFF ? 2 : 1)) return charWidth(cluster);
|
|
237
|
+
|
|
238
|
+
const cps = [];
|
|
239
|
+
for (let i = 0; i < cluster.length;) {
|
|
240
|
+
const cp = cluster.codePointAt(i);
|
|
241
|
+
cps.push(cp);
|
|
242
|
+
i += cp > 0xFFFF ? 2 : 1;
|
|
243
|
+
}
|
|
208
244
|
// ZWJ sequence / flag pair / VS16-presented symbol => 2 columns.
|
|
209
245
|
if (cps.includes(0x200D)) return 2;
|
|
210
|
-
if (cps.
|
|
211
|
-
const base = cps[0];
|
|
246
|
+
if (cps.some(isRegional)) return 2;
|
|
212
247
|
if (cps.includes(0xFE0F)) return 2; // explicit emoji presentation
|
|
213
|
-
|
|
214
|
-
let
|
|
215
|
-
|
|
216
|
-
return baseW + extra;
|
|
248
|
+
let width = charWidth(String.fromCodePoint(cps[0]));
|
|
249
|
+
for (let k = 1; k < cps.length; k++) width += charWidth(String.fromCodePoint(cps[k]));
|
|
250
|
+
return width;
|
|
217
251
|
}
|
|
218
252
|
|
|
219
253
|
// Estimate the LLM token count of a plain string, matching kimi-code's
|
|
@@ -264,7 +298,11 @@ export function expandTabs(s, tabSize = 8) {
|
|
|
264
298
|
continue;
|
|
265
299
|
}
|
|
266
300
|
out += ch;
|
|
267
|
-
|
|
301
|
+
// Per CHARACTER, `charWidth` is the right measure and is far cheaper than
|
|
302
|
+
// clusterWidth here: the loop already iterates code points (so a surrogate
|
|
303
|
+
// pair arrives as a single `ch`), and clusterWidth would spin up the
|
|
304
|
+
// segmenter for every single character.
|
|
305
|
+
col += charWidth(ch);
|
|
268
306
|
}
|
|
269
307
|
return out;
|
|
270
308
|
}
|